From: Job Snijders <job@openbsd.org>
Subject: sysupgrade/ftp: use a 'needle' to poke through caching layers
To: tech@openbsd.org
Date: Thu, 2 May 2024 13:59:19 +0000

Dear all,

If a HTTP caching layer is in use (either a proxy, or if you're fetching
via a CDN), ideally - the TTL for SHA256.sig is low (we all wants the
latest snapshots) and the TTL on the *.tgz files higher (if nothing
changed, serve up cached objects).

The below changeset could be a step towards making the best of any
potential caching layers and also getting fresh sets as soon as they
appear at the origin, while providing increased chances of consistency
between *.tgz files and the SHA256.sig. In other words, you don't want
to end up with a new SHA256.sig and old *.tgz files.

The trick is to append the SHA256 hash of the SHA256.sig file as query
string when fetching the set files. This 'needle' should prompt the
caching layer to do an origin fetch. New SHA256.sig file == new URLs for
the set files == new cache fill.

I also added a new -Q option to ftp(1) to visually hide the query string
so that things continue to look the same as they did before.

Thoughts?

Kind regards,

Job

Index: usr.sbin/sysupgrade/sysupgrade.sh
===================================================================
RCS file: /cvs/src/usr.sbin/sysupgrade/sysupgrade.sh,v
diff -u -p -r1.49 sysupgrade.sh
--- usr.sbin/sysupgrade/sysupgrade.sh	12 Oct 2023 12:31:15 -0000	1.49
+++ usr.sbin/sysupgrade/sysupgrade.sh	2 May 2024 13:30:20 -0000
@@ -149,6 +149,12 @@ esac
 [[ -f ${SIGNIFY_KEY} ]] || err "cannot find ${SIGNIFY_KEY}"
 
 unpriv -f SHA256 signify -Ve -p "${SIGNIFY_KEY}" -x SHA256.sig -m SHA256
+if [[ $MIRROR == @(http|https)://* ]] && $SNAP; then
+	_NEEDLE="?$(cksum -q -a sha256 SHA256.sig)"
+else
+	_NEEDLE=""
+fi
+
 rm SHA256.sig
 
 if cmp -s /var/db/installed.SHA256 SHA256 && ! $FORCE; then
@@ -174,7 +180,7 @@ done
 
 [[ -n ${OLD_FILES} ]] && rm ${OLD_FILES}
 for f in ${DL}; do
-	unpriv -f $f ftp -N sysupgrade -Vmo ${f} ${URL}${f}
+	unpriv -f $f ftp -N sysupgrade -QVmo ${f} ${URL}${f}${_NEEDLE}
 done
 
 if [[ -n ${DL} ]]; then
Index: usr.bin/ftp/ftp.1
===================================================================
RCS file: /cvs/src/usr.bin/ftp/ftp.1,v
diff -u -p -r1.124 ftp.1
--- usr.bin/ftp/ftp.1	15 Sep 2022 12:47:10 -0000	1.124
+++ usr.bin/ftp/ftp.1	2 May 2024 13:30:20 -0000
@@ -38,7 +38,7 @@
 .Nd Internet file transfer program
 .Sh SYNOPSIS
 .Nm ftp
-.Op Fl 46AadEegiMmnptVv
+.Op Fl 46AadEegiMmnpQtVv
 .Op Fl D Ar title
 .Op Fl k Ar seconds
 .Op Fl P Ar port
@@ -228,6 +228,10 @@ This option has been deprecated as
 .Nm
 now tries to use passive mode by default, falling back to active mode
 if the server does not support passive connections.
+.It Fl Q
+Cause
+.Nm
+to not display any HTTP query strings in the progress bar.
 .It Fl r Ar seconds
 Retry to connect if failed, pausing for number of
 .Ar seconds .
Index: usr.bin/ftp/ftp_var.h
===================================================================
RCS file: /cvs/src/usr.bin/ftp/ftp_var.h,v
diff -u -p -r1.46 ftp_var.h
--- usr.bin/ftp/ftp_var.h	2 Feb 2021 12:58:42 -0000	1.46
+++ usr.bin/ftp/ftp_var.h	2 May 2024 13:30:20 -0000
@@ -148,6 +148,7 @@ extern unsigned int retry_connect;	/* re
 extern int	ttywidth;	/* width of tty */
 extern int	epsv4;		/* use EPSV/EPRT on IPv4 connections */
 extern int	epsv4bad;	/* EPSV doesn't work on the current server */
+extern int	clipqs;		/* clip query string when showing progress */
 
 #ifndef SMALL
 extern int	  editing;	/* command line editing enabled */
Index: usr.bin/ftp/main.c
===================================================================
RCS file: /cvs/src/usr.bin/ftp/main.c,v
diff -u -p -r1.146 main.c
--- usr.bin/ftp/main.c	23 Dec 2023 23:03:00 -0000	1.146
+++ usr.bin/ftp/main.c	2 May 2024 13:30:20 -0000
@@ -108,6 +108,7 @@ int	ntflag;
 int	mapflag;
 int	preserve;
 int	progress;
+int	clipqs;
 int	code;
 int	crflag;
 char	pasv[BUFSIZ];
@@ -354,6 +355,7 @@ main(volatile int argc, char *argv[])
 	mark = HASHBYTES;
 	epsv4 = 1;
 	epsv4bad = 0;
+	clipqs = 0;
 
 	/* Set default operation mode based on FTPMODE environment variable */
 	if ((cp = getenv("FTPMODE")) != NULL && *cp != '\0') {
@@ -420,7 +422,7 @@ main(volatile int argc, char *argv[])
 	httpuseragent = NULL;
 
 	while ((ch = getopt(argc, argv,
-		    "46AaCc:dD:EeN:gik:Mmno:pP:r:S:s:TtU:uvVw:")) != -1) {
+		    "46AaCc:dD:EeN:gik:Mmno:pP:Qr:S:s:TtU:uvVw:")) != -1) {
 		switch (ch) {
 		case '4':
 			family = PF_INET;
@@ -514,7 +516,9 @@ main(volatile int argc, char *argv[])
 		case 'P':
 			ftpport = optarg;
 			break;
-
+		case 'Q':
+			clipqs = 1;
+			break;
 		case 'r':
 			retry_connect = strtonum(optarg, 0, INT_MAX, &errstr);
 			if (errstr != NULL) {
@@ -1064,7 +1068,7 @@ usage(void)
 {
 	fprintf(stderr, "usage: "
 #ifndef SMALL
-	    "ftp [-46AadEegiMmnptVv] [-D title] [-k seconds] [-P port] "
+	    "ftp [-46AadEegiMmnpQtVv] [-D title] [-k seconds] [-P port] "
 	    "[-r seconds]\n"
 	    "           [-s sourceaddr] [host [port]]\n"
 	    "       ftp [-C] [-N name] [-o output] [-s sourceaddr]\n"
Index: usr.bin/ftp/util.c
===================================================================
RCS file: /cvs/src/usr.bin/ftp/util.c,v
diff -u -p -r1.98 util.c
--- usr.bin/ftp/util.c	8 Mar 2023 04:43:11 -0000	1.98
+++ usr.bin/ftp/util.c	2 May 2024 13:30:20 -0000
@@ -786,6 +786,8 @@ progressmeter(int flag, const char *file
 			free(title);
 			title = strdup(filename);
 		}
+		if (clipqs)
+			title[strcspn(title, "?")] = '\0';
 		free(filenamebuf);
 	}