From: Job Snijders <job@openbsd.org>
Subject: Re: sysupgrade/ftp: use a 'needle' to poke through caching layers
To: tech@openbsd.org
Date: Thu, 2 May 2024 18:40:52 +0000

Theo suggested to clip the query string by default, and introduce the -Q
option for those that want to see the query string in the progress bar.
Thanks to Kirill Korinsky for spotting that title can be NULL.

Updated:

Index: usr.sbin/sysupgrade/sysupgrade.sh
===================================================================
RCS file: /cvs/src/usr.sbin/sysupgrade/sysupgrade.sh,v
diff -u -p -r1.49 sysupgrade.sh
--- usr.sbin/sysupgrade/sysupgrade.sh	12 Oct 2023 12:31:15 -0000	1.49
+++ usr.sbin/sysupgrade/sysupgrade.sh	2 May 2024 18:38:23 -0000
@@ -149,6 +149,12 @@ esac
 [[ -f ${SIGNIFY_KEY} ]] || err "cannot find ${SIGNIFY_KEY}"
 
 unpriv -f SHA256 signify -Ve -p "${SIGNIFY_KEY}" -x SHA256.sig -m SHA256
+if [[ $MIRROR == @(http|https)://* ]] && $SNAP; then
+	_NEEDLE="?$(cksum -q -a sha256 SHA256.sig)"
+else
+	_NEEDLE=""
+fi
+
 rm SHA256.sig
 
 if cmp -s /var/db/installed.SHA256 SHA256 && ! $FORCE; then
@@ -174,7 +180,7 @@ done
 
 [[ -n ${OLD_FILES} ]] && rm ${OLD_FILES}
 for f in ${DL}; do
-	unpriv -f $f ftp -N sysupgrade -Vmo ${f} ${URL}${f}
+	unpriv -f $f ftp -N sysupgrade -Vmo ${f} ${URL}${f}${_NEEDLE}
 done
 
 if [[ -n ${DL} ]]; then
Index: usr.bin/ftp/ftp.1
===================================================================
RCS file: /cvs/src/usr.bin/ftp/ftp.1,v
diff -u -p -r1.124 ftp.1
--- usr.bin/ftp/ftp.1	15 Sep 2022 12:47:10 -0000	1.124
+++ usr.bin/ftp/ftp.1	2 May 2024 18:38:23 -0000
@@ -38,7 +38,7 @@
 .Nd Internet file transfer program
 .Sh SYNOPSIS
 .Nm ftp
-.Op Fl 46AadEegiMmnptVv
+.Op Fl 46AadEegiMmnpQtVv
 .Op Fl D Ar title
 .Op Fl k Ar seconds
 .Op Fl P Ar port
@@ -228,6 +228,10 @@ This option has been deprecated as
 .Nm
 now tries to use passive mode by default, falling back to active mode
 if the server does not support passive connections.
+.It Fl Q
+Cause
+.Nm
+to display any HTTP query strings in the progress bar.
 .It Fl r Ar seconds
 Retry to connect if failed, pausing for number of
 .Ar seconds .
Index: usr.bin/ftp/ftp_var.h
===================================================================
RCS file: /cvs/src/usr.bin/ftp/ftp_var.h,v
diff -u -p -r1.46 ftp_var.h
--- usr.bin/ftp/ftp_var.h	2 Feb 2021 12:58:42 -0000	1.46
+++ usr.bin/ftp/ftp_var.h	2 May 2024 18:38:23 -0000
@@ -148,6 +148,7 @@ extern unsigned int retry_connect;	/* re
 extern int	ttywidth;	/* width of tty */
 extern int	epsv4;		/* use EPSV/EPRT on IPv4 connections */
 extern int	epsv4bad;	/* EPSV doesn't work on the current server */
+extern int	clipqs;		/* clip query string when showing progress */
 
 #ifndef SMALL
 extern int	  editing;	/* command line editing enabled */
Index: usr.bin/ftp/main.c
===================================================================
RCS file: /cvs/src/usr.bin/ftp/main.c,v
diff -u -p -r1.146 main.c
--- usr.bin/ftp/main.c	23 Dec 2023 23:03:00 -0000	1.146
+++ usr.bin/ftp/main.c	2 May 2024 18:38:23 -0000
@@ -108,6 +108,7 @@ int	ntflag;
 int	mapflag;
 int	preserve;
 int	progress;
+int	clipqs;
 int	code;
 int	crflag;
 char	pasv[BUFSIZ];
@@ -354,6 +355,7 @@ main(volatile int argc, char *argv[])
 	mark = HASHBYTES;
 	epsv4 = 1;
 	epsv4bad = 0;
+	clipqs = 1;
 
 	/* Set default operation mode based on FTPMODE environment variable */
 	if ((cp = getenv("FTPMODE")) != NULL && *cp != '\0') {
@@ -420,7 +422,7 @@ main(volatile int argc, char *argv[])
 	httpuseragent = NULL;
 
 	while ((ch = getopt(argc, argv,
-		    "46AaCc:dD:EeN:gik:Mmno:pP:r:S:s:TtU:uvVw:")) != -1) {
+		    "46AaCc:dD:EeN:gik:Mmno:pP:Qr:S:s:TtU:uvVw:")) != -1) {
 		switch (ch) {
 		case '4':
 			family = PF_INET;
@@ -514,7 +516,9 @@ main(volatile int argc, char *argv[])
 		case 'P':
 			ftpport = optarg;
 			break;
-
+		case 'Q':
+			clipqs = 0;
+			break;
 		case 'r':
 			retry_connect = strtonum(optarg, 0, INT_MAX, &errstr);
 			if (errstr != NULL) {
@@ -1064,7 +1068,7 @@ usage(void)
 {
 	fprintf(stderr, "usage: "
 #ifndef SMALL
-	    "ftp [-46AadEegiMmnptVv] [-D title] [-k seconds] [-P port] "
+	    "ftp [-46AadEegiMmnpQtVv] [-D title] [-k seconds] [-P port] "
 	    "[-r seconds]\n"
 	    "           [-s sourceaddr] [host [port]]\n"
 	    "       ftp [-C] [-N name] [-o output] [-s sourceaddr]\n"
Index: usr.bin/ftp/util.c
===================================================================
RCS file: /cvs/src/usr.bin/ftp/util.c,v
diff -u -p -r1.98 util.c
--- usr.bin/ftp/util.c	8 Mar 2023 04:43:11 -0000	1.98
+++ usr.bin/ftp/util.c	2 May 2024 18:38:24 -0000
@@ -786,6 +786,8 @@ progressmeter(int flag, const char *file
 			free(title);
 			title = strdup(filename);
 		}
+		if (clipqs && title != NULL)
+			title[strcspn(title, "?")] = '\0';
 		free(filenamebuf);
 	}