From: hshoexer <hshoexer@yerbouti.franken.de>
Subject: Re: rewrite wbinvd_on_all_cpus_acked() using cpu_xcall(9)
To: tech@openbsd.org
Date: Mon, 10 Nov 2025 20:33:33 +0100

Hi,

now with cpu_xcall(9) being commited here's the the diff for
wbinvd_on_all_cpus_acked.  Right now, wbinvd_on_all_cpus_acked()
is only used by psp(4).  So I'd suggest to implement it in directly
in psp(4).

As wbinvd is only used during vm launch and shutdown I use the
"broadcast and sleep" approach.

ok?

Removing wbinvd_on_all_cpus_acked() would be a separat diff.

---------------------------------------------------------------
Index: sys//conf/files
===================================================================
RCS file: /cvs/src/sys/conf/files,v
diff -u -p -u -p -r1.746 files
--- sys//conf/files	13 Jul 2025 05:45:21 -0000	1.746
+++ sys//conf/files	10 Nov 2025 19:29:37 -0000
@@ -475,7 +475,7 @@ device	ccp {}
 file	dev/ic/ccp.c			ccp
 
 # AMD Platform Security Processor
-device	psp
+device	psp: xcall
 attach	psp at ccp
 file	dev/ic/psp.c			psp	needs-flag
 
Index: sys//dev/ic/psp.c
===================================================================
RCS file: /cvs/src/sys/dev/ic/psp.c,v
diff -u -p -u -p -r1.20 psp.c
--- sys//dev/ic/psp.c	14 Aug 2025 11:18:11 -0000	1.20
+++ sys//dev/ic/psp.c	10 Nov 2025 19:29:37 -0000
@@ -24,6 +24,7 @@
 #include <sys/pledge.h>
 #include <sys/proc.h>
 #include <sys/rwlock.h>
+#include <sys/xcall.h>
 
 #include <machine/bus.h>
 
@@ -260,6 +261,34 @@ ccp_docmd(struct psp_softc *sc, int cmd,
 	return (0);
 }
 
+void
+psp_xc_wbinvd(void *arg)
+{
+	struct refcnt *r = arg;
+
+	wbinvd();
+
+	refcnt_rele_wake(r);
+}
+
+void
+psp_wbinvd(void)
+{
+	struct cpu_info *ci;
+	CPU_INFO_ITERATOR cii;
+	struct refcnt r = REFCNT_INITIALIZER();
+	struct xcall xc = XCALL_INITIALIZER(psp_xc_wbinvd, &r);
+
+	CPU_INFO_FOREACH(cii, ci) {
+		if (!ISSET(ci->ci_flags, CPUF_RUNNING))
+			continue;
+		refcnt_take(&r);
+		cpu_xcall(ci, &xc);
+	}
+
+	refcnt_finalize(&r, "psp_xc_wbinvd");
+}
+
 int
 psp_init(struct psp_softc *sc, struct psp_init *uinit)
 {
@@ -277,7 +306,7 @@ psp_init(struct psp_softc *sc, struct ps
 	if (error)
 		return (error);
 
-	wbinvd_on_all_cpus_acked();
+	psp_wbinvd();
 
 	sc->sc_flags |= PSPF_INITIALIZED;
 
@@ -359,7 +388,7 @@ psp_shutdown(struct psp_softc *sc)
 		return (error);
 
 	/* wbinvd right after SHUTDOWN */
-	wbinvd_on_all_cpus_acked();
+	psp_wbinvd();
 
 	/* release TMR */
 	bus_dmamap_unload(sc->sc_dmat, sc->sc_tmr_map);
@@ -398,7 +427,7 @@ psp_df_flush(struct psp_softc *sc)
 {
 	int error;
 
-	wbinvd_on_all_cpus_acked();
+	psp_wbinvd();
 
 	error = ccp_docmd(sc, PSP_CMD_DF_FLUSH, 0x0);
 
@@ -490,7 +519,7 @@ psp_launch_update_data(struct psp_softc 
 	ludata->handle = ulud->handle;
 
 	/* Drain caches before we encrypt memory. */
-	wbinvd_on_all_cpus_acked();
+	psp_wbinvd();
 
 	/*
 	 * Launch update one physical page at a time.  We could
@@ -554,7 +583,7 @@ psp_launch_update_vmsa(struct psp_softc 
 	luvmsa->length = PAGE_SIZE;
 
 	/* Drain caches before we encrypt the VMSA. */
-	wbinvd_on_all_cpus_acked();
+	psp_wbinvd();
 
 	error = ccp_docmd(sc, PSP_CMD_LAUNCH_UPDATE_VMSA,
 	    sc->sc_cmd_map->dm_segs[0].ds_addr);