From: hshoexer <hshoexer@yerbouti.franken.de>
Subject: Re: rewrite wbinvd_on_all_cpus_acked() using cpu_xcall(9)
To: tech@openbsd.org
Date: Tue, 11 Nov 2025 12:28:40 +0100

On Tue, Nov 11, 2025 at 06:42:33AM +1000, David Gwynne wrote:
> On Mon, Nov 10, 2025 at 08:33:33PM +0100, hshoexer wrote:
> > Hi,
> > 
> > now with cpu_xcall(9) being commited here's the the diff for
> > wbinvd_on_all_cpus_acked.  Right now, wbinvd_on_all_cpus_acked()
> > is only used by psp(4).  So I'd suggest to implement it in directly
> > in psp(4).
> > 
> > As wbinvd is only used during vm launch and shutdown I use the
> > "broadcast and sleep" approach.
> > 
> > ok?
> 
> id make the wmesg in refcnt_finalize shorter. iirc only 8ish characters
> from a wait message make it up to userland for top/ps to look at.

good point!  I will tweak the wmesg.

> ok by me either way.

thx!

> > 
> > Removing wbinvd_on_all_cpus_acked() would be a separat diff.
> > 
> > ---------------------------------------------------------------
> > Index: sys//conf/files
> > ===================================================================
> > RCS file: /cvs/src/sys/conf/files,v
> > diff -u -p -u -p -r1.746 files
> > --- sys//conf/files	13 Jul 2025 05:45:21 -0000	1.746
> > +++ sys//conf/files	10 Nov 2025 19:29:37 -0000
> > @@ -475,7 +475,7 @@ device	ccp {}
> >  file	dev/ic/ccp.c			ccp
> >  
> >  # AMD Platform Security Processor
> > -device	psp
> > +device	psp: xcall
> >  attach	psp at ccp
> >  file	dev/ic/psp.c			psp	needs-flag
> >  
> > Index: sys//dev/ic/psp.c
> > ===================================================================
> > RCS file: /cvs/src/sys/dev/ic/psp.c,v
> > diff -u -p -u -p -r1.20 psp.c
> > --- sys//dev/ic/psp.c	14 Aug 2025 11:18:11 -0000	1.20
> > +++ sys//dev/ic/psp.c	10 Nov 2025 19:29:37 -0000
> > @@ -24,6 +24,7 @@
> >  #include <sys/pledge.h>
> >  #include <sys/proc.h>
> >  #include <sys/rwlock.h>
> > +#include <sys/xcall.h>
> >  
> >  #include <machine/bus.h>
> >  
> > @@ -260,6 +261,34 @@ ccp_docmd(struct psp_softc *sc, int cmd,
> >  	return (0);
> >  }
> >  
> > +void
> > +psp_xc_wbinvd(void *arg)
> > +{
> > +	struct refcnt *r = arg;
> > +
> > +	wbinvd();
> > +
> > +	refcnt_rele_wake(r);
> > +}
> > +
> > +void
> > +psp_wbinvd(void)
> > +{
> > +	struct cpu_info *ci;
> > +	CPU_INFO_ITERATOR cii;
> > +	struct refcnt r = REFCNT_INITIALIZER();
> > +	struct xcall xc = XCALL_INITIALIZER(psp_xc_wbinvd, &r);
> > +
> > +	CPU_INFO_FOREACH(cii, ci) {
> > +		if (!ISSET(ci->ci_flags, CPUF_RUNNING))
> > +			continue;
> > +		refcnt_take(&r);
> > +		cpu_xcall(ci, &xc);
> > +	}
> > +
> > +	refcnt_finalize(&r, "psp_xc_wbinvd");
> > +}
> > +
> >  int
> >  psp_init(struct psp_softc *sc, struct psp_init *uinit)
> >  {
> > @@ -277,7 +306,7 @@ psp_init(struct psp_softc *sc, struct ps
> >  	if (error)
> >  		return (error);
> >  
> > -	wbinvd_on_all_cpus_acked();
> > +	psp_wbinvd();
> >  
> >  	sc->sc_flags |= PSPF_INITIALIZED;
> >  
> > @@ -359,7 +388,7 @@ psp_shutdown(struct psp_softc *sc)
> >  		return (error);
> >  
> >  	/* wbinvd right after SHUTDOWN */
> > -	wbinvd_on_all_cpus_acked();
> > +	psp_wbinvd();
> >  
> >  	/* release TMR */
> >  	bus_dmamap_unload(sc->sc_dmat, sc->sc_tmr_map);
> > @@ -398,7 +427,7 @@ psp_df_flush(struct psp_softc *sc)
> >  {
> >  	int error;
> >  
> > -	wbinvd_on_all_cpus_acked();
> > +	psp_wbinvd();
> >  
> >  	error = ccp_docmd(sc, PSP_CMD_DF_FLUSH, 0x0);
> >  
> > @@ -490,7 +519,7 @@ psp_launch_update_data(struct psp_softc 
> >  	ludata->handle = ulud->handle;
> >  
> >  	/* Drain caches before we encrypt memory. */
> > -	wbinvd_on_all_cpus_acked();
> > +	psp_wbinvd();
> >  
> >  	/*
> >  	 * Launch update one physical page at a time.  We could
> > @@ -554,7 +583,7 @@ psp_launch_update_vmsa(struct psp_softc 
> >  	luvmsa->length = PAGE_SIZE;
> >  
> >  	/* Drain caches before we encrypt the VMSA. */
> > -	wbinvd_on_all_cpus_acked();
> > +	psp_wbinvd();
> >  
> >  	error = ccp_docmd(sc, PSP_CMD_LAUNCH_UPDATE_VMSA,
> >  	    sc->sc_cmd_map->dm_segs[0].ds_addr);
> > 
>