From: David Gwynne <david@gwynne.id.au>
Subject: Re: rewrite wbinvd_on_all_cpus_acked() using cpu_xcall(9)
To: tech@openbsd.org
Date: Tue, 11 Nov 2025 06:42:33 +1000

On Mon, Nov 10, 2025 at 08:33:33PM +0100, hshoexer wrote:
> Hi,
> 
> now with cpu_xcall(9) being commited here's the the diff for
> wbinvd_on_all_cpus_acked.  Right now, wbinvd_on_all_cpus_acked()
> is only used by psp(4).  So I'd suggest to implement it in directly
> in psp(4).
> 
> As wbinvd is only used during vm launch and shutdown I use the
> "broadcast and sleep" approach.
> 
> ok?

id make the wmesg in refcnt_finalize shorter. iirc only 8ish characters
from a wait message make it up to userland for top/ps to look at.

ok by me either way.

> 
> Removing wbinvd_on_all_cpus_acked() would be a separat diff.
> 
> ---------------------------------------------------------------
> Index: sys//conf/files
> ===================================================================
> RCS file: /cvs/src/sys/conf/files,v
> diff -u -p -u -p -r1.746 files
> --- sys//conf/files	13 Jul 2025 05:45:21 -0000	1.746
> +++ sys//conf/files	10 Nov 2025 19:29:37 -0000
> @@ -475,7 +475,7 @@ device	ccp {}
>  file	dev/ic/ccp.c			ccp
>  
>  # AMD Platform Security Processor
> -device	psp
> +device	psp: xcall
>  attach	psp at ccp
>  file	dev/ic/psp.c			psp	needs-flag
>  
> Index: sys//dev/ic/psp.c
> ===================================================================
> RCS file: /cvs/src/sys/dev/ic/psp.c,v
> diff -u -p -u -p -r1.20 psp.c
> --- sys//dev/ic/psp.c	14 Aug 2025 11:18:11 -0000	1.20
> +++ sys//dev/ic/psp.c	10 Nov 2025 19:29:37 -0000
> @@ -24,6 +24,7 @@
>  #include <sys/pledge.h>
>  #include <sys/proc.h>
>  #include <sys/rwlock.h>
> +#include <sys/xcall.h>
>  
>  #include <machine/bus.h>
>  
> @@ -260,6 +261,34 @@ ccp_docmd(struct psp_softc *sc, int cmd,
>  	return (0);
>  }
>  
> +void
> +psp_xc_wbinvd(void *arg)
> +{
> +	struct refcnt *r = arg;
> +
> +	wbinvd();
> +
> +	refcnt_rele_wake(r);
> +}
> +
> +void
> +psp_wbinvd(void)
> +{
> +	struct cpu_info *ci;
> +	CPU_INFO_ITERATOR cii;
> +	struct refcnt r = REFCNT_INITIALIZER();
> +	struct xcall xc = XCALL_INITIALIZER(psp_xc_wbinvd, &r);
> +
> +	CPU_INFO_FOREACH(cii, ci) {
> +		if (!ISSET(ci->ci_flags, CPUF_RUNNING))
> +			continue;
> +		refcnt_take(&r);
> +		cpu_xcall(ci, &xc);
> +	}
> +
> +	refcnt_finalize(&r, "psp_xc_wbinvd");
> +}
> +
>  int
>  psp_init(struct psp_softc *sc, struct psp_init *uinit)
>  {
> @@ -277,7 +306,7 @@ psp_init(struct psp_softc *sc, struct ps
>  	if (error)
>  		return (error);
>  
> -	wbinvd_on_all_cpus_acked();
> +	psp_wbinvd();
>  
>  	sc->sc_flags |= PSPF_INITIALIZED;
>  
> @@ -359,7 +388,7 @@ psp_shutdown(struct psp_softc *sc)
>  		return (error);
>  
>  	/* wbinvd right after SHUTDOWN */
> -	wbinvd_on_all_cpus_acked();
> +	psp_wbinvd();
>  
>  	/* release TMR */
>  	bus_dmamap_unload(sc->sc_dmat, sc->sc_tmr_map);
> @@ -398,7 +427,7 @@ psp_df_flush(struct psp_softc *sc)
>  {
>  	int error;
>  
> -	wbinvd_on_all_cpus_acked();
> +	psp_wbinvd();
>  
>  	error = ccp_docmd(sc, PSP_CMD_DF_FLUSH, 0x0);
>  
> @@ -490,7 +519,7 @@ psp_launch_update_data(struct psp_softc 
>  	ludata->handle = ulud->handle;
>  
>  	/* Drain caches before we encrypt memory. */
> -	wbinvd_on_all_cpus_acked();
> +	psp_wbinvd();
>  
>  	/*
>  	 * Launch update one physical page at a time.  We could
> @@ -554,7 +583,7 @@ psp_launch_update_vmsa(struct psp_softc 
>  	luvmsa->length = PAGE_SIZE;
>  
>  	/* Drain caches before we encrypt the VMSA. */
> -	wbinvd_on_all_cpus_acked();
> +	psp_wbinvd();
>  
>  	error = ccp_docmd(sc, PSP_CMD_LAUNCH_UPDATE_VMSA,
>  	    sc->sc_cmd_map->dm_segs[0].ds_addr);
>