From: Mark Kettenis <mark.kettenis@xs4all.nl>
Subject: Re: cpu_xcall glue for arm64
To: David Gwynne <david@gwynne.id.au>
Cc: tech@openbsd.org
Date: Tue, 22 Jul 2025 11:28:34 +0200

> Date: Sun, 13 Jul 2025 16:22:03 +1000
> From: David Gwynne <david@gwynne.id.au>
> 
> a driver can depend on xcall and get this glue code compiled so it will
> work on arm64.
> 
> ok?

So what's the way forward here.  Theo objected to the extent of the
'#ifdef NXCALL > 0' and I think and then the discussion died?  I'm not
a fan of that either, but we also don't want that code in RAMDISK
kernels if we don't need it.  So perhaps just drop those here and make
the xcall stuff non-optional for MULTIPROCESSOR kernels?

> Index: arm64/cpu.c
> ===================================================================
> RCS file: /cvs/src/sys/arch/arm64/arm64/cpu.c,v
> diff -u -p -r1.142 cpu.c
> --- arm64/cpu.c	1 Jul 2025 11:10:36 -0000	1.142
> +++ arm64/cpu.c	13 Jul 2025 06:01:15 -0000
> @@ -1595,6 +1595,10 @@ cpu_attach(struct device *parent, struct
>  		}
>  #ifdef MULTIPROCESSOR
>  	}
> +
> +#if NXCALL > 0
> +	cpu_xcall_establish(ci);
> +#endif
>  #endif
>  
>  #if NKSTAT > 0
> Index: dev/agintc.c
> ===================================================================
> RCS file: /cvs/src/sys/arch/arm64/dev/agintc.c,v
> diff -u -p -r1.63 agintc.c
> --- dev/agintc.c	6 Jul 2025 12:22:31 -0000	1.63
> +++ dev/agintc.c	13 Jul 2025 06:01:15 -0000
> @@ -1428,6 +1428,10 @@ agintc_ipi_handler(void *v)
>  			agintc_ipi_ddb(v);
>  		if (ISSET(reasons, 1 << ARM_IPI_HALT))
>  			agintc_ipi_halt(v);
> +#if NXCALL > 0
> +		if (ISSET(reasons, 1 << ARM_IPI_XCALL))
> +			cpu_xcall_dispatch(ci);
> +#endif
>  	}
>  
>  	return (1);
> Index: dev/ampintc.c
> ===================================================================
> RCS file: /cvs/src/sys/arch/arm64/dev/ampintc.c,v
> diff -u -p -r1.33 ampintc.c
> --- dev/ampintc.c	6 Jul 2025 12:22:31 -0000	1.33
> +++ dev/ampintc.c	13 Jul 2025 06:01:15 -0000
> @@ -1033,6 +1033,10 @@ ampintc_ipi_handler(void *v)
>  			ampintc_ipi_ddb(v);
>  		if (ISSET(reasons, 1 << ARM_IPI_HALT))
>  			ampintc_ipi_halt(v);
> +#if NXCALL > 0
> +		if (ISSET(reasons, 1 << ARM_IPI_XCALL))
> +			cpu_xcall_dispatch(ci);
> +#endif
>  	}
>  
>  	return (1);
> Index: dev/aplintc.c
> ===================================================================
> RCS file: /cvs/src/sys/arch/arm64/dev/aplintc.c,v
> diff -u -p -r1.19 aplintc.c
> --- dev/aplintc.c	6 Jul 2025 12:22:31 -0000	1.19
> +++ dev/aplintc.c	13 Jul 2025 06:01:15 -0000
> @@ -689,6 +689,10 @@ aplintc_handle_ipi(struct aplintc_softc 
>  #endif
>  		if (ISSET(reasons, 1 << ARM_IPI_HALT))
>  			cpu_halt();
> +#if NXCALL > 0
> +		if (ISSET(reasons, 1 << ARM_IPI_XCALL))
> +			cpu_xcall_dispatch(ci);
> +#endif
>  	}
>  
>  	sc->sc_ipi_count.ec_count++;
> Index: dev/bcm2836_intr.c
> ===================================================================
> RCS file: /cvs/src/sys/arch/arm64/dev/bcm2836_intr.c,v
> diff -u -p -r1.16 bcm2836_intr.c
> --- dev/bcm2836_intr.c	6 Jul 2025 12:22:31 -0000	1.16
> +++ dev/bcm2836_intr.c	13 Jul 2025 06:01:15 -0000
> @@ -638,6 +638,10 @@ bcm_intc_handle_ipi(void)
>  		db_enter();
>  	}
>  #endif
> +#if NXCALL > 0
> +	if (ISSET(mbox_val, 1 << ARM_IPI_XCALL))
> +		cpu_xcall_dispatch(curcpu());
> +#endif
>  }
>  
>  void
> Index: include/cpu.h
> ===================================================================
> RCS file: /cvs/src/sys/arch/arm64/include/cpu.h,v
> diff -u -p -r1.51 cpu.h
> --- include/cpu.h	11 Feb 2025 22:27:09 -0000	1.51
> +++ include/cpu.h	13 Jul 2025 06:01:15 -0000
> @@ -57,6 +57,8 @@
>  
>  #ifdef _KERNEL
>  
> +#include "xcall.h"
> +
>  /*
>   * Kernel-only definitions
>   */
> @@ -112,6 +114,9 @@ void cpu_identify_cleanup(void);
>  #include <sys/sched.h>
>  #include <sys/srp.h>
>  #include <uvm/uvm_percpu.h>
> +#if NXCALL > 0
> +#include <sys/xcall.h>
> +#endif
>  
>  struct cpu_info {
>  	struct device		*ci_dev; /* Device corresponding to this CPU */
> @@ -167,6 +172,9 @@ struct cpu_info {
>  
>  #ifdef MULTIPROCESSOR
>  	struct srp_hazard	ci_srp_hazards[SRP_HAZARD_NUM];
> +#if NXCALL > 0
> +	struct xcall_cpu	ci_xcall;
> +#endif
>  #define __HAVE_UVM_PERCPU
>  	struct uvm_pmr_cache	ci_uvm;
>  	volatile int		ci_flags;
> Index: include/intr.h
> ===================================================================
> RCS file: /cvs/src/sys/arch/arm64/include/intr.h,v
> diff -u -p -r1.25 intr.h
> --- include/intr.h	30 Jun 2025 14:19:20 -0000	1.25
> +++ include/intr.h	13 Jul 2025 06:01:15 -0000
> @@ -199,6 +199,9 @@ extern void (*intr_send_ipi_func)(struct
>  #define ARM_IPI_NOP	0
>  #define ARM_IPI_DDB	1
>  #define ARM_IPI_HALT	2
> +#define ARM_IPI_XCALL	3
> +
> +#define cpu_xcall_ipi(_ci) arm_send_ipi((_ci), ARM_IPI_XCALL)

Maybe implement this as a static inline function instead?

>  
>  #ifdef DIAGNOSTIC
>  /*
> 
>