From: Scott Cheloha Subject: clockintr_queue: add "cq_cpu", a back-pointer to the dispatch CPU To: tech@openbsd.org Date: Fri, 23 Feb 2024 20:31:14 -0600 Thank you for your comments on the tickless timeout roadmap. I have an idea of where things may get tricky, but now have ideas for managing it. -- Part (1) is to extend struct intrclock with a new function, ic_send(). ic_send() will allow arbitrary CPUs to poke the dispatch CPU for a given clockintr_queue to tell that CPU to reprogram its interrupt clock. On the backend, ic_send() will usually be implemented with an IPI. We need this to handle the situation where a newly scheduled clockintr has the most imminent expiration time on a given queue, but the caller is not the dispatch CPU for that queue. In order for ic_send() to work we first need to add a cpu_info* back-pointer, "cq_cpu", to clockintr_queue. This is so callers know which CPU to poke. This should be set during initialization, so make it a clockqueue_init() parameter and update all callers. ok? Index: sys/clockintr.h =================================================================== RCS file: /cvs/src/sys/sys/clockintr.h,v diff -u -p -r1.26 clockintr.h --- sys/clockintr.h 9 Feb 2024 16:52:58 -0000 1.26 +++ sys/clockintr.h 24 Feb 2024 02:05:56 -0000 @@ -104,6 +104,7 @@ struct clockintr_queue { struct clockintr *cq_running; /* [m] running clockintr */ struct clockintr cq_hardclock; /* [o] hardclock handle */ struct intrclock cq_intrclock; /* [I] local interrupt clock */ + struct cpu_info *cq_cpu; /* [I] dispatch CPU */ struct clockintr_stat cq_stat; /* [o] dispatch statistics */ volatile uint32_t cq_gen; /* [o] cq_stat update generation */ volatile uint32_t cq_dispatch; /* [o] dispatch is running */ @@ -136,7 +137,7 @@ void clockintr_stagger(struct clockintr void clockintr_unbind(struct clockintr *, uint32_t); uint64_t clockrequest_advance(struct clockrequest *, uint64_t); uint64_t clockrequest_advance_random(struct clockrequest *, uint64_t, uint32_t); -void clockqueue_init(struct clockintr_queue *); +void clockqueue_init(struct clockintr_queue *, struct cpu_info *); int sysctl_clockintr(int *, u_int, void *, size_t *, void *, size_t); #endif /* _KERNEL */ Index: kern/kern_clockintr.c =================================================================== RCS file: /cvs/src/sys/kern/kern_clockintr.c,v diff -u -p -r1.68 kern_clockintr.c --- kern/kern_clockintr.c 24 Feb 2024 01:43:32 -0000 1.68 +++ kern/kern_clockintr.c 24 Feb 2024 02:05:57 -0000 @@ -337,7 +337,7 @@ clockintr_cancel_locked(struct clockintr clockqueue_pend_delete(cq, cl); if (ISSET(cq->cq_flags, CQ_INTRCLOCK)) { if (was_next && !TAILQ_EMPTY(&cq->cq_pend)) { - if (cq == &curcpu()->ci_queue) + if (curcpu() == cq->cq_cpu) clockqueue_intrclock_reprogram(cq); } } @@ -409,7 +409,7 @@ clockintr_schedule_locked(struct clockin clockqueue_pend_insert(cq, cl, expiration); if (ISSET(cq->cq_flags, CQ_INTRCLOCK)) { if (cl == TAILQ_FIRST(&cq->cq_pend)) { - if (cq == &curcpu()->ci_queue) + if (curcpu() == cq->cq_cpu) clockqueue_intrclock_reprogram(cq); } } @@ -443,7 +443,7 @@ clockintr_hardclock(struct clockrequest } void -clockqueue_init(struct clockintr_queue *cq) +clockqueue_init(struct clockintr_queue *cq, struct cpu_info *ci) { if (ISSET(cq->cq_flags, CQ_INIT)) return; @@ -452,6 +452,7 @@ clockqueue_init(struct clockintr_queue * mtx_init(&cq->cq_mtx, IPL_CLOCK); TAILQ_INIT(&cq->cq_all); TAILQ_INIT(&cq->cq_pend); + cq->cq_cpu = ci; cq->cq_gen = 1; SET(cq->cq_flags, CQ_INIT); } Index: kern/init_main.c =================================================================== RCS file: /cvs/src/sys/kern/init_main.c,v diff -u -p -r1.325 init_main.c --- kern/init_main.c 14 Feb 2024 06:17:51 -0000 1.325 +++ kern/init_main.c 24 Feb 2024 02:05:58 -0000 @@ -314,7 +314,7 @@ main(void *framep) /* Initialize run queues */ sched_init_runqueues(); sleep_queue_init(); - clockqueue_init(&curcpu()->ci_queue); + clockqueue_init(&curcpu()->ci_queue, curcpu()); sched_init_cpu(curcpu()); p->p_cpu->ci_randseed = (arc4random() & 0x7fffffff) + 1; Index: arch/alpha/alpha/cpu.c =================================================================== RCS file: /cvs/src/sys/arch/alpha/alpha/cpu.c,v diff -u -p -r1.48 cpu.c --- arch/alpha/alpha/cpu.c 24 Oct 2023 13:20:09 -0000 1.48 +++ arch/alpha/alpha/cpu.c 24 Feb 2024 02:05:58 -0000 @@ -596,7 +596,7 @@ cpu_hatch(struct cpu_info *ci) ALPHA_TBIA(); alpha_pal_imb(); - clockqueue_init(&ci->ci_queue); + clockqueue_init(&ci->ci_queue, ci); KERNEL_LOCK(); sched_init_cpu(ci); ci->ci_curproc = ci->ci_fpcurproc = NULL; Index: arch/amd64/amd64/cpu.c =================================================================== RCS file: /cvs/src/sys/arch/amd64/amd64/cpu.c,v diff -u -p -r1.181 cpu.c --- arch/amd64/amd64/cpu.c 18 Feb 2024 05:42:50 -0000 1.181 +++ arch/amd64/amd64/cpu.c 24 Feb 2024 02:06:06 -0000 @@ -699,7 +699,7 @@ cpu_attach(struct device *parent, struct #if defined(MULTIPROCESSOR) cpu_intr_init(ci); cpu_start_secondary(ci); - clockqueue_init(&ci->ci_queue); + clockqueue_init(&ci->ci_queue, ci); sched_init_cpu(ci); ncpus++; if (ci->ci_flags & CPUF_PRESENT) { Index: arch/arm/arm/cpu.c =================================================================== RCS file: /cvs/src/sys/arch/arm/arm/cpu.c,v diff -u -p -r1.59 cpu.c --- arch/arm/arm/cpu.c 24 Oct 2023 13:20:09 -0000 1.59 +++ arch/arm/arm/cpu.c 24 Feb 2024 02:06:07 -0000 @@ -391,7 +391,7 @@ cpu_attach(struct device *parent, struct "cpu-release-addr", 0); } - clockqueue_init(&ci->ci_queue); + clockqueue_init(&ci->ci_queue, ci); sched_init_cpu(ci); if (cpu_hatch_secondary(ci, spinup_method, spinup_data)) { atomic_setbits_int(&ci->ci_flags, CPUF_IDENTIFY); Index: arch/arm64/arm64/cpu.c =================================================================== RCS file: /cvs/src/sys/arch/arm64/arm64/cpu.c,v diff -u -p -r1.104 cpu.c --- arch/arm64/arm64/cpu.c 21 Feb 2024 21:50:17 -0000 1.104 +++ arch/arm64/arm64/cpu.c 24 Feb 2024 02:06:10 -0000 @@ -890,7 +890,7 @@ cpu_attach(struct device *parent, struct "cpu-release-addr", 0); } - clockqueue_init(&ci->ci_queue); + clockqueue_init(&ci->ci_queue, ci); sched_init_cpu(ci); if (cpu_start_secondary(ci, spinup_method, spinup_data)) { atomic_setbits_int(&ci->ci_flags, CPUF_IDENTIFY); Index: arch/hppa/dev/cpu.c =================================================================== RCS file: /cvs/src/sys/arch/hppa/dev/cpu.c,v diff -u -p -r1.47 cpu.c --- arch/hppa/dev/cpu.c 24 Oct 2023 13:20:09 -0000 1.47 +++ arch/hppa/dev/cpu.c 24 Feb 2024 02:06:10 -0000 @@ -202,7 +202,7 @@ cpu_boot_secondary_processors(void) ci->ci_randseed = (arc4random() & 0x7fffffff) + 1; - clockqueue_init(&ci->ci_queue); + clockqueue_init(&ci->ci_queue, ci); sched_init_cpu(ci); /* Release the specified CPU by triggering an EIR{0}. */ Index: arch/i386/i386/cpu.c =================================================================== RCS file: /cvs/src/sys/arch/i386/i386/cpu.c,v diff -u -p -r1.114 cpu.c --- arch/i386/i386/cpu.c 24 Oct 2023 13:20:10 -0000 1.114 +++ arch/i386/i386/cpu.c 24 Feb 2024 02:06:13 -0000 @@ -360,7 +360,7 @@ cpu_attach(struct device *parent, struct #endif cpu_tsx_disable(ci); identifycpu(ci); - clockqueue_init(&ci->ci_queue); + clockqueue_init(&ci->ci_queue, ci); sched_init_cpu(ci); ci->ci_next = cpu_info_list->ci_next; cpu_info_list->ci_next = ci; Index: arch/luna88k/luna88k/machdep.c =================================================================== RCS file: /cvs/src/sys/arch/luna88k/luna88k/machdep.c,v diff -u -p -r1.144 machdep.c --- arch/luna88k/luna88k/machdep.c 24 Oct 2023 13:20:10 -0000 1.144 +++ arch/luna88k/luna88k/machdep.c 24 Feb 2024 02:06:13 -0000 @@ -761,7 +761,7 @@ secondary_main() cpu_configuration_print(0); ncpus++; - clockqueue_init(&ci->ci_queue); + clockqueue_init(&ci->ci_queue, ci); sched_init_cpu(ci); ci->ci_curproc = NULL; ci->ci_randseed = (arc4random() & 0x7fffffff) + 1; Index: arch/macppc/macppc/cpu.c =================================================================== RCS file: /cvs/src/sys/arch/macppc/macppc/cpu.c,v diff -u -p -r1.87 cpu.c --- arch/macppc/macppc/cpu.c 24 Oct 2023 13:20:10 -0000 1.87 +++ arch/macppc/macppc/cpu.c 24 Feb 2024 02:06:19 -0000 @@ -670,7 +670,7 @@ cpu_boot_secondary_processors(void) continue; ci->ci_randseed = (arc4random() & 0x7fffffff) + 1; - clockqueue_init(&ci->ci_queue); + clockqueue_init(&ci->ci_queue, ci); sched_init_cpu(ci); cpu_spinup(NULL, ci); Index: arch/mips64/mips64/cpu.c =================================================================== RCS file: /cvs/src/sys/arch/mips64/mips64/cpu.c,v diff -u -p -r1.83 cpu.c --- arch/mips64/mips64/cpu.c 15 Jun 2023 22:18:07 -0000 1.83 +++ arch/mips64/mips64/cpu.c 24 Feb 2024 02:06:19 -0000 @@ -395,7 +395,7 @@ cpu_boot_secondary_processors(void) continue; ci->ci_randseed = (arc4random() & 0x7fffffff) + 1; - clockqueue_init(&ci->ci_queue); + clockqueue_init(&ci->ci_queue, ci); sched_init_cpu(ci); cpu_boot_secondary(ci); } Index: arch/powerpc64/powerpc64/cpu.c =================================================================== RCS file: /cvs/src/sys/arch/powerpc64/powerpc64/cpu.c,v diff -u -p -r1.28 cpu.c --- arch/powerpc64/powerpc64/cpu.c 24 Oct 2023 13:20:10 -0000 1.28 +++ arch/powerpc64/powerpc64/cpu.c 24 Feb 2024 02:06:20 -0000 @@ -189,7 +189,7 @@ cpu_attach(struct device *parent, struct if (dev->dv_unit != 0) { int timeout = 10000; - clockqueue_init(&ci->ci_queue); + clockqueue_init(&ci->ci_queue, ci); sched_init_cpu(ci); ncpus++; Index: arch/riscv64/riscv64/cpu.c =================================================================== RCS file: /cvs/src/sys/arch/riscv64/riscv64/cpu.c,v diff -u -p -r1.18 cpu.c --- arch/riscv64/riscv64/cpu.c 26 Jan 2024 16:59:47 -0000 1.18 +++ arch/riscv64/riscv64/cpu.c 24 Feb 2024 02:06:20 -0000 @@ -215,7 +215,7 @@ cpu_attach(struct device *parent, struct if (ci->ci_flags & CPUF_AP) { int timeout = 10000; - clockqueue_init(&ci->ci_queue); + clockqueue_init(&ci->ci_queue, ci); sched_init_cpu(ci); if (cpu_hatch_secondary(ci)) { atomic_setbits_int(&ci->ci_flags, CPUF_IDENTIFY); Index: arch/sparc64/sparc64/cpu.c =================================================================== RCS file: /cvs/src/sys/arch/sparc64/sparc64/cpu.c,v diff -u -p -r1.75 cpu.c --- arch/sparc64/sparc64/cpu.c 24 Oct 2023 13:20:10 -0000 1.75 +++ arch/sparc64/sparc64/cpu.c 24 Feb 2024 02:06:23 -0000 @@ -185,7 +185,7 @@ alloc_cpuinfo(struct mainbus_attach_args cpi->ci_self = cpi; cpi->ci_node = ma->ma_node; - clockqueue_init(&cpi->ci_queue); + clockqueue_init(&cpi->ci_queue, cpi); sched_init_cpu(cpi); /*