From: Mike Larkin Subject: Re: limit softnet threads to number of cpu To: Alexander Bluhm Cc: tech@openbsd.org Date: Mon, 8 Sep 2025 12:10:34 -0700 On Mon, Sep 08, 2025 at 04:35:49PM +0200, Alexander Bluhm wrote: > On Sat, Aug 02, 2025 at 10:18:11PM +0200, Alexander Bluhm wrote: > > Hi, > > > > Currently always 8 softnet threads are startet, but only up to > > number of CPU are used. I would like to remove useless threads. > > > > Problem is that softnet tasks must be initialized before autoconf > > is running. Drivers need them to attach queues. But number of CPU > > is known only after autoconf has discovert them. > > > > So I split the code into softnet_init() and softnet_percpu(). The > > latter removes task queues that are not needed. Threads have not > > been created yet. > > > > I have a follow up diff that pins softnet threads to specific CPU. > > But that needs more testing. I would like to commit this thread > > handling diff first. > > > > ok? > > I did not find a better solution to limit softnet tasks and threads > to number of CPU. Note that thread creation is deferred, they are > destroyed before they get started. > > dhill@ has an alternative diff which puts the task creation into > first call to softnet_count(). It works, but looks fragile. > > Can we get my diff commited now? ok? > > bluhm I had a long reply typed up about the fact that we probably know the number of CPUs by the time the first network driver attaches but then realized that doesn't then solve the question of when the softnet tasks would be created. It would have to be shimmed in somewhere after cpuX but before any network driver, and that would be less clean than your approach below. The diff looks ok to me, so ok mlarkin (but i did not test) > > Index: kern/init_main.c > =================================================================== > RCS file: /data/mirror/openbsd/cvs/src/sys/kern/init_main.c,v > diff -u -p -r1.329 init_main.c > --- kern/init_main.c 9 Jun 2025 10:57:46 -0000 1.329 > +++ kern/init_main.c 8 Sep 2025 11:41:56 -0000 > @@ -328,6 +328,7 @@ main(void *framep) > > /* Initialize the interface/address trees */ > ifinit(); > + softnet_init(); > > /* Lock the kernel on behalf of proc0. */ > KERNEL_LOCK(); > @@ -345,6 +346,9 @@ main(void *framep) > > /* Per CPU memory allocation */ > percpu_init(); > + > + /* Reduce softnet threads to number of CPU */ > + softnet_percpu(); > > /* Initialize the file systems. */ > #if defined(NFSSERVER) || defined(NFSCLIENT) > Index: net/if.c > =================================================================== > RCS file: /data/mirror/openbsd/cvs/src/sys/net/if.c,v > diff -u -p -r1.740 if.c > --- net/if.c 21 Jul 2025 20:36:41 -0000 1.740 > +++ net/if.c 8 Sep 2025 11:43:00 -0000 > @@ -237,7 +237,11 @@ struct softnet { > struct taskq *sn_taskq; > struct netstack sn_netstack; > } __aligned(64); > +#ifdef MULTIPROCESSOR > #define NET_TASKQ 8 > +#else > +#define NET_TASKQ 1 > +#endif > struct softnet softnets[NET_TASKQ]; > > struct task if_input_task_locked = TASK_INITIALIZER(if_netisr, NULL); > @@ -255,16 +259,22 @@ struct rwlock netlock = RWLOCK_INITIALIZ > void > ifinit(void) > { > - unsigned int i; > - > /* > * most machines boot with 4 or 5 interfaces, so size the initial map > * to accommodate this > */ > if_idxmap_init(8); /* 8 is a nice power of 2 for malloc */ > +} > + > +void > +softnet_init(void) > +{ > + unsigned int i; > > + /* Number of CPU is unknown, but driver attach needs softnet tasks. */ > for (i = 0; i < NET_TASKQ; i++) { > struct softnet *sn = &softnets[i]; > + > snprintf(sn->sn_name, sizeof(sn->sn_name), "softnet%u", i); > sn->sn_taskq = taskq_create(sn->sn_name, 1, IPL_NET, > TASKQ_MPSAFE); > @@ -273,6 +283,22 @@ ifinit(void) > } > } > > +void > +softnet_percpu(void) > +{ > +#ifdef MULTIPROCESSOR > + unsigned int i; > + > + /* After attaching all CPUs and interfaces, remove useless threads. */ > + for (i = softnet_count(); i < NET_TASKQ; i++) { > + struct softnet *sn = &softnets[i]; > + > + taskq_destroy(sn->sn_taskq); > + sn->sn_taskq = NULL; > + } > +#endif /* MULTIPROCESSOR */ > +} > + > static struct if_idxmap if_idxmap; > > /* > @@ -3642,10 +3668,10 @@ unhandled_af(int af) > panic("unhandled af %d", af); > } > > -int > -net_sn_count(void) > +unsigned int > +softnet_count(void) > { > - static int nsoftnets; > + static unsigned int nsoftnets; > > if (nsoftnets == 0) > nsoftnets = min(NET_TASKQ, ncpus); > @@ -3656,7 +3682,7 @@ net_sn_count(void) > struct softnet * > net_sn(unsigned int ifindex) > { > - return (&softnets[ifindex % net_sn_count()]); > + return (&softnets[ifindex % softnet_count()]); > } > > struct taskq * > @@ -3672,7 +3698,7 @@ net_tq_barriers(const char *wmesg) > struct refcnt r = REFCNT_INITIALIZER(); > int i; > > - for (i = 0; i < nitems(barriers); i++) { > + for (i = 0; i < softnet_count(); i++) { > task_set(&barriers[i], (void (*)(void *))refcnt_rele_wake, &r); > refcnt_take(&r); > task_add(softnets[i].sn_taskq, &barriers[i]); > Index: net/if.h > =================================================================== > RCS file: /data/mirror/openbsd/cvs/src/sys/net/if.h,v > diff -u -p -r1.220 if.h > --- net/if.h 19 Jul 2025 16:40:40 -0000 1.220 > +++ net/if.h 8 Sep 2025 11:41:56 -0000 > @@ -561,10 +561,13 @@ void if_congestion(void); > int if_congested(void); > __dead void unhandled_af(int); > int if_setlladdr(struct ifnet *, const uint8_t *); > +void softnet_init(void); > +void softnet_percpu(void); > +unsigned int > + softnet_count(void); > struct taskq * > net_tq(unsigned int); > void net_tq_barriers(const char *); > -int net_sn_count(void); > > #endif /* _KERNEL */ > > Index: net/if_loop.c > =================================================================== > RCS file: /data/mirror/openbsd/cvs/src/sys/net/if_loop.c,v > diff -u -p -r1.102 if_loop.c > --- net/if_loop.c 7 Jul 2025 02:28:50 -0000 1.102 > +++ net/if_loop.c 8 Sep 2025 11:41:56 -0000 > @@ -178,8 +178,8 @@ loop_clone_create(struct if_clone *ifc, > rtable_l2set(0, 0, ifp->if_index); > } else > if_attach(ifp); > - if_attach_queues(ifp, net_sn_count()); > - if_attach_iqueues(ifp, net_sn_count()); > + if_attach_queues(ifp, softnet_count()); > + if_attach_iqueues(ifp, softnet_count()); > if_alloc_sadl(ifp); > #if NBPFILTER > 0 > bpfattach(&ifp->if_bpf, ifp, DLT_LOOP, sizeof(u_int32_t)); >