From: Alexander Bluhm <bluhm@openbsd.org>
Subject: Re: limit softnet threads to number of cpu
To: tech@openbsd.org
Date: Mon, 8 Sep 2025 16:35:49 +0200

On Sat, Aug 02, 2025 at 10:18:11PM +0200, Alexander Bluhm wrote:
> Hi,
> 
> Currently always 8 softnet threads are startet, but only up to
> number of CPU are used.  I would like to remove useless threads.
> 
> Problem is that softnet tasks must be initialized before autoconf
> is running.  Drivers need them to attach queues.  But number of CPU
> is known only after autoconf has discovert them.
> 
> So I split the code into softnet_init() and softnet_percpu().  The
> latter removes task queues that are not needed.  Threads have not
> been created yet.
> 
> I have a follow up diff that pins softnet threads to specific CPU.
> But that needs more testing.  I would like to commit this thread
> handling diff first.
> 
> ok?

I did not find a better solution to limit softnet tasks and threads
to number of CPU.  Note that thread creation is deferred, they are
destroyed before they get started.

dhill@ has an alternative diff which puts the task creation into
first call to softnet_count().  It works, but looks fragile.

Can we get my diff commited now?  ok?

bluhm

Index: kern/init_main.c
===================================================================
RCS file: /data/mirror/openbsd/cvs/src/sys/kern/init_main.c,v
diff -u -p -r1.329 init_main.c
--- kern/init_main.c	9 Jun 2025 10:57:46 -0000	1.329
+++ kern/init_main.c	8 Sep 2025 11:41:56 -0000
@@ -328,6 +328,7 @@ main(void *framep)
 
 	/* Initialize the interface/address trees */
 	ifinit();
+	softnet_init();
 
 	/* Lock the kernel on behalf of proc0. */
 	KERNEL_LOCK();
@@ -345,6 +346,9 @@ main(void *framep)
 
 	/* Per CPU memory allocation */
 	percpu_init();
+
+	/* Reduce softnet threads to number of CPU */
+	softnet_percpu();
 
 	/* Initialize the file systems. */
 #if defined(NFSSERVER) || defined(NFSCLIENT)
Index: net/if.c
===================================================================
RCS file: /data/mirror/openbsd/cvs/src/sys/net/if.c,v
diff -u -p -r1.740 if.c
--- net/if.c	21 Jul 2025 20:36:41 -0000	1.740
+++ net/if.c	8 Sep 2025 11:43:00 -0000
@@ -237,7 +237,11 @@ struct softnet {
 	struct taskq	*sn_taskq;
 	struct netstack	 sn_netstack;
 } __aligned(64);
+#ifdef MULTIPROCESSOR
 #define NET_TASKQ	8
+#else
+#define NET_TASKQ	1
+#endif
 struct softnet	softnets[NET_TASKQ];
 
 struct task	if_input_task_locked = TASK_INITIALIZER(if_netisr, NULL);
@@ -255,16 +259,22 @@ struct rwlock netlock = RWLOCK_INITIALIZ
 void
 ifinit(void)
 {
-	unsigned int	i;
-
 	/*
 	 * most machines boot with 4 or 5 interfaces, so size the initial map
 	 * to accommodate this
 	 */
 	if_idxmap_init(8); /* 8 is a nice power of 2 for malloc */
+}
+
+void
+softnet_init(void)
+{
+	unsigned int i;
 
+	/* Number of CPU is unknown, but driver attach needs softnet tasks. */
 	for (i = 0; i < NET_TASKQ; i++) {
 		struct softnet *sn = &softnets[i];
+
 		snprintf(sn->sn_name, sizeof(sn->sn_name), "softnet%u", i);
 		sn->sn_taskq = taskq_create(sn->sn_name, 1, IPL_NET,
 		    TASKQ_MPSAFE);
@@ -273,6 +283,22 @@ ifinit(void)
 	}
 }
 
+void
+softnet_percpu(void)
+{
+#ifdef MULTIPROCESSOR
+	unsigned int i;
+
+	/* After attaching all CPUs and interfaces, remove useless threads. */
+	for (i = softnet_count(); i < NET_TASKQ; i++) {
+		struct softnet *sn = &softnets[i];
+
+		taskq_destroy(sn->sn_taskq);
+		sn->sn_taskq = NULL;
+	}
+#endif /* MULTIPROCESSOR */
+}
+
 static struct if_idxmap if_idxmap;
 
 /*
@@ -3642,10 +3668,10 @@ unhandled_af(int af)
 	panic("unhandled af %d", af);
 }
 
-int
-net_sn_count(void)
+unsigned int
+softnet_count(void)
 {
-	static int nsoftnets;
+	static unsigned int nsoftnets;
 
 	if (nsoftnets == 0)
 		nsoftnets = min(NET_TASKQ, ncpus);
@@ -3656,7 +3682,7 @@ net_sn_count(void)
 struct softnet *
 net_sn(unsigned int ifindex)
 {
-	return (&softnets[ifindex % net_sn_count()]);
+	return (&softnets[ifindex % softnet_count()]);
 }
 
 struct taskq *
@@ -3672,7 +3698,7 @@ net_tq_barriers(const char *wmesg)
 	struct refcnt r = REFCNT_INITIALIZER();
 	int i;
 
-	for (i = 0; i < nitems(barriers); i++) {
+	for (i = 0; i < softnet_count(); i++) {
 		task_set(&barriers[i], (void (*)(void *))refcnt_rele_wake, &r);
 		refcnt_take(&r);
 		task_add(softnets[i].sn_taskq, &barriers[i]);
Index: net/if.h
===================================================================
RCS file: /data/mirror/openbsd/cvs/src/sys/net/if.h,v
diff -u -p -r1.220 if.h
--- net/if.h	19 Jul 2025 16:40:40 -0000	1.220
+++ net/if.h	8 Sep 2025 11:41:56 -0000
@@ -561,10 +561,13 @@ void	if_congestion(void);
 int	if_congested(void);
 __dead void	unhandled_af(int);
 int	if_setlladdr(struct ifnet *, const uint8_t *);
+void	softnet_init(void);
+void	softnet_percpu(void);
+unsigned int
+	softnet_count(void);
 struct taskq *
 	net_tq(unsigned int);
 void	net_tq_barriers(const char *);
-int	net_sn_count(void);
 
 #endif /* _KERNEL */
 
Index: net/if_loop.c
===================================================================
RCS file: /data/mirror/openbsd/cvs/src/sys/net/if_loop.c,v
diff -u -p -r1.102 if_loop.c
--- net/if_loop.c	7 Jul 2025 02:28:50 -0000	1.102
+++ net/if_loop.c	8 Sep 2025 11:41:56 -0000
@@ -178,8 +178,8 @@ loop_clone_create(struct if_clone *ifc, 
 		rtable_l2set(0, 0, ifp->if_index);
 	} else
 		if_attach(ifp);
-	if_attach_queues(ifp, net_sn_count());
-	if_attach_iqueues(ifp, net_sn_count());
+	if_attach_queues(ifp, softnet_count());
+	if_attach_iqueues(ifp, softnet_count());
 	if_alloc_sadl(ifp);
 #if NBPFILTER > 0
 	bpfattach(&ifp->if_bpf, ifp, DLT_LOOP, sizeof(u_int32_t));