From: Alexander Bluhm <bluhm@openbsd.org>
Subject: Re: Relax sockets splicing locking
To: Vitaliy Makkoveev <mvs@openbsd.org>
Cc: tech@openbsd.org
Date: Mon, 6 Jan 2025 01:17:44 +0100

On Sat, Jan 04, 2025 at 08:07:14PM +0300, Vitaliy Makkoveev wrote:
> On Sat, Jan 04, 2025 at 06:39:26PM +0300, Vitaliy Makkoveev wrote:
> > > On 4 Jan 2025, at 16:34, Alexander Bluhm <bluhm@openbsd.org> wrote:
> > > 
> 
> > > [skip]
> 
> > > I have tested it with my regress and netlink setup.  My perform
> > > testing machines are currently busy with other tests.
> > > 
> > > We have no propper regression tests for splicing and unsplicing
> > > under heavy load.  I am currently trying to write a test.
> > > 
> > 
> > We need something like multithreaded attempts of simultaneous
> > splicing and unsplicing one socket to itself. Or the same but
> > for two sockets.
> > 
> 
> The simplest test to concurrently splice/unsplice socket.

I have commited a more exhaustive test in regress/sys/netinet/tcpthread.
It does all kind of TCP socket operations in parallel.

But your test also has value as it does splice/unsplice very fast.
If you want, I can integrate it somewhere in regress/sys/kern/sosplice
with a proper regress Makefile.

Currently the kernel does not complain about unsplicing sockets
that are not spliced.  I think it should report an error.  That
would help counting success in our tests.

bluhm

> #include <sys/types.h>
> #include <sys/socket.h>
> #include <sys/stat.h>
> #include <sys/sysctl.h>
> #include <stdio.h>
> #include <err.h>
> #include <errno.h>
> #include <pthread.h>
> #include <string.h>
> #include <time.h>
> #include <unistd.h>
> 
> #include <netinet/in.h>
> 
> static int s;
> 
> static int spliced = 0;
> static int unspliced = 0;
> 
> static void *
> thr_splice(void *arg)
> {
> 	while (1) {
> 		if (setsockopt(s, SOL_SOCKET, SO_SPLICE, &s, sizeof(s)) == 0)
> 			spliced = 1;
> 	}
> 
> 	return NULL;
> }
> 
> static void *
> thr_unsplice(void *arg)
> {
> 	while (1) {
> 		if (setsockopt(s, SOL_SOCKET, SO_SPLICE, NULL, 0) == 0)
> 			unspliced = 1;
> 	}
> 
> 	return NULL;
> }
> 
> int
> main(int argc, char *argv[])
> {
> 	struct timespec testtime = {
> 		.tv_sec = 60,
> 		.tv_nsec = 0,
> 	};
> 
> 	int mib[2], ncpu;
> 	size_t len;
> 
> 	struct sockaddr_in sin;
> 	socklen_t slen;
> 
> 	pthread_t thr;
> 	int i, error;
> 
> 	if (argc == 2 && !strcmp(argv[1], "--infinite"))
> 		testtime.tv_sec = (10 * 365 * 86400);
> 
> 	mib[0] = CTL_HW;
> 	mib[1] = HW_NCPUONLINE;
> 	len = sizeof(ncpu);
> 
> 	if (sysctl(mib, 2, &ncpu, &len, NULL, 0) < 0)
> 		err(1, "sysctl");
> 	if (ncpu <= 0)
> 		errx(1, "Wrong number of CPUs online: %d", ncpu);
> 	if (ncpu == 1)
> 		ncpu = 2;
> 
> 	if ((s = socket(AF_INET, SOCK_DGRAM, 0)) < 0)
> 		err(1, "socket");
> 
> 	memset(&sin, 0, sizeof(sin));
> 	sin.sin_family = AF_INET;
> 	sin.sin_addr.s_addr = INADDR_ANY;
> 
> 	if (bind(s, (struct sockaddr *)&sin, sizeof(sin)) < 0)
> 		err(1, "bind");
> 
> 	slen = sizeof(sin);
> 	if (getsockname(s, (struct sockaddr *)&sin, &slen) < 0)
> 		err(1, "getsockname");
> 
> 	if (connect(s, (struct sockaddr *)&sin, sizeof(sin)) < 0)
> 		err(1, "connect");
> 
> 	for (i = 0; i < ncpu; ++i) {
> 		error = pthread_create(&thr, NULL, thr_splice, NULL);
> 		if (error)
> 			errc(1, error, "pthread_create");
> 		error = pthread_create(&thr, NULL, thr_unsplice, NULL);
> 		if (error)
> 			errc(1, error, "pthread_create");
> 	}
> 
> 	nanosleep(&testtime, NULL);
> 
> 	if (!(spliced && unspliced))
> 		errx(1, "was %sspliced, was %sunspliced",
> 		    spliced ? "" : "not ", unspliced ? "" : "not ");
> 
> 	return 0;
> }