From: Vitaliy Makkoveev <otto@bsdbox.dev>
Subject: Re: remove inpcb mutex
To: Alexander Bluhm <bluhm@openbsd.org>
Cc: tech@openbsd.org
Date: Fri, 12 Jul 2024 20:28:37 +0300

> On 12 Jul 2024, at 20:05, Alexander Bluhm <bluhm@openbsd.org> wrote:
> 
> Hi,
> 
> mvs@ has converted all the incpb locking into socket receive buffer
> mutex.  So we don't need per PCB mutex inp_mtx anymore.  Also delete
> related locking functions.  A flag PR_MPSOCKET indicates whether
> protocol functions support parallel access with per socket rw-lock.
> 
> TCP is the only protocol that is not MP capable and needs exclusive
> netlock.
> 
> Also IPv4 GRE does not have the flag, but it may be added.  Next
> diff.
> 
> ok?
> 

Yes please.

> bluhm
> 
> Index: kern/uipc_socket2.c
> ===================================================================
> RCS file: /data/mirror/openbsd/cvs/src/sys/kern/uipc_socket2.c,v
> diff -u -p -r1.156 uipc_socket2.c
> --- kern/uipc_socket2.c	28 Jun 2024 21:30:24 -0000	1.156
> +++ kern/uipc_socket2.c	12 Jul 2024 15:21:16 -0000
> @@ -365,7 +365,7 @@ solock_shared(struct socket *so)
> 	switch (so->so_proto->pr_domain->dom_family) {
> 	case PF_INET:
> 	case PF_INET6:
> -		if (so->so_proto->pr_usrreqs->pru_lock != NULL) {
> +		if (ISSET(so->so_proto->pr_flags, PR_MPSOCKET)) {
> 			NET_LOCK_SHARED();
> 			rw_enter_write(&so->so_lock);
> 		} else
> @@ -425,7 +425,7 @@ sounlock_shared(struct socket *so)
> 	switch (so->so_proto->pr_domain->dom_family) {
> 	case PF_INET:
> 	case PF_INET6:
> -		if (so->so_proto->pr_usrreqs->pru_unlock != NULL) {
> +		if (ISSET(so->so_proto->pr_flags, PR_MPSOCKET)) {
> 			rw_exit_write(&so->so_lock);
> 			NET_UNLOCK_SHARED();
> 		} else
> @@ -460,7 +460,7 @@ soassertlocked(struct socket *so)
> 		if (rw_status(&netlock) == RW_READ) {
> 			NET_ASSERT_LOCKED();
> 
> -			if (splassert_ctl > 0 && pru_locked(so) == 0 &&
> +			if (splassert_ctl > 0 &&
> 			    rw_status(&so->so_lock) != RW_WRITE)
> 				splassert_fail(0, RW_WRITE, __func__);
> 		} else
> @@ -481,12 +481,12 @@ sosleep_nsec(struct socket *so, void *id
> 	switch (so->so_proto->pr_domain->dom_family) {
> 	case PF_INET:
> 	case PF_INET6:
> -		if (so->so_proto->pr_usrreqs->pru_unlock != NULL &&
> +		if (ISSET(so->so_proto->pr_flags, PR_MPSOCKET) &&
> 		    rw_status(&netlock) == RW_READ) {
> 			rw_exit_write(&so->so_lock);
> 		}
> 		ret = rwsleep_nsec(ident, &netlock, prio, wmesg, nsecs);
> -		if (so->so_proto->pr_usrreqs->pru_lock != NULL &&
> +		if (ISSET(so->so_proto->pr_flags, PR_MPSOCKET) &&
> 		    rw_status(&netlock) == RW_READ) {
> 			rw_enter_write(&so->so_lock);
> 		}
> Index: netinet/in_pcb.c
> ===================================================================
> RCS file: /data/mirror/openbsd/cvs/src/sys/netinet/in_pcb.c,v
> diff -u -p -r1.302 in_pcb.c
> --- netinet/in_pcb.c	19 Apr 2024 10:13:58 -0000	1.302
> +++ netinet/in_pcb.c	12 Jul 2024 15:05:18 -0000
> @@ -239,7 +239,6 @@ in_pcballoc(struct socket *so, struct in
> 	inp->inp_table = table;
> 	inp->inp_socket = so;
> 	refcnt_init_trace(&inp->inp_refcnt, DT_REFCNT_IDX_INPCB);
> -	mtx_init(&inp->inp_mtx, IPL_SOFTNET);
> 	inp->inp_seclevel.sl_auth = IPSEC_AUTH_LEVEL_DEFAULT;
> 	inp->inp_seclevel.sl_esp_trans = IPSEC_ESP_TRANS_LEVEL_DEFAULT;
> 	inp->inp_seclevel.sl_esp_network = IPSEC_ESP_NETWORK_LEVEL_DEFAULT;
> Index: netinet/in_pcb.h
> ===================================================================
> RCS file: /data/mirror/openbsd/cvs/src/sys/netinet/in_pcb.h,v
> diff -u -p -r1.157 in_pcb.h
> --- netinet/in_pcb.h	19 Apr 2024 10:13:58 -0000	1.157
> +++ netinet/in_pcb.h	12 Jul 2024 15:31:46 -0000
> @@ -80,7 +80,6 @@
>  *	N	net lock
>  *	t	inpt_mtx		pcb table mutex
>  *	y	inpt_notify		pcb table rwlock for notify
> - *	p	inpcb_mtx		pcb mutex
>  *	L	pf_inp_mtx		link pf to inp mutex
>  *	s	so_lock			socket rwlock
>  */
> @@ -94,8 +93,8 @@
>  * needed, so that socket layer input have a consistent view at these
>  * values.
>  *
> - * In soconnect() and sosend() pcb mutex cannot be used.  They eventually
> - * can call IP output which takes pf lock which is a sleeping lock.
> + * In soconnect() and sosend() a per pcb mutex cannot be used.  They
> + * eventually call IP output which takes pf lock which is a sleeping lock.
>  * Also connect(2) does a route lookup for source selection.  There
>  * route resolve happens, which creates a route, which sends a route
>  * message, which needs route lock, which is a rw-lock.
> @@ -106,15 +105,9 @@
>  *
>  * So there are three locks.  Table mutex is for writing inp_[lf]addr/port
>  * and lookup, socket rw-lock to separate sockets in system calls, and
> - * pcb mutex to protect socket receive buffer.  Changing inp_[lf]addr/port
> - * takes both per socket rw-lock and global table mutex.  Protocol
> - * input only reads inp_[lf]addr/port during lookup and is safe.  System
> - * call only reads when holding socket rw-lock and is safe.  The socket
> - * layer needs pcb mutex only in soreceive().
> - *
> - * Function pru_lock() grabs the pcb mutex and its existence indicates
> - * that a protocol is MP safe.  Otherwise the exclusive net lock is
> - * used.
> + * socket buffer mutex to protect socket receive buffer.  Changing
> + * inp_[lf]addr/port takes both per socket rw-lock and global table mutex.
> + * Protocol input only reads inp_[lf]addr/port during lookup and is safe.
>  */
> 
> struct pf_state_key;
> @@ -149,7 +142,6 @@ struct inpcb {
> 	caddr_t	  inp_ppcb;		/* pointer to per-protocol pcb */
> 	struct    route inp_route;	/* cached route */
> 	struct    refcnt inp_refcnt;	/* refcount PCB, delay memory free */
> -	struct	  mutex inp_mtx;	/* protect PCB and socket members */
> 	int	  inp_flags;		/* generic IP/datagram flags */
> 	union {				/* Header prototype. */
> 		struct ip hu_ip;
> Index: netinet/in_proto.c
> ===================================================================
> RCS file: /data/mirror/openbsd/cvs/src/sys/netinet/in_proto.c,v
> diff -u -p -r1.104 in_proto.c
> --- netinet/in_proto.c	14 Apr 2024 20:46:27 -0000	1.104
> +++ netinet/in_proto.c	12 Jul 2024 15:26:38 -0000
> @@ -185,7 +185,7 @@ const struct protosw inetsw[] = {
>   .pr_type	= SOCK_DGRAM,
>   .pr_domain	= &inetdomain,
>   .pr_protocol	= IPPROTO_UDP,
> -  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_SPLICE,
> +  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_SPLICE|PR_MPSOCKET,
>   .pr_input	= udp_input,
>   .pr_ctlinput	= udp_ctlinput,
>   .pr_ctloutput	= ip_ctloutput,
> @@ -210,7 +210,7 @@ const struct protosw inetsw[] = {
>   .pr_type	= SOCK_RAW,
>   .pr_domain	= &inetdomain,
>   .pr_protocol	= IPPROTO_RAW,
> -  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_MPINPUT,
> +  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_MPINPUT|PR_MPSOCKET,
>   .pr_input	= rip_input,
>   .pr_ctloutput	= rip_ctloutput,
>   .pr_usrreqs	= &rip_usrreqs,
> @@ -219,7 +219,7 @@ const struct protosw inetsw[] = {
>   .pr_type	= SOCK_RAW,
>   .pr_domain	= &inetdomain,
>   .pr_protocol	= IPPROTO_ICMP,
> -  .pr_flags	= PR_ATOMIC|PR_ADDR,
> +  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_MPSOCKET,
>   .pr_input	= icmp_input,
>   .pr_ctloutput	= rip_ctloutput,
>   .pr_usrreqs	= &rip_usrreqs,
> @@ -230,7 +230,7 @@ const struct protosw inetsw[] = {
>   .pr_type	= SOCK_RAW,
>   .pr_domain	= &inetdomain,
>   .pr_protocol	= IPPROTO_IPV4,
> -  .pr_flags	= PR_ATOMIC|PR_ADDR,
> +  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_MPSOCKET,
> #if NGIF > 0
>   .pr_input	= in_gif_input,
> #else
> @@ -246,7 +246,7 @@ const struct protosw inetsw[] = {
>   .pr_type	= SOCK_RAW,
>   .pr_domain	= &inetdomain,
>   .pr_protocol	= IPPROTO_IPV6,
> -  .pr_flags	= PR_ATOMIC|PR_ADDR,
> +  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_MPSOCKET,
> #if NGIF > 0
>   .pr_input	= in_gif_input,
> #else
> @@ -261,7 +261,7 @@ const struct protosw inetsw[] = {
>   .pr_type	= SOCK_RAW,
>   .pr_domain	= &inetdomain,
>   .pr_protocol	= IPPROTO_MPLS,
> -  .pr_flags	= PR_ATOMIC|PR_ADDR,
> +  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_MPSOCKET,
>   .pr_input	= in_gif_input,
>   .pr_usrreqs	= &rip_usrreqs,
> },
> @@ -270,7 +270,7 @@ const struct protosw inetsw[] = {
>   .pr_type	= SOCK_RAW,
>   .pr_domain	= &inetdomain,
>   .pr_protocol	= IPPROTO_IGMP,
> -  .pr_flags	= PR_ATOMIC|PR_ADDR,
> +  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_MPSOCKET,
>   .pr_input	= igmp_input,
>   .pr_ctloutput	= rip_ctloutput,
>   .pr_usrreqs	= &rip_usrreqs,
> @@ -284,7 +284,7 @@ const struct protosw inetsw[] = {
>   .pr_type	= SOCK_RAW,
>   .pr_domain	= &inetdomain,
>   .pr_protocol	= IPPROTO_AH,
> -  .pr_flags	= PR_ATOMIC|PR_ADDR,
> +  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_MPSOCKET,
>   .pr_input	= ah46_input,
>   .pr_ctlinput	= ah4_ctlinput,
>   .pr_ctloutput	= rip_ctloutput,
> @@ -295,7 +295,7 @@ const struct protosw inetsw[] = {
>   .pr_type	= SOCK_RAW,
>   .pr_domain	= &inetdomain,
>   .pr_protocol	= IPPROTO_ESP,
> -  .pr_flags	= PR_ATOMIC|PR_ADDR,
> +  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_MPSOCKET,
>   .pr_input	= esp46_input,
>   .pr_ctlinput	= esp4_ctlinput,
>   .pr_ctloutput	= rip_ctloutput,
> @@ -306,7 +306,7 @@ const struct protosw inetsw[] = {
>   .pr_type	= SOCK_RAW,
>   .pr_domain	= &inetdomain,
>   .pr_protocol	= IPPROTO_IPCOMP,
> -  .pr_flags	= PR_ATOMIC|PR_ADDR,
> +  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_MPSOCKET,
>   .pr_input	= ipcomp46_input,
>   .pr_ctloutput	= rip_ctloutput,
>   .pr_usrreqs	= &rip_usrreqs,
> @@ -330,7 +330,7 @@ const struct protosw inetsw[] = {
>   .pr_type	= SOCK_RAW,
>   .pr_domain	= &inetdomain,
>   .pr_protocol	= IPPROTO_CARP,
> -  .pr_flags	= PR_ATOMIC|PR_ADDR,
> +  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_MPSOCKET,
>   .pr_input	= carp_proto_input,
>   .pr_ctloutput	= rip_ctloutput,
>   .pr_usrreqs	= &rip_usrreqs,
> @@ -342,7 +342,7 @@ const struct protosw inetsw[] = {
>   .pr_type	= SOCK_RAW,
>   .pr_domain	= &inetdomain,
>   .pr_protocol	= IPPROTO_PFSYNC,
> -  .pr_flags	= PR_ATOMIC|PR_ADDR,
> +  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_MPSOCKET,
>   .pr_input	= pfsync_input4,
>   .pr_ctloutput	= rip_ctloutput,
>   .pr_usrreqs	= &rip_usrreqs,
> @@ -354,7 +354,7 @@ const struct protosw inetsw[] = {
>   .pr_type	= SOCK_RAW,
>   .pr_domain	= &inetdomain,
>   .pr_protocol	= IPPROTO_DIVERT,
> -  .pr_flags	= PR_ATOMIC|PR_ADDR,
> +  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_MPSOCKET,
>   .pr_ctloutput	= rip_ctloutput,
>   .pr_usrreqs	= &divert_usrreqs,
>   .pr_init	= divert_init,
> @@ -366,7 +366,7 @@ const struct protosw inetsw[] = {
>   .pr_type	= SOCK_RAW,
>   .pr_domain	= &inetdomain,
>   .pr_protocol	= IPPROTO_ETHERIP,
> -  .pr_flags	= PR_ATOMIC|PR_ADDR,
> +  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_MPSOCKET,
>   .pr_input	= ip_etherip_input,
>   .pr_ctloutput	= rip_ctloutput,
>   .pr_usrreqs	= &rip_usrreqs,
> @@ -377,7 +377,7 @@ const struct protosw inetsw[] = {
>   /* raw wildcard */
>   .pr_type	= SOCK_RAW,
>   .pr_domain	= &inetdomain,
> -  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_MPINPUT,
> +  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_MPINPUT|PR_MPSOCKET,
>   .pr_input	= rip_input,
>   .pr_ctloutput	= rip_ctloutput,
>   .pr_usrreqs	= &rip_usrreqs,
> Index: netinet/ip_divert.c
> ===================================================================
> RCS file: /data/mirror/openbsd/cvs/src/sys/netinet/ip_divert.c,v
> diff -u -p -r1.95 ip_divert.c
> --- netinet/ip_divert.c	5 Mar 2024 09:45:13 -0000	1.95
> +++ netinet/ip_divert.c	12 Jul 2024 15:12:30 -0000
> @@ -65,9 +65,6 @@ const struct sysctl_bounded_args divertc
> const struct pr_usrreqs divert_usrreqs = {
> 	.pru_attach	= divert_attach,
> 	.pru_detach	= divert_detach,
> -	.pru_lock	= divert_lock,
> -	.pru_unlock	= divert_unlock,
> -	.pru_locked	= divert_locked,
> 	.pru_bind	= divert_bind,
> 	.pru_shutdown	= divert_shutdown,
> 	.pru_send	= divert_send,
> @@ -294,32 +291,6 @@ divert_detach(struct socket *so)
> 
> 	in_pcbdetach(inp);
> 	return (0);
> -}
> -
> -void
> -divert_lock(struct socket *so)
> -{
> -	struct inpcb *inp = sotoinpcb(so);
> -
> -	NET_ASSERT_LOCKED();
> -	mtx_enter(&inp->inp_mtx);
> -}
> -
> -void
> -divert_unlock(struct socket *so)
> -{
> -	struct inpcb *inp = sotoinpcb(so);
> -
> -	NET_ASSERT_LOCKED();
> -	mtx_leave(&inp->inp_mtx);
> -}
> -
> -int
> -divert_locked(struct socket *so)
> -{
> -	struct inpcb *inp = sotoinpcb(so);
> -
> -	return mtx_owned(&inp->inp_mtx);
> }
> 
> int
> Index: netinet/ip_divert.h
> ===================================================================
> RCS file: /data/mirror/openbsd/cvs/src/sys/netinet/ip_divert.h,v
> diff -u -p -r1.25 ip_divert.h
> --- netinet/ip_divert.h	3 Feb 2024 22:50:09 -0000	1.25
> +++ netinet/ip_divert.h	12 Jul 2024 16:56:44 -0000
> @@ -72,9 +72,6 @@ void	 divert_packet(struct mbuf *, int, 
> int	 divert_sysctl(int *, u_int, void *, size_t *, void *, size_t);
> int	 divert_attach(struct socket *, int, int);
> int	 divert_detach(struct socket *);
> -void	 divert_lock(struct socket *);
> -void	 divert_unlock(struct socket *);
> -int	 divert_locked(struct socket *);
> int	 divert_bind(struct socket *, struct mbuf *, struct proc *);
> int	 divert_shutdown(struct socket *);
> int	 divert_send(struct socket *, struct mbuf *, struct mbuf *,
> Index: netinet/ip_var.h
> ===================================================================
> RCS file: /data/mirror/openbsd/cvs/src/sys/netinet/ip_var.h,v
> diff -u -p -r1.119 ip_var.h
> --- netinet/ip_var.h	2 Jul 2024 18:33:47 -0000	1.119
> +++ netinet/ip_var.h	12 Jul 2024 16:58:57 -0000
> @@ -274,9 +274,6 @@ struct mbuf *
> 	 rip_chkhdr(struct mbuf *, struct mbuf *);
> int	 rip_attach(struct socket *, int, int);
> int	 rip_detach(struct socket *);
> -void	 rip_lock(struct socket *);
> -void	 rip_unlock(struct socket *);
> -int	 rip_locked(struct socket *);
> int	 rip_bind(struct socket *, struct mbuf *, struct proc *);
> int	 rip_connect(struct socket *, struct mbuf *);
> int	 rip_disconnect(struct socket *);
> Index: netinet/raw_ip.c
> ===================================================================
> RCS file: /data/mirror/openbsd/cvs/src/sys/netinet/raw_ip.c,v
> diff -u -p -r1.159 raw_ip.c
> --- netinet/raw_ip.c	17 Apr 2024 20:48:51 -0000	1.159
> +++ netinet/raw_ip.c	12 Jul 2024 15:21:27 -0000
> @@ -106,9 +106,6 @@ struct inpcbtable rawcbtable;
> const struct pr_usrreqs rip_usrreqs = {
> 	.pru_attach	= rip_attach,
> 	.pru_detach	= rip_detach,
> -	.pru_lock	= rip_lock,
> -	.pru_unlock	= rip_unlock,
> -	.pru_locked	= rip_locked,
> 	.pru_bind	= rip_bind,
> 	.pru_connect	= rip_connect,
> 	.pru_disconnect	= rip_disconnect,
> @@ -512,32 +509,6 @@ rip_detach(struct socket *so)
> 	in_pcbdetach(inp);
> 
> 	return (0);
> -}
> -
> -void
> -rip_lock(struct socket *so)
> -{
> -	struct inpcb *inp = sotoinpcb(so);
> -
> -	NET_ASSERT_LOCKED();
> -	mtx_enter(&inp->inp_mtx);
> -}
> -
> -void
> -rip_unlock(struct socket *so)
> -{
> -	struct inpcb *inp = sotoinpcb(so);
> -
> -	NET_ASSERT_LOCKED();
> -	mtx_leave(&inp->inp_mtx);
> -}
> -
> -int
> -rip_locked(struct socket *so)
> -{
> -	struct inpcb *inp = sotoinpcb(so);
> -
> -	return mtx_owned(&inp->inp_mtx);
> }
> 
> int
> Index: netinet/udp_usrreq.c
> ===================================================================
> RCS file: /data/mirror/openbsd/cvs/src/sys/netinet/udp_usrreq.c,v
> diff -u -p -r1.320 udp_usrreq.c
> --- netinet/udp_usrreq.c	17 Apr 2024 20:48:51 -0000	1.320
> +++ netinet/udp_usrreq.c	12 Jul 2024 15:22:07 -0000
> @@ -125,9 +125,6 @@ u_int	udp_recvspace = 40 * (1024 + sizeo
> const struct pr_usrreqs udp_usrreqs = {
> 	.pru_attach	= udp_attach,
> 	.pru_detach	= udp_detach,
> -	.pru_lock	= udp_lock,
> -	.pru_unlock	= udp_unlock,
> -	.pru_locked	= udp_locked,
> 	.pru_bind	= udp_bind,
> 	.pru_connect	= udp_connect,
> 	.pru_disconnect	= udp_disconnect,
> @@ -142,9 +139,6 @@ const struct pr_usrreqs udp_usrreqs = {
> const struct pr_usrreqs udp6_usrreqs = {
> 	.pru_attach	= udp_attach,
> 	.pru_detach	= udp_detach,
> -	.pru_lock	= udp_lock,
> -	.pru_unlock	= udp_unlock,
> -	.pru_locked	= udp_locked,
> 	.pru_bind	= udp_bind,
> 	.pru_connect	= udp_connect,
> 	.pru_disconnect	= udp_disconnect,
> @@ -1138,32 +1132,6 @@ udp_detach(struct socket *so)
> 
> 	in_pcbdetach(inp);
> 	return (0);
> -}
> -
> -void
> -udp_lock(struct socket *so)
> -{
> -	struct inpcb *inp = sotoinpcb(so);
> -
> -	NET_ASSERT_LOCKED();
> -	mtx_enter(&inp->inp_mtx);
> -}
> -
> -void
> -udp_unlock(struct socket *so)
> -{
> -	struct inpcb *inp = sotoinpcb(so);
> -
> -	NET_ASSERT_LOCKED();
> -	mtx_leave(&inp->inp_mtx);
> -}
> -
> -int
> -udp_locked(struct socket *so)
> -{
> -	struct inpcb *inp = sotoinpcb(so);
> -
> -	return mtx_owned(&inp->inp_mtx);
> }
> 
> int
> Index: netinet/udp_var.h
> ===================================================================
> RCS file: /data/mirror/openbsd/cvs/src/sys/netinet/udp_var.h,v
> diff -u -p -r1.51 udp_var.h
> --- netinet/udp_var.h	3 Feb 2024 22:50:09 -0000	1.51
> +++ netinet/udp_var.h	12 Jul 2024 16:57:41 -0000
> @@ -145,9 +145,6 @@ int	 udp6_output(struct inpcb *, struct 
> int	 udp_sysctl(int *, u_int, void *, size_t *, void *, size_t);
> int	 udp_attach(struct socket *, int, int);
> int	 udp_detach(struct socket *);
> -void	 udp_lock(struct socket *);
> -void	 udp_unlock(struct socket *);
> -int	 udp_locked(struct socket *);
> int	 udp_bind(struct socket *, struct mbuf *, struct proc *);
> int	 udp_connect(struct socket *, struct mbuf *);
> int	 udp_disconnect(struct socket *);
> Index: netinet6/in6_proto.c
> ===================================================================
> RCS file: /data/mirror/openbsd/cvs/src/sys/netinet6/in6_proto.c,v
> diff -u -p -r1.114 in6_proto.c
> --- netinet6/in6_proto.c	16 Apr 2024 12:40:40 -0000	1.114
> +++ netinet6/in6_proto.c	12 Jul 2024 16:55:40 -0000
> @@ -136,7 +136,7 @@ const struct protosw inet6sw[] = {
>   .pr_type	= SOCK_DGRAM,
>   .pr_domain	= &inet6domain,
>   .pr_protocol	= IPPROTO_UDP,
> -  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_SPLICE,
> +  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_SPLICE|PR_MPSOCKET,
>   .pr_input	= udp_input,
>   .pr_ctlinput	= udp6_ctlinput,
>   .pr_ctloutput	= ip6_ctloutput,
> @@ -158,7 +158,7 @@ const struct protosw inet6sw[] = {
>   .pr_type	= SOCK_RAW,
>   .pr_domain	= &inet6domain,
>   .pr_protocol	= IPPROTO_RAW,
> -  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_MPINPUT,
> +  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_MPINPUT|PR_MPSOCKET,
>   .pr_input	= rip6_input,
>   .pr_ctlinput	= rip6_ctlinput,
>   .pr_ctloutput	= rip6_ctloutput,
> @@ -169,7 +169,7 @@ const struct protosw inet6sw[] = {
>   .pr_type	= SOCK_RAW,
>   .pr_domain	= &inet6domain,
>   .pr_protocol	= IPPROTO_ICMPV6,
> -  .pr_flags	= PR_ATOMIC|PR_ADDR,
> +  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_MPSOCKET,
>   .pr_input	= icmp6_input,
>   .pr_ctlinput	= rip6_ctlinput,
>   .pr_ctloutput	= rip6_ctloutput,
> @@ -204,7 +204,7 @@ const struct protosw inet6sw[] = {
>   .pr_type	= SOCK_RAW,
>   .pr_domain	= &inet6domain,
>   .pr_protocol	= IPPROTO_AH,
> -  .pr_flags	= PR_ATOMIC|PR_ADDR,
> +  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_MPSOCKET,
>   .pr_input	= ah46_input,
>   .pr_ctloutput	= rip6_ctloutput,
>   .pr_usrreqs	= &rip6_usrreqs,
> @@ -214,7 +214,7 @@ const struct protosw inet6sw[] = {
>   .pr_type	= SOCK_RAW,
>   .pr_domain	= &inet6domain,
>   .pr_protocol	= IPPROTO_ESP,
> -  .pr_flags	= PR_ATOMIC|PR_ADDR,
> +  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_MPSOCKET,
>   .pr_input	= esp46_input,
>   .pr_ctloutput	= rip6_ctloutput,
>   .pr_usrreqs	= &rip6_usrreqs,
> @@ -224,7 +224,7 @@ const struct protosw inet6sw[] = {
>   .pr_type	= SOCK_RAW,
>   .pr_domain	= &inet6domain,
>   .pr_protocol	= IPPROTO_IPCOMP,
> -  .pr_flags	= PR_ATOMIC|PR_ADDR,
> +  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_MPSOCKET,
>   .pr_input	= ipcomp46_input,
>   .pr_ctloutput	= rip6_ctloutput,
>   .pr_usrreqs	= &rip6_usrreqs,
> @@ -235,7 +235,7 @@ const struct protosw inet6sw[] = {
>   .pr_type	= SOCK_RAW,
>   .pr_domain	= &inet6domain,
>   .pr_protocol	= IPPROTO_IPV4,
> -  .pr_flags	= PR_ATOMIC|PR_ADDR,
> +  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_MPSOCKET,
> #if NGIF > 0
>   .pr_input	= in6_gif_input,
> #else
> @@ -248,7 +248,7 @@ const struct protosw inet6sw[] = {
>   .pr_type	= SOCK_RAW,
>   .pr_domain	= &inet6domain,
>   .pr_protocol	= IPPROTO_IPV6,
> -  .pr_flags	= PR_ATOMIC|PR_ADDR,
> +  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_MPSOCKET,
> #if NGIF > 0
>   .pr_input	= in6_gif_input,
> #else
> @@ -262,7 +262,7 @@ const struct protosw inet6sw[] = {
>   .pr_type	= SOCK_RAW,
>   .pr_domain	= &inet6domain,
>   .pr_protocol	= IPPROTO_MPLS,
> -  .pr_flags	= PR_ATOMIC|PR_ADDR,
> +  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_MPSOCKET,
> #if NGIF > 0
>   .pr_input	= in6_gif_input,
> #else
> @@ -277,7 +277,7 @@ const struct protosw inet6sw[] = {
>   .pr_type	= SOCK_RAW,
>   .pr_domain	= &inet6domain,
>   .pr_protocol	= IPPROTO_CARP,
> -  .pr_flags	= PR_ATOMIC|PR_ADDR,
> +  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_MPSOCKET,
>   .pr_input	= carp6_proto_input,
>   .pr_ctloutput = rip6_ctloutput,
>   .pr_usrreqs	= &rip6_usrreqs,
> @@ -289,7 +289,7 @@ const struct protosw inet6sw[] = {
>   .pr_type	= SOCK_RAW,
>   .pr_domain	= &inet6domain,
>   .pr_protocol	= IPPROTO_DIVERT,
> -  .pr_flags	= PR_ATOMIC|PR_ADDR,
> +  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_MPSOCKET,
>   .pr_ctloutput	= rip6_ctloutput,
>   .pr_usrreqs	= &divert6_usrreqs,
>   .pr_init	= divert6_init,
> @@ -301,7 +301,7 @@ const struct protosw inet6sw[] = {
>   .pr_type	= SOCK_RAW,
>   .pr_domain	= &inet6domain,
>   .pr_protocol	= IPPROTO_ETHERIP,
> -  .pr_flags	= PR_ATOMIC|PR_ADDR,
> +  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_MPSOCKET,
>   .pr_input	= ip6_etherip_input,
>   .pr_ctloutput	= rip6_ctloutput,
>   .pr_usrreqs	= &rip6_usrreqs,
> @@ -312,7 +312,7 @@ const struct protosw inet6sw[] = {
>   .pr_type	= SOCK_RAW,
>   .pr_domain	= &inet6domain,
>   .pr_protocol	= IPPROTO_GRE,
> -  .pr_flags	= PR_ATOMIC|PR_ADDR,
> +  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_MPSOCKET,
>   .pr_input	= gre_input6,
>   .pr_ctloutput	= rip6_ctloutput,
>   .pr_usrreqs	= &rip6_usrreqs,
> @@ -322,7 +322,7 @@ const struct protosw inet6sw[] = {
>   /* raw wildcard */
>   .pr_type	= SOCK_RAW,
>   .pr_domain	= &inet6domain,
> -  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_MPINPUT,
> +  .pr_flags	= PR_ATOMIC|PR_ADDR|PR_MPINPUT|PR_MPSOCKET,
>   .pr_input	= rip6_input,
>   .pr_ctloutput	= rip6_ctloutput,
>   .pr_usrreqs	= &rip6_usrreqs,
> Index: netinet6/ip6_divert.c
> ===================================================================
> RCS file: /data/mirror/openbsd/cvs/src/sys/netinet6/ip6_divert.c,v
> diff -u -p -r1.95 ip6_divert.c
> --- netinet6/ip6_divert.c	13 Feb 2024 12:22:09 -0000	1.95
> +++ netinet6/ip6_divert.c	12 Jul 2024 15:25:30 -0000
> @@ -68,9 +68,6 @@ const struct sysctl_bounded_args divert6
> const struct pr_usrreqs divert6_usrreqs = {
> 	.pru_attach	= divert6_attach,
> 	.pru_detach	= divert_detach,
> -	.pru_lock	= divert_lock,
> -	.pru_unlock	= divert_unlock,
> -	.pru_locked	= divert_locked,
> 	.pru_bind	= divert_bind,
> 	.pru_shutdown	= divert_shutdown,
> 	.pru_send	= divert6_send,
> Index: netinet6/ip6_var.h
> ===================================================================
> RCS file: /data/mirror/openbsd/cvs/src/sys/netinet6/ip6_var.h,v
> diff -u -p -r1.119 ip6_var.h
> --- netinet6/ip6_var.h	4 Jul 2024 12:50:08 -0000	1.119
> +++ netinet6/ip6_var.h	12 Jul 2024 16:58:52 -0000
> @@ -354,9 +354,6 @@ int	rip6_output(struct mbuf *, struct so
> 	    struct mbuf *);
> int	rip6_attach(struct socket *, int, int);
> int	rip6_detach(struct socket *);
> -void	rip6_lock(struct socket *);
> -void	rip6_unlock(struct socket *);
> -int	rip6_locked(struct socket *);
> int	rip6_bind(struct socket *, struct mbuf *, struct proc *);
> int	rip6_connect(struct socket *, struct mbuf *);
> int	rip6_disconnect(struct socket *);
> Index: netinet6/raw_ip6.c
> ===================================================================
> RCS file: /data/mirror/openbsd/cvs/src/sys/netinet6/raw_ip6.c,v
> diff -u -p -r1.184 raw_ip6.c
> --- netinet6/raw_ip6.c	17 Apr 2024 20:48:51 -0000	1.184
> +++ netinet6/raw_ip6.c	12 Jul 2024 15:23:34 -0000
> @@ -108,9 +108,6 @@ struct cpumem *rip6counters;
> const struct pr_usrreqs rip6_usrreqs = {
> 	.pru_attach	= rip6_attach,
> 	.pru_detach	= rip6_detach,
> -	.pru_lock	= rip6_lock,
> -	.pru_unlock	= rip6_unlock,
> -	.pru_locked	= rip6_locked,
> 	.pru_bind	= rip6_bind,
> 	.pru_connect	= rip6_connect,
> 	.pru_disconnect	= rip6_disconnect,
> @@ -642,32 +639,6 @@ rip6_detach(struct socket *so)
> 	in_pcbdetach(inp);
> 
> 	return (0);
> -}
> -
> -void
> -rip6_lock(struct socket *so)
> -{
> -	struct inpcb *inp = sotoinpcb(so);
> -
> -	NET_ASSERT_LOCKED();
> -	mtx_enter(&inp->inp_mtx);
> -}
> -
> -void
> -rip6_unlock(struct socket *so)
> -{
> -	struct inpcb *inp = sotoinpcb(so);
> -
> -	NET_ASSERT_LOCKED();
> -	mtx_leave(&inp->inp_mtx);
> -}
> -
> -int
> -rip6_locked(struct socket *so)
> -{
> -	struct inpcb *inp = sotoinpcb(so);
> -
> -	return mtx_owned(&inp->inp_mtx);
> }
> 
> int
> Index: sys/protosw.h
> ===================================================================
> RCS file: /data/mirror/openbsd/cvs/src/sys/sys/protosw.h,v
> diff -u -p -r1.66 protosw.h
> --- sys/protosw.h	14 Apr 2024 20:46:27 -0000	1.66
> +++ sys/protosw.h	12 Jul 2024 15:26:18 -0000
> @@ -67,9 +67,6 @@ struct ifnet;
> struct pr_usrreqs {
> 	int	(*pru_attach)(struct socket *, int, int);
> 	int	(*pru_detach)(struct socket *);
> -	void	(*pru_lock)(struct socket *);
> -	void	(*pru_unlock)(struct socket *);
> -	int	(*pru_locked)(struct socket *so);
> 	int	(*pru_bind)(struct socket *, struct mbuf *, struct proc *);
> 	int	(*pru_listen)(struct socket *);
> 	int	(*pru_connect)(struct socket *, struct mbuf *);
> @@ -133,6 +130,7 @@ struct protosw {
> 					   socket */
> #define PR_SPLICE	0x0040		/* socket splicing is possible */
> #define PR_MPINPUT	0x0080		/* input runs with shared netlock */
> +#define PR_MPSOCKET	0x0100		/* socket uses shared netlock */
> 
> /*
>  * The arguments to usrreq are:
> @@ -282,28 +280,6 @@ static inline int
> pru_detach(struct socket *so)
> {
> 	return (*so->so_proto->pr_usrreqs->pru_detach)(so);
> -}
> -
> -static inline void
> -pru_lock(struct socket *so)
> -{
> -	if (so->so_proto->pr_usrreqs->pru_lock)
> -		(*so->so_proto->pr_usrreqs->pru_lock)(so);
> -}
> -
> -static inline void
> -pru_unlock(struct socket *so)
> -{
> -	if (so->so_proto->pr_usrreqs->pru_unlock)
> -		(*so->so_proto->pr_usrreqs->pru_unlock)(so);
> -}
> -
> -static inline int
> -pru_locked(struct socket *so)
> -{
> -	if (so->so_proto->pr_usrreqs->pru_locked)
> -		return (*so->so_proto->pr_usrreqs->pru_locked)(so);
> -	return (0);
> }
> 
> static inline int
>