From: Vitaliy Makkoveev <mvs@openbsd.org>
Subject: Switch AF_ROUTE sockets to the new locking scheme
To: tech@openbsd.org
Date: Sat, 25 May 2024 19:26:25 +0300

At sockets layer only mark buffers as SB_MTXLOCK. At PCB layer only
protect `so_rcv' with corresponding `sb_mtx' mutex(9).

Please note, SS_ISCONNECTED and SS_CANTRCVMORE bits are redundant for
AF_ROUTE, and SS_CANTRCVMORE modifications performed with both solock()
and `sb_mtx' held. This makes unlocked SS_CANTRCVMORE safe.

Index: sys/kern/uipc_socket.c
===================================================================
RCS file: /cvs/src/sys/kern/uipc_socket.c,v
diff -u -p -r1.335 uipc_socket.c
--- sys/kern/uipc_socket.c	17 May 2024 19:11:14 -0000	1.335
+++ sys/kern/uipc_socket.c	25 May 2024 16:11:30 -0000
@@ -166,6 +166,7 @@ soalloc(const struct protosw *prp, int w
 		}
 		break;
 	case AF_KEY:
+	case AF_ROUTE:
 	case AF_UNIX:
 		so->so_snd.sb_flags |= SB_MTXLOCK;
 		so->so_rcv.sb_flags |= SB_MTXLOCK;
Index: sys/net/rtsock.c
===================================================================
RCS file: /cvs/src/sys/net/rtsock.c,v
diff -u -p -r1.373 rtsock.c
--- sys/net/rtsock.c	3 Dec 2023 10:51:17 -0000	1.373
+++ sys/net/rtsock.c	25 May 2024 16:11:30 -0000
@@ -313,10 +313,12 @@ route_rcvd(struct socket *so)
 	 * If we are in a FLUSH state, check if the buffer is
 	 * empty so that we can clear the flag.
 	 */
+
+	mtx_enter(&so->so_rcv.sb_mtx);
 	if (((rop->rop_flags & ROUTECB_FLAG_FLUSH) != 0) &&
-	    ((sbspace(rop->rop_socket, &rop->rop_socket->so_rcv) ==
-	    rop->rop_socket->so_rcv.sb_hiwat)))
+	    ((sbspace(so, &so->so_rcv) == so->so_rcv.sb_hiwat)))
 		rop->rop_flags &= ~ROUTECB_FLAG_FLUSH;
+	mtx_leave(&so->so_rcv.sb_mtx);
 }
 
 int
@@ -478,8 +480,14 @@ rtm_senddesync(struct socket *so)
 	 */
 	desync_mbuf = rtm_msg1(RTM_DESYNC, NULL);
 	if (desync_mbuf != NULL) {
-		if (sbappendaddr(so, &so->so_rcv, &route_src,
-		    desync_mbuf, NULL) != 0) {
+		int ret;
+
+		mtx_enter(&so->so_rcv.sb_mtx);
+		ret = sbappendaddr(so, &so->so_rcv, &route_src,
+		    desync_mbuf, NULL);
+		mtx_leave(&so->so_rcv.sb_mtx);
+
+		if (ret != 0) {
 			rop->rop_flags &= ~ROUTECB_FLAG_DESYNC;
 			sorwakeup(rop->rop_socket);
 			return;
@@ -586,6 +594,7 @@ rtm_sendup(struct socket *so, struct mbu
 {
 	struct rtpcb *rop = sotortpcb(so);
 	struct mbuf *m;
+	int send_desync = 0;
 
 	soassertlocked(so);
 
@@ -593,8 +602,13 @@ rtm_sendup(struct socket *so, struct mbu
 	if (m == NULL)
 		return (ENOMEM);
 
+	mtx_enter(&so->so_rcv.sb_mtx);
 	if (sbspace(so, &so->so_rcv) < (2 * MSIZE) ||
-	    sbappendaddr(so, &so->so_rcv, &route_src, m, NULL) == 0) {
+	    sbappendaddr(so, &so->so_rcv, &route_src, m, NULL) == 0)
+		send_desync = 1;
+	mtx_leave(&so->so_rcv.sb_mtx);
+
+	if (send_desync) {
 		/* Flag socket as desync'ed and flush required */
 		rop->rop_flags |= ROUTECB_FLAG_DESYNC | ROUTECB_FLAG_FLUSH;
 		rtm_senddesync(so);