From: Alexander Bluhm <bluhm@openbsd.org>
Subject: Re: ice(4): SoftLRO
To: Jan Klemkow <jan@openbsd.org>
Cc: tech@openbsd.org
Date: Fri, 27 Jun 2025 16:37:59 +0200

On Thu, Jun 26, 2025 at 11:33:16PM +0200, Jan Klemkow wrote:
> Hi,
> 
> This diff add SoftLRO support to ice(4) like its done in ixl(4).  It
> doesn't use the ice_decode_rx_desc_ptype() function, because we just
> want two specific cases of TCP inner protocols for our SoftLRO
> mechanism.
> 
> ok?

Depending on the test, I get up to twice the receive performance.
Best is iperf3 with 15 threads receiving TCP streams from Linux.
It goes from 18 GBit to 42 GBit.
http://bluhm.genua.de/netlink/results/2025-06-27T00:42:37Z/netlink.html

OK bluhm@

> Index: dev/pci/if_ice.c
> ===================================================================
> RCS file: /cvs/src/sys/dev/pci/if_ice.c,v
> diff -u -p -r1.51 if_ice.c
> --- dev/pci/if_ice.c	23 Jun 2025 08:03:22 -0000	1.51
> +++ dev/pci/if_ice.c	26 Jun 2025 20:43:15 -0000
> @@ -29152,6 +29152,7 @@ ice_rxeof(struct ice_softc *sc, struct i
>  	struct ice_rx_map *rxm;
>  	bus_dmamap_t map;
>  	unsigned int cons, prod;
> +	struct mbuf_list mltcp = MBUF_LIST_INITIALIZER();
>  	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
>  	struct mbuf *m;
>  	uint16_t status0, ptype;
> @@ -29235,7 +29236,14 @@ ice_rxeof(struct ice_softc *sc, struct i
>  				ICE_RX_FLEX_DESC_PTYPE_M;
>  			ice_rx_checksum(m, status0, ptype);
>  
> -			ml_enqueue(&ml, m);
> +#ifndef SMALL_KERNEL
> +			if (ISSET(ifp->if_xflags, IFXF_LRO) &&
> +			    (ptype == ICE_RX_FLEX_DECS_PTYPE_MAC_IPV4_TCP ||
> +			     ptype == ICE_RX_FLEX_DECS_PTYPE_MAC_IPV6_TCP))
> +				tcp_softlro_glue(&mltcp, m, ifp);
> +			else
> +#endif
> +				ml_enqueue(&ml, m);
>  
>  			rxq->rxq_m_head = NULL;
>  			rxq->rxq_m_tail = &rxq->rxq_m_head;
> @@ -29248,8 +29256,15 @@ ice_rxeof(struct ice_softc *sc, struct i
>  	} while (cons != prod);
>  
>  	if (done) {
> +		int livelocked = 0;
> +
>  		rxq->rxq_cons = cons;
> +		if (ifiq_input(ifiq, &mltcp))
> +			livelocked = 1;
>  		if (ifiq_input(ifiq, &ml))
> +			livelocked = 1;
> +
> +		if (livelocked)
>  			if_rxr_livelocked(&rxq->rxq_acct);
>  		ice_rxfill(sc, rxq);
>  	}
> @@ -30419,6 +30434,11 @@ ice_attach_hook(struct device *self)
>  	    IFCAP_CSUM_TCPv4 | IFCAP_CSUM_UDPv4 |
>  	    IFCAP_CSUM_TCPv6 | IFCAP_CSUM_UDPv6 |
>  	    IFCAP_TSOv4 | IFCAP_TSOv6;
> +	ifp->if_capabilities |= IFCAP_LRO;
> +#if notyet
> +	/* for now tcplro at ice(4) is default off */
> +	ifp->if_xflags |= IFXF_LRO;
> +#endif
>  
>  	if_attach(ifp);
>  	ether_ifattach(ifp);
> Index: dev/pci/if_icereg.h
> ===================================================================
> RCS file: /cvs/src/sys/dev/pci/if_icereg.h,v
> diff -u -p -r1.2 if_icereg.h
> --- dev/pci/if_icereg.h	25 Nov 2024 12:50:47 -0000	1.2
> +++ dev/pci/if_icereg.h	26 Jun 2025 20:34:50 -0000
> @@ -13676,6 +13676,8 @@ enum ice_umbcast_dest_addr_types {
>  
>  /* for ice_32byte_rx_flex_desc.ptype_flexi_flags0 member */
>  #define ICE_RX_FLEX_DESC_PTYPE_M	(0x3FF) /* 10-bits */
> +#define ICE_RX_FLEX_DECS_PTYPE_MAC_IPV4_TCP 26
> +#define ICE_RX_FLEX_DECS_PTYPE_MAC_IPV6_TCP 92
>  
>  enum ice_rx_flex_desc_flexi_flags0_bits { /* field is 6 bits long */
>  	ICE_RX_FLEX_DESC_FLEXI_FLAGS0_S = 10,