On Mon, Jan 01, 2024 at 07:12:31PM +0100, Mark Kettenis wrote:
> On amd64, we only have 240 interrupt vectors, which means they're a
> limited.  And unless I'm terribly confused, we're not using 48 of
> those because of the way we've assigned the priority levels.  And with
> more and more devices supporting multiple interrupt vectors, we can
> put these to good use.  So this diff rearranges the levels to make the
> additonal vectors available for use.  Since it most likely network
> devices that need these, I leave a gap between between IPL_NET and
> IPL_SOFTTTY.  That way, it will take longer for IPL_NET interrupts to
> spill over into the IPL_SOFTTTY range and mess up the interrupt
> priorities.
> 
> Now I vagelue remember something about trying something like this
> before.  So I want to make sure this doesn't cause any problems.
> Therefore I'd appreciate it if people can test this, especially on
> machines with lots of network devices.

On one of my machine with many interfaces and interrupt issues it got
two more working interfaces.  Looks like ix8 and ix9 are working now.

Thanks,
Jan

--- old.log	Tue Jan  2 19:44:05 2024
+++ new.log	Tue Jan  2 19:44:17 2024
@@ -1,7 +1,7 @@
-OpenBSD 7.4-current (GENERIC.MP) #8: Tue Jan  2 19:14:06 CET 2024
+OpenBSD 7.4-current (GENERIC.MP) #9: Tue Jan  2 19:32:58 CET 2024
     root@ot51.obsd-lab.genua.de:/usr/src/sys/arch/amd64/compile/GENERIC.MP
 real mem = 68338536448 (65172MB)
-avail mem = 66246852608 (63177MB)
+avail mem = 66246856704 (63177MB)
 random: good seed from bootblocks
 mpath0 at root
 scsibus0 at mpath0: 256 targets

... skip cpu attachment diffs ...

@@ -256,16 +256,14 @@
 pchb3 at pci15 dev 0 function 4 vendor "Intel", unknown product 0x0998 rev 0x00
 ppb12 at pci15 dev 2 function 0 vendor "Intel", unknown product 0x347a rev 0x04: msi
 pci16 at ppb12 bus 138
-ix8 at pci16 dev 0 function 0 "Intel 82598EB" rev 0x01failed to allocate interrupt slot for PIC msix pin -2138439677
-ixgbe_allocate_msix: pci_intr_establish vec 3 failed
-ix9 at pci16 dev 0 function 1 "Intel 82598EB" rev 0x01failed to allocate interrupt slot for PIC msix pin -2138439421
-ixgbe_allocate_msix: pci_intr_establish vec 3 failed
+ix8 at pci16 dev 0 function 0 "Intel 82598EB" rev 0x01, msix, 8 queues, address 68:05:ca:1d:e3:0b
+ix9 at pci16 dev 0 function 1 "Intel 82598EB" rev 0x01, msix, 8 queues, address 68:05:ca:1d:e3:0a
 ppb13 at pci15 dev 4 function 0 vendor "Intel", unknown product 0x347c rev 0x04: msi
 pci17 at ppb13 bus 139
-ix10 at pci17 dev 0 function 0 "Intel 82598EB" rev 0x01failed to allocate interrupt slot for PIC msix pin -2138374141
-ixgbe_allocate_msix: pci_intr_establish vec 3 failed
-ix11 at pci17 dev 0 function 1 "Intel 82598EB" rev 0x01failed to allocate interrupt slot for PIC msix pin -2138373885
-ixgbe_allocate_msix: pci_intr_establish vec 3 failed
+ix10 at pci17 dev 0 function 0 "Intel 82598EB" rev 0x01failed to allocate interrupt slot for PIC msix pin -2138374144
+ixgbe_allocate_msix: pci_intr_establish vec 0 failed
+ix11 at pci17 dev 0 function 1 "Intel 82598EB" rev 0x01failed to allocate interrupt slot for PIC msix pin -2138373888
+ixgbe_allocate_msix: pci_intr_establish vec 0 failed
 pci18 at mainbus0 bus 194
 vendor "Intel", unknown product 0x09a2 (class system subclass miscellaneous, rev 0x04) at pci18 dev 0 function 0 not configured
 vendor "Intel", unknown product 0x09a4 (class system subclass miscellaneous, rev 0x04) at pci18 dev 0 function 1 not configured
@@ -273,10 +271,10 @@
 pchb4 at pci18 dev 0 function 4 vendor "Intel", unknown product 0x0998 rev 0x00
 ppb14 at pci18 dev 2 function 0 vendor "Intel", unknown product 0x347a rev 0x04: msi
 pci19 at ppb14 bus 195
-ix12 at pci19 dev 0 function 0 "Intel 82599" rev 0x01failed to allocate interrupt slot for PIC msix pin -2134704126
-ixgbe_allocate_msix: pci_intr_establish vec 2 failed
-ix13 at pci19 dev 0 function 1 "Intel 82599" rev 0x01failed to allocate interrupt slot for PIC msix pin -2134703870
-ixgbe_allocate_msix: pci_intr_establish vec 2 failed
+ix12 at pci19 dev 0 function 0 "Intel 82599" rev 0x01failed to allocate interrupt slot for PIC msix pin -2134704128
+ixgbe_allocate_msix: pci_intr_establish vec 0 failed
+ix13 at pci19 dev 0 function 1 "Intel 82599" rev 0x01failed to allocate interrupt slot for PIC msix pin -2134703872
+ixgbe_allocate_msix: pci_intr_establish vec 0 failed
 pci20 at mainbus0 bus 254
 vendor "Intel", unknown product 0x3450 (class system subclass miscellaneous, rev 0x00) at pci20 dev 0 function 0 not configured
 vendor "Intel", unknown product 0x3451 (class system subclass miscellaneous, rev 0x00) at pci20 dev 0 function 1 not configured

> Index: arch/amd64/include/intrdefs.h
> ===================================================================
> RCS file: /cvs/src/sys/arch/amd64/include/intrdefs.h,v
> retrieving revision 1.22
> diff -u -p -r1.22 intrdefs.h
> --- arch/amd64/include/intrdefs.h	31 Aug 2021 17:40:59 -0000	1.22
> +++ arch/amd64/include/intrdefs.h	1 Jan 2024 17:54:53 -0000
> @@ -19,10 +19,10 @@
>   *
>   */
>  #define	IPL_NONE	0x0	/* nothing */
> -#define	IPL_SOFTCLOCK	0x4	/* timeouts */
> -#define	IPL_SOFTNET	0x5	/* protocol stacks */
> -#define	IPL_BIO		0x6	/* block I/O */
> -#define	IPL_NET		0x7	/* network */
> +#define	IPL_SOFTCLOCK	0x1	/* timeouts */
> +#define	IPL_SOFTNET	0x2	/* protocol stacks */
> +#define	IPL_BIO		0x3	/* block I/O */
> +#define	IPL_NET		0x4	/* network */
>  #define	IPL_SOFTTTY	0x8	/* delayed terminal handling */
>  #define	IPL_TTY		0x9	/* terminal */
>  #define	IPL_VM		0xa	/* memory allocation */
>