From: Stefan Fritsch <sf@openbsd.org>
Subject: vio(4) multi-queue V9
To: tech@openbsd.org
Date: Mon, 25 Nov 2024 20:50:26 +0100

Hi,

this is the vio multi-queue diff, updated to -current and with a bug fix 
for a crash found by Mark Patruck if the number of cpus was lower than the 
number of available queues.

Cheers,
Stefan

diff --git a/share/man/man4/vio.4 b/share/man/man4/vio.4
index e3c713941a0..ce4ba88241d 100644
--- a/share/man/man4/vio.4
+++ b/share/man/man4/vio.4
@@ -34,11 +34,6 @@ Setting the bit 0x2 in the flags disables the RingEventIndex feature.
 This can be tried as a workaround for possible bugs in host implementations of
 .Nm
 at the cost of slightly reduced performance.
-.Pp
-Setting the bit 0x100 in the flags forces the interface to be always in
-promiscuous mode.
-This can be used as a workaround for a bug in QEMU before version 1.7.2 that
-prevents packets with a VLAN tag from being sent to the guest.
 .Sh SEE ALSO
 .Xr intro 4 ,
 .Xr virtio 4
diff --git a/sys/dev/fdt/virtio_mmio.c b/sys/dev/fdt/virtio_mmio.c
index 5d7a6db6465..b7aa80b5d77 100644
--- a/sys/dev/fdt/virtio_mmio.c
+++ b/sys/dev/fdt/virtio_mmio.c
@@ -98,12 +98,15 @@ void		virtio_mmio_write_device_config_8(struct virtio_softc *, int, uint64_t);
 uint16_t	virtio_mmio_read_queue_size(struct virtio_softc *, uint16_t);
 void		virtio_mmio_setup_queue(struct virtio_softc *, struct virtqueue *, uint64_t);
 void		virtio_mmio_setup_intrs(struct virtio_softc *);
+int		virtio_mmio_attach_finish(struct virtio_softc *, struct virtio_attach_args *);
 int		virtio_mmio_get_status(struct virtio_softc *);
 void		virtio_mmio_set_status(struct virtio_softc *, int);
 int		virtio_mmio_negotiate_features(struct virtio_softc *,
     const struct virtio_feature_name *);
 int		virtio_mmio_intr(void *);
 void		virtio_mmio_intr_barrier(struct virtio_softc *);
+int		virtio_mmio_intr_establish(struct virtio_softc *, struct virtio_attach_args *,
+    int, struct cpu_info *, int (*)(void *), void *);
 
 struct virtio_mmio_softc {
 	struct virtio_softc	sc_sc;
@@ -119,6 +122,11 @@ struct virtio_mmio_softc {
 	uint32_t		sc_version;
 };
 
+struct virtio_mmio_attach_args {
+	struct virtio_attach_args	 vma_va;
+	struct fdt_attach_args          *vma_fa;
+};
+
 const struct cfattach virtio_mmio_ca = {
 	sizeof(struct virtio_mmio_softc),
 	virtio_mmio_match,
@@ -151,8 +159,10 @@ const struct virtio_ops virtio_mmio_ops = {
 	virtio_mmio_get_status,
 	virtio_mmio_set_status,
 	virtio_mmio_negotiate_features,
+	virtio_mmio_attach_finish,
 	virtio_mmio_intr,
 	virtio_mmio_intr_barrier,
+	virtio_mmio_intr_establish,
 };
 
 uint16_t
@@ -250,7 +260,7 @@ virtio_mmio_attach(struct device *parent, struct device *self, void *aux)
 	struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)self;
 	struct virtio_softc *vsc = &sc->sc_sc;
 	uint32_t id, magic;
-	struct virtio_attach_args va = { 0 };
+	struct virtio_mmio_attach_args vma = { { 0 }, faa };
 
 	if (faa->fa_nreg < 1) {
 		printf(": no register data\n");
@@ -299,36 +309,43 @@ virtio_mmio_attach(struct device *parent, struct device *self, void *aux)
 	virtio_mmio_set_status(vsc, VIRTIO_CONFIG_DEVICE_STATUS_ACK);
 	virtio_mmio_set_status(vsc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER);
 
-	va.va_devid = id;
-	va.va_nintr = 1;
+	vma.vma_va.va_devid = id;
+	vma.vma_va.va_nintr = 1;
 	vsc->sc_child = NULL;
-	config_found(self, &va, NULL);
+	config_found(self, &vma, NULL);
 	if (vsc->sc_child == NULL) {
 		printf("%s: no matching child driver; not configured\n",
 		    vsc->sc_dev.dv_xname);
-		goto fail_1;
+		goto fail;
 	}
 	if (vsc->sc_child == VIRTIO_CHILD_ERROR) {
 		printf("%s: virtio configuration failed\n",
 		    vsc->sc_dev.dv_xname);
-		goto fail_1;
+		goto fail;
 	}
 
-	sc->sc_ih = fdt_intr_establish(faa->fa_node, vsc->sc_ipl,
+	return;
+
+fail:
+	virtio_set_status(vsc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED);
+}
+
+int
+virtio_mmio_attach_finish(struct virtio_softc *vsc,
+    struct virtio_attach_args *va)
+{
+	struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)vsc;
+	struct virtio_mmio_attach_args *vma =
+	    (struct virtio_mmio_attach_args *)va;
+
+	sc->sc_ih = fdt_intr_establish(vma->vma_fa->fa_node, vsc->sc_ipl,
 	    virtio_mmio_intr, sc, vsc->sc_dev.dv_xname);
 	if (sc->sc_ih == NULL) {
 		printf("%s: couldn't establish interrupt\n",
 		    vsc->sc_dev.dv_xname);
-		goto fail_2;
+		return -EIO;
 	}
-
-	return;
-
-fail_2:
-	config_detach(vsc->sc_child, 0);
-fail_1:
-	/* no mmio_mapreg_unmap() or mmio_intr_unmap() */
-	virtio_set_status(vsc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED);
+	return 0;
 }
 
 int
@@ -532,3 +549,11 @@ virtio_mmio_intr_barrier(struct virtio_softc *vsc)
 	if (sc->sc_ih)
 		intr_barrier(sc->sc_ih);
 }
+
+int
+virtio_mmio_intr_establish(struct virtio_softc *vsc,
+    struct virtio_attach_args *va, int vec, struct cpu_info *ci,
+    int (*func)(void *), void *arg)
+{
+	return ENXIO;
+}
diff --git a/sys/dev/pci/virtio_pci.c b/sys/dev/pci/virtio_pci.c
index ad9ce8a1254..1b7654537a4 100644
--- a/sys/dev/pci/virtio_pci.c
+++ b/sys/dev/pci/virtio_pci.c
@@ -50,7 +50,7 @@
  * XXX: PCI-endian while the device specific registers are native endian.
  */
 
-#define MAX_MSIX_VECS	8
+#define MAX_MSIX_VECS	16
 
 struct virtio_pci_softc;
 struct virtio_pci_attach_args;
@@ -62,7 +62,7 @@ int		virtio_pci_attach_10(struct virtio_pci_softc *sc, struct pci_attach_args *p
 int		virtio_pci_detach(struct device *, int);
 
 void		virtio_pci_kick(struct virtio_softc *, uint16_t);
-int		virtio_pci_adjust_config_region(struct virtio_pci_softc *);
+int		virtio_pci_adjust_config_region(struct virtio_pci_softc *, int offset);
 uint8_t		virtio_pci_read_device_config_1(struct virtio_softc *, int);
 uint16_t	virtio_pci_read_device_config_2(struct virtio_softc *, int);
 uint32_t	virtio_pci_read_device_config_4(struct virtio_softc *, int);
@@ -74,15 +74,17 @@ void		virtio_pci_write_device_config_8(struct virtio_softc *, int, uint64_t);
 uint16_t	virtio_pci_read_queue_size(struct virtio_softc *, uint16_t);
 void		virtio_pci_setup_queue(struct virtio_softc *, struct virtqueue *, uint64_t);
 void		virtio_pci_setup_intrs(struct virtio_softc *);
+int		virtio_pci_attach_finish(struct virtio_softc *, struct virtio_attach_args *);
 int		virtio_pci_get_status(struct virtio_softc *);
 void		virtio_pci_set_status(struct virtio_softc *, int);
 int		virtio_pci_negotiate_features(struct virtio_softc *, const struct virtio_feature_name *);
 int		virtio_pci_negotiate_features_10(struct virtio_softc *, const struct virtio_feature_name *);
 void		virtio_pci_set_msix_queue_vector(struct virtio_pci_softc *, uint32_t, uint16_t);
 void		virtio_pci_set_msix_config_vector(struct virtio_pci_softc *, uint16_t);
-int		virtio_pci_msix_establish(struct virtio_pci_softc *, struct virtio_pci_attach_args *, int, int (*)(void *), void *);
+int		virtio_pci_msix_establish(struct virtio_pci_softc *, struct virtio_pci_attach_args *, int, struct cpu_info *, int (*)(void *), void *);
 int		virtio_pci_setup_msix(struct virtio_pci_softc *, struct virtio_pci_attach_args *, int);
 void		virtio_pci_intr_barrier(struct virtio_softc *);
+int		virtio_pci_intr_establish(struct virtio_softc *, struct virtio_attach_args *, int, struct cpu_info *, int (*)(void *), void *);
 void		virtio_pci_free_irqs(struct virtio_pci_softc *);
 int		virtio_pci_poll_intr(void *);
 int		virtio_pci_legacy_intr(void *);
@@ -99,6 +101,7 @@ enum irq_type {
 	IRQ_NO_MSIX,
 	IRQ_MSIX_SHARED, /* vec 0: config irq, vec 1 shared by all vqs */
 	IRQ_MSIX_PER_VQ, /* vec 0: config irq, vec n: irq of vq[n-1] */
+	IRQ_MSIX_CHILD,  /* assigned by child driver */
 };
 
 struct virtio_pci_intr {
@@ -175,8 +178,10 @@ const struct virtio_ops virtio_pci_ops = {
 	virtio_pci_get_status,
 	virtio_pci_set_status,
 	virtio_pci_negotiate_features,
+	virtio_pci_attach_finish,
 	virtio_pci_poll_intr,
 	virtio_pci_intr_barrier,
+	virtio_pci_intr_establish,
 };
 
 static inline uint64_t
@@ -357,8 +362,7 @@ virtio_pci_match(struct device *parent, void *match, void *aux)
 		return 1;
 	/* virtio 1.0 */
 	if (PCI_PRODUCT(pa->pa_id) >= 0x1040 &&
-	    PCI_PRODUCT(pa->pa_id) <= 0x107f &&
-	    PCI_REVISION(pa->pa_class) == 1)
+	    PCI_PRODUCT(pa->pa_id) <= 0x107f)
 		return 1;
 	return 0;
 }
@@ -590,23 +594,20 @@ virtio_pci_attach(struct device *parent, struct device *self, void *aux)
 	struct pci_attach_args *pa = (struct pci_attach_args *)aux;
 	pci_chipset_tag_t pc = pa->pa_pc;
 	pcitag_t tag = pa->pa_tag;
-	int revision, ret = ENODEV;
+	int revision, product, ret = ENODEV;
 	pcireg_t id;
-	char const *intrstr;
-	pci_intr_handle_t ih;
 	struct virtio_pci_attach_args vpa = { { 0 }, pa };
 
 	revision = PCI_REVISION(pa->pa_class);
-	switch (revision) {
-	case 0:
+	product = PCI_PRODUCT(pa->pa_id);
+	if (revision == 0) {
 		/* subsystem ID shows what I am */
 		id = PCI_PRODUCT(pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG));
-		break;
-	case 1:
-		id = PCI_PRODUCT(pa->pa_id) - 0x1040;
-		break;
-	default:
-		printf("unknown revision 0x%02x; giving up\n", revision);
+	} else if (product >= 0x1040 && product <= 0x107f) {
+		id = product - 0x1040;
+	} else {
+		printf("unknown device prod 0x%04x rev 0x%02x; giving up\n",
+		    product, revision);
 		return;
 	}
 
@@ -635,7 +636,7 @@ virtio_pci_attach(struct device *parent, struct device *self, void *aux)
 
 	vsc->sc_ops = &virtio_pci_ops;
 	if ((vsc->sc_dev.dv_cfdata->cf_flags & VIRTIO_CF_NO_VERSION_1) == 0 &&
-	    (revision == 1 ||
+	    (revision >= 1 ||
 	     (vsc->sc_dev.dv_cfdata->cf_flags & VIRTIO_CF_PREFER_VERSION_1))) {
 		ret = virtio_pci_attach_10(sc, pa);
 	}
@@ -645,13 +646,15 @@ virtio_pci_attach(struct device *parent, struct device *self, void *aux)
 	}
 	if (ret != 0) {
 		printf(": Cannot attach (%d)\n", ret);
-		goto fail_0;
+		goto free;
 	}
 
-	sc->sc_devcfg_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI;
 	sc->sc_irq_type = IRQ_NO_MSIX;
-	if (virtio_pci_adjust_config_region(sc) != 0)
-		goto fail_0;
+	if (virtio_pci_adjust_config_region(sc,
+	    VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI) != 0)
+	{
+		goto free;
+	}
 
 	virtio_device_reset(vsc);
 	virtio_set_status(vsc, VIRTIO_CONFIG_DEVICE_STATUS_ACK);
@@ -664,25 +667,49 @@ virtio_pci_attach(struct device *parent, struct device *self, void *aux)
 	if (vsc->sc_child == NULL) {
 		printf("%s: no matching child driver; not configured\n",
 		    vsc->sc_dev.dv_xname);
-		goto fail_1;
+		goto err;
 	}
 	if (vsc->sc_child == VIRTIO_CHILD_ERROR) {
 		printf("%s: virtio configuration failed\n",
 		    vsc->sc_dev.dv_xname);
-		goto fail_1;
+		goto err;
 	}
 
-	if (virtio_pci_setup_msix(sc, &vpa, 0) == 0) {
+	return;
+
+err:
+	/* no pci_mapreg_unmap() or pci_intr_unmap() */
+	virtio_set_status(vsc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED);
+free:
+	free(sc->sc_intr, M_DEVBUF, sc->sc_nintr * sizeof(*sc->sc_intr));
+}
+
+int
+virtio_pci_attach_finish(struct virtio_softc *vsc,
+    struct virtio_attach_args *va)
+{
+	struct virtio_pci_softc *sc = (struct virtio_pci_softc *)vsc;
+	struct virtio_pci_attach_args *vpa =
+	    (struct virtio_pci_attach_args *)va;
+	pci_intr_handle_t ih;
+	pci_chipset_tag_t pc = vpa->vpa_pa->pa_pc;
+	char const *intrstr;
+
+	if (sc->sc_irq_type == IRQ_MSIX_CHILD) {
+		intrstr = "msix";
+	} else if (virtio_pci_setup_msix(sc, vpa, 0) == 0) {
 		sc->sc_irq_type = IRQ_MSIX_PER_VQ;
 		intrstr = "msix per-VQ";
-	} else if (virtio_pci_setup_msix(sc, &vpa, 1) == 0) {
+	} else if (virtio_pci_setup_msix(sc, vpa, 1) == 0) {
 		sc->sc_irq_type = IRQ_MSIX_SHARED;
 		intrstr = "msix shared";
 	} else {
 		int (*ih_func)(void *) = virtio_pci_legacy_intr;
-		if (pci_intr_map_msi(pa, &ih) != 0 && pci_intr_map(pa, &ih) != 0) {
-			printf("%s: couldn't map interrupt\n", vsc->sc_dev.dv_xname);
-			goto fail_2;
+		if (pci_intr_map_msi(vpa->vpa_pa, &ih) != 0 &&
+		    pci_intr_map(vpa->vpa_pa, &ih) != 0) {
+			printf("%s: couldn't map interrupt\n",
+			    vsc->sc_dev.dv_xname);
+			return -EIO;
 		}
 		intrstr = pci_intr_string(pc, ih);
 		/*
@@ -696,25 +723,17 @@ virtio_pci_attach(struct device *parent, struct device *self, void *aux)
 		    vsc->sc_ipl | IPL_MPSAFE, ih_func, sc,
 		    vsc->sc_child->dv_xname);
 		if (sc->sc_intr[0].ih == NULL) {
-			printf("%s: couldn't establish interrupt", vsc->sc_dev.dv_xname);
+			printf("%s: couldn't establish interrupt",
+			    vsc->sc_dev.dv_xname);
 			if (intrstr != NULL)
 				printf(" at %s", intrstr);
 			printf("\n");
-			goto fail_2;
+			return -EIO;
 		}
 	}
-	virtio_pci_setup_intrs(vsc);
-	printf("%s: %s\n", vsc->sc_dev.dv_xname, intrstr);
 
-	return;
-
-fail_2:
-	config_detach(vsc->sc_child, 0);
-fail_1:
-	/* no pci_mapreg_unmap() or pci_intr_unmap() */
-	virtio_set_status(vsc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED);
-fail_0:
-	free(sc->sc_intr, M_DEVBUF, sc->sc_nintr * sizeof(*sc->sc_intr));
+	printf("%s: %s\n", vsc->sc_dev.dv_xname, intrstr);
+	return 0;
 }
 
 int
@@ -740,11 +759,14 @@ virtio_pci_detach(struct device *self, int flags)
 }
 
 int
-virtio_pci_adjust_config_region(struct virtio_pci_softc *sc)
+virtio_pci_adjust_config_region(struct virtio_pci_softc *sc, int offset)
 {
 	if (sc->sc_sc.sc_version_1)
 		return 0;
-	sc->sc_devcfg_iosize = sc->sc_iosize - sc->sc_devcfg_offset;
+	if (sc->sc_devcfg_offset == offset)
+		return 0;
+	sc->sc_devcfg_offset = offset;
+	sc->sc_devcfg_iosize = sc->sc_iosize - offset;
 	sc->sc_devcfg_iot = sc->sc_iot;
 	if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, sc->sc_devcfg_offset,
 	    sc->sc_devcfg_iosize, &sc->sc_devcfg_ioh) != 0) {
@@ -944,30 +966,33 @@ virtio_pci_write_device_config_8(struct virtio_softc *vsc,
 
 int
 virtio_pci_msix_establish(struct virtio_pci_softc *sc,
-    struct virtio_pci_attach_args *vpa, int idx,
+    struct virtio_pci_attach_args *vpa, int idx, struct cpu_info *ci,
     int (*handler)(void *), void *ih_arg)
 {
 	struct virtio_softc *vsc = &sc->sc_sc;
 	pci_intr_handle_t ih;
+	int r;
 
 	KASSERT(idx < sc->sc_nintr);
 
-	if (pci_intr_map_msix(vpa->vpa_pa, idx, &ih) != 0) {
+	r = pci_intr_map_msix(vpa->vpa_pa, idx, &ih);
+	if (r != 0) {
 #if VIRTIO_DEBUG
 		printf("%s[%d]: pci_intr_map_msix failed\n",
 		    vsc->sc_dev.dv_xname, idx);
 #endif
-		return 1;
+		return r;
 	}
 	snprintf(sc->sc_intr[idx].name, sizeof(sc->sc_intr[idx].name), "%s:%d",
 	    vsc->sc_child->dv_xname, idx);
-	sc->sc_intr[idx].ih = pci_intr_establish(sc->sc_pc, ih, vsc->sc_ipl,
-	    handler, ih_arg, sc->sc_intr[idx].name);
+	sc->sc_intr[idx].ih = pci_intr_establish_cpu(sc->sc_pc, ih, vsc->sc_ipl,
+	    ci, handler, ih_arg, sc->sc_intr[idx].name);
 	if (sc->sc_intr[idx].ih == NULL) {
 		printf("%s[%d]: couldn't establish msix interrupt\n",
-		    vsc->sc_dev.dv_xname, idx);
-		return 1;
+		    vsc->sc_child->dv_xname, idx);
+		return ENOMEM;
 	}
+	virtio_pci_adjust_config_region(sc, VIRTIO_CONFIG_DEVICE_CONFIG_MSI);
 	return 0;
 }
 
@@ -1017,8 +1042,8 @@ virtio_pci_free_irqs(struct virtio_pci_softc *sc)
 		}
 	}
 
-	sc->sc_devcfg_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI;
-	virtio_pci_adjust_config_region(sc);
+	/* XXX msix_delroute does not unset PCI_MSIX_MC_MSIXE -> leave alone? */
+	virtio_pci_adjust_config_region(sc, VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI);
 }
 
 int
@@ -1026,34 +1051,33 @@ virtio_pci_setup_msix(struct virtio_pci_softc *sc,
     struct virtio_pci_attach_args *vpa, int shared)
 {
 	struct virtio_softc *vsc = &sc->sc_sc;
-	int i;
+	int i, r = 0;
 
 	/* Shared needs config + queue */
 	if (shared && vpa->vpa_va.va_nintr < 1 + 1)
-		return 1;
+		return ERANGE;
 	/* Per VQ needs config + N * queue */
 	if (!shared && vpa->vpa_va.va_nintr < 1 + vsc->sc_nvqs)
-		return 1;
+		return ERANGE;
 
-	if (virtio_pci_msix_establish(sc, vpa, 0, virtio_pci_config_intr, vsc))
-		return 1;
-	sc->sc_devcfg_offset = VIRTIO_CONFIG_DEVICE_CONFIG_MSI;
-	virtio_pci_adjust_config_region(sc);
+	r = virtio_pci_msix_establish(sc, vpa, 0, NULL, virtio_pci_config_intr, vsc);
+	if (r != 0)
+		return r;
 
 	if (shared) {
-		if (virtio_pci_msix_establish(sc, vpa, 1,
-		    virtio_pci_shared_queue_intr, vsc)) {
+		r = virtio_pci_msix_establish(sc, vpa, 1, NULL,
+		    virtio_pci_shared_queue_intr, vsc);
+		if (r != 0)
 			goto fail;
-		}
 
 		for (i = 0; i < vsc->sc_nvqs; i++)
 			vsc->sc_vqs[i].vq_intr_vec = 1;
 	} else {
 		for (i = 0; i < vsc->sc_nvqs; i++) {
-			if (virtio_pci_msix_establish(sc, vpa, i + 1,
-			    virtio_pci_queue_intr, &vsc->sc_vqs[i])) {
+			r = virtio_pci_msix_establish(sc, vpa, i + 1, NULL,
+			    virtio_pci_queue_intr, &vsc->sc_vqs[i]);
+			if (r != 0)
 				goto fail;
-			}
 			vsc->sc_vqs[i].vq_intr_vec = i + 1;
 		}
 	}
@@ -1061,7 +1085,28 @@ virtio_pci_setup_msix(struct virtio_pci_softc *sc,
 	return 0;
 fail:
 	virtio_pci_free_irqs(sc);
-	return 1;
+	return r;
+}
+
+int
+virtio_pci_intr_establish(struct virtio_softc *vsc,
+    struct virtio_attach_args *va, int vec, struct cpu_info *ci,
+    int (*func)(void *), void *arg)
+{
+	struct virtio_pci_attach_args *vpa;
+	struct virtio_pci_softc *sc;
+
+	if (vsc->sc_ops != &virtio_pci_ops)
+		return ENXIO;
+
+	vpa = (struct virtio_pci_attach_args *)va;
+	sc = (struct virtio_pci_softc *)vsc;
+
+	if (vec >= sc->sc_nintr || sc->sc_nintr <= 1)
+		return ERANGE;
+
+	sc->sc_irq_type = IRQ_MSIX_CHILD;
+	return virtio_pci_msix_establish(sc, vpa, vec, ci, func, arg);
 }
 
 void
diff --git a/sys/dev/pv/if_vio.c b/sys/dev/pv/if_vio.c
index 3d4d728e02d..6233f2cb142 100644
--- a/sys/dev/pv/if_vio.c
+++ b/sys/dev/pv/if_vio.c
@@ -32,8 +32,10 @@
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/device.h>
+#include <sys/intrmap.h>
 #include <sys/mbuf.h>
 #include <sys/mutex.h>
+#include <sys/percpu.h>	/* for CACHELINESIZE */
 #include <sys/sockio.h>
 #include <sys/timeout.h>
 
@@ -64,8 +66,15 @@
  * if_vioreg.h:
  */
 /* Configuration registers */
-#define VIRTIO_NET_CONFIG_MAC		0 /* 8bit x 6byte */
-#define VIRTIO_NET_CONFIG_STATUS	6 /* 16bit */
+#define VIRTIO_NET_CONFIG_MAC		 0 /*  8 bit x 6 byte */
+#define VIRTIO_NET_CONFIG_STATUS	 6 /* 16 bit */
+#define VIRTIO_NET_CONFIG_MAX_QUEUES	 8 /* 16 bit */
+#define VIRTIO_NET_CONFIG_MTU		10 /* 16 bit */
+#define VIRTIO_NET_CONFIG_SPEED		12 /* 32 bit */
+#define VIRTIO_NET_CONFIG_DUPLEX	16 /*  8 bit */
+#define VIRTIO_NET_CONFIG_RSS_SIZE	17 /*  8 bit */
+#define VIRTIO_NET_CONFIG_RSS_LEN	18 /* 16 bit */
+#define VIRTIO_NET_CONFIG_HASH_TYPES	20 /* 16 bit */
 
 /* Feature bits */
 #define VIRTIO_NET_F_CSUM			(1ULL<<0)
@@ -98,12 +107,6 @@
 #define VIRTIO_NET_F_RSC_EXT			(1ULL<<61)
 #define VIRTIO_NET_F_STANDBY			(1ULL<<62)
 #define VIRTIO_NET_F_SPEED_DUPLEX		(1ULL<<63)
-/*
- * Config(8) flags. The lowest byte is reserved for generic virtio stuff.
- */
-
-/* Workaround for vlan related bug in qemu < version 2.0 */
-#define CONFFLAG_QEMU_VLAN_BUG		(1<<8)
 
 static const struct virtio_feature_name virtio_net_feature_names[] = {
 #if VIRTIO_DEBUG
@@ -183,6 +186,11 @@ struct virtio_net_ctrl_cmd {
 # define VIRTIO_NET_CTRL_VLAN_ADD	0
 # define VIRTIO_NET_CTRL_VLAN_DEL	1
 
+#define VIRTIO_NET_CTRL_MQ		4
+# define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET	0
+# define VIRTIO_NET_CTRL_MQ_RSS_CONFIG		1
+# define VIRTIO_NET_CTRL_MQ_HASH_CONFIG		2
+
 #define VIRTIO_NET_CTRL_GUEST_OFFLOADS	5
 # define VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET	0
 
@@ -196,6 +204,12 @@ struct virtio_net_ctrl_rx {
 	uint8_t	onoff;
 } __packed;
 
+struct virtio_net_ctrl_mq_pairs_set {
+	uint16_t virtqueue_pairs;
+};
+#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN	1
+#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX	0x8000
+
 struct virtio_net_ctrl_guest_offloads {
 	uint64_t offloads;
 } __packed;
@@ -231,7 +245,7 @@ struct vio_queue {
 	struct virtqueue	 *viq_txvq;
 	struct mutex		  viq_txmtx, viq_rxmtx;
 	int			  viq_txfree_slots;
-};
+} __aligned(CACHELINESIZE);
 
 struct vio_softc {
 	struct device		sc_dev;
@@ -251,14 +265,16 @@ struct vio_softc {
 	caddr_t			sc_dma_kva;
 
 	int			sc_hdr_size;
-	struct virtio_net_ctrl_cmd *sc_ctrl_cmd;
-	struct virtio_net_ctrl_status *sc_ctrl_status;
-	struct virtio_net_ctrl_rx *sc_ctrl_rx;
-	struct virtio_net_ctrl_guest_offloads *sc_ctrl_guest_offloads;
-	struct virtio_net_ctrl_mac_tbl *sc_ctrl_mac_tbl_uc;
+	struct virtio_net_ctrl_cmd		*sc_ctrl_cmd;
+	struct virtio_net_ctrl_status		*sc_ctrl_status;
+	struct virtio_net_ctrl_rx		*sc_ctrl_rx;
+	struct virtio_net_ctrl_mq_pairs_set	*sc_ctrl_mq_pairs;
+	struct virtio_net_ctrl_guest_offloads	*sc_ctrl_guest_offloads;
+	struct virtio_net_ctrl_mac_tbl		*sc_ctrl_mac_tbl_uc;
 #define sc_ctrl_mac_info sc_ctrl_mac_tbl_uc
-	struct virtio_net_ctrl_mac_tbl *sc_ctrl_mac_tbl_mc;
+	struct virtio_net_ctrl_mac_tbl		*sc_ctrl_mac_tbl_mc;
 
+	struct intrmap		*sc_intrmap;
 	struct vio_queue	*sc_q;
 	uint16_t		sc_nqueues;
 	int			sc_tx_slots_per_req;
@@ -317,10 +333,15 @@ void	vio_tx_drain(struct vio_softc *);
 int	vio_encap(struct vio_queue *, int, struct mbuf *);
 void	vio_txtick(void *);
 
+int	vio_queue_intr(void *);
+int	vio_config_intr(void *);
+int	vio_ctrl_intr(void *);
+
 /* other control */
 void	vio_link_state(struct ifnet *);
 int	vio_config_change(struct virtio_softc *);
 int	vio_ctrl_rx(struct vio_softc *, int, int);
+int	vio_ctrl_mq(struct vio_softc *);
 int	vio_ctrl_guest_offloads(struct vio_softc *, uint64_t);
 int	vio_set_rx_filter(struct vio_softc *);
 void	vio_iff(struct vio_softc *);
@@ -408,6 +429,8 @@ vio_free_dmamem(struct vio_softc *sc)
  *   sc_ctrl_status:	 return value for a command via ctrl vq (READ)
  *   sc_ctrl_rx:	 parameter for a VIRTIO_NET_CTRL_RX class command
  *			 (WRITE)
+ *   sc_ctrl_mq_pairs_set: set number of rx/tx queue pais (WRITE)
+ *   sc_ctrl_guest_offloads: configure offload features (WRITE)
  *   sc_ctrl_mac_tbl_uc: unicast MAC address filter for a VIRTIO_NET_CTRL_MAC
  *			 class command (WRITE)
  *   sc_ctrl_mac_tbl_mc: multicast MAC address filter for a VIRTIO_NET_CTRL_MAC
@@ -449,6 +472,7 @@ vio_alloc_mem(struct vio_softc *sc, int tx_max_segments)
 		allocsize += sizeof(struct virtio_net_ctrl_cmd) * 1;
 		allocsize += sizeof(struct virtio_net_ctrl_status) * 1;
 		allocsize += sizeof(struct virtio_net_ctrl_rx) * 1;
+		allocsize += sizeof(struct virtio_net_ctrl_mq_pairs_set) * 1;
 		allocsize += sizeof(struct virtio_net_ctrl_guest_offloads) * 1;
 		allocsize += VIO_CTRL_MAC_INFO_SIZE;
 	}
@@ -474,6 +498,8 @@ vio_alloc_mem(struct vio_softc *sc, int tx_max_segments)
 		offset += sizeof(*sc->sc_ctrl_status);
 		sc->sc_ctrl_rx = (void *)(kva + offset);
 		offset += sizeof(*sc->sc_ctrl_rx);
+		sc->sc_ctrl_mq_pairs = (void *)(kva + offset);
+		offset += sizeof(*sc->sc_ctrl_mq_pairs);
 		sc->sc_ctrl_guest_offloads = (void *)(kva + offset);
 		offset += sizeof(*sc->sc_ctrl_guest_offloads);
 		sc->sc_ctrl_mac_tbl_uc = (void *)(kva + offset);
@@ -597,7 +623,8 @@ vio_attach(struct device *parent, struct device *self, void *aux)
 {
 	struct vio_softc *sc = (struct vio_softc *)self;
 	struct virtio_softc *vsc = (struct virtio_softc *)parent;
-	int i, tx_max_segments;
+	struct virtio_attach_args *va = aux;
+	int i, r, tx_max_segments;
 	struct ifnet *ifp = &sc->sc_ac.ac_if;
 
 	if (vsc->sc_child != NULL) {
@@ -610,12 +637,14 @@ vio_attach(struct device *parent, struct device *self, void *aux)
 
 	vsc->sc_child = self;
 	vsc->sc_ipl = IPL_NET | IPL_MPSAFE;
-	vsc->sc_config_change = NULL;
 	vsc->sc_driver_features = VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS |
 	    VIRTIO_NET_F_CTRL_VQ | VIRTIO_NET_F_CTRL_RX |
 	    VIRTIO_NET_F_MRG_RXBUF | VIRTIO_NET_F_CSUM |
 	    VIRTIO_F_RING_EVENT_IDX | VIRTIO_NET_F_GUEST_CSUM;
 
+	if (va->va_nintr > 3 && ncpus > 1)
+		vsc->sc_driver_features |= VIRTIO_NET_F_MQ;
+
 	vsc->sc_driver_features |= VIRTIO_NET_F_HOST_TSO4;
 	vsc->sc_driver_features |= VIRTIO_NET_F_HOST_TSO6;
 
@@ -623,12 +652,26 @@ vio_attach(struct device *parent, struct device *self, void *aux)
 	vsc->sc_driver_features |= VIRTIO_NET_F_GUEST_TSO4;
 	vsc->sc_driver_features |= VIRTIO_NET_F_GUEST_TSO6;
 
-	virtio_negotiate_features(vsc, virtio_net_feature_names);
+	if (virtio_negotiate_features(vsc, virtio_net_feature_names) != 0)
+		goto err;
 
-	sc->sc_nqueues = 1;
-	vsc->sc_nvqs = 2 * sc->sc_nqueues;
-	if (virtio_has_feature(vsc, VIRTIO_NET_F_CTRL_VQ))
-		vsc->sc_nvqs++;
+	if (virtio_has_feature(vsc, VIRTIO_NET_F_MQ)) {
+		i = virtio_read_device_config_2(vsc,
+		    VIRTIO_NET_CONFIG_MAX_QUEUES);
+		vsc->sc_nvqs = 2 * i + 1;
+		i = MIN(i, VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX);
+		sc->sc_intrmap = intrmap_create(&sc->sc_dev, i,
+		    va->va_nintr - 2, 0);
+		sc->sc_nqueues = intrmap_count(sc->sc_intrmap);
+		printf(": %u queue%s", sc->sc_nqueues,
+		    sc->sc_nqueues > 1 ? "s"  : "");
+	} else {
+		sc->sc_nqueues = 1;
+		printf(": 1 queue");
+		vsc->sc_nvqs = 2;
+		if (virtio_has_feature(vsc, VIRTIO_NET_F_CTRL_VQ))
+			vsc->sc_nvqs++;
+	}
 
 	vsc->sc_vqs = mallocarray(vsc->sc_nvqs, sizeof(*vsc->sc_vqs), M_DEVBUF,
 	    M_WAITOK|M_ZERO);
@@ -728,18 +771,66 @@ vio_attach(struct device *parent, struct device *self, void *aux)
 		else
 			virtio_stop_vq_intr(vsc, vioq->viq_txvq);
 		vioq->viq_txfree_slots = vioq->viq_txvq->vq_num - 1;
+		KASSERT(vioq->viq_txfree_slots > sc->sc_tx_slots_per_req);
+		if (vioq->viq_txvq->vq_num != sc->sc_q[0].viq_txvq->vq_num) {
+			printf("inequal tx queue size %d: %d != %d\n", i,
+			    vioq->viq_txvq->vq_num,
+			    sc->sc_q[0].viq_txvq->vq_num);
+			goto err;
+		}
+		DPRINTF("%d: q %p rx %p tx %p\n", i, vioq, vioq->viq_rxvq,
+		    vioq->viq_txvq);
+
+		if (sc->sc_intrmap != NULL) {
+			vioq->viq_rxvq->vq_intr_vec = i + 2;
+			vioq->viq_txvq->vq_intr_vec = i + 2;
+		}
 	}
 
 	/* control queue */
 	if (virtio_has_feature(vsc, VIRTIO_NET_F_CTRL_VQ)) {
-		sc->sc_ctl_vq = &vsc->sc_vqs[2];
-		if (virtio_alloc_vq(vsc, sc->sc_ctl_vq, 2, 1,
-		    "control") != 0)
+		i = 2;
+		if (virtio_has_feature(vsc, VIRTIO_NET_F_MQ)) {
+			i = 2 * virtio_read_device_config_2(vsc,
+			    VIRTIO_NET_CONFIG_MAX_QUEUES);
+		}
+		sc->sc_ctl_vq =  &vsc->sc_vqs[i];
+		if (virtio_alloc_vq(vsc, sc->sc_ctl_vq, i, 1, "control") != 0)
 			goto err;
 		sc->sc_ctl_vq->vq_done = vio_ctrleof;
+		if (sc->sc_intrmap != NULL)
+			sc->sc_ctl_vq->vq_intr_vec = 1;
 		virtio_start_vq_intr(vsc, sc->sc_ctl_vq);
 	}
 
+	if (sc->sc_intrmap) {
+		r = virtio_intr_establish(vsc, va, 0, NULL, vio_config_intr,
+		    vsc);
+		if (r != 0) {
+			printf("%s: cannot alloc config intr: %d\n",
+			    sc->sc_dev.dv_xname, r);
+			goto err;
+		}
+		r = virtio_intr_establish(vsc, va, 1, NULL, vio_ctrl_intr,
+		    sc->sc_ctl_vq);
+		if (r != 0) {
+			printf("%s: cannot alloc ctrl intr: %d\n",
+			    sc->sc_dev.dv_xname, r);
+			goto err;
+		}
+		for (i = 0; i < sc->sc_nqueues; i++) {
+			struct cpu_info *ci = NULL;
+			ci = intrmap_cpu(sc->sc_intrmap, i);
+			r = virtio_intr_establish(vsc, va, i + 2, ci,
+			    vio_queue_intr, &sc->sc_q[i]);
+			if (r != 0) {
+				printf("%s: cannot alloc q%d intr: %d\n",
+				    sc->sc_dev.dv_xname, i, r);
+				goto err;
+			}
+		}
+	}
+
 	if (vio_alloc_mem(sc, tx_max_segments) < 0)
 		goto err;
 
@@ -753,10 +844,17 @@ vio_attach(struct device *parent, struct device *self, void *aux)
 	ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL);
 	ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO);
 	vsc->sc_config_change = vio_config_change;
-	timeout_set(&sc->sc_txtick, vio_txtick, sc->sc_q[0].viq_txvq);
-	timeout_set(&sc->sc_rxtick, vio_rxtick, sc->sc_q[0].viq_rxvq);
+	timeout_set(&sc->sc_txtick, vio_txtick, sc);
+	timeout_set(&sc->sc_rxtick, vio_rxtick, sc);
+
+	if (virtio_attach_finish(vsc, va) != 0)
+		goto err;
+
+	if (virtio_has_feature(vsc, VIRTIO_NET_F_MQ)) {
+		/* ctrl queue works only after DRIVER_OK */
+		vio_ctrl_mq(sc);
+	}
 
-	virtio_set_status(vsc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK);
 	if_attach(ifp);
 	ether_ifattach(ifp);
 	vio_link_state(ifp);
@@ -802,6 +900,33 @@ vio_link_state(struct ifnet *ifp)
 	}
 }
 
+/* interrupt handlers for multi-queue */
+int
+vio_queue_intr(void *arg)
+{
+	struct vio_queue *vioq = arg;
+	struct virtio_softc *vsc = vioq->viq_sc->sc_virtio;
+	int r;
+	r = virtio_check_vq(vsc, vioq->viq_txvq);
+	r |= virtio_check_vq(vsc, vioq->viq_rxvq);
+	return r;
+}
+
+int
+vio_config_intr(void *arg)
+{
+	struct virtio_softc *vsc = arg;
+	return vio_config_change(vsc);
+}
+
+int
+vio_ctrl_intr(void *arg)
+{
+	struct virtqueue *vq = arg;
+	return virtio_check_vq(vq->vq_owner, vq);
+}
+
+
 int
 vio_config_change(struct virtio_softc *vsc)
 {
@@ -910,6 +1035,8 @@ vio_stop(struct ifnet *ifp, int disable)
 	if (virtio_has_feature(vsc, VIRTIO_NET_F_CTRL_VQ))
 		virtio_start_vq_intr(vsc, sc->sc_ctl_vq);
 	virtio_reinit_end(vsc);
+	if (virtio_has_feature(vsc, VIRTIO_NET_F_MQ))
+		vio_ctrl_mq(sc);
 	if (virtio_has_feature(vsc, VIRTIO_NET_F_CTRL_VQ))
 		vio_ctrl_wakeup(sc, FREE);
 }
@@ -1053,7 +1180,8 @@ again:
 		r = virtio_enqueue_reserve(vq, slot,
 		    vioq->viq_txdmamaps[slot]->dm_nsegs + 1);
 		if (r != 0) {
-			printf("%s: virtio_enqueue_reserve failed?\n", __func__);
+			printf("%s: virtio_enqueue_reserve failed?\n",
+			    __func__);
 			m_freem(m);
 			viq_ifq->ifq_errors++;
 			bus_dmamap_unload(vsc->sc_dmat,
@@ -1081,8 +1209,9 @@ again:
 	}
 	if (used_slots > 0) {
 		if (used_slots > vioq->viq_txfree_slots)
-			printf("%s: used_slots %d viq_txfree_slots %d free_slots %d\n",
-			    __func__, used_slots, vioq->viq_txfree_slots, free_slots);
+			printf("%s: used_slots %d viq_txfree_slots %d "
+			    "free_slots %d\n", __func__, used_slots,
+			    vioq->viq_txfree_slots, free_slots);
 		vioq->viq_txfree_slots -= used_slots;
 		KASSERT(vioq->viq_txfree_slots >= 0);
 	}
@@ -1132,6 +1261,33 @@ vio_dump(struct vio_softc *sc)
 }
 #endif
 
+static int
+vio_rxr_info(struct vio_softc *sc, struct if_rxrinfo *ifri)
+{
+	struct if_rxring_info *ifrs, *ifr;
+	int error;
+	unsigned int i;
+
+	ifrs = mallocarray(sc->sc_nqueues, sizeof(*ifrs),
+	    M_TEMP, M_WAITOK|M_ZERO|M_CANFAIL);
+	if (ifrs == NULL)
+		return (ENOMEM);
+
+	for (i = 0; i < sc->sc_nqueues; i++) {
+		ifr = &ifrs[i];
+
+		ifr->ifr_size = sc->sc_rx_mbuf_size;
+		snprintf(ifr->ifr_name, sizeof(ifr->ifr_name), "%u", i);
+		ifr->ifr_info = sc->sc_q[i].viq_rxring;
+	}
+
+	error = if_rxr_info_ioctl(ifri, i, ifrs);
+
+	free(ifrs, M_TEMP, i * sizeof(*ifrs));
+
+	return (error);
+}
+
 int
 vio_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 {
@@ -1166,8 +1322,7 @@ vio_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 		r = ifmedia_ioctl(ifp, ifr, &sc->sc_media, cmd);
 		break;
 	case SIOCGIFRXR:
-		r = if_rxr_ioctl((struct if_rxrinfo *)ifr->ifr_data,
-		    NULL, sc->sc_rx_mbuf_size, &sc->sc_q[0].viq_rxring);
+		r = vio_rxr_info(sc, (struct if_rxrinfo *)ifr->ifr_data);
 		break;
 	default:
 		r = ether_ioctl(ifp, &sc->sc_ac, cmd, data);
@@ -1661,6 +1816,8 @@ vio_ctrl_submit(struct vio_softc *sc, int slot)
 			vio_ctrl_wakeup(sc, RESET);
 			return ENXIO;
 		}
+		if (cold)
+			virtio_check_vq(sc->sc_virtio, sc->sc_ctl_vq);
 	}
 
 	VIO_DMAMEM_SYNC(vsc, sc, sc->sc_ctrl_cmd,
@@ -1718,6 +1875,41 @@ vio_ctrl_rx(struct vio_softc *sc, int cmd, int onoff)
 	return r;
 }
 
+/* issue a VIRTIO_NET_CTRL_MQ class command and wait for completion */
+int
+vio_ctrl_mq(struct vio_softc *sc)
+{
+	struct virtio_softc *vsc = sc->sc_virtio;
+	struct virtqueue *vq = sc->sc_ctl_vq;
+	int r, slot;
+
+
+	r = vio_ctrl_start(sc, VIRTIO_NET_CTRL_MQ,
+	    VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, 1, &slot);
+	if (r != 0)
+		return r;
+
+	sc->sc_ctrl_mq_pairs->virtqueue_pairs = sc->sc_nqueues;
+
+	vio_dmamem_enqueue(vsc, sc, vq, slot, sc->sc_ctrl_mq_pairs,
+	    sizeof(*sc->sc_ctrl_mq_pairs), 1);
+
+	r = vio_ctrl_submit(sc, slot);
+
+	VIO_DMAMEM_SYNC(vsc, sc, sc->sc_ctrl_mq_pairs,
+	    sizeof(*sc->sc_ctrl_mq_pairs), BUS_DMASYNC_POSTWRITE);
+
+	if (r != 0)
+		printf("%s: ctrl cmd %d failed\n", sc->sc_dev.dv_xname,
+		    VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET);
+
+	DPRINTF("%s: cmd %d %d: %d\n", __func__,
+	    VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, sc->sc_nqueues, r);
+
+	vio_ctrl_finish(sc);
+	return r;
+}
+
 int
 vio_ctrl_guest_offloads(struct vio_softc *sc, uint64_t features)
 {
@@ -1845,9 +2037,6 @@ vio_iff(struct vio_softc *sc)
 		return;
 	}
 
-	if (sc->sc_dev.dv_cfdata->cf_flags & CONFFLAG_QEMU_VLAN_BUG)
-		ifp->if_flags |= IFF_PROMISC;
-
 	if (ifp->if_flags & IFF_PROMISC || ac->ac_multirangecnt > 0 ||
 	    ac->ac_multicnt >= VIRTIO_NET_CTRL_MAC_MC_ENTRIES) {
 		ifp->if_flags |= IFF_ALLMULTI;
diff --git a/sys/dev/pv/vioblk.c b/sys/dev/pv/vioblk.c
index 153a2762170..0bf8fdd1ac3 100644
--- a/sys/dev/pv/vioblk.c
+++ b/sys/dev/pv/vioblk.c
@@ -170,12 +170,12 @@ vioblk_attach(struct device *parent, struct device *self, void *aux)
 {
 	struct vioblk_softc *sc = (struct vioblk_softc *)self;
 	struct virtio_softc *vsc = (struct virtio_softc *)parent;
+	struct virtio_attach_args *va = aux;
 	struct scsibus_attach_args saa;
 	int qsize;
 
 	vsc->sc_vqs = &sc->sc_vq[0];
 	vsc->sc_nvqs = 1;
-	vsc->sc_config_change = NULL;
 	if (vsc->sc_child)
 		panic("already attached to something else");
 	vsc->sc_child = self;
@@ -184,7 +184,8 @@ vioblk_attach(struct device *parent, struct device *self, void *aux)
 	vsc->sc_driver_features = VIRTIO_BLK_F_RO | VIRTIO_F_NOTIFY_ON_EMPTY |
 	     VIRTIO_BLK_F_SIZE_MAX | VIRTIO_BLK_F_SEG_MAX | VIRTIO_BLK_F_FLUSH;
 
-        virtio_negotiate_features(vsc, vioblk_feature_names);
+        if (virtio_negotiate_features(vsc, vioblk_feature_names) != 0)
+		goto err;
 
 	if (virtio_has_feature(vsc, VIRTIO_BLK_F_SIZE_MAX)) {
 		uint32_t size_max = virtio_read_device_config_4(vsc,
@@ -252,10 +253,11 @@ vioblk_attach(struct device *parent, struct device *self, void *aux)
 	saa.saa_quirks = 0;
 	saa.saa_wwpn = saa.saa_wwnn = 0;
 
-	virtio_set_status(vsc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK);
+	if (virtio_attach_finish(vsc, va) != 0)
+		goto err;
 	config_found(self, &saa, scsiprint);
-
 	return;
+
 err:
 	vsc->sc_child = VIRTIO_CHILD_ERROR;
 	return;
diff --git a/sys/dev/pv/viocon.c b/sys/dev/pv/viocon.c
index 681a842cd93..6f097a3309b 100644
--- a/sys/dev/pv/viocon.c
+++ b/sys/dev/pv/viocon.c
@@ -179,7 +179,6 @@ viocon_attach(struct device *parent, struct device *self, void *aux)
 		panic("already attached to something else");
 	vsc->sc_child = self;
 	vsc->sc_ipl = IPL_TTY;
-	vsc->sc_config_change = NULL;
 	sc->sc_virtio = vsc;
 	sc->sc_max_ports = maxports;
 
@@ -193,7 +192,8 @@ viocon_attach(struct device *parent, struct device *self, void *aux)
 	}
 
 	vsc->sc_driver_features = VIRTIO_CONSOLE_F_SIZE;
-	virtio_negotiate_features(vsc, viocon_feature_names);
+	if (virtio_negotiate_features(vsc, viocon_feature_names) != 0)
+		goto err;
 
 	printf("\n");
 	DPRINTF("%s: softc: %p\n", __func__, sc);
@@ -201,10 +201,11 @@ viocon_attach(struct device *parent, struct device *self, void *aux)
 		printf("\n%s: viocon_port_create failed\n", __func__);
 		goto err;
 	}
+	if (virtio_attach_finish(vsc, va) != 0)
+		goto err;
 	viocon_rx_fill(sc->sc_ports[0]);
-	virtio_set_status(vsc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK);
-
 	return;
+
 err:
 	vsc->sc_child = VIRTIO_CHILD_ERROR;
 	free(vsc->sc_vqs, M_DEVBUF, 2 * (maxports + 1) * sizeof(struct virtqueue));
diff --git a/sys/dev/pv/viogpu.c b/sys/dev/pv/viogpu.c
index cce31d8780e..1bf45b7caa7 100644
--- a/sys/dev/pv/viogpu.c
+++ b/sys/dev/pv/viogpu.c
@@ -150,6 +150,7 @@ viogpu_attach(struct device *parent, struct device *self, void *aux)
 {
 	struct viogpu_softc *sc = (struct viogpu_softc *)self;
 	struct virtio_softc *vsc = (struct virtio_softc *)parent;
+	struct virtio_attach_args *va = aux;
 	struct wsemuldisplaydev_attach_args waa;
 	struct rasops_info *ri = &sc->sc_ri;
 	uint32_t defattr;
@@ -161,10 +162,11 @@ viogpu_attach(struct device *parent, struct device *self, void *aux)
 	}
 	vsc->sc_child = self;
 
-	virtio_negotiate_features(vsc, viogpu_feature_names);
+	if (virtio_negotiate_features(vsc, viogpu_feature_names) != 0)
+		goto err;
 	if (!vsc->sc_version_1) {
 		printf(": requires virtio version 1\n");
-		return;
+		goto err;
 	}
 
 	vsc->sc_ipl = IPL_TTY;
@@ -175,13 +177,13 @@ viogpu_attach(struct device *parent, struct device *self, void *aux)
 	vsc->sc_vqs = sc->sc_vqs;
 	if (virtio_alloc_vq(vsc, &sc->sc_vqs[VQCTRL], VQCTRL, 1, "control")) {
 		printf(": alloc_vq failed\n");
-		return;
+		goto err;
 	}
 	sc->sc_vqs[VQCTRL].vq_done = viogpu_vq_done;
 
 	if (virtio_alloc_vq(vsc, &sc->sc_vqs[VQCURS], VQCURS, 1, "cursor")) {
 		printf(": alloc_vq failed\n");
-		return;
+		goto err;
 	}
 	vsc->sc_nvqs = nitems(sc->sc_vqs);
 
@@ -191,7 +193,7 @@ viogpu_attach(struct device *parent, struct device *self, void *aux)
 	    sc->sc_dma_size, 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW,
 	    &sc->sc_dma_map) != 0) {
 		printf(": create failed");
-		goto err;
+		goto errdma;
 	}
 	if (bus_dmamem_alloc(vsc->sc_dmat, sc->sc_dma_size, 16, 0,
 	    &sc->sc_dma_seg, 1, &nsegs, BUS_DMA_NOWAIT | BUS_DMA_ZERO) != 0) {
@@ -209,7 +211,8 @@ viogpu_attach(struct device *parent, struct device *self, void *aux)
 		goto unmap;
 	}
 
-	virtio_set_status(vsc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK);
+	if (virtio_attach_finish(vsc, va) != 0)
+		goto unmap;
 
 	if (viogpu_get_display_info(sc) != 0)
 		goto unmap;
@@ -302,8 +305,10 @@ free:
 	bus_dmamem_free(vsc->sc_dmat, &sc->sc_dma_seg, 1);
 destroy:
 	bus_dmamap_destroy(vsc->sc_dmat, sc->sc_dma_map);
-err:
+errdma:
 	printf(": DMA setup failed\n");
+err:
+	vsc->sc_child = VIRTIO_CHILD_ERROR;
 	return;
 }
 
diff --git a/sys/dev/pv/viomb.c b/sys/dev/pv/viomb.c
index e8d16c58102..52eddc30a76 100644
--- a/sys/dev/pv/viomb.c
+++ b/sys/dev/pv/viomb.c
@@ -135,6 +135,7 @@ viomb_attach(struct device *parent, struct device *self, void *aux)
 {
 	struct viomb_softc *sc = (struct viomb_softc *)self;
 	struct virtio_softc *vsc = (struct virtio_softc *)parent;
+	struct virtio_attach_args *va = aux;
 	int i;
 
 	if (vsc->sc_child != NULL) {
@@ -219,8 +220,10 @@ viomb_attach(struct device *parent, struct device *self, void *aux)
 	sensordev_install(&sc->sc_sensdev);
 
 	printf("\n");
-	virtio_set_status(vsc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK);
+	if (virtio_attach_finish(vsc, va) != 0)
+		goto err_dmamap;
 	return;
+
 err_dmamap:
 	bus_dmamap_destroy(vsc->sc_dmat, sc->sc_req.bl_dmamap);
 err:
diff --git a/sys/dev/pv/viornd.c b/sys/dev/pv/viornd.c
index 8139b4a6a27..f349cc63257 100644
--- a/sys/dev/pv/viornd.c
+++ b/sys/dev/pv/viornd.c
@@ -83,18 +83,19 @@ viornd_attach(struct device *parent, struct device *self, void *aux)
 {
 	struct viornd_softc *sc = (struct viornd_softc *)self;
 	struct virtio_softc *vsc = (struct virtio_softc *)parent;
+	struct virtio_attach_args *va = aux;
 	unsigned int shift;
 
 	vsc->sc_vqs = &sc->sc_vq;
 	vsc->sc_nvqs = 1;
-	vsc->sc_config_change = NULL;
 	if (vsc->sc_child != NULL)
 		panic("already attached to something else");
 	vsc->sc_child = self;
 	vsc->sc_ipl = IPL_NET;
 	sc->sc_virtio = vsc;
 
-	virtio_negotiate_features(vsc, NULL);
+	if (virtio_negotiate_features(vsc, NULL) != 0)
+		goto err;
 
 	if (sc->sc_dev.dv_cfdata->cf_flags & VIORND_ONESHOT) {
 		sc->sc_interval = 0;
@@ -136,7 +137,8 @@ viornd_attach(struct device *parent, struct device *self, void *aux)
 	timeout_add(&sc->sc_tick, 1);
 
 	printf("\n");
-	virtio_set_status(vsc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK);
+	if (virtio_attach_finish(vsc, va) != 0)
+		goto err2;
 	return;
 err2:
 	bus_dmamap_destroy(vsc->sc_dmat, sc->sc_dmamap);
diff --git a/sys/dev/pv/vioscsi.c b/sys/dev/pv/vioscsi.c
index 41dbc113d0d..0e8f9b41a78 100644
--- a/sys/dev/pv/vioscsi.c
+++ b/sys/dev/pv/vioscsi.c
@@ -105,6 +105,7 @@ vioscsi_attach(struct device *parent, struct device *self, void *aux)
 {
 	struct virtio_softc *vsc = (struct virtio_softc *)parent;
 	struct vioscsi_softc *sc = (struct vioscsi_softc *)self;
+	struct virtio_attach_args *va = aux;
 	struct scsibus_attach_args saa;
 	int i, rv;
 
@@ -120,7 +121,8 @@ vioscsi_attach(struct device *parent, struct device *self, void *aux)
 	vsc->sc_vqs = sc->sc_vqs;
 	vsc->sc_nvqs = nitems(sc->sc_vqs);
 
-	virtio_negotiate_features(vsc, NULL);
+	if (virtio_negotiate_features(vsc, NULL) != 0)
+		goto err;
 	uint32_t cmd_per_lun = virtio_read_device_config_4(vsc,
 	    VIRTIO_SCSI_CONFIG_CMD_PER_LUN);
 	uint32_t seg_max = virtio_read_device_config_4(vsc,
@@ -166,7 +168,8 @@ vioscsi_attach(struct device *parent, struct device *self, void *aux)
 	saa.saa_quirks = saa.saa_flags = 0;
 	saa.saa_wwpn = saa.saa_wwnn = 0;
 
-	virtio_set_status(vsc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK);
+	if (virtio_attach_finish(vsc, va) != 0)
+		goto err;
 	config_found(self, &saa, scsiprint);
 	return;
 
diff --git a/sys/dev/pv/virtio.c b/sys/dev/pv/virtio.c
index 53e75db9a23..c9cfe95a71a 100644
--- a/sys/dev/pv/virtio.c
+++ b/sys/dev/pv/virtio.c
@@ -154,6 +154,27 @@ virtio_reset(struct virtio_softc *sc)
 	sc->sc_active_features = 0;
 }
 
+int
+virtio_attach_finish(struct virtio_softc *sc, struct virtio_attach_args *va)
+{
+	int i, ret;
+
+	ret = sc->sc_ops->attach_finish(sc, va);
+	if (ret != 0)
+		return ret;
+
+	sc->sc_ops->setup_intrs(sc);
+	for (i = 0; i < sc->sc_nvqs; i++) {
+		struct virtqueue *vq = &sc->sc_vqs[i];
+
+		if (vq->vq_num == 0)
+			continue;
+		virtio_setup_queue(sc, vq, vq->vq_dmamap->dm_segs[0].ds_addr);
+	}
+	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK);
+	return 0;
+}
+
 void
 virtio_reinit_start(struct virtio_softc *sc)
 {
@@ -162,12 +183,13 @@ virtio_reinit_start(struct virtio_softc *sc)
 	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK);
 	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER);
 	virtio_negotiate_features(sc, NULL);
+	sc->sc_ops->setup_intrs(sc);
 	for (i = 0; i < sc->sc_nvqs; i++) {
 		int n;
 		struct virtqueue *vq = &sc->sc_vqs[i];
-		n = virtio_read_queue_size(sc, vq->vq_index);
-		if (n == 0)	/* vq disappeared */
+		if (vq->vq_num == 0)	/* not used */
 			continue;
+		n = virtio_read_queue_size(sc, vq->vq_index);
 		if (n != vq->vq_num) {
 			panic("%s: virtqueue size changed, vq index %d",
 			    sc->sc_dev.dv_xname, vq->vq_index);
@@ -175,7 +197,6 @@ virtio_reinit_start(struct virtio_softc *sc)
 		virtio_init_vq(sc, vq);
 		virtio_setup_queue(sc, vq, vq->vq_dmamap->dm_segs[0].ds_addr);
 	}
-	sc->sc_ops->setup_intrs(sc);
 }
 
 void
@@ -255,8 +276,11 @@ virtio_check_vqs(struct virtio_softc *sc)
 	int i, r = 0;
 
 	/* going backwards is better for if_vio */
-	for (i = sc->sc_nvqs - 1; i >= 0; i--)
+	for (i = sc->sc_nvqs - 1; i >= 0; i--) {
+		if (sc->sc_vqs[i].vq_num == 0)	/* not used */
+			continue;
 		r |= virtio_check_vq(sc, &sc->sc_vqs[i]);
+	}
 
 	return r;
 }
@@ -421,7 +445,6 @@ virtio_alloc_vq(struct virtio_softc *sc, struct virtqueue *vq, int index,
 	}
 
 	virtio_init_vq(sc, vq);
-	virtio_setup_queue(sc, vq, vq->vq_dmamap->dm_segs[0].ds_addr);
 
 #if VIRTIO_DEBUG
 	printf("\nallocated %u byte for virtqueue %d for %s, size %d\n",
@@ -450,6 +473,11 @@ virtio_free_vq(struct virtio_softc *sc, struct virtqueue *vq)
 	struct vq_entry *qe;
 	int i = 0;
 
+	if (vq->vq_num == 0) {
+		/* virtio_alloc_vq() was never called */
+		return 0;
+	}
+
 	/* device must be already deactivated */
 	/* confirm the vq is empty */
 	SLIST_FOREACH(qe, &vq->vq_freelist, qe_list) {
@@ -874,6 +902,10 @@ virtio_dequeue_commit(struct virtqueue *vq, int slot)
  * Returns 0 on success; returns 1 if the used ring has already advanced
  * too far, and the caller must process the queue again (otherwise, no
  * more interrupts will happen).
+ *
+ * The next nslots entries in the used ring will receive no interrupt.
+ * The next interrupt will be triggered when nslots+1 slots have been
+ * used.
  */
 int
 virtio_postpone_intr(struct virtqueue *vq, uint16_t nslots)
@@ -912,13 +944,16 @@ virtio_postpone_intr_smart(struct virtqueue *vq)
 /*
  * Postpone interrupt until all of the available descriptors have been
  * consumed.
+ *
+ * If there is no descriptor available right now, interrupt will be
+ * disabled.
  */
 int
 virtio_postpone_intr_far(struct virtqueue *vq)
 {
 	uint16_t	nslots;
 
-	nslots = (uint16_t)(vq->vq_avail->idx - vq->vq_used_idx);
+	nslots = (uint16_t)(vq->vq_avail->idx - vq->vq_used_idx - 1);
 
 	return virtio_postpone_intr(vq, nslots);
 }
@@ -947,6 +982,11 @@ virtio_stop_vq_intr(struct virtio_softc *sc, struct virtqueue *vq)
 	vq->vq_queued++;
 }
 
+/*
+ * For event_idx, there will be an interrupt for the next used descriptor,
+ * regardless if that descriptor is in the available ring now or if
+ * the available ring is now empty and a descriptor is put there later.
+ */
 int
 virtio_start_vq_intr(struct virtio_softc *sc, struct virtqueue *vq)
 {
@@ -999,6 +1039,10 @@ virtio_vq_dump(struct virtqueue *vq)
 #endif
 	/* Common fields */
 	printf(" + addr: %p\n", vq);
+	if (vq->vq_num == 0) {
+		printf(" + vq is unused\n");
+		return;
+	}
 	printf(" + vq num: %d\n", vq->vq_num);
 	printf(" + vq mask: 0x%X\n", vq->vq_mask);
 	printf(" + vq index: %d\n", vq->vq_index);
diff --git a/sys/dev/pv/virtiovar.h b/sys/dev/pv/virtiovar.h
index eb15dace89a..819849d01bf 100644
--- a/sys/dev/pv/virtiovar.h
+++ b/sys/dev/pv/virtiovar.h
@@ -103,7 +103,8 @@ struct vq_entry {
 
 struct virtqueue {
 	struct virtio_softc	*vq_owner;
-	unsigned int		vq_num;  /* queue size (# of entries) */
+	unsigned int		vq_num;  /* queue size (# of entries),
+					  * 0 if unused/non-existant */
 	unsigned int		vq_mask; /* (1 << vq_num - 1) */
 	int			vq_index; /* queue number (0, 1, ...) */
 
@@ -161,8 +162,11 @@ struct virtio_ops {
 	int		(*get_status)(struct virtio_softc *);
 	void		(*set_status)(struct virtio_softc *, int);
 	int		(*neg_features)(struct virtio_softc *, const struct virtio_feature_name *);
+	int		(*attach_finish)(struct virtio_softc *, struct virtio_attach_args *);
 	int		(*poll_intr)(void *);
 	void		(*intr_barrier)(struct virtio_softc *);
+	int		(*intr_establish)(struct virtio_softc *, struct virtio_attach_args *,
+			    int, struct cpu_info *, int (*)(void *), void *);
 };
 
 #define VIRTIO_CHILD_ERROR	((void*)1)
@@ -179,7 +183,7 @@ struct virtio_softc {
 	int			 sc_indirect;
 	int			 sc_version_1;
 
-	int			 sc_nvqs;	/* set by child */
+	int			 sc_nvqs;	/* size of sc_vqs, set by child */
 	struct virtqueue	*sc_vqs;	/* set by child */
 
 	struct device		*sc_child;	/* set by child,
@@ -206,6 +210,14 @@ struct virtio_softc {
 #define	virtio_set_status(sc, i)		(sc)->sc_ops->set_status(sc, i)
 #define	virtio_intr_barrier(sc)			(sc)->sc_ops->intr_barrier(sc)
 
+/*
+ * virtio_intr_establish() only works if va_nintr > 1. If it is called by a
+ * child driver, the transport driver will skip automatic intr allocation and
+ * the child driver must allocate all required interrupts itself. Vector 0 is
+ * always used for the config change interrupt.
+ */
+#define	virtio_intr_establish(sc, va, v, ci, fn, a)	(sc)->sc_ops->intr_establish(sc, va, v, ci, fn, a)
+
 /* only for transport drivers */
 #define	virtio_device_reset(sc)			virtio_set_status((sc), 0)
 
@@ -220,6 +232,7 @@ virtio_has_feature(struct virtio_softc *sc, uint64_t fbit)
 int virtio_alloc_vq(struct virtio_softc*, struct virtqueue*, int, int,
 		    const char*);
 int virtio_free_vq(struct virtio_softc*, struct virtqueue*);
+int virtio_attach_finish(struct virtio_softc *, struct virtio_attach_args *);
 void virtio_reset(struct virtio_softc *);
 void virtio_reinit_start(struct virtio_softc *);
 void virtio_reinit_end(struct virtio_softc *);
diff --git a/sys/dev/pv/vmmci.c b/sys/dev/pv/vmmci.c
index 580228bdb4c..e91e9f7e53e 100644
--- a/sys/dev/pv/vmmci.c
+++ b/sys/dev/pv/vmmci.c
@@ -89,6 +89,7 @@ vmmci_attach(struct device *parent, struct device *self, void *aux)
 {
 	struct vmmci_softc *sc = (struct vmmci_softc *)self;
 	struct virtio_softc *vsc = (struct virtio_softc *)parent;
+	struct virtio_attach_args *va = aux;
 
 	if (vsc->sc_child != NULL)
 		panic("already attached to something else");
@@ -101,7 +102,8 @@ vmmci_attach(struct device *parent, struct device *self, void *aux)
 
 	vsc->sc_driver_features = VMMCI_F_TIMESYNC | VMMCI_F_ACK |
 	    VMMCI_F_SYNCRTC;
-	virtio_negotiate_features(vsc, NULL);
+	if (virtio_negotiate_features(vsc, NULL) != 0)
+		goto err;
 
 	if (virtio_has_feature(vsc, VMMCI_F_TIMESYNC)) {
 		strlcpy(sc->sc_sensordev.xname, sc->sc_dev.dv_xname,
@@ -115,7 +117,12 @@ vmmci_attach(struct device *parent, struct device *self, void *aux)
 	}
 
 	printf("\n");
-	virtio_set_status(vsc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK);
+	if (virtio_attach_finish(vsc, va) != 0)
+		goto err;
+	return;
+
+err:
+	vsc->sc_child = VIRTIO_CHILD_ERROR;
 }
 
 int