From: Alexander Bluhm Subject: Re: kstats for igc(4) To: David Gwynne Cc: tech@openbsd.org Date: Tue, 26 Mar 2024 17:32:38 +0100 On Tue, Mar 26, 2024 at 01:52:50PM +1000, David Gwynne wrote: > the subject says it all. they work pretty much the same as the em(4) > counters, but with less quirks (so far). OK bluhm@ > the interesting ones for me are the rx no bufs, tx discards, rxd min > thresh and tx tso ctx counters. > > igc0:0:igc-stats:0 > crc errs: 0 > alignment errs: 0 > rx errs: 0 > missed pkts: 0 > single colls: 0 > excessive colls: 0 > multiple colls: 0 > late colls: 0 > collisions: 0 > recv errs: 0 > defers: 0 > tx no crs: 0 > host tx discard: 0 > recv len errs: 0 > xon rx: 0 > xon tx: 0 > xoff rx: 0 > xoff tx: 0 > fc rx unsupp: 0 > rx 64B: 204 packets > rx 65-127B: 497 packets > rx 128-255B: 200 packets > rx 256-511B: 35 packets > rx 512-1023B: 5 packets > rx 1024-maxB: 11 packets > rx good: 952 packets > rx bcast: 103 packets > rx mcast: 0 packets > tx good: 859 packets > rx good bytes: 125964 bytes > tx good bytes: 374936 bytes > rx no bufs: 0 > rx undersize: 0 > rx frags: 0 > rx oversize: 0 > rx jabbers: 0 > rx mgmt: 0 packets > rx mgmt drops: 0 packets > tx mgmt: 0 packets > rx total bytes: 179498 bytes > tx total bytes: 374936 bytes > rx total: 1207 packets > tx total: 859 packets > tx 64B: 88 packets > tx 65-127B: 366 packets > tx 128-255B: 111 packets > tx 256-511B: 62 packets > tx 512-1023B: 69 packets > tx 1024-maxB: 163 packets > tx mcast: 0 packets > tx bcast: 9 packets > tx tso ctx: 0 > interrupts: 1424 > rx to host: 952 packets > eee tx lpi: 0 > eee rx lpi: 0 > host rx: 859 packets > rxd min thresh: 0 > host good rx: 125964 bytes > host good tx: 374936 bytes > len errs: 0 > > Index: igc_regs.h > =================================================================== > RCS file: /cvs/src/sys/dev/pci/igc_regs.h,v > retrieving revision 1.2 > diff -u -p -r1.2 igc_regs.h > --- igc_regs.h 15 Aug 2023 08:27:30 -0000 1.2 > +++ igc_regs.h 26 Mar 2024 03:49:37 -0000 > @@ -164,6 +164,7 @@ > /* Statistics Register Descriptions */ > #define IGC_CRCERRS 0x04000 /* CRC Error Count - R/clr */ > #define IGC_ALGNERRC 0x04004 /* Alignment Error Count - R/clr */ > +#define IGC_RXERRC 0x04004 /* Receive Error Count - R/clr */ > #define IGC_MPC 0x04010 /* Missed Packet Count - R/clr */ > #define IGC_SCC 0x04014 /* Single Collision Count - R/clr */ > #define IGC_ECOL 0x04018 /* Excessive Collision Count - R/clr */ > @@ -218,7 +219,14 @@ > #define IGC_BPTC 0x040F4 /* Broadcast Packets Tx Count - R/clr */ > #define IGC_TSCTC 0x040F8 /* TCP Segmentation Context Tx - R/clr */ > #define IGC_IAC 0x04100 /* Interrupt Assertion Count */ > +#define IGC_RPTHC 0x04104 /* Rx Packets To Host */ > +#define IGC_HGPTC 0x04118 /* Host Good Packets Tx Count */ > #define IGC_RXDMTC 0x04120 /* Rx Descriptor Minimum Threshold Count */ > +#define IGC_HGORCL 0x04128 /* Host Good Octets Received Count Low */ > +#define IGC_HGORCH 0x0412C /* Host Good Octets Received Count High */ > +#define IGC_HGOTCL 0x04130 /* Host Good Octets Transmit Count Low */ > +#define IGC_HGOTCH 0x04134 /* Host Good Octets Transmit Count High */ > +#define IGC_LENERRS 0x04138 /* Length Errors Count */ > > #define IGC_VFGPRC 0x00F10 > #define IGC_VFGORC 0x00F18 > @@ -229,11 +237,7 @@ > #define IGC_VFGPTLBC 0x00F44 > #define IGC_VFGORLBC 0x00F48 > #define IGC_VFGPRLBC 0x00F40 > -#define IGC_HGORCL 0x04128 /* Host Good Octets Received Count Low */ > -#define IGC_HGORCH 0x0412C /* Host Good Octets Received Count High */ > -#define IGC_HGOTCL 0x04130 /* Host Good Octets Transmit Count Low */ > -#define IGC_HGOTCH 0x04134 /* Host Good Octets Transmit Count High */ > -#define IGC_LENERRS 0x04138 /* Length Errors Count */ > + > #define IGC_PCS_ANADV 0x04218 /* AN advertisement - RW */ > #define IGC_PCS_LPAB 0x0421C /* Link Partner Ability - RW */ > #define IGC_RXCSUM 0x05000 /* Rx Checksum Control - RW */ > Index: if_igc.c > =================================================================== > RCS file: /cvs/src/sys/dev/pci/if_igc.c,v > retrieving revision 1.18 > diff -u -p -r1.18 if_igc.c > --- if_igc.c 23 Feb 2024 01:06:18 -0000 1.18 > +++ if_igc.c 26 Mar 2024 03:49:37 -0000 > @@ -30,6 +30,7 @@ > > #include "bpfilter.h" > #include "vlan.h" > +#include "kstat.h" > > #include > #include > @@ -41,6 +42,7 @@ > #include > #include > #include > +#include > > #include > #include > @@ -145,6 +147,10 @@ void igc_get_hw_control(struct igc_softc > void igc_release_hw_control(struct igc_softc *); > int igc_is_valid_ether_addr(uint8_t *); > > +#if NKSTAT > 0 > +void igc_kstat_attach(struct igc_softc *); > +#endif > + > /********************************************************************* > * OpenBSD Device Interface Entry Points > *********************************************************************/ > @@ -279,6 +285,10 @@ igc_attach(struct device *parent, struct > igc_get_hw_control(sc); > > printf(", address %s\n", ether_sprintf(sc->hw.mac.addr)); > + > +#if NKSTAT > 0 > + igc_kstat_attach(sc); > +#endif > return; > > err_late: > @@ -2453,3 +2463,328 @@ igc_is_valid_ether_addr(uint8_t *addr) > > return 1; > } > + > +#if NKSTAT > 0 > + > +/* > + * the below are read to clear, so they need to be accumulated for > + * userland to see counters. periodically fetch the counters from a > + * timeout to avoid a 32 roll-over between kstat reads. > + */ > + > +enum igc_stat { > + igc_stat_crcerrs, > + igc_stat_algnerrc, > + igc_stat_rxerrc, > + igc_stat_mpc, > + igc_stat_scc, > + igc_stat_ecol, > + igc_stat_mcc, > + igc_stat_latecol, > + igc_stat_colc, > + igc_stat_rerc, > + igc_stat_dc, > + igc_stat_tncrs, > + igc_stat_htdpmc, > + igc_stat_rlec, > + igc_stat_xonrxc, > + igc_stat_xontxc, > + igc_stat_xoffrxc, > + igc_stat_xofftxc, > + igc_stat_fcruc, > + igc_stat_prc64, > + igc_stat_prc127, > + igc_stat_prc255, > + igc_stat_prc511, > + igc_stat_prc1023, > + igc_stat_prc1522, > + igc_stat_gprc, > + igc_stat_bprc, > + igc_stat_mprc, > + igc_stat_gptc, > + igc_stat_gorc, > + igc_stat_gotc, > + igc_stat_rnbc, > + igc_stat_ruc, > + igc_stat_rfc, > + igc_stat_roc, > + igc_stat_rjc, > + igc_stat_mgtprc, > + igc_stat_mgtpdc, > + igc_stat_mgtptc, > + igc_stat_tor, > + igc_stat_tot, > + igc_stat_tpr, > + igc_stat_tpt, > + igc_stat_ptc64, > + igc_stat_ptc127, > + igc_stat_ptc255, > + igc_stat_ptc511, > + igc_stat_ptc1023, > + igc_stat_ptc1522, > + igc_stat_mptc, > + igc_stat_bptc, > + igc_stat_tsctc, > + > + igc_stat_iac, > + igc_stat_rpthc, > + igc_stat_tlpic, > + igc_stat_rlpic, > + igc_stat_hgptc, > + igc_stat_rxdmtc, > + igc_stat_hgorc, > + igc_stat_hgotc, > + igc_stat_lenerrs, > + > + igc_stat_count > +}; > + > +struct igc_counter { > + const char *name; > + enum kstat_kv_unit unit; > + uint32_t reg; > +}; > + > +static const struct igc_counter igc_counters[igc_stat_count] = { > + [igc_stat_crcerrs] = > + { "crc errs", KSTAT_KV_U_NONE, IGC_CRCERRS }, > + [igc_stat_algnerrc] = > + { "alignment errs", KSTAT_KV_U_NONE, IGC_ALGNERRC }, > + [igc_stat_rxerrc] = > + { "rx errs", KSTAT_KV_U_NONE, IGC_RXERRC }, > + [igc_stat_mpc] = > + { "missed pkts", KSTAT_KV_U_NONE, IGC_MPC }, > + [igc_stat_scc] = > + { "single colls", KSTAT_KV_U_NONE, IGC_SCC }, > + [igc_stat_ecol] = > + { "excessive colls", KSTAT_KV_U_NONE, IGC_ECOL }, > + [igc_stat_mcc] = > + { "multiple colls", KSTAT_KV_U_NONE, IGC_MCC }, > + [igc_stat_latecol] = > + { "late colls", KSTAT_KV_U_NONE, IGC_LATECOL }, > + [igc_stat_colc] = > + { "collisions", KSTAT_KV_U_NONE, IGC_COLC }, > + [igc_stat_rerc] = > + { "recv errs", KSTAT_KV_U_NONE, IGC_RERC }, > + [igc_stat_dc] = > + { "defers", KSTAT_KV_U_NONE, IGC_DC }, > + [igc_stat_tncrs] = > + { "tx no crs", KSTAT_KV_U_NONE, IGC_TNCRS}, > + [igc_stat_htdpmc] = > + { "host tx discards", KSTAT_KV_U_NONE, IGC_HTDPMC }, > + [igc_stat_rlec] = > + { "recv len errs", KSTAT_KV_U_NONE, IGC_RLEC }, > + [igc_stat_xonrxc] = > + { "xon rx", KSTAT_KV_U_NONE, IGC_XONRXC }, > + [igc_stat_xontxc] = > + { "xon tx", KSTAT_KV_U_NONE, IGC_XONTXC }, > + [igc_stat_xoffrxc] = > + { "xoff rx", KSTAT_KV_U_NONE, IGC_XOFFRXC }, > + [igc_stat_xofftxc] = > + { "xoff tx", KSTAT_KV_U_NONE, IGC_XOFFTXC }, > + [igc_stat_fcruc] = > + { "fc rx unsupp", KSTAT_KV_U_NONE, IGC_FCRUC }, > + [igc_stat_prc64] = > + { "rx 64B", KSTAT_KV_U_PACKETS, IGC_PRC64 }, > + [igc_stat_prc127] = > + { "rx 65-127B", KSTAT_KV_U_PACKETS, IGC_PRC127 }, > + [igc_stat_prc255] = > + { "rx 128-255B", KSTAT_KV_U_PACKETS, IGC_PRC255 }, > + [igc_stat_prc511] = > + { "rx 256-511B", KSTAT_KV_U_PACKETS, IGC_PRC511 }, > + [igc_stat_prc1023] = > + { "rx 512-1023B", KSTAT_KV_U_PACKETS, IGC_PRC1023 }, > + [igc_stat_prc1522] = > + { "rx 1024-maxB", KSTAT_KV_U_PACKETS, IGC_PRC1522 }, > + [igc_stat_gprc] = > + { "rx good", KSTAT_KV_U_PACKETS, IGC_GPRC }, > + [igc_stat_bprc] = > + { "rx bcast", KSTAT_KV_U_PACKETS, IGC_BPRC }, > + [igc_stat_mprc] = > + { "rx mcast", KSTAT_KV_U_PACKETS, IGC_MPRC }, > + [igc_stat_gptc] = > + { "tx good", KSTAT_KV_U_PACKETS, IGC_GPTC }, > + [igc_stat_gorc] = > + { "rx good bytes", KSTAT_KV_U_BYTES, 0 }, > + [igc_stat_gotc] = > + { "tx good bytes", KSTAT_KV_U_BYTES, 0 }, > + [igc_stat_rnbc] = > + { "rx no bufs", KSTAT_KV_U_NONE, IGC_RNBC }, > + [igc_stat_ruc] = > + { "rx undersize", KSTAT_KV_U_NONE, IGC_RUC }, > + [igc_stat_rfc] = > + { "rx frags", KSTAT_KV_U_NONE, IGC_RFC }, > + [igc_stat_roc] = > + { "rx oversize", KSTAT_KV_U_NONE, IGC_ROC }, > + [igc_stat_rjc] = > + { "rx jabbers", KSTAT_KV_U_NONE, IGC_RJC }, > + [igc_stat_mgtprc] = > + { "rx mgmt", KSTAT_KV_U_PACKETS, IGC_MGTPRC }, > + [igc_stat_mgtpdc] = > + { "rx mgmt drops", KSTAT_KV_U_PACKETS, IGC_MGTPDC }, > + [igc_stat_mgtptc] = > + { "tx mgmt", KSTAT_KV_U_PACKETS, IGC_MGTPTC }, > + [igc_stat_tor] = > + { "rx total bytes", KSTAT_KV_U_BYTES, 0 }, > + [igc_stat_tot] = > + { "tx total bytes", KSTAT_KV_U_BYTES, 0 }, > + [igc_stat_tpr] = > + { "rx total", KSTAT_KV_U_PACKETS, IGC_TPR }, > + [igc_stat_tpt] = > + { "tx total", KSTAT_KV_U_PACKETS, IGC_TPT }, > + [igc_stat_ptc64] = > + { "tx 64B", KSTAT_KV_U_PACKETS, IGC_PTC64 }, > + [igc_stat_ptc127] = > + { "tx 65-127B", KSTAT_KV_U_PACKETS, IGC_PTC127 }, > + [igc_stat_ptc255] = > + { "tx 128-255B", KSTAT_KV_U_PACKETS, IGC_PTC255 }, > + [igc_stat_ptc511] = > + { "tx 256-511B", KSTAT_KV_U_PACKETS, IGC_PTC511 }, > + [igc_stat_ptc1023] = > + { "tx 512-1023B", KSTAT_KV_U_PACKETS, IGC_PTC1023 }, > + [igc_stat_ptc1522] = > + { "tx 1024-maxB", KSTAT_KV_U_PACKETS, IGC_PTC1522 }, > + [igc_stat_mptc] = > + { "tx mcast", KSTAT_KV_U_PACKETS, IGC_MPTC }, > + [igc_stat_bptc] = > + { "tx bcast", KSTAT_KV_U_PACKETS, IGC_BPTC }, > + [igc_stat_tsctc] = > + { "tx tso ctx", KSTAT_KV_U_NONE, IGC_TSCTC }, > + > + [igc_stat_iac] = > + { "interrupts", KSTAT_KV_U_NONE, IGC_IAC }, > + [igc_stat_rpthc] = > + { "rx to host", KSTAT_KV_U_PACKETS, IGC_RPTHC }, > + [igc_stat_tlpic] = > + { "eee tx lpi", KSTAT_KV_U_NONE, IGC_TLPIC }, > + [igc_stat_rlpic] = > + { "eee rx lpi", KSTAT_KV_U_NONE, IGC_RLPIC }, > + [igc_stat_hgptc] = > + { "host rx", KSTAT_KV_U_PACKETS, IGC_HGPTC }, > + [igc_stat_rxdmtc] = > + { "rxd min thresh", KSTAT_KV_U_NONE, IGC_RXDMTC }, > + [igc_stat_hgorc] = > + { "host good rx", KSTAT_KV_U_BYTES, 0 }, > + [igc_stat_hgotc] = > + { "host good tx", KSTAT_KV_U_BYTES, 0 }, > + [igc_stat_lenerrs] = > + { "len errs", KSTAT_KV_U_NONE, IGC_LENERRS }, > +}; > + > +static void > +igc_stat_read(struct igc_softc *sc) > +{ > + struct igc_hw *hw = &sc->hw; > + struct kstat *ks = sc->ks; > + struct kstat_kv *kvs = ks->ks_data; > + uint32_t hi, lo; > + unsigned int i; > + > + for (i = 0; i < nitems(igc_counters); i++) { > + const struct igc_counter *c = &igc_counters[i]; > + if (c->reg == 0) > + continue; > + > + kstat_kv_u64(&kvs[i]) += IGC_READ_REG(hw, c->reg); > + } > + > + lo = IGC_READ_REG(hw, IGC_GORCL); > + hi = IGC_READ_REG(hw, IGC_GORCH); > + kstat_kv_u64(&kvs[igc_stat_gorc]) += > + ((uint64_t)hi << 32) | ((uint64_t)lo << 0); > + > + lo = IGC_READ_REG(hw, IGC_GOTCL); > + hi = IGC_READ_REG(hw, IGC_GOTCH); > + kstat_kv_u64(&kvs[igc_stat_gotc]) += > + ((uint64_t)hi << 32) | ((uint64_t)lo << 0); > + > + lo = IGC_READ_REG(hw, IGC_TORL); > + hi = IGC_READ_REG(hw, IGC_TORH); > + kstat_kv_u64(&kvs[igc_stat_tor]) += > + ((uint64_t)hi << 32) | ((uint64_t)lo << 0); > + > + lo = IGC_READ_REG(hw, IGC_TOTL); > + hi = IGC_READ_REG(hw, IGC_TOTH); > + kstat_kv_u64(&kvs[igc_stat_tot]) += > + ((uint64_t)hi << 32) | ((uint64_t)lo << 0); > + > + lo = IGC_READ_REG(hw, IGC_HGORCL); > + hi = IGC_READ_REG(hw, IGC_HGORCH); > + kstat_kv_u64(&kvs[igc_stat_hgorc]) += > + ((uint64_t)hi << 32) | ((uint64_t)lo << 0); > + > + lo = IGC_READ_REG(hw, IGC_HGOTCL); > + hi = IGC_READ_REG(hw, IGC_HGOTCH); > + kstat_kv_u64(&kvs[igc_stat_hgotc]) += > + ((uint64_t)hi << 32) | ((uint64_t)lo << 0); > +} > + > +static void > +igc_kstat_tick(void *arg) > +{ > + struct igc_softc *sc = arg; > + > + if (mtx_enter_try(&sc->ks_mtx)) { > + igc_stat_read(sc); > + mtx_leave(&sc->ks_mtx); > + } > + > + timeout_add_sec(&sc->ks_tmo, 4); > +} > + > +static int > +igc_kstat_read(struct kstat *ks) > +{ > + struct igc_softc *sc = ks->ks_softc; > + > + igc_stat_read(sc); > + nanouptime(&ks->ks_updated); > + > + return (0); > +} > + > +void > +igc_kstat_attach(struct igc_softc *sc) > +{ > + struct kstat *ks; > + struct kstat_kv *kvs; > + size_t len; > + unsigned int i; > + > + mtx_init(&sc->ks_mtx, IPL_SOFTCLOCK); > + timeout_set(&sc->ks_tmo, igc_kstat_tick, sc); > + > + kvs = mallocarray(sizeof(*kvs), nitems(igc_counters), M_DEVBUF, > + M_WAITOK|M_ZERO|M_CANFAIL); > + if (kvs == NULL) { > + printf("%s: unable to allocate igc kstats\n", DEVNAME(sc)); > + return; > + } > + len = sizeof(*kvs) * nitems(igc_counters); > + > + ks = kstat_create(DEVNAME(sc), 0, "igc-stats", 0, KSTAT_T_KV, 0); > + if (ks == NULL) { > + printf("%s: unable to create igc kstats\n", DEVNAME(sc)); > + free(kvs, M_DEVBUF, len); > + return; > + } > + > + for (i = 0; i < nitems(igc_counters); i++) { > + const struct igc_counter *c = &igc_counters[i]; > + kstat_kv_unit_init(&kvs[i], c->name, > + KSTAT_KV_T_COUNTER64, c->unit); > + } > + > + ks->ks_softc = sc; > + ks->ks_data = kvs; > + ks->ks_datalen = len; > + ks->ks_read = igc_kstat_read; > + kstat_set_mutex(ks, &sc->ks_mtx); > + > + kstat_install(ks); > + > + sc->ks = ks; > + > + igc_kstat_tick(sc); /* let's gooo */ > +} > +#endif /* NKSTAT > 0 */ > Index: if_igc.h > =================================================================== > RCS file: /cvs/src/sys/dev/pci/if_igc.h,v > retrieving revision 1.2 > diff -u -p -r1.2 if_igc.h > --- if_igc.h 9 Jan 2022 05:42:50 -0000 1.2 > +++ if_igc.h 26 Mar 2024 03:49:37 -0000 > @@ -199,6 +199,7 @@ > > /* Forward declaration. */ > struct igc_hw; > +struct kstat; > > struct igc_osdep { > bus_dma_tag_t os_dmat; > @@ -320,6 +321,11 @@ struct igc_softc { > > /* Multicast array memory */ > uint8_t *mta; > + > + /* Counters */ > + struct mutex ks_mtx; > + struct timeout ks_tmo; > + struct kstat *ks; > }; > > #define DEVNAME(_sc) ((_sc)->sc_dev.dv_xname)