From: David Gwynne Subject: kstats for igc(4) To: tech@openbsd.org Date: Tue, 26 Mar 2024 13:52:50 +1000 the subject says it all. they work pretty much the same as the em(4) counters, but with less quirks (so far). the interesting ones for me are the rx no bufs, tx discards, rxd min thresh and tx tso ctx counters. igc0:0:igc-stats:0 crc errs: 0 alignment errs: 0 rx errs: 0 missed pkts: 0 single colls: 0 excessive colls: 0 multiple colls: 0 late colls: 0 collisions: 0 recv errs: 0 defers: 0 tx no crs: 0 host tx discard: 0 recv len errs: 0 xon rx: 0 xon tx: 0 xoff rx: 0 xoff tx: 0 fc rx unsupp: 0 rx 64B: 204 packets rx 65-127B: 497 packets rx 128-255B: 200 packets rx 256-511B: 35 packets rx 512-1023B: 5 packets rx 1024-maxB: 11 packets rx good: 952 packets rx bcast: 103 packets rx mcast: 0 packets tx good: 859 packets rx good bytes: 125964 bytes tx good bytes: 374936 bytes rx no bufs: 0 rx undersize: 0 rx frags: 0 rx oversize: 0 rx jabbers: 0 rx mgmt: 0 packets rx mgmt drops: 0 packets tx mgmt: 0 packets rx total bytes: 179498 bytes tx total bytes: 374936 bytes rx total: 1207 packets tx total: 859 packets tx 64B: 88 packets tx 65-127B: 366 packets tx 128-255B: 111 packets tx 256-511B: 62 packets tx 512-1023B: 69 packets tx 1024-maxB: 163 packets tx mcast: 0 packets tx bcast: 9 packets tx tso ctx: 0 interrupts: 1424 rx to host: 952 packets eee tx lpi: 0 eee rx lpi: 0 host rx: 859 packets rxd min thresh: 0 host good rx: 125964 bytes host good tx: 374936 bytes len errs: 0 Index: igc_regs.h =================================================================== RCS file: /cvs/src/sys/dev/pci/igc_regs.h,v retrieving revision 1.2 diff -u -p -r1.2 igc_regs.h --- igc_regs.h 15 Aug 2023 08:27:30 -0000 1.2 +++ igc_regs.h 26 Mar 2024 03:49:37 -0000 @@ -164,6 +164,7 @@ /* Statistics Register Descriptions */ #define IGC_CRCERRS 0x04000 /* CRC Error Count - R/clr */ #define IGC_ALGNERRC 0x04004 /* Alignment Error Count - R/clr */ +#define IGC_RXERRC 0x04004 /* Receive Error Count - R/clr */ #define IGC_MPC 0x04010 /* Missed Packet Count - R/clr */ #define IGC_SCC 0x04014 /* Single Collision Count - R/clr */ #define IGC_ECOL 0x04018 /* Excessive Collision Count - R/clr */ @@ -218,7 +219,14 @@ #define IGC_BPTC 0x040F4 /* Broadcast Packets Tx Count - R/clr */ #define IGC_TSCTC 0x040F8 /* TCP Segmentation Context Tx - R/clr */ #define IGC_IAC 0x04100 /* Interrupt Assertion Count */ +#define IGC_RPTHC 0x04104 /* Rx Packets To Host */ +#define IGC_HGPTC 0x04118 /* Host Good Packets Tx Count */ #define IGC_RXDMTC 0x04120 /* Rx Descriptor Minimum Threshold Count */ +#define IGC_HGORCL 0x04128 /* Host Good Octets Received Count Low */ +#define IGC_HGORCH 0x0412C /* Host Good Octets Received Count High */ +#define IGC_HGOTCL 0x04130 /* Host Good Octets Transmit Count Low */ +#define IGC_HGOTCH 0x04134 /* Host Good Octets Transmit Count High */ +#define IGC_LENERRS 0x04138 /* Length Errors Count */ #define IGC_VFGPRC 0x00F10 #define IGC_VFGORC 0x00F18 @@ -229,11 +237,7 @@ #define IGC_VFGPTLBC 0x00F44 #define IGC_VFGORLBC 0x00F48 #define IGC_VFGPRLBC 0x00F40 -#define IGC_HGORCL 0x04128 /* Host Good Octets Received Count Low */ -#define IGC_HGORCH 0x0412C /* Host Good Octets Received Count High */ -#define IGC_HGOTCL 0x04130 /* Host Good Octets Transmit Count Low */ -#define IGC_HGOTCH 0x04134 /* Host Good Octets Transmit Count High */ -#define IGC_LENERRS 0x04138 /* Length Errors Count */ + #define IGC_PCS_ANADV 0x04218 /* AN advertisement - RW */ #define IGC_PCS_LPAB 0x0421C /* Link Partner Ability - RW */ #define IGC_RXCSUM 0x05000 /* Rx Checksum Control - RW */ Index: if_igc.c =================================================================== RCS file: /cvs/src/sys/dev/pci/if_igc.c,v retrieving revision 1.18 diff -u -p -r1.18 if_igc.c --- if_igc.c 23 Feb 2024 01:06:18 -0000 1.18 +++ if_igc.c 26 Mar 2024 03:49:37 -0000 @@ -30,6 +30,7 @@ #include "bpfilter.h" #include "vlan.h" +#include "kstat.h" #include #include @@ -41,6 +42,7 @@ #include #include #include +#include #include #include @@ -145,6 +147,10 @@ void igc_get_hw_control(struct igc_softc void igc_release_hw_control(struct igc_softc *); int igc_is_valid_ether_addr(uint8_t *); +#if NKSTAT > 0 +void igc_kstat_attach(struct igc_softc *); +#endif + /********************************************************************* * OpenBSD Device Interface Entry Points *********************************************************************/ @@ -279,6 +285,10 @@ igc_attach(struct device *parent, struct igc_get_hw_control(sc); printf(", address %s\n", ether_sprintf(sc->hw.mac.addr)); + +#if NKSTAT > 0 + igc_kstat_attach(sc); +#endif return; err_late: @@ -2453,3 +2463,328 @@ igc_is_valid_ether_addr(uint8_t *addr) return 1; } + +#if NKSTAT > 0 + +/* + * the below are read to clear, so they need to be accumulated for + * userland to see counters. periodically fetch the counters from a + * timeout to avoid a 32 roll-over between kstat reads. + */ + +enum igc_stat { + igc_stat_crcerrs, + igc_stat_algnerrc, + igc_stat_rxerrc, + igc_stat_mpc, + igc_stat_scc, + igc_stat_ecol, + igc_stat_mcc, + igc_stat_latecol, + igc_stat_colc, + igc_stat_rerc, + igc_stat_dc, + igc_stat_tncrs, + igc_stat_htdpmc, + igc_stat_rlec, + igc_stat_xonrxc, + igc_stat_xontxc, + igc_stat_xoffrxc, + igc_stat_xofftxc, + igc_stat_fcruc, + igc_stat_prc64, + igc_stat_prc127, + igc_stat_prc255, + igc_stat_prc511, + igc_stat_prc1023, + igc_stat_prc1522, + igc_stat_gprc, + igc_stat_bprc, + igc_stat_mprc, + igc_stat_gptc, + igc_stat_gorc, + igc_stat_gotc, + igc_stat_rnbc, + igc_stat_ruc, + igc_stat_rfc, + igc_stat_roc, + igc_stat_rjc, + igc_stat_mgtprc, + igc_stat_mgtpdc, + igc_stat_mgtptc, + igc_stat_tor, + igc_stat_tot, + igc_stat_tpr, + igc_stat_tpt, + igc_stat_ptc64, + igc_stat_ptc127, + igc_stat_ptc255, + igc_stat_ptc511, + igc_stat_ptc1023, + igc_stat_ptc1522, + igc_stat_mptc, + igc_stat_bptc, + igc_stat_tsctc, + + igc_stat_iac, + igc_stat_rpthc, + igc_stat_tlpic, + igc_stat_rlpic, + igc_stat_hgptc, + igc_stat_rxdmtc, + igc_stat_hgorc, + igc_stat_hgotc, + igc_stat_lenerrs, + + igc_stat_count +}; + +struct igc_counter { + const char *name; + enum kstat_kv_unit unit; + uint32_t reg; +}; + +static const struct igc_counter igc_counters[igc_stat_count] = { + [igc_stat_crcerrs] = + { "crc errs", KSTAT_KV_U_NONE, IGC_CRCERRS }, + [igc_stat_algnerrc] = + { "alignment errs", KSTAT_KV_U_NONE, IGC_ALGNERRC }, + [igc_stat_rxerrc] = + { "rx errs", KSTAT_KV_U_NONE, IGC_RXERRC }, + [igc_stat_mpc] = + { "missed pkts", KSTAT_KV_U_NONE, IGC_MPC }, + [igc_stat_scc] = + { "single colls", KSTAT_KV_U_NONE, IGC_SCC }, + [igc_stat_ecol] = + { "excessive colls", KSTAT_KV_U_NONE, IGC_ECOL }, + [igc_stat_mcc] = + { "multiple colls", KSTAT_KV_U_NONE, IGC_MCC }, + [igc_stat_latecol] = + { "late colls", KSTAT_KV_U_NONE, IGC_LATECOL }, + [igc_stat_colc] = + { "collisions", KSTAT_KV_U_NONE, IGC_COLC }, + [igc_stat_rerc] = + { "recv errs", KSTAT_KV_U_NONE, IGC_RERC }, + [igc_stat_dc] = + { "defers", KSTAT_KV_U_NONE, IGC_DC }, + [igc_stat_tncrs] = + { "tx no crs", KSTAT_KV_U_NONE, IGC_TNCRS}, + [igc_stat_htdpmc] = + { "host tx discards", KSTAT_KV_U_NONE, IGC_HTDPMC }, + [igc_stat_rlec] = + { "recv len errs", KSTAT_KV_U_NONE, IGC_RLEC }, + [igc_stat_xonrxc] = + { "xon rx", KSTAT_KV_U_NONE, IGC_XONRXC }, + [igc_stat_xontxc] = + { "xon tx", KSTAT_KV_U_NONE, IGC_XONTXC }, + [igc_stat_xoffrxc] = + { "xoff rx", KSTAT_KV_U_NONE, IGC_XOFFRXC }, + [igc_stat_xofftxc] = + { "xoff tx", KSTAT_KV_U_NONE, IGC_XOFFTXC }, + [igc_stat_fcruc] = + { "fc rx unsupp", KSTAT_KV_U_NONE, IGC_FCRUC }, + [igc_stat_prc64] = + { "rx 64B", KSTAT_KV_U_PACKETS, IGC_PRC64 }, + [igc_stat_prc127] = + { "rx 65-127B", KSTAT_KV_U_PACKETS, IGC_PRC127 }, + [igc_stat_prc255] = + { "rx 128-255B", KSTAT_KV_U_PACKETS, IGC_PRC255 }, + [igc_stat_prc511] = + { "rx 256-511B", KSTAT_KV_U_PACKETS, IGC_PRC511 }, + [igc_stat_prc1023] = + { "rx 512-1023B", KSTAT_KV_U_PACKETS, IGC_PRC1023 }, + [igc_stat_prc1522] = + { "rx 1024-maxB", KSTAT_KV_U_PACKETS, IGC_PRC1522 }, + [igc_stat_gprc] = + { "rx good", KSTAT_KV_U_PACKETS, IGC_GPRC }, + [igc_stat_bprc] = + { "rx bcast", KSTAT_KV_U_PACKETS, IGC_BPRC }, + [igc_stat_mprc] = + { "rx mcast", KSTAT_KV_U_PACKETS, IGC_MPRC }, + [igc_stat_gptc] = + { "tx good", KSTAT_KV_U_PACKETS, IGC_GPTC }, + [igc_stat_gorc] = + { "rx good bytes", KSTAT_KV_U_BYTES, 0 }, + [igc_stat_gotc] = + { "tx good bytes", KSTAT_KV_U_BYTES, 0 }, + [igc_stat_rnbc] = + { "rx no bufs", KSTAT_KV_U_NONE, IGC_RNBC }, + [igc_stat_ruc] = + { "rx undersize", KSTAT_KV_U_NONE, IGC_RUC }, + [igc_stat_rfc] = + { "rx frags", KSTAT_KV_U_NONE, IGC_RFC }, + [igc_stat_roc] = + { "rx oversize", KSTAT_KV_U_NONE, IGC_ROC }, + [igc_stat_rjc] = + { "rx jabbers", KSTAT_KV_U_NONE, IGC_RJC }, + [igc_stat_mgtprc] = + { "rx mgmt", KSTAT_KV_U_PACKETS, IGC_MGTPRC }, + [igc_stat_mgtpdc] = + { "rx mgmt drops", KSTAT_KV_U_PACKETS, IGC_MGTPDC }, + [igc_stat_mgtptc] = + { "tx mgmt", KSTAT_KV_U_PACKETS, IGC_MGTPTC }, + [igc_stat_tor] = + { "rx total bytes", KSTAT_KV_U_BYTES, 0 }, + [igc_stat_tot] = + { "tx total bytes", KSTAT_KV_U_BYTES, 0 }, + [igc_stat_tpr] = + { "rx total", KSTAT_KV_U_PACKETS, IGC_TPR }, + [igc_stat_tpt] = + { "tx total", KSTAT_KV_U_PACKETS, IGC_TPT }, + [igc_stat_ptc64] = + { "tx 64B", KSTAT_KV_U_PACKETS, IGC_PTC64 }, + [igc_stat_ptc127] = + { "tx 65-127B", KSTAT_KV_U_PACKETS, IGC_PTC127 }, + [igc_stat_ptc255] = + { "tx 128-255B", KSTAT_KV_U_PACKETS, IGC_PTC255 }, + [igc_stat_ptc511] = + { "tx 256-511B", KSTAT_KV_U_PACKETS, IGC_PTC511 }, + [igc_stat_ptc1023] = + { "tx 512-1023B", KSTAT_KV_U_PACKETS, IGC_PTC1023 }, + [igc_stat_ptc1522] = + { "tx 1024-maxB", KSTAT_KV_U_PACKETS, IGC_PTC1522 }, + [igc_stat_mptc] = + { "tx mcast", KSTAT_KV_U_PACKETS, IGC_MPTC }, + [igc_stat_bptc] = + { "tx bcast", KSTAT_KV_U_PACKETS, IGC_BPTC }, + [igc_stat_tsctc] = + { "tx tso ctx", KSTAT_KV_U_NONE, IGC_TSCTC }, + + [igc_stat_iac] = + { "interrupts", KSTAT_KV_U_NONE, IGC_IAC }, + [igc_stat_rpthc] = + { "rx to host", KSTAT_KV_U_PACKETS, IGC_RPTHC }, + [igc_stat_tlpic] = + { "eee tx lpi", KSTAT_KV_U_NONE, IGC_TLPIC }, + [igc_stat_rlpic] = + { "eee rx lpi", KSTAT_KV_U_NONE, IGC_RLPIC }, + [igc_stat_hgptc] = + { "host rx", KSTAT_KV_U_PACKETS, IGC_HGPTC }, + [igc_stat_rxdmtc] = + { "rxd min thresh", KSTAT_KV_U_NONE, IGC_RXDMTC }, + [igc_stat_hgorc] = + { "host good rx", KSTAT_KV_U_BYTES, 0 }, + [igc_stat_hgotc] = + { "host good tx", KSTAT_KV_U_BYTES, 0 }, + [igc_stat_lenerrs] = + { "len errs", KSTAT_KV_U_NONE, IGC_LENERRS }, +}; + +static void +igc_stat_read(struct igc_softc *sc) +{ + struct igc_hw *hw = &sc->hw; + struct kstat *ks = sc->ks; + struct kstat_kv *kvs = ks->ks_data; + uint32_t hi, lo; + unsigned int i; + + for (i = 0; i < nitems(igc_counters); i++) { + const struct igc_counter *c = &igc_counters[i]; + if (c->reg == 0) + continue; + + kstat_kv_u64(&kvs[i]) += IGC_READ_REG(hw, c->reg); + } + + lo = IGC_READ_REG(hw, IGC_GORCL); + hi = IGC_READ_REG(hw, IGC_GORCH); + kstat_kv_u64(&kvs[igc_stat_gorc]) += + ((uint64_t)hi << 32) | ((uint64_t)lo << 0); + + lo = IGC_READ_REG(hw, IGC_GOTCL); + hi = IGC_READ_REG(hw, IGC_GOTCH); + kstat_kv_u64(&kvs[igc_stat_gotc]) += + ((uint64_t)hi << 32) | ((uint64_t)lo << 0); + + lo = IGC_READ_REG(hw, IGC_TORL); + hi = IGC_READ_REG(hw, IGC_TORH); + kstat_kv_u64(&kvs[igc_stat_tor]) += + ((uint64_t)hi << 32) | ((uint64_t)lo << 0); + + lo = IGC_READ_REG(hw, IGC_TOTL); + hi = IGC_READ_REG(hw, IGC_TOTH); + kstat_kv_u64(&kvs[igc_stat_tot]) += + ((uint64_t)hi << 32) | ((uint64_t)lo << 0); + + lo = IGC_READ_REG(hw, IGC_HGORCL); + hi = IGC_READ_REG(hw, IGC_HGORCH); + kstat_kv_u64(&kvs[igc_stat_hgorc]) += + ((uint64_t)hi << 32) | ((uint64_t)lo << 0); + + lo = IGC_READ_REG(hw, IGC_HGOTCL); + hi = IGC_READ_REG(hw, IGC_HGOTCH); + kstat_kv_u64(&kvs[igc_stat_hgotc]) += + ((uint64_t)hi << 32) | ((uint64_t)lo << 0); +} + +static void +igc_kstat_tick(void *arg) +{ + struct igc_softc *sc = arg; + + if (mtx_enter_try(&sc->ks_mtx)) { + igc_stat_read(sc); + mtx_leave(&sc->ks_mtx); + } + + timeout_add_sec(&sc->ks_tmo, 4); +} + +static int +igc_kstat_read(struct kstat *ks) +{ + struct igc_softc *sc = ks->ks_softc; + + igc_stat_read(sc); + nanouptime(&ks->ks_updated); + + return (0); +} + +void +igc_kstat_attach(struct igc_softc *sc) +{ + struct kstat *ks; + struct kstat_kv *kvs; + size_t len; + unsigned int i; + + mtx_init(&sc->ks_mtx, IPL_SOFTCLOCK); + timeout_set(&sc->ks_tmo, igc_kstat_tick, sc); + + kvs = mallocarray(sizeof(*kvs), nitems(igc_counters), M_DEVBUF, + M_WAITOK|M_ZERO|M_CANFAIL); + if (kvs == NULL) { + printf("%s: unable to allocate igc kstats\n", DEVNAME(sc)); + return; + } + len = sizeof(*kvs) * nitems(igc_counters); + + ks = kstat_create(DEVNAME(sc), 0, "igc-stats", 0, KSTAT_T_KV, 0); + if (ks == NULL) { + printf("%s: unable to create igc kstats\n", DEVNAME(sc)); + free(kvs, M_DEVBUF, len); + return; + } + + for (i = 0; i < nitems(igc_counters); i++) { + const struct igc_counter *c = &igc_counters[i]; + kstat_kv_unit_init(&kvs[i], c->name, + KSTAT_KV_T_COUNTER64, c->unit); + } + + ks->ks_softc = sc; + ks->ks_data = kvs; + ks->ks_datalen = len; + ks->ks_read = igc_kstat_read; + kstat_set_mutex(ks, &sc->ks_mtx); + + kstat_install(ks); + + sc->ks = ks; + + igc_kstat_tick(sc); /* let's gooo */ +} +#endif /* NKSTAT > 0 */ Index: if_igc.h =================================================================== RCS file: /cvs/src/sys/dev/pci/if_igc.h,v retrieving revision 1.2 diff -u -p -r1.2 if_igc.h --- if_igc.h 9 Jan 2022 05:42:50 -0000 1.2 +++ if_igc.h 26 Mar 2024 03:49:37 -0000 @@ -199,6 +199,7 @@ /* Forward declaration. */ struct igc_hw; +struct kstat; struct igc_osdep { bus_dma_tag_t os_dmat; @@ -320,6 +321,11 @@ struct igc_softc { /* Multicast array memory */ uint8_t *mta; + + /* Counters */ + struct mutex ks_mtx; + struct timeout ks_tmo; + struct kstat *ks; }; #define DEVNAME(_sc) ((_sc)->sc_dev.dv_xname)