diff --git a/agent/agent.go b/agent/agent.go index 8e5a75c1f..757afdb87 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -32,7 +32,7 @@ type Agent struct { diskBaseline map[string]prevDisk // Latest disk I/O counters of any interval, seeds a new interval diskUsageCacheDuration time.Duration // How long to cache disk usage (to avoid waking sleeping disks) lastDiskUsageUpdate time.Time // Last time disk usage was collected - netInterfaces map[string]struct{} // Stores all valid network interfaces + netInterfaces map[string]bool // Valid network interfaces; true if byte counters come from MAC stats (Jetson nvethernet) netIoStats map[uint16]system.NetIoStats // Keeps track of bandwidth usage per cache interval netInterfaceDeltaTrackers map[uint16]*deltatracker.DeltaTracker[string, uint64] // Per-cache-time NIC delta trackers dockerManager *dockerManager // Manages Docker API requests diff --git a/agent/network.go b/agent/network.go index 933ecc5e4..c980a277a 100644 --- a/agent/network.go +++ b/agent/network.go @@ -92,7 +92,7 @@ func (a *Agent) updateNetworkStats(cacheTimeMs uint16, systemStats *system.Stats func (a *Agent) initializeNetIoStats() { // reset valid network interfaces - a.netInterfaces = make(map[string]struct{}, 0) + a.netInterfaces = make(map[string]bool, 0) // parse NICS env var for whitelist / blacklist nicsEnvVal, nicsEnvExists := utils.GetEnv("NICS") @@ -107,9 +107,14 @@ func (a *Agent) initializeNetIoStats() { if skipNetworkInterface(v, nicCfg) { continue } + // driver is checked only here so updates don't pay for it on non-Jetson systems + useMacCounters := isNvidiaEthernet(v.Name) + if useMacCounters { + correctNvethernetCounters(&v) + } slog.Info("Detected network interface", "name", v.Name, "sent", v.BytesSent, "recv", v.BytesRecv) // store as a valid network interface - a.netInterfaces[v.Name] = struct{}{} + a.netInterfaces[v.Name] = useMacCounters } } @@ -159,9 +164,13 @@ func (a *Agent) sumAndTrackPerNicDeltas(cacheTimeMs uint16, msElapsed uint64, ne tracker.Cycle() for _, v := range netIO { - if _, exists := a.netInterfaces[v.Name]; !exists { + useMacCounters, exists := a.netInterfaces[v.Name] + if !exists { continue } + if useMacCounters { + correctNvethernetCounters(&v) + } totalBytesSent += v.BytesSent totalBytesRecv += v.BytesRecv diff --git a/agent/network_counters_linux.go b/agent/network_counters_linux.go new file mode 100644 index 000000000..4cb68d6f5 --- /dev/null +++ b/agent/network_counters_linux.go @@ -0,0 +1,71 @@ +//go:build linux + +package agent + +import ( + "log/slog" + "math" + "os" + "path/filepath" + "strings" + + "github.com/safchain/ethtool" + psutilNet "github.com/shirou/gopsutil/v4/net" +) + +// correctNvethernetCounters replaces the inflated sysfs byte counters of an +// nvethernet NIC with its MAC octet counters. +func correctNvethernetCounters(v *psutilNet.IOCountersStat) { + tx, rx, ok := readEthtoolMACOctets(v.Name) + if !ok { + return + } + v.BytesSent = tx + v.BytesRecv = rx +} + +func isNvidiaEthernet(name string) bool { + if name == "" || strings.Contains(name, "/") { + return false + } + + driverPath := filepath.Join("/sys/class/net", name, "device/driver") + target, err := os.Readlink(driverPath) + if err != nil { + return false + } + return filepath.Base(target) == "nvethernet" +} + +func readEthtoolMACOctets(name string) (tx, rx uint64, ok bool) { + stats, err := ethtool.Stats(name) + if err != nil { + slog.Debug("Failed to read ethtool network counters", "interface", name, "err", err) + return 0, 0, false + } + + tx, okTx := ethtoolCounter(stats, "mmc_tx_octetcount_gb", "mmc_tx_octetcount_gb_h") + rx, okRx := ethtoolCounter(stats, "mmc_rx_octetcount_gb", "mmc_rx_octetcount_gb_h") + if !okTx || !okRx { + return 0, 0, false + } + return tx, rx, true +} + +// ethtoolCounter combines nvethernet's split MMC counters. The driver accumulates +// the low and high registers into independent 64-bit fields, so the low value can +// exceed 32 bits and must be added rather than OR'd into the shifted high word. +func ethtoolCounter(stats map[string]uint64, lowKey, highKey string) (uint64, bool) { + low, ok := stats[lowKey] + if !ok { + return 0, false + } + high, hasHigh := stats[highKey] + if !hasHigh { + return low, true + } + if high > (math.MaxUint64-low)>>32 { + return 0, false + } + return high<<32 + low, true +} diff --git a/agent/network_counters_linux_test.go b/agent/network_counters_linux_test.go new file mode 100644 index 000000000..e057438ad --- /dev/null +++ b/agent/network_counters_linux_test.go @@ -0,0 +1,53 @@ +//go:build linux && testing + +package agent + +import ( + "math" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestEthtoolCounterCombinesHighWord(t *testing.T) { + stats := map[string]uint64{ + "counter": 123, + "counter_h": 2, + } + + value, ok := ethtoolCounter(stats, "counter", "counter_h") + require.True(t, ok) + assert.Equal(t, uint64(2<<32+123), value) +} + +func TestEthtoolCounterAddsLowWordAbove32Bits(t *testing.T) { + // nvethernet accumulates each register in 64-bit software fields, so the low + // word can carry past 32 bits; OR would drop the overlapping bit. + stats := map[string]uint64{ + "counter": 1<<32 + 5, + "counter_h": 1, + } + + value, ok := ethtoolCounter(stats, "counter", "counter_h") + require.True(t, ok) + assert.Equal(t, uint64(2<<32+5), value) +} + +func TestEthtoolCounterRejectsOverflow(t *testing.T) { + stats := map[string]uint64{ + "counter": math.MaxUint64, + "counter_h": 1, + } + + _, ok := ethtoolCounter(stats, "counter", "counter_h") + assert.False(t, ok) +} + +func TestEthtoolCounterFallsBackToLowWord(t *testing.T) { + stats := map[string]uint64{"counter": 456} + + value, ok := ethtoolCounter(stats, "counter", "counter_h") + require.True(t, ok) + assert.Equal(t, uint64(456), value) +} diff --git a/agent/network_counters_unsupported.go b/agent/network_counters_unsupported.go new file mode 100644 index 000000000..59f70a926 --- /dev/null +++ b/agent/network_counters_unsupported.go @@ -0,0 +1,11 @@ +//go:build !linux + +package agent + +import psutilNet "github.com/shirou/gopsutil/v4/net" + +func isNvidiaEthernet(name string) bool { + return false +} + +func correctNvethernetCounters(v *psutilNet.IOCountersStat) {} diff --git a/agent/network_test.go b/agent/network_test.go index 6f7ecb403..f04910992 100644 --- a/agent/network_test.go +++ b/agent/network_test.go @@ -344,7 +344,7 @@ func TestComputeBytesPerSecond(t *testing.T) { func TestSumAndTrackPerNicDeltas(t *testing.T) { a := &Agent{ - netInterfaces: map[string]struct{}{"eth0": {}, "wlan0": {}}, + netInterfaces: map[string]bool{"eth0": false, "wlan0": false}, netInterfaceDeltaTrackers: make(map[uint16]*deltatracker.DeltaTracker[string, uint64]), } @@ -373,7 +373,7 @@ func TestSumAndTrackPerNicDeltas(t *testing.T) { func TestSumAndTrackPerNicDeltasHandlesCounterReset(t *testing.T) { a := &Agent{ - netInterfaces: map[string]struct{}{"eth0": {}}, + netInterfaces: map[string]bool{"eth0": false}, netInterfaceDeltaTrackers: make(map[uint16]*deltatracker.DeltaTracker[string, uint64]), } @@ -469,7 +469,7 @@ func TestApplyNetworkTotals(t *testing.T) { t.Run(tt.name, func(t *testing.T) { // Setup agent with initialized maps a := &Agent{ - netInterfaces: make(map[string]struct{}), + netInterfaces: make(map[string]bool), netIoStats: make(map[uint16]system.NetIoStats), netInterfaceDeltaTrackers: make(map[uint16]*deltatracker.DeltaTracker[string, uint64]), } @@ -511,3 +511,22 @@ func TestApplyNetworkTotals(t *testing.T) { }) } } + +func TestSumAndTrackPerNicDeltasKeepsCountersWhenMacReadFails(t *testing.T) { + a := &Agent{ + // missing0 is flagged for MAC counters but has no ethtool stats to read + netInterfaces: map[string]bool{"eth0": false, "missing0": true}, + netInterfaceDeltaTrackers: make(map[uint16]*deltatracker.DeltaTracker[string, uint64]), + } + netIO := []psutilNet.IOCountersStat{ + {Name: "eth0", BytesSent: 100, BytesRecv: 200}, + {Name: "missing0", BytesSent: 300, BytesRecv: 400}, + } + stats := &system.Stats{} + a.ensureNetworkInterfacesMap(stats) + + tx, rx := a.sumAndTrackPerNicDeltas(1, 0, netIO, stats) + assert.Equal(t, uint64(400), tx) + assert.Equal(t, uint64(600), rx) + assert.Equal(t, [4]uint64{0, 0, 300, 400}, stats.NetworkInterfaces["missing0"]) +} diff --git a/go.mod b/go.mod index d128fc0b4..2ae169617 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 github.com/pocketbase/dbx v1.12.0 github.com/pocketbase/pocketbase v0.40.4 + github.com/safchain/ethtool v0.7.0 github.com/shirou/gopsutil/v4 v4.26.8 github.com/spf13/cast v1.10.0 github.com/spf13/cobra v1.10.2 diff --git a/go.sum b/go.sum index 19b9d209d..07eb5e6a3 100644 --- a/go.sum +++ b/go.sum @@ -115,6 +115,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qq github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/safchain/ethtool v0.7.0 h1:rlJzfDetsVvT61uz8x1YIcFn12akMfuPulHtZjtb7Is= +github.com/safchain/ethtool v0.7.0/go.mod h1:MenQKEjXdfkjD3mp2QdCk8B/hwvkrlOTm/FD4gTpFxQ= github.com/shirou/gopsutil/v4 v4.26.8 h1:YQMTF/1J50B5+Y0vlo1eDRf5DoR7Gk69hY+8wjYkQeo= github.com/shirou/gopsutil/v4 v4.26.8/go.mod h1:5O9FjBiXoTDFatIWjZZosqj4pV0DRtLx598xGbBehzM= github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=