fix(agent): use nvethernet MAC counters (#2090)

Co-authored-by: hank <hank@henrygd.me>

agent/agent.go

@@ -32,7 +32,7 @@ type Agent struct {
 	diskBaseline              map[string]prevDisk                                   // Latest disk I/O counters of any interval, seeds a new interval
 	diskUsageCacheDuration    time.Duration                                         // How long to cache disk usage (to avoid waking sleeping disks)
 	lastDiskUsageUpdate       time.Time                                             // Last time disk usage was collected
-	netInterfaces             map[string]struct{}                                   // Stores all valid network interfaces
+	netInterfaces             map[string]bool                                       // Valid network interfaces; true if byte counters come from MAC stats (Jetson nvethernet)
 	netIoStats                map[uint16]system.NetIoStats                          // Keeps track of bandwidth usage per cache interval
 	netInterfaceDeltaTrackers map[uint16]*deltatracker.DeltaTracker[string, uint64] // Per-cache-time NIC delta trackers
 	dockerManager             *dockerManager                                        // Manages Docker API requests

agent/network.go

@@ -92,7 +92,7 @@ func (a *Agent) updateNetworkStats(cacheTimeMs uint16, systemStats *system.Stats
 
 func (a *Agent) initializeNetIoStats() {
 	// reset valid network interfaces
-	a.netInterfaces = make(map[string]struct{}, 0)
+	a.netInterfaces = make(map[string]bool, 0)
 
 	// parse NICS env var for whitelist / blacklist
 	nicsEnvVal, nicsEnvExists := utils.GetEnv("NICS")
@@ -107,9 +107,14 @@ func (a *Agent) initializeNetIoStats() {
 			if skipNetworkInterface(v, nicCfg) {
 				continue
 			}
+			// driver is checked only here so updates don't pay for it on non-Jetson systems
+			useMacCounters := isNvidiaEthernet(v.Name)
+			if useMacCounters {
+				correctNvethernetCounters(&v)
+			}
 			slog.Info("Detected network interface", "name", v.Name, "sent", v.BytesSent, "recv", v.BytesRecv)
 			// store as a valid network interface
-			a.netInterfaces[v.Name] = struct{}{}
+			a.netInterfaces[v.Name] = useMacCounters
 		}
 	}
 
@@ -159,9 +164,13 @@ func (a *Agent) sumAndTrackPerNicDeltas(cacheTimeMs uint16, msElapsed uint64, ne
 	tracker.Cycle()
 
 	for _, v := range netIO {
-		if _, exists := a.netInterfaces[v.Name]; !exists {
+		useMacCounters, exists := a.netInterfaces[v.Name]
+		if !exists {
 			continue
 		}
+		if useMacCounters {
+			correctNvethernetCounters(&v)
+		}
 		totalBytesSent += v.BytesSent
 		totalBytesRecv += v.BytesRecv
 

agent/network_counters_linux.go

@@ -0,0 +1,71 @@
+//go:build linux
+
+package agent
+
+import (
+	"log/slog"
+	"math"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/safchain/ethtool"
+	psutilNet "github.com/shirou/gopsutil/v4/net"
+)
+
+// correctNvethernetCounters replaces the inflated sysfs byte counters of an
+// nvethernet NIC with its MAC octet counters.
+func correctNvethernetCounters(v *psutilNet.IOCountersStat) {
+	tx, rx, ok := readEthtoolMACOctets(v.Name)
+	if !ok {
+		return
+	}
+	v.BytesSent = tx
+	v.BytesRecv = rx
+}
+
+func isNvidiaEthernet(name string) bool {
+	if name == "" || strings.Contains(name, "/") {
+		return false
+	}
+
+	driverPath := filepath.Join("/sys/class/net", name, "device/driver")
+	target, err := os.Readlink(driverPath)
+	if err != nil {
+		return false
+	}
+	return filepath.Base(target) == "nvethernet"
+}
+
+func readEthtoolMACOctets(name string) (tx, rx uint64, ok bool) {
+	stats, err := ethtool.Stats(name)
+	if err != nil {
+		slog.Debug("Failed to read ethtool network counters", "interface", name, "err", err)
+		return 0, 0, false
+	}
+
+	tx, okTx := ethtoolCounter(stats, "mmc_tx_octetcount_gb", "mmc_tx_octetcount_gb_h")
+	rx, okRx := ethtoolCounter(stats, "mmc_rx_octetcount_gb", "mmc_rx_octetcount_gb_h")
+	if !okTx || !okRx {
+		return 0, 0, false
+	}
+	return tx, rx, true
+}
+
+// ethtoolCounter combines nvethernet's split MMC counters. The driver accumulates
+// the low and high registers into independent 64-bit fields, so the low value can
+// exceed 32 bits and must be added rather than OR'd into the shifted high word.
+func ethtoolCounter(stats map[string]uint64, lowKey, highKey string) (uint64, bool) {
+	low, ok := stats[lowKey]
+	if !ok {
+		return 0, false
+	}
+	high, hasHigh := stats[highKey]
+	if !hasHigh {
+		return low, true
+	}
+	if high > (math.MaxUint64-low)>>32 {
+		return 0, false
+	}
+	return high<<32 + low, true
+}

agent/network_counters_linux_test.go

@@ -0,0 +1,53 @@
+//go:build linux && testing
+
+package agent
+
+import (
+	"math"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestEthtoolCounterCombinesHighWord(t *testing.T) {
+	stats := map[string]uint64{
+		"counter":   123,
+		"counter_h": 2,
+	}
+
+	value, ok := ethtoolCounter(stats, "counter", "counter_h")
+	require.True(t, ok)
+	assert.Equal(t, uint64(2<<32+123), value)
+}
+
+func TestEthtoolCounterAddsLowWordAbove32Bits(t *testing.T) {
+	// nvethernet accumulates each register in 64-bit software fields, so the low
+	// word can carry past 32 bits; OR would drop the overlapping bit.
+	stats := map[string]uint64{
+		"counter":   1<<32 + 5,
+		"counter_h": 1,
+	}
+
+	value, ok := ethtoolCounter(stats, "counter", "counter_h")
+	require.True(t, ok)
+	assert.Equal(t, uint64(2<<32+5), value)
+}
+
+func TestEthtoolCounterRejectsOverflow(t *testing.T) {
+	stats := map[string]uint64{
+		"counter":   math.MaxUint64,
+		"counter_h": 1,
+	}
+
+	_, ok := ethtoolCounter(stats, "counter", "counter_h")
+	assert.False(t, ok)
+}
+
+func TestEthtoolCounterFallsBackToLowWord(t *testing.T) {
+	stats := map[string]uint64{"counter": 456}
+
+	value, ok := ethtoolCounter(stats, "counter", "counter_h")
+	require.True(t, ok)
+	assert.Equal(t, uint64(456), value)
+}

agent/network_counters_unsupported.go

@@ -0,0 +1,11 @@
+//go:build !linux
+
+package agent
+
+import psutilNet "github.com/shirou/gopsutil/v4/net"
+
+func isNvidiaEthernet(name string) bool {
+	return false
+}
+
+func correctNvethernetCounters(v *psutilNet.IOCountersStat) {}

agent/network_test.go

@@ -344,7 +344,7 @@ func TestComputeBytesPerSecond(t *testing.T) {
 
 func TestSumAndTrackPerNicDeltas(t *testing.T) {
 	a := &Agent{
-		netInterfaces:             map[string]struct{}{"eth0": {}, "wlan0": {}},
+		netInterfaces:             map[string]bool{"eth0": false, "wlan0": false},
 		netInterfaceDeltaTrackers: make(map[uint16]*deltatracker.DeltaTracker[string, uint64]),
 	}
 
@@ -373,7 +373,7 @@ func TestSumAndTrackPerNicDeltas(t *testing.T) {
 
 func TestSumAndTrackPerNicDeltasHandlesCounterReset(t *testing.T) {
 	a := &Agent{
-		netInterfaces:             map[string]struct{}{"eth0": {}},
+		netInterfaces:             map[string]bool{"eth0": false},
 		netInterfaceDeltaTrackers: make(map[uint16]*deltatracker.DeltaTracker[string, uint64]),
 	}
 
@@ -469,7 +469,7 @@ func TestApplyNetworkTotals(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			// Setup agent with initialized maps
 			a := &Agent{
-				netInterfaces:             make(map[string]struct{}),
+				netInterfaces:             make(map[string]bool),
 				netIoStats:                make(map[uint16]system.NetIoStats),
 				netInterfaceDeltaTrackers: make(map[uint16]*deltatracker.DeltaTracker[string, uint64]),
 			}
@@ -511,3 +511,22 @@ func TestApplyNetworkTotals(t *testing.T) {
 		})
 	}
 }
+
+func TestSumAndTrackPerNicDeltasKeepsCountersWhenMacReadFails(t *testing.T) {
+	a := &Agent{
+		// missing0 is flagged for MAC counters but has no ethtool stats to read
+		netInterfaces:             map[string]bool{"eth0": false, "missing0": true},
+		netInterfaceDeltaTrackers: make(map[uint16]*deltatracker.DeltaTracker[string, uint64]),
+	}
+	netIO := []psutilNet.IOCountersStat{
+		{Name: "eth0", BytesSent: 100, BytesRecv: 200},
+		{Name: "missing0", BytesSent: 300, BytesRecv: 400},
+	}
+	stats := &system.Stats{}
+	a.ensureNetworkInterfacesMap(stats)
+
+	tx, rx := a.sumAndTrackPerNicDeltas(1, 0, netIO, stats)
+	assert.Equal(t, uint64(400), tx)
+	assert.Equal(t, uint64(600), rx)
+	assert.Equal(t, [4]uint64{0, 0, 300, 400}, stats.NetworkInterfaces["missing0"])
+}

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/pocketbase/dbx v1.12.0
 	github.com/pocketbase/pocketbase v0.40.4
+	github.com/safchain/ethtool v0.7.0
 	github.com/shirou/gopsutil/v4 v4.26.8
 	github.com/spf13/cast v1.10.0
 	github.com/spf13/cobra v1.10.2

go.sum

@@ -115,6 +115,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qq
 github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/safchain/ethtool v0.7.0 h1:rlJzfDetsVvT61uz8x1YIcFn12akMfuPulHtZjtb7Is=
+github.com/safchain/ethtool v0.7.0/go.mod h1:MenQKEjXdfkjD3mp2QdCk8B/hwvkrlOTm/FD4gTpFxQ=
 github.com/shirou/gopsutil/v4 v4.26.8 h1:YQMTF/1J50B5+Y0vlo1eDRf5DoR7Gk69hY+8wjYkQeo=
 github.com/shirou/gopsutil/v4 v4.26.8/go.mod h1:5O9FjBiXoTDFatIWjZZosqj4pV0DRtLx598xGbBehzM=
 github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=