From 98e86b4c9ce3e8a98002632d2ac4c5c8cc764ed1 Mon Sep 17 00:00:00 2001 From: Ryan Chou <88779759+ryanchou1994@users.noreply.github.com> Date: Fri, 14 Aug 2026 06:14:30 +0800 Subject: [PATCH] fix(agent): treat a backwards container CPU counter as a new baseline (#2205) CalculateCpuPercentLinux subtracted the stored previous counters from the current ones without checking direction. When a stats response is processed after a newer one for the same container, or an accounting counter resets, the current total reads lower and the unsigned subtraction wraps to ~2^64 instead of going negative. On the container counter that surfaces as the reported error, and the sample is discarded along with the container's network stats: cpu pct greater than 100: 1.15292150348562e+13 On the system counter it is quieter and worse: the wrapped value inflates the divisor, so the percentage collapses toward zero and is stored as a healthy sample rather than rejected. A synthetic rollback measures 2.7e-12 percent. Both directions are now treated as a new baseline (0% for one sample), which matches how the function already handles the first-run case. CalculateCpuPercentWindows had the same unguarded subtraction and is fixed the same way. Fixes #2149 Co-authored-by: Ryan Chou --- agent/docker_test.go | 38 ++++++++++++++++++++++++ internal/entities/container/container.go | 17 ++++++++++- 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/agent/docker_test.go b/agent/docker_test.go index d4485084..ea76b8a0 100644 --- a/agent/docker_test.go +++ b/agent/docker_test.go @@ -1021,6 +1021,44 @@ func TestCpuPercentageCalculationWithRealData(t *testing.T) { assert.InDelta(t, expectedPct, actualPct, 0.01) } +func TestCpuPercentageHandlesCounterRollback(t *testing.T) { + // If a stats response is processed after a newer one for the same container, + // or an accounting counter resets, the current total can be lower than the + // stored previous value. Unsigned subtraction wraps to ~2^64 instead of + // going negative, so the percentage explodes, validateCpuPercentage rejects + // the sample, and the whole collection is discarded - network stats too. + stats := &container.ApiStats{ + CPUStats: container.CPUStats{ + CPUUsage: container.CPUUsage{TotalUsage: 1_000_000}, + SystemUsage: 20_000_000, + }, + } + + // Container counter went backwards. + assert.Equal(t, 0.0, stats.CalculateCpuPercentLinux(2_000_000, 10_000_000)) + // System counter went backwards. + assert.Equal(t, 0.0, stats.CalculateCpuPercentLinux(500_000, 30_000_000)) + // A normal forward sample is unaffected: 500000 / 10000000 * 100 = 5%. + assert.InDelta(t, 5.0, stats.CalculateCpuPercentLinux(500_000, 10_000_000), 0.001) +} + +func TestCpuPercentageWindowsHandlesCounterRollback(t *testing.T) { + now := time.Now() + stats := &container.ApiStats{ + Read: now, + NumProcs: 4, + CPUStats: container.CPUStats{ + CPUUsage: container.CPUUsage{TotalUsage: 1_000_000}, + }, + } + prevRead := now.Add(-time.Second) + + // Container counter went backwards. + assert.Equal(t, 0.0, stats.CalculateCpuPercentWindows(2_000_000, prevRead)) + // A normal forward sample is unaffected. + assert.Greater(t, stats.CalculateCpuPercentWindows(500_000, prevRead), 0.0) +} + func TestNetworkStatsCalculationWithRealData(t *testing.T) { // Create synthetic test data to avoid timing issues apiStats1 := &container.ApiStats{ diff --git a/internal/entities/container/container.go b/internal/entities/container/container.go index cd3b704c..989d4b2b 100644 --- a/internal/entities/container/container.go +++ b/internal/entities/container/container.go @@ -52,6 +52,17 @@ type HostInfo struct { } func (s *ApiStats) CalculateCpuPercentLinux(prevCpuContainer uint64, prevCpuSystem uint64) float64 { + // A counter can read lower than the stored previous value when a stats + // response is processed after a newer one for the same container, or when an + // accounting counter resets. Unsigned subtraction wraps to ~2^64 instead of + // going negative: on the container counter that surfaces as an absurd + // percentage the caller rejects, discarding the whole sample; on the system + // counter it inflates the divisor and silently reports near-zero CPU. + // Treat either direction as a new baseline. + if s.CPUStats.CPUUsage.TotalUsage < prevCpuContainer || s.CPUStats.SystemUsage < prevCpuSystem { + return 0.0 + } + cpuDelta := s.CPUStats.CPUUsage.TotalUsage - prevCpuContainer systemDelta := s.CPUStats.SystemUsage - prevCpuSystem @@ -70,7 +81,11 @@ func (s *ApiStats) CalculateCpuPercentWindows(prevCpuUsage uint64, prevRead time possIntervals /= 100 // Convert to number of 100ns intervals possIntervals *= uint64(s.NumProcs) // Multiple by the number of processors - // Intervals used + // Intervals used. Same rollback guard as the Linux path: an out-of-order or + // reset counter would wrap the subtraction to ~2^64. + if s.CPUStats.CPUUsage.TotalUsage < prevCpuUsage { + return 0.0 + } intervalsUsed := s.CPUStats.CPUUsage.TotalUsage - prevCpuUsage // Percentage avoiding divide-by-zero