fix(agent): treat a backwards container CPU counter as a new baseline (#2205)

CalculateCpuPercentLinux subtracted the stored previous counters from the
current ones without checking direction. When a stats response is processed
after a newer one for the same container, or an accounting counter resets,
the current total reads lower and the unsigned subtraction wraps to ~2^64
instead of going negative.

On the container counter that surfaces as the reported error, and the sample
is discarded along with the container's network stats:

    cpu pct greater than 100: 1.15292150348562e+13

On the system counter it is quieter and worse: the wrapped value inflates the
divisor, so the percentage collapses toward zero and is stored as a healthy
sample rather than rejected. A synthetic rollback measures 2.7e-12 percent.

Both directions are now treated as a new baseline (0% for one sample), which
matches how the function already handles the first-run case.

CalculateCpuPercentWindows had the same unguarded subtraction and is fixed
the same way.

Fixes #2149

Co-authored-by: Ryan Chou <ryanchou1994@users.noreply.github.com>
This commit is contained in:
Ryan Chou
2026-08-14 06:14:30 +08:00
committed by GitHub
parent d40372842b
commit 98e86b4c9c
2 changed files with 54 additions and 1 deletions

View File

@@ -1021,6 +1021,44 @@ func TestCpuPercentageCalculationWithRealData(t *testing.T) {
assert.InDelta(t, expectedPct, actualPct, 0.01)
}
func TestCpuPercentageHandlesCounterRollback(t *testing.T) {
// If a stats response is processed after a newer one for the same container,
// or an accounting counter resets, the current total can be lower than the
// stored previous value. Unsigned subtraction wraps to ~2^64 instead of
// going negative, so the percentage explodes, validateCpuPercentage rejects
// the sample, and the whole collection is discarded - network stats too.
stats := &container.ApiStats{
CPUStats: container.CPUStats{
CPUUsage: container.CPUUsage{TotalUsage: 1_000_000},
SystemUsage: 20_000_000,
},
}
// Container counter went backwards.
assert.Equal(t, 0.0, stats.CalculateCpuPercentLinux(2_000_000, 10_000_000))
// System counter went backwards.
assert.Equal(t, 0.0, stats.CalculateCpuPercentLinux(500_000, 30_000_000))
// A normal forward sample is unaffected: 500000 / 10000000 * 100 = 5%.
assert.InDelta(t, 5.0, stats.CalculateCpuPercentLinux(500_000, 10_000_000), 0.001)
}
func TestCpuPercentageWindowsHandlesCounterRollback(t *testing.T) {
now := time.Now()
stats := &container.ApiStats{
Read: now,
NumProcs: 4,
CPUStats: container.CPUStats{
CPUUsage: container.CPUUsage{TotalUsage: 1_000_000},
},
}
prevRead := now.Add(-time.Second)
// Container counter went backwards.
assert.Equal(t, 0.0, stats.CalculateCpuPercentWindows(2_000_000, prevRead))
// A normal forward sample is unaffected.
assert.Greater(t, stats.CalculateCpuPercentWindows(500_000, prevRead), 0.0)
}
func TestNetworkStatsCalculationWithRealData(t *testing.T) {
// Create synthetic test data to avoid timing issues
apiStats1 := &container.ApiStats{