mirror of
https://github.com/henrygd/beszel.git
synced 2026-08-18 08:17:47 +02:00
fix(agent): treat a backwards container CPU counter as a new baseline (#2205)
CalculateCpuPercentLinux subtracted the stored previous counters from the
current ones without checking direction. When a stats response is processed
after a newer one for the same container, or an accounting counter resets,
the current total reads lower and the unsigned subtraction wraps to ~2^64
instead of going negative.
On the container counter that surfaces as the reported error, and the sample
is discarded along with the container's network stats:
cpu pct greater than 100: 1.15292150348562e+13
On the system counter it is quieter and worse: the wrapped value inflates the
divisor, so the percentage collapses toward zero and is stored as a healthy
sample rather than rejected. A synthetic rollback measures 2.7e-12 percent.
Both directions are now treated as a new baseline (0% for one sample), which
matches how the function already handles the first-run case.
CalculateCpuPercentWindows had the same unguarded subtraction and is fixed
the same way.
Fixes #2149
Co-authored-by: Ryan Chou <ryanchou1994@users.noreply.github.com>
This commit is contained in:
@@ -1021,6 +1021,44 @@ func TestCpuPercentageCalculationWithRealData(t *testing.T) {
|
|||||||
assert.InDelta(t, expectedPct, actualPct, 0.01)
|
assert.InDelta(t, expectedPct, actualPct, 0.01)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCpuPercentageHandlesCounterRollback(t *testing.T) {
|
||||||
|
// If a stats response is processed after a newer one for the same container,
|
||||||
|
// or an accounting counter resets, the current total can be lower than the
|
||||||
|
// stored previous value. Unsigned subtraction wraps to ~2^64 instead of
|
||||||
|
// going negative, so the percentage explodes, validateCpuPercentage rejects
|
||||||
|
// the sample, and the whole collection is discarded - network stats too.
|
||||||
|
stats := &container.ApiStats{
|
||||||
|
CPUStats: container.CPUStats{
|
||||||
|
CPUUsage: container.CPUUsage{TotalUsage: 1_000_000},
|
||||||
|
SystemUsage: 20_000_000,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// Container counter went backwards.
|
||||||
|
assert.Equal(t, 0.0, stats.CalculateCpuPercentLinux(2_000_000, 10_000_000))
|
||||||
|
// System counter went backwards.
|
||||||
|
assert.Equal(t, 0.0, stats.CalculateCpuPercentLinux(500_000, 30_000_000))
|
||||||
|
// A normal forward sample is unaffected: 500000 / 10000000 * 100 = 5%.
|
||||||
|
assert.InDelta(t, 5.0, stats.CalculateCpuPercentLinux(500_000, 10_000_000), 0.001)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCpuPercentageWindowsHandlesCounterRollback(t *testing.T) {
|
||||||
|
now := time.Now()
|
||||||
|
stats := &container.ApiStats{
|
||||||
|
Read: now,
|
||||||
|
NumProcs: 4,
|
||||||
|
CPUStats: container.CPUStats{
|
||||||
|
CPUUsage: container.CPUUsage{TotalUsage: 1_000_000},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
prevRead := now.Add(-time.Second)
|
||||||
|
|
||||||
|
// Container counter went backwards.
|
||||||
|
assert.Equal(t, 0.0, stats.CalculateCpuPercentWindows(2_000_000, prevRead))
|
||||||
|
// A normal forward sample is unaffected.
|
||||||
|
assert.Greater(t, stats.CalculateCpuPercentWindows(500_000, prevRead), 0.0)
|
||||||
|
}
|
||||||
|
|
||||||
func TestNetworkStatsCalculationWithRealData(t *testing.T) {
|
func TestNetworkStatsCalculationWithRealData(t *testing.T) {
|
||||||
// Create synthetic test data to avoid timing issues
|
// Create synthetic test data to avoid timing issues
|
||||||
apiStats1 := &container.ApiStats{
|
apiStats1 := &container.ApiStats{
|
||||||
|
|||||||
@@ -52,6 +52,17 @@ type HostInfo struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *ApiStats) CalculateCpuPercentLinux(prevCpuContainer uint64, prevCpuSystem uint64) float64 {
|
func (s *ApiStats) CalculateCpuPercentLinux(prevCpuContainer uint64, prevCpuSystem uint64) float64 {
|
||||||
|
// A counter can read lower than the stored previous value when a stats
|
||||||
|
// response is processed after a newer one for the same container, or when an
|
||||||
|
// accounting counter resets. Unsigned subtraction wraps to ~2^64 instead of
|
||||||
|
// going negative: on the container counter that surfaces as an absurd
|
||||||
|
// percentage the caller rejects, discarding the whole sample; on the system
|
||||||
|
// counter it inflates the divisor and silently reports near-zero CPU.
|
||||||
|
// Treat either direction as a new baseline.
|
||||||
|
if s.CPUStats.CPUUsage.TotalUsage < prevCpuContainer || s.CPUStats.SystemUsage < prevCpuSystem {
|
||||||
|
return 0.0
|
||||||
|
}
|
||||||
|
|
||||||
cpuDelta := s.CPUStats.CPUUsage.TotalUsage - prevCpuContainer
|
cpuDelta := s.CPUStats.CPUUsage.TotalUsage - prevCpuContainer
|
||||||
systemDelta := s.CPUStats.SystemUsage - prevCpuSystem
|
systemDelta := s.CPUStats.SystemUsage - prevCpuSystem
|
||||||
|
|
||||||
@@ -70,7 +81,11 @@ func (s *ApiStats) CalculateCpuPercentWindows(prevCpuUsage uint64, prevRead time
|
|||||||
possIntervals /= 100 // Convert to number of 100ns intervals
|
possIntervals /= 100 // Convert to number of 100ns intervals
|
||||||
possIntervals *= uint64(s.NumProcs) // Multiple by the number of processors
|
possIntervals *= uint64(s.NumProcs) // Multiple by the number of processors
|
||||||
|
|
||||||
// Intervals used
|
// Intervals used. Same rollback guard as the Linux path: an out-of-order or
|
||||||
|
// reset counter would wrap the subtraction to ~2^64.
|
||||||
|
if s.CPUStats.CPUUsage.TotalUsage < prevCpuUsage {
|
||||||
|
return 0.0
|
||||||
|
}
|
||||||
intervalsUsed := s.CPUStats.CPUUsage.TotalUsage - prevCpuUsage
|
intervalsUsed := s.CPUStats.CPUUsage.TotalUsage - prevCpuUsage
|
||||||
|
|
||||||
// Percentage avoiding divide-by-zero
|
// Percentage avoiding divide-by-zero
|
||||||
|
|||||||
Reference in New Issue
Block a user