mirror of
https://github.com/henrygd/beszel.git
synced 2026-08-18 08:17:47 +02:00
fix(agent): use time-based CPU calc for Podman containers (#2131)
This commit is contained in:
Pavel Artsishevsky
committed by
GitHub
GitHub
parent
2054b276a7
commit
0a9cad31d9
@@ -544,11 +544,18 @@ func (dm *dockerManager) updateContainerStats(ctr *container.ApiInfo, cacheTimeM
|
||||
// Get previous CPU values
|
||||
prevCpuContainer, prevCpuSystem := dm.getCpuPreviousValues(cacheTimeMs, ctr.IdShort)
|
||||
|
||||
// Calculate CPU percentage based on platform
|
||||
// Calculate CPU percentage based on platform.
|
||||
// Podman reports system_cpu_usage from cgroup cpu.stat (not /proc/stat), so it reflects
|
||||
// only cgroup-tracked activity rather than total host capacity. Use a time-based method
|
||||
// instead so the result is comparable to host CPU utilization. See:
|
||||
// https://github.com/henrygd/beszel/issues/2049
|
||||
var cpuPct float64
|
||||
if dm.isWindows {
|
||||
prevRead := dm.lastCpuReadTime[cacheTimeMs][ctr.IdShort]
|
||||
cpuPct = res.CalculateCpuPercentWindows(prevCpuContainer, prevRead)
|
||||
} else if dm.usingPodman && res.CPUStats.OnlineCPUs > 0 {
|
||||
prevRead := dm.lastCpuReadTime[cacheTimeMs][ctr.IdShort]
|
||||
cpuPct = res.CalculateCpuPercentPodman(prevCpuContainer, prevRead)
|
||||
} else {
|
||||
cpuPct = res.CalculateCpuPercentLinux(prevCpuContainer, prevCpuSystem)
|
||||
}
|
||||
|
||||
@@ -1059,6 +1059,162 @@ func TestCpuPercentageWindowsHandlesCounterRollback(t *testing.T) {
|
||||
assert.Greater(t, stats.CalculateCpuPercentWindows(500_000, prevRead), 0.0)
|
||||
}
|
||||
|
||||
func TestCalculateCpuPercentPodman(t *testing.T) {
|
||||
baseTime := time.Date(2026, 3, 15, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
prevCpuContainer uint64
|
||||
prevRead time.Time
|
||||
currentUsage uint64
|
||||
currentRead time.Time
|
||||
onlineCPUs uint32
|
||||
expectedPct float64
|
||||
}{
|
||||
{
|
||||
name: "normal calculation",
|
||||
// container used 2ms of CPU over 1s with 2 CPUs → 0.1%
|
||||
prevCpuContainer: 1_000_000_000,
|
||||
prevRead: baseTime,
|
||||
currentUsage: 1_002_000_000, // +2ms CPU time
|
||||
currentRead: baseTime.Add(time.Second),
|
||||
onlineCPUs: 2,
|
||||
expectedPct: 0.1, // 2e6 / (1e9 * 2) * 100
|
||||
},
|
||||
{
|
||||
name: "first run returns zero",
|
||||
prevCpuContainer: 0,
|
||||
prevRead: baseTime,
|
||||
currentUsage: 5_000_000,
|
||||
currentRead: baseTime.Add(time.Second),
|
||||
onlineCPUs: 4,
|
||||
expectedPct: 0.0,
|
||||
},
|
||||
{
|
||||
name: "zero online cpus returns zero",
|
||||
prevCpuContainer: 1_000_000_000,
|
||||
prevRead: baseTime,
|
||||
currentUsage: 1_010_000_000,
|
||||
currentRead: baseTime.Add(time.Second),
|
||||
onlineCPUs: 0,
|
||||
expectedPct: 0.0,
|
||||
},
|
||||
{
|
||||
name: "same read time returns zero",
|
||||
prevCpuContainer: 1_000_000_000,
|
||||
prevRead: baseTime,
|
||||
currentUsage: 1_010_000_000,
|
||||
currentRead: baseTime, // no elapsed time
|
||||
onlineCPUs: 2,
|
||||
expectedPct: 0.0,
|
||||
},
|
||||
{
|
||||
name: "counter rollback returns zero",
|
||||
prevCpuContainer: 2_000_000_000,
|
||||
prevRead: baseTime,
|
||||
currentUsage: 1_000_000_000,
|
||||
currentRead: baseTime.Add(time.Second),
|
||||
onlineCPUs: 2,
|
||||
expectedPct: 0.0,
|
||||
},
|
||||
{
|
||||
name: "100% single cpu",
|
||||
// container consumed a full CPU-second over 1s on a 1-CPU host → 100%
|
||||
prevCpuContainer: 1_000_000_000,
|
||||
prevRead: baseTime,
|
||||
currentUsage: 2_000_000_000, // +1s CPU time
|
||||
currentRead: baseTime.Add(time.Second),
|
||||
onlineCPUs: 1,
|
||||
expectedPct: 100.0, // 1e9 / (1e9 * 1) * 100
|
||||
},
|
||||
{
|
||||
name: "high utilization on multi-cpu host",
|
||||
// container used 800ms on a 4-CPU host over 1s → 20%
|
||||
prevCpuContainer: 10_000_000_000,
|
||||
prevRead: baseTime,
|
||||
currentUsage: 10_800_000_000,
|
||||
currentRead: baseTime.Add(time.Second),
|
||||
onlineCPUs: 4,
|
||||
expectedPct: 20.0, // 800e6 / (1e9 * 4) * 100
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := &container.ApiStats{
|
||||
Read: tt.currentRead,
|
||||
CPUStats: container.CPUStats{
|
||||
CPUUsage: container.CPUUsage{TotalUsage: tt.currentUsage},
|
||||
OnlineCPUs: tt.onlineCPUs,
|
||||
},
|
||||
}
|
||||
got := s.CalculateCpuPercentPodman(tt.prevCpuContainer, tt.prevRead)
|
||||
assert.InDelta(t, tt.expectedPct, got, 0.001, "test %q", tt.name)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateContainerStatsPodmanCpuCalculation(t *testing.T) {
|
||||
// Verify that Podman containers use the time-based CPU calculation
|
||||
// when online_cpus is provided in the stats response.
|
||||
// container used 20ms CPU over 1s with 2 CPUs → 1%
|
||||
prevReadTime := time.Date(2026, 3, 15, 21, 26, 58, 0, time.UTC) // 1 second before stats read
|
||||
const prevCpuUsage = uint64(5_000_000_000)
|
||||
|
||||
dm := &dockerManager{
|
||||
client: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
switch req.URL.EscapedPath() {
|
||||
case "/containers/0123456789ab/stats":
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Status: "200 OK",
|
||||
Header: make(http.Header),
|
||||
Body: io.NopCloser(strings.NewReader(`{
|
||||
"read":"2026-03-15T21:26:59Z",
|
||||
"cpu_stats":{"cpu_usage":{"total_usage":5020000000},"system_cpu_usage":9999999,"online_cpus":2},
|
||||
"memory_stats":{"usage":1048576,"stats":{"inactive_file":262144}},
|
||||
"networks":{"eth0":{"rx_bytes":0,"tx_bytes":0}}
|
||||
}`)),
|
||||
Request: req,
|
||||
}, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected path: %s", req.URL.EscapedPath())
|
||||
}
|
||||
})},
|
||||
containerStatsMap: make(map[string]*container.Stats),
|
||||
apiStats: &container.ApiStats{},
|
||||
usingPodman: true,
|
||||
lastCpuContainer: map[uint16]map[string]uint64{
|
||||
defaultCacheTimeMs: {"0123456789ab": prevCpuUsage},
|
||||
},
|
||||
lastCpuSystem: map[uint16]map[string]uint64{
|
||||
defaultCacheTimeMs: {"0123456789ab": 1}, // intentionally tiny — should NOT be used
|
||||
},
|
||||
lastCpuReadTime: map[uint16]map[string]time.Time{
|
||||
defaultCacheTimeMs: {"0123456789ab": prevReadTime},
|
||||
},
|
||||
networkSentTrackers: make(map[uint16]*deltatracker.DeltaTracker[string, uint64]),
|
||||
networkRecvTrackers: make(map[uint16]*deltatracker.DeltaTracker[string, uint64]),
|
||||
lastNetworkReadTime: make(map[uint16]map[string]time.Time),
|
||||
}
|
||||
|
||||
ctr := &container.ApiInfo{
|
||||
IdShort: "0123456789ab",
|
||||
Names: []string{"/myapp"},
|
||||
Status: "Up 5 minutes",
|
||||
Image: "myapp:latest",
|
||||
}
|
||||
|
||||
err := dm.updateContainerStats(ctr, defaultCacheTimeMs)
|
||||
require.NoError(t, err)
|
||||
|
||||
// cpu delta = 5020000000 - 5000000000 = 20000000 ns (20ms)
|
||||
// elapsed = 1s = 1000000000 ns, online_cpus = 2
|
||||
// expected = 20000000 / (1000000000 * 2) * 100 = 1.0%
|
||||
expectedCpu := 1.0
|
||||
assert.InDelta(t, expectedCpu, dm.containerStatsMap[ctr.IdShort].Cpu, 0.01)
|
||||
}
|
||||
|
||||
func TestNetworkStatsCalculationWithRealData(t *testing.T) {
|
||||
// Create synthetic test data to avoid timing issues
|
||||
apiStats1 := &container.ApiStats{
|
||||
|
||||
@@ -74,6 +74,30 @@ func (s *ApiStats) CalculateCpuPercentLinux(prevCpuContainer uint64, prevCpuSyst
|
||||
return float64(cpuDelta) / float64(systemDelta) * 100.0
|
||||
}
|
||||
|
||||
// CalculateCpuPercentPodman calculates CPU percentage for Podman containers.
|
||||
// Podman populates system_cpu_usage from cgroup cpu.stat rather than /proc/stat, so it
|
||||
// represents only cgroup-accounted activity, not total host CPU capacity. Using it as
|
||||
// a denominator inflates the result. Instead we use elapsed wall-clock time × online_cpus,
|
||||
// matching the approach used for Windows and recommended in:
|
||||
// https://github.com/henrygd/beszel/issues/2049
|
||||
func (s *ApiStats) CalculateCpuPercentPodman(prevCpuContainer uint64, prevRead time.Time) float64 {
|
||||
if prevCpuContainer == 0 || s.CPUStats.OnlineCPUs == 0 {
|
||||
return 0.0
|
||||
}
|
||||
// Treat a reset or out-of-order counter as a new baseline instead of
|
||||
// allowing unsigned subtraction to wrap to an enormous percentage.
|
||||
if s.CPUStats.CPUUsage.TotalUsage < prevCpuContainer {
|
||||
return 0.0
|
||||
}
|
||||
cpuDelta := s.CPUStats.CPUUsage.TotalUsage - prevCpuContainer
|
||||
elapsedNs := uint64(s.Read.Sub(prevRead).Nanoseconds())
|
||||
systemCapacity := elapsedNs * uint64(s.CPUStats.OnlineCPUs)
|
||||
if systemCapacity == 0 {
|
||||
return 0.0
|
||||
}
|
||||
return float64(cpuDelta) / float64(systemCapacity) * 100.0
|
||||
}
|
||||
|
||||
// from: https://github.com/docker/cli/blob/master/cli/command/container/stats_helpers.go#L185
|
||||
func (s *ApiStats) CalculateCpuPercentWindows(prevCpuUsage uint64, prevRead time.Time) float64 {
|
||||
// Max number of 100ns intervals between the previous time read and now
|
||||
@@ -98,8 +122,10 @@ func (s *ApiStats) CalculateCpuPercentWindows(prevCpuUsage uint64, prevRead time
|
||||
type CPUStats struct {
|
||||
// CPU Usage. Linux and Windows.
|
||||
CPUUsage CPUUsage `json:"cpu_usage"`
|
||||
// System Usage. Linux only.
|
||||
// System Usage. Linux only. Populated from /proc/stat on Docker; from cgroup cpu.stat on Podman.
|
||||
SystemUsage uint64 `json:"system_cpu_usage,omitempty"`
|
||||
// Number of online CPUs. Linux only. Used by Podman for time-based CPU calculation.
|
||||
OnlineCPUs uint32 `json:"online_cpus,omitempty"`
|
||||
}
|
||||
|
||||
type CPUUsage struct {
|
||||
|
||||
Reference in New Issue
Block a user