Compare commits

...

2 Commits

Author SHA1 Message Date
henrygd
38f2ba3984 Small refactoring of docker manager
- Add isWindows flag to dockerManager
- `CalculateCpuPercentWindows` and `CalculateCpuPercentLinux` methods added to container.ApiStats
- Remove prevNetStats.Time in favor of Stats.PrevRead
- Replace regex Windows check with strings.Contains, and check the `/containers/json` response
2025-04-25 18:39:24 -04:00
ViryBe
1a7d897bdc compute cpu and memory stats for docker on windows (#653)
The Docker daemon's API returns different values on Windows and non-Windows systems. This impacts
the calculation of `cpuPct` and `usedMemory` for each container. The systems are disciminate on the
`Server` header send by the server. Uses the unix stats calculation in case the header is not set.

`docker stats` implementation for reference:
https://github.com/docker/cli/blob/master/cli/command/container/stats_helpers.go#L100

Co-authored-by: Benoit VIRY <benoit.viry@cgx-group.com>
2025-04-25 18:27:36 -04:00
2 changed files with 54 additions and 71 deletions

View File

@@ -26,6 +26,7 @@ type dockerManager struct {
containerStatsMap map[string]*container.Stats // Keeps track of container stats
validIds map[string]struct{} // Map of valid container ids, used to prune invalid containers from containerStatsMap
goodDockerVersion bool // Whether docker version is at least 25.0.0 (one-shot works correctly)
isWindows bool // Whether the Docker Engine API is running on Windows
}
// userAgentRoundTripper is a custom http.RoundTripper that adds a User-Agent header to all requests
@@ -69,6 +70,8 @@ func (dm *dockerManager) getDockerStats() ([]*container.Stats, error) {
return nil, err
}
dm.isWindows = strings.Contains(resp.Header.Get("Server"), "windows")
containersLength := len(dm.apiContainerList)
// store valid ids to clean up old container ids from map
@@ -80,8 +83,7 @@ func (dm *dockerManager) getDockerStats() ([]*container.Stats, error) {
var failedContainers []*container.ApiInfo
for i := range dm.apiContainerList {
ctr := dm.apiContainerList[i]
for _, ctr := range dm.apiContainerList {
ctr.IdShort = ctr.Id[:12]
dm.validIds[ctr.IdShort] = struct{}{}
// check if container is less than 1 minute old (possible restart)
@@ -167,22 +169,27 @@ func (dm *dockerManager) updateContainerStats(ctr *container.ApiInfo) error {
return err
}
// check if container has valid data, otherwise may be in restart loop (#103)
if res.MemoryStats.Usage == 0 {
return fmt.Errorf("%s - no memory stats - see https://github.com/henrygd/beszel/issues/144", name)
// calculate cpu and memory stats
var usedMemory uint64
var cpuPct float64
if dm.isWindows {
usedMemory = res.MemoryStats.PrivateWorkingSet
cpuPct = res.CalculateCpuPercentWindows(stats.PrevCpu[0], stats.PrevRead)
} else {
// check if container has valid data, otherwise may be in restart loop (#103)
if res.MemoryStats.Usage == 0 {
return fmt.Errorf("%s - no memory stats - see https://github.com/henrygd/beszel/issues/144", name)
}
memCache := res.MemoryStats.Stats.InactiveFile
if memCache == 0 {
memCache = res.MemoryStats.Stats.Cache
}
usedMemory = res.MemoryStats.Usage - memCache
cpuPct = res.CalculateCpuPercentLinux(stats.PrevCpu)
}
// memory (https://docs.docker.com/reference/cli/docker/container/stats/)
memCache := res.MemoryStats.Stats.InactiveFile
if memCache == 0 {
memCache = res.MemoryStats.Stats.Cache
}
usedMemory := res.MemoryStats.Usage - memCache
// cpu
cpuDelta := res.CPUStats.CPUUsage.TotalUsage - stats.PrevCpu[0]
systemDelta := res.CPUStats.SystemUsage - stats.PrevCpu[1]
cpuPct := float64(cpuDelta) / float64(systemDelta) * 100
if cpuPct > 100 {
return fmt.Errorf("%s cpu pct greater than 100: %+v", name, cpuPct)
}
@@ -197,18 +204,18 @@ func (dm *dockerManager) updateContainerStats(ctr *container.ApiInfo) error {
var sent_delta, recv_delta float64
// prevent first run from sending all prev sent/recv bytes
if initialized {
secondsElapsed := time.Since(stats.PrevNet.Time).Seconds()
secondsElapsed := time.Since(stats.PrevRead).Seconds()
sent_delta = float64(total_sent-stats.PrevNet.Sent) / secondsElapsed
recv_delta = float64(total_recv-stats.PrevNet.Recv) / secondsElapsed
}
stats.PrevNet.Sent = total_sent
stats.PrevNet.Recv = total_recv
stats.PrevNet.Time = time.Now()
stats.Cpu = twoDecimals(cpuPct)
stats.Mem = bytesToMegabytes(float64(usedMemory))
stats.NetworkSent = bytesToMegabytes(sent_delta)
stats.NetworkRecv = bytesToMegabytes(recv_delta)
stats.PrevRead = res.Read
return nil
}

View File

@@ -27,38 +27,41 @@ type ApiInfo struct {
// Docker container resources from /containers/{id}/stats
type ApiStats struct {
// Common stats
// Read time.Time `json:"read"`
// PreRead time.Time `json:"preread"`
Read time.Time `json:"read"` // Time of stats generation
NumProcs uint32 `json:"num_procs,omitzero"` // Windows specific, not populated on Linux.
Networks map[string]NetworkStats
CPUStats CPUStats `json:"cpu_stats"`
MemoryStats MemoryStats `json:"memory_stats"`
}
// Linux specific stats, not populated on Windows.
// PidsStats PidsStats `json:"pids_stats,omitempty"`
// BlkioStats BlkioStats `json:"blkio_stats,omitempty"`
func (s *ApiStats) CalculateCpuPercentLinux(prevCpuUsage [2]uint64) float64 {
cpuDelta := s.CPUStats.CPUUsage.TotalUsage - prevCpuUsage[0]
systemDelta := s.CPUStats.SystemUsage - prevCpuUsage[1]
return float64(cpuDelta) / float64(systemDelta) * 100
}
// Windows specific stats, not populated on Linux.
// NumProcs uint32 `json:"num_procs"`
// StorageStats StorageStats `json:"storage_stats,omitempty"`
// Networks request version >=1.21
Networks map[string]NetworkStats
// from: https://github.com/docker/cli/blob/master/cli/command/container/stats_helpers.go#L185
func (s *ApiStats) CalculateCpuPercentWindows(prevCpuUsage uint64, prevRead time.Time) float64 {
// Max number of 100ns intervals between the previous time read and now
possIntervals := uint64(s.Read.Sub(prevRead).Nanoseconds())
possIntervals /= 100 // Convert to number of 100ns intervals
possIntervals *= uint64(s.NumProcs) // Multiple by the number of processors
// Shared stats
CPUStats CPUStats `json:"cpu_stats,omitempty"`
// PreCPUStats CPUStats `json:"precpu_stats,omitempty"` // "Pre"="Previous"
MemoryStats MemoryStats `json:"memory_stats,omitempty"`
// Intervals used
intervalsUsed := s.CPUStats.CPUUsage.TotalUsage - prevCpuUsage
// Percentage avoiding divide-by-zero
if possIntervals > 0 {
return float64(intervalsUsed) / float64(possIntervals) * 100.0
}
return 0.00
}
type CPUStats struct {
// CPU Usage. Linux and Windows.
CPUUsage CPUUsage `json:"cpu_usage"`
// System Usage. Linux only.
SystemUsage uint64 `json:"system_cpu_usage,omitempty"`
// Online CPUs. Linux only.
// OnlineCPUs uint32 `json:"online_cpus,omitempty"`
// Throttling Data. Linux only.
// ThrottlingData ThrottlingData `json:"throttling_data,omitempty"`
}
type CPUUsage struct {
@@ -66,42 +69,15 @@ type CPUUsage struct {
// Units: nanoseconds (Linux)
// Units: 100's of nanoseconds (Windows)
TotalUsage uint64 `json:"total_usage"`
// Total CPU time consumed per core (Linux). Not used on Windows.
// Units: nanoseconds.
// PercpuUsage []uint64 `json:"percpu_usage,omitempty"`
// Time spent by tasks of the cgroup in kernel mode (Linux).
// Time spent by all container processes in kernel mode (Windows).
// Units: nanoseconds (Linux).
// Units: 100's of nanoseconds (Windows). Not populated for Hyper-V Containers.
// UsageInKernelmode uint64 `json:"usage_in_kernelmode"`
// Time spent by tasks of the cgroup in user mode (Linux).
// Time spent by all container processes in user mode (Windows).
// Units: nanoseconds (Linux).
// Units: 100's of nanoseconds (Windows). Not populated for Hyper-V Containers
// UsageInUsermode uint64 `json:"usage_in_usermode"`
}
type MemoryStats struct {
// current res_counter usage for memory
Usage uint64 `json:"usage,omitempty"`
// all the stats exported via memory.stat.
Stats MemoryStatsStats `json:"stats,omitempty"`
// maximum usage ever recorded.
// MaxUsage uint64 `json:"max_usage,omitempty"`
// TODO(vishh): Export these as stronger types.
// number of times memory usage hits limits.
// Failcnt uint64 `json:"failcnt,omitempty"`
// Limit uint64 `json:"limit,omitempty"`
// // committed bytes
// Commit uint64 `json:"commitbytes,omitempty"`
// // peak committed bytes
// CommitPeak uint64 `json:"commitpeakbytes,omitempty"`
// // private working set
// PrivateWorkingSet uint64 `json:"privateworkingset,omitempty"`
Stats MemoryStatsStats `json:"stats"`
// private working set (Windows only)
PrivateWorkingSet uint64 `json:"privateworkingset,omitempty"`
}
type MemoryStatsStats struct {
@@ -119,7 +95,6 @@ type NetworkStats struct {
type prevNetStats struct {
Sent uint64
Recv uint64
Time time.Time
}
// Docker container stats
@@ -131,4 +106,5 @@ type Stats struct {
NetworkRecv float64 `json:"nr"`
PrevCpu [2]uint64 `json:"-"`
PrevNet prevNetStats `json:"-"`
PrevRead time.Time `json:"-"`
}