Compare commits

..

2 Commits

Author SHA1 Message Date
Sven van Ginkel
2230097dc7 chore: update inactivity-actions (#1742) 2026-02-12 12:29:22 -05:00
henrygd
25c77c5664 make: auto-apply glibc tag for agent on linux/amd64 glibc 2026-02-11 13:49:29 -05:00
3 changed files with 43 additions and 60 deletions

View File

@@ -6,6 +6,7 @@ on:
workflow_dispatch:
permissions:
actions: write
issues: write
pull-requests: write
@@ -48,6 +49,9 @@ jobs:
# Action can not skip PRs, set it to 100 years to cover it.
days-before-pr-stale: 36524
# Max issues to process before early exit. Next run resumes from cache. GH API limit: 5000.
operations-per-run: 1500
# Labels
stale-issue-label: 'stale'
remove-stale-when-updated: true
@@ -56,4 +60,5 @@ jobs:
# Exemptions
exempt-assignees: true
exempt-milestones: true
exempt-milestones: true

View File

@@ -3,6 +3,40 @@ OS ?= $(shell go env GOOS)
ARCH ?= $(shell go env GOARCH)
# Skip building the web UI if true
SKIP_WEB ?= false
# Controls NVML/glibc agent build tag behavior:
# - auto (default): enable on linux/amd64 glibc hosts
# - true: always enable
# - false: always disable
NVML ?= auto
# Detect glibc host for local linux/amd64 builds.
HOST_GLIBC := $(shell \
if [ "$(OS)" = "linux" ] && [ "$(ARCH)" = "amd64" ]; then \
for p in /lib64/ld-linux-x86-64.so.2 /lib/x86_64-linux-gnu/ld-linux-x86-64.so.2 /lib/ld-linux-x86-64.so.2; do \
[ -e "$$p" ] && { echo true; exit 0; }; \
done; \
if command -v ldd >/dev/null 2>&1; then \
if ldd --version 2>&1 | tr '[:upper:]' '[:lower:]' | awk '/gnu libc|glibc/{found=1} END{exit !found}'; then \
echo true; \
else \
echo false; \
fi; \
else \
echo false; \
fi; \
else \
echo false; \
fi)
# Enable glibc build tag for NVML on supported Linux builds.
AGENT_GO_TAGS :=
ifeq ($(NVML),true)
AGENT_GO_TAGS := -tags glibc
else ifeq ($(NVML),auto)
ifeq ($(HOST_GLIBC),true)
AGENT_GO_TAGS := -tags glibc
endif
endif
# Set executable extension based on target OS
EXE_EXT := $(if $(filter windows,$(OS)),.exe,)
@@ -54,7 +88,7 @@ fetch-smartctl-conditional:
# Update build-agent to include conditional .NET build
build-agent: tidy build-dotnet-conditional fetch-smartctl-conditional
GOOS=$(OS) GOARCH=$(ARCH) go build -o ./build/beszel-agent_$(OS)_$(ARCH)$(EXE_EXT) -ldflags "-w -s" ./internal/cmd/agent
GOOS=$(OS) GOARCH=$(ARCH) go build $(AGENT_GO_TAGS) -o ./build/beszel-agent_$(OS)_$(ARCH)$(EXE_EXT) -ldflags "-w -s" ./internal/cmd/agent
build-hub: tidy $(if $(filter false,$(SKIP_WEB)),build-web-ui)
GOOS=$(OS) GOARCH=$(ARCH) go build -o ./build/beszel_$(OS)_$(ARCH)$(EXE_EXT) -ldflags "-w -s" ./internal/cmd/hub
@@ -90,9 +124,9 @@ dev-hub:
dev-agent:
@if command -v entr >/dev/null 2>&1; then \
find ./internal/cmd/agent/*.go ./agent/*.go | entr -r go run github.com/henrygd/beszel/internal/cmd/agent; \
find ./internal/cmd/agent/*.go ./agent/*.go | entr -r go run $(AGENT_GO_TAGS) github.com/henrygd/beszel/internal/cmd/agent; \
else \
go run github.com/henrygd/beszel/internal/cmd/agent; \
go run $(AGENT_GO_TAGS) github.com/henrygd/beszel/internal/cmd/agent; \
fi
build-dotnet:

View File

@@ -32,10 +32,6 @@ var ansiEscapePattern = regexp.MustCompile(`\x1b\[[0-9;]*[a-zA-Z]|\x1b\][^\x07]*
const (
// Docker API timeout in milliseconds
dockerTimeoutMs = 2100
// Number of consecutive /containers/json failures before forcing a client reset on old Docker versions
dockerClientResetFailureThreshold = 3
// Minimum time between Docker client resets to avoid reset flapping
dockerClientResetCooldown = 30 * time.Second
// Maximum realistic network speed (5 GB/s) to detect bad deltas
maxNetworkSpeedBps uint64 = 5e9
// Maximum conceivable memory usage of a container (100TB) to detect bad memory stats
@@ -59,16 +55,12 @@ type dockerManager struct {
containerStatsMap map[string]*container.Stats // Keeps track of container stats
validIds map[string]struct{} // Map of valid container ids, used to prune invalid containers from containerStatsMap
goodDockerVersion bool // Whether docker version is at least 25.0.0 (one-shot works correctly)
versionChecked bool // Whether docker version detection completed successfully
isWindows bool // Whether the Docker Engine API is running on Windows
buf *bytes.Buffer // Buffer to store and read response bodies
decoder *json.Decoder // Reusable JSON decoder that reads from buf
apiStats *container.ApiStats // Reusable API stats object
excludeContainers []string // Patterns to exclude containers by name
usingPodman bool // Whether the Docker Engine API is running on Podman
transport *http.Transport // Base transport used by client for connection resets
consecutiveListFailures int // Number of consecutive /containers/json request failures
lastClientReset time.Time // Last time the Docker client connections were reset
// Cache-time-aware tracking for CPU stats (similar to cpu.go)
// Maps cache time intervals to container-specific CPU usage tracking
@@ -127,10 +119,8 @@ func (dm *dockerManager) shouldExcludeContainer(name string) bool {
func (dm *dockerManager) getDockerStats(cacheTimeMs uint16) ([]*container.Stats, error) {
resp, err := dm.client.Get("http://localhost/containers/json")
if err != nil {
dm.handleContainerListError(err)
return nil, err
}
dm.consecutiveListFailures = 0
dm.apiContainerList = dm.apiContainerList[:0]
if err := dm.decode(resp, &dm.apiContainerList); err != nil {
@@ -214,50 +204,6 @@ func (dm *dockerManager) getDockerStats(cacheTimeMs uint16) ([]*container.Stats,
return stats, nil
}
func (dm *dockerManager) handleContainerListError(err error) {
dm.consecutiveListFailures++
if !dm.shouldResetDockerClient(err) {
return
}
dm.resetDockerClientConnections()
}
func (dm *dockerManager) shouldResetDockerClient(err error) bool {
if !dm.versionChecked || dm.goodDockerVersion {
return false
}
if dm.consecutiveListFailures < dockerClientResetFailureThreshold {
return false
}
if !dm.lastClientReset.IsZero() && time.Since(dm.lastClientReset) < dockerClientResetCooldown {
return false
}
return isDockerApiOverloadError(err)
}
func isDockerApiOverloadError(err error) bool {
if err == nil {
return false
}
if errors.Is(err, context.DeadlineExceeded) {
return true
}
msg := err.Error()
return strings.Contains(msg, "Client.Timeout exceeded") ||
strings.Contains(msg, "request canceled") ||
strings.Contains(msg, "context deadline exceeded") ||
strings.Contains(msg, "EOF")
}
func (dm *dockerManager) resetDockerClientConnections() {
if dm.transport == nil {
return
}
dm.transport.CloseIdleConnections()
dm.lastClientReset = time.Now()
slog.Warn("Reset Docker client connections after repeated /containers/json failures", "failures", dm.consecutiveListFailures)
}
// initializeCpuTracking initializes CPU tracking maps for a specific cache time interval
func (dm *dockerManager) initializeCpuTracking(cacheTimeMs uint16) {
// Initialize cache time maps if they don't exist
@@ -607,7 +553,6 @@ func newDockerManager() *dockerManager {
Timeout: timeout,
Transport: userAgentTransport,
},
transport: transport,
containerStatsMap: make(map[string]*container.Stats),
sem: make(chan struct{}, 5),
apiContainerList: []*container.ApiInfo{},
@@ -666,7 +611,6 @@ func (dm *dockerManager) checkDockerVersion() {
if err := dm.decode(resp, &versionInfo); err != nil {
return
}
dm.versionChecked = true
// if version > 24, one-shot works correctly and we can limit concurrent operations
if dockerVersion, err := semver.Parse(versionInfo.Version); err == nil && dockerVersion.Major > 24 {
dm.goodDockerVersion = true