diff --git a/agent/client_test.go b/agent/client_test.go index 0478ac7cb..f7df455f5 100644 --- a/agent/client_test.go +++ b/agent/client_test.go @@ -17,6 +17,7 @@ import ( "net/url" "os" "path/filepath" + "strconv" "strings" "testing" "time" @@ -316,7 +317,7 @@ func TestGetTLSConfigErrors(t *testing.T) { require.Error(t, err) assert.Nil(t, tlsConfig) assert.Contains(t, err.Error(), tc.errorMatch) - assert.Contains(t, err.Error(), tc.path) + assert.Contains(t, err.Error(), strconv.Quote(tc.path)) }) } } @@ -681,12 +682,12 @@ func TestGetToken(t *testing.T) { t.Run("error when TOKEN_FILE points to non-existent file", func(t *testing.T) { // Set TOKEN_FILE to a non-existent file - t.Setenv("TOKEN_FILE", "/non/existent/file.txt") + t.Setenv("TOKEN_FILE", filepath.Join(t.TempDir(), "missing.txt")) token, err := getToken() assert.Error(t, err) assert.Equal(t, "", token) - assert.Contains(t, err.Error(), "no such file or directory") + assert.ErrorIs(t, err, os.ErrNotExist) }) t.Run("handles empty token file", func(t *testing.T) { diff --git a/agent/disk_test.go b/agent/disk_test.go index 487f16973..b36f91b41 100644 --- a/agent/disk_test.go +++ b/agent/disk_test.go @@ -5,6 +5,7 @@ package agent import ( "math" "os" + "path/filepath" "runtime" "strings" "testing" @@ -13,6 +14,7 @@ import ( "github.com/henrygd/beszel/internal/entities/system" "github.com/shirou/gopsutil/v4/disk" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestParseFilesystemEntry(t *testing.T) { @@ -506,13 +508,13 @@ func TestAddConfiguredExtraFilesystems(t *testing.T) { func TestAddExtraFilesystemFolders(t *testing.T) { t.Run("adds missing folders and skips existing mountpoints", func(t *testing.T) { agent := &Agent{fsStats: map[string]*system.FsStats{ - "existing": {Mountpoint: "/extra-filesystems/existing"}, + "existing": {Mountpoint: filepath.FromSlash("/extra-filesystems/existing")}, }} discovery := diskDiscovery{ agent: agent, ctx: fsRegistrationContext{ isWindows: false, - efPath: "/extra-filesystems", + efPath: filepath.FromSlash("/extra-filesystems"), diskIoCounters: map[string]disk.IOCountersStat{ "newdisk": {Name: "newdisk"}, }, @@ -521,10 +523,10 @@ func TestAddExtraFilesystemFolders(t *testing.T) { discovery.addExtraFilesystemFolders([]string{"existing", "newdisk__Archive"}) - assert.Len(t, agent.fsStats, 2) + require.Len(t, agent.fsStats, 2) stats, exists := agent.fsStats["newdisk"] - assert.True(t, exists) - assert.Equal(t, "/extra-filesystems/newdisk__Archive", stats.Mountpoint) + require.True(t, exists) + assert.Equal(t, filepath.FromSlash("/extra-filesystems/newdisk__Archive"), stats.Mountpoint) assert.Equal(t, "Archive", stats.Name) }) } @@ -535,7 +537,7 @@ func TestAddPartitionExtraFs(t *testing.T) { agent: agent, ctx: fsRegistrationContext{ isWindows: false, - efPath: "/extra-filesystems", + efPath: filepath.FromSlash("/extra-filesystems"), diskIoCounters: map[string]disk.IOCountersStat{ "nvme0n1p1": {Name: "nvme0n1p1"}, "nvme1n1": {Name: "nvme1n1"}, @@ -550,12 +552,12 @@ func TestAddPartitionExtraFs(t *testing.T) { d.addPartitionExtraFs(disk.PartitionStat{ Device: "/dev/nvme0n1p1", - Mountpoint: "/extra-filesystems/nvme0n1p1__caddy1-root", + Mountpoint: filepath.FromSlash("/extra-filesystems/nvme0n1p1__caddy1-root"), }) stats, exists := agent.fsStats["nvme0n1p1"] - assert.True(t, exists) - assert.Equal(t, "/extra-filesystems/nvme0n1p1__caddy1-root", stats.Mountpoint) + require.True(t, exists) + assert.Equal(t, filepath.FromSlash("/extra-filesystems/nvme0n1p1__caddy1-root"), stats.Mountpoint) assert.Equal(t, "caddy1-root", stats.Name) }) @@ -566,10 +568,10 @@ func TestAddPartitionExtraFs(t *testing.T) { // These simulate the virtual mounts that appear when host / is bind-mounted // with disk.Partitions(all=true) — e.g. /proc, /sys, /dev visible under the mount. for _, nested := range []string{ - "/extra-filesystems/nvme0n1p1__caddy1-root/proc", - "/extra-filesystems/nvme0n1p1__caddy1-root/sys", - "/extra-filesystems/nvme0n1p1__caddy1-root/dev", - "/extra-filesystems/nvme0n1p1__caddy1-root/run", + filepath.FromSlash("/extra-filesystems/nvme0n1p1__caddy1-root/proc"), + filepath.FromSlash("/extra-filesystems/nvme0n1p1__caddy1-root/sys"), + filepath.FromSlash("/extra-filesystems/nvme0n1p1__caddy1-root/dev"), + filepath.FromSlash("/extra-filesystems/nvme0n1p1__caddy1-root/run"), } { d.addPartitionExtraFs(disk.PartitionStat{Device: "tmpfs", Mountpoint: nested}) } @@ -582,18 +584,20 @@ func TestAddPartitionExtraFs(t *testing.T) { d := makeDiscovery(agent) partitions := []disk.PartitionStat{ - {Device: "/dev/nvme0n1p1", Mountpoint: "/extra-filesystems/nvme0n1p1__caddy1-root"}, - {Device: "/dev/nvme1n1", Mountpoint: "/extra-filesystems/nvme1n1__caddy1-docker"}, - {Device: "proc", Mountpoint: "/extra-filesystems/nvme0n1p1__caddy1-root/proc"}, - {Device: "sysfs", Mountpoint: "/extra-filesystems/nvme0n1p1__caddy1-root/sys"}, - {Device: "overlay", Mountpoint: "/extra-filesystems/nvme0n1p1__caddy1-root/var/lib/docker"}, + {Device: "/dev/nvme0n1p1", Mountpoint: filepath.FromSlash("/extra-filesystems/nvme0n1p1__caddy1-root")}, + {Device: "/dev/nvme1n1", Mountpoint: filepath.FromSlash("/extra-filesystems/nvme1n1__caddy1-docker")}, + {Device: "proc", Mountpoint: filepath.FromSlash("/extra-filesystems/nvme0n1p1__caddy1-root/proc")}, + {Device: "sysfs", Mountpoint: filepath.FromSlash("/extra-filesystems/nvme0n1p1__caddy1-root/sys")}, + {Device: "overlay", Mountpoint: filepath.FromSlash("/extra-filesystems/nvme0n1p1__caddy1-root/var/lib/docker")}, } for _, p := range partitions { d.addPartitionExtraFs(p) } - assert.Len(t, agent.fsStats, 2) + require.Len(t, agent.fsStats, 2) + require.Contains(t, agent.fsStats, "nvme0n1p1") assert.Equal(t, "caddy1-root", agent.fsStats["nvme0n1p1"].Name) + require.Contains(t, agent.fsStats, "nvme1n1") assert.Equal(t, "caddy1-docker", agent.fsStats["nvme1n1"].Name) }) @@ -1081,7 +1085,7 @@ func TestNormalizeDeviceName(t *testing.T) { assert.Equal(t, "", normalizeDeviceName(" ")) // A drive-relative path is a path, not a volume. - assert.Equal(t, `C:data`, normalizeDeviceName(`C:data`)) + assert.Equal(t, filepath.Base(`C:data`), normalizeDeviceName(`C:data`)) } func TestFindIoDeviceWindowsVolumeNames(t *testing.T) { diff --git a/agent/gpu_fixture_test.go b/agent/gpu_fixture_test.go new file mode 100644 index 000000000..ed7ab801b --- /dev/null +++ b/agent/gpu_fixture_test.go @@ -0,0 +1,85 @@ +//go:build testing + +package agent + +import ( + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// Run a copy of the test binary as a GPU command so fixtures do not need a shell. +func TestMain(m *testing.M) { + executable, err := os.Executable() + if err != nil { + panic(err) + } + switch strings.TrimSuffix(filepath.Base(executable), ".exe") { + case nvidiaSmiCmd, rocmSmiCmd, tegraStatsCmd, nvtopCmd, intelGpuStatsCmd: + output, err := os.ReadFile(executable + ".stdout") + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + // Only the parent creates files; a late collector must not undo cleanup. + args, err := os.OpenFile(executable+".args", os.O_WRONLY|os.O_TRUNC, 0) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + _, err = io.WriteString(args, strings.Join(os.Args[1:], " ")) + closeErr := args.Close() + if err == nil { + err = closeErr + } + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + fmt.Print(string(output)) + os.Exit(0) + } + os.Exit(m.Run()) +} + +func gpuCommandFixture(t *testing.T, dir, name, output string) string { + t.Helper() + executable, err := os.Executable() + require.NoError(t, err) + if runtime.GOOS == "windows" { + name += ".exe" + } + path := filepath.Join(dir, name) + if err := os.Link(executable, path); err != nil { + src, err := os.Open(executable) + require.NoError(t, err) + defer src.Close() + dst, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0755) + require.NoError(t, err) + _, err = io.Copy(dst, src) + closeErr := dst.Close() + require.NoError(t, err) + require.NoError(t, closeErr) + } + require.NoError(t, os.WriteFile(path+".stdout", []byte(output), 0600)) + require.NoError(t, os.WriteFile(path+".args", nil, 0600)) + return path + ".args" +} + +func TestGPUFixtureDoesNotRecreateRemovedArgs(t *testing.T) { + argsFile := gpuCommandFixture(t, t.TempDir(), nvidiaSmiCmd, "fixture output\n") + require.NoError(t, os.WriteFile(argsFile, nil, 0600)) + require.NoError(t, os.Remove(argsFile)) + + cmd := exec.Command(strings.TrimSuffix(argsFile, ".args")) + err := cmd.Run() + require.NoFileExists(t, argsFile, "a late fixture process must not recreate files removed by cleanup") + require.Error(t, err, "the fixture must report a missing argument-capture file") +} diff --git a/agent/gpu_nvml_unsupported_test.go b/agent/gpu_nvml_unsupported_test.go new file mode 100644 index 000000000..9313953ff --- /dev/null +++ b/agent/gpu_nvml_unsupported_test.go @@ -0,0 +1,30 @@ +//go:build testing && !(amd64 && (windows || (linux && glibc))) + +package agent + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// This fallback requires NVML initialisation to fail, as guaranteed by the unsupported implementation. +func TestNewGPUManagerPriorityNvmlFallbackToNvidiaSmi(t *testing.T) { + dir := t.TempDir() + t.Setenv("PATH", dir) + t.Setenv("BESZEL_AGENT_GPU_COLLECTOR", "nvml,nvidia-smi") + + gpuCommandFixture(t, dir, "nvidia-smi", `0, NVIDIA Fallback GPU, 41, 256, 1024, 8, 14`+"\n") + + gm, err := NewGPUManager() + require.NoError(t, err) + require.NotNil(t, gm) + + waitGPUs(t, gm, "0") + gm.Lock() + defer gm.Unlock() + gpu, ok := gm.GpuDataMap["0"] + require.True(t, ok) + assert.Equal(t, "Fallback GPU", gpu.Name) +} diff --git a/agent/gpu_test.go b/agent/gpu_test.go index 82a83a733..0636dc259 100644 --- a/agent/gpu_test.go +++ b/agent/gpu_test.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" "os" - "path/filepath" "slices" "strings" "testing" @@ -1125,7 +1124,7 @@ func TestGPUCapabilitiesAndLegacyPriority(t *testing.T) { tests := []struct { name string - setupCommands func(string) error + setupCommands func(*testing.T, string) wantNvidiaSmi bool wantRocmSmi bool wantTegrastats bool @@ -1133,10 +1132,8 @@ func TestGPUCapabilitiesAndLegacyPriority(t *testing.T) { wantErr bool }{ { - name: "nvidia-smi not available", - setupCommands: func(_ string) error { - return nil - }, + name: "nvidia-smi not available", + setupCommands: func(*testing.T, string) {}, wantNvidiaSmi: false, wantRocmSmi: false, wantTegrastats: false, @@ -1145,14 +1142,8 @@ func TestGPUCapabilitiesAndLegacyPriority(t *testing.T) { }, { name: "nvidia-smi available", - setupCommands: func(tempDir string) error { - path := filepath.Join(tempDir, "nvidia-smi") - script := `#!/bin/sh -echo "test"` - if err := os.WriteFile(path, []byte(script), 0755); err != nil { - return err - } - return nil + setupCommands: func(t *testing.T, tempDir string) { + gpuCommandFixture(t, tempDir, "nvidia-smi", "test\n") }, wantNvidiaSmi: true, wantTegrastats: false, @@ -1162,14 +1153,8 @@ echo "test"` }, { name: "rocm-smi available", - setupCommands: func(tempDir string) error { - path := filepath.Join(tempDir, "rocm-smi") - script := `#!/bin/sh -echo "test"` - if err := os.WriteFile(path, []byte(script), 0755); err != nil { - return err - } - return nil + setupCommands: func(t *testing.T, tempDir string) { + gpuCommandFixture(t, tempDir, "rocm-smi", "test\n") }, wantNvidiaSmi: false, wantRocmSmi: true, @@ -1179,14 +1164,8 @@ echo "test"` }, { name: "tegrastats available", - setupCommands: func(tempDir string) error { - path := filepath.Join(tempDir, "tegrastats") - script := `#!/bin/sh -echo "test"` - if err := os.WriteFile(path, []byte(script), 0755); err != nil { - return err - } - return nil + setupCommands: func(t *testing.T, tempDir string) { + gpuCommandFixture(t, tempDir, "tegrastats", "test\n") }, wantNvidiaSmi: false, wantRocmSmi: false, @@ -1196,14 +1175,8 @@ echo "test"` }, { name: "nvtop available", - setupCommands: func(tempDir string) error { - path := filepath.Join(tempDir, "nvtop") - script := `#!/bin/sh -echo "[]"` - if err := os.WriteFile(path, []byte(script), 0755); err != nil { - return err - } - return nil + setupCommands: func(t *testing.T, tempDir string) { + gpuCommandFixture(t, tempDir, "nvtop", "test\n") }, wantNvidiaSmi: false, wantRocmSmi: false, @@ -1212,12 +1185,9 @@ echo "[]"` wantErr: false, }, { - name: "no gpu tools available", - setupCommands: func(_ string) error { - // The subtest already restricts PATH to its empty temporary directory. - return nil - }, - wantErr: true, + name: "no gpu tools available", + setupCommands: func(*testing.T, string) {}, + wantErr: true, }, } @@ -1225,9 +1195,7 @@ echo "[]"` t.Run(tt.name, func(t *testing.T) { tempDir := t.TempDir() t.Setenv("PATH", tempDir) - if err := tt.setupCommands(tempDir); err != nil { - t.Fatal(err) - } + tt.setupCommands(t, tempDir) gm := &GPUManager{} caps := gm.discoverGpuCapabilities() @@ -1269,6 +1237,21 @@ echo "[]"` } } +func waitGPUs(t *testing.T, gm *GPUManager, ids ...string) { + t.Helper() + require.Eventually(t, func() bool { + gm.Lock() + defer gm.Unlock() + for _, id := range ids { + gpu := gm.GpuDataMap[id] + if gpu == nil || gpu.Count == 0 { + return false + } + } + return true + }, 5*time.Second, 10*time.Millisecond, "GPU collectors did not produce data for %v", ids) +} + func TestCollectorStartHelpers(t *testing.T) { // Set up temp dir with the commands dir := t.TempDir() @@ -1277,21 +1260,17 @@ func TestCollectorStartHelpers(t *testing.T) { tests := []struct { name string command string - setup func(t *testing.T) error + gpuID string + setup func(t *testing.T) validate func(t *testing.T, gm *GPUManager) gm *GPUManager }{ { name: "nvidia-smi collector", command: "nvidia-smi", - setup: func(t *testing.T) error { - path := filepath.Join(dir, "nvidia-smi") - script := `#!/bin/sh -echo "0, NVIDIA Test GPU, 50, 1024, 4096, 25, 100"` - if err := os.WriteFile(path, []byte(script), 0755); err != nil { - return err - } - return nil + gpuID: "0", + setup: func(t *testing.T) { + gpuCommandFixture(t, dir, "nvidia-smi", `0, NVIDIA Test GPU, 50, 1024, 4096, 25, 100`+"\n") }, validate: func(t *testing.T, gm *GPUManager) { gpu, exists := gm.GpuDataMap["0"] @@ -1306,14 +1285,9 @@ echo "0, NVIDIA Test GPU, 50, 1024, 4096, 25, 100"` { name: "rocm-smi collector", command: "rocm-smi", - setup: func(t *testing.T) error { - path := filepath.Join(dir, "rocm-smi") - script := `#!/bin/sh -echo '{"card0": {"Temperature (Sensor edge) (C)": "49.0", "Current Socket Graphics Package Power (W)": "28.159", "GPU use (%)": "0", "VRAM Total Memory (B)": "536870912", "VRAM Total Used Memory (B)": "445550592", "Card Series": "Rembrandt [Radeon 680M]", "Card Model": "0x1681", "Card Vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", "Card SKU": "REMBRANDT", "Subsystem ID": "0x8a22", "Device Rev": "0xc8", "Node ID": "1", "GUID": "34756", "GFX Version": "gfx1035"}}'` - if err := os.WriteFile(path, []byte(script), 0755); err != nil { - return err - } - return nil + gpuID: "34756", + setup: func(t *testing.T) { + gpuCommandFixture(t, dir, "rocm-smi", `{"card0": {"Temperature (Sensor edge) (C)": "49.0", "Current Socket Graphics Package Power (W)": "28.159", "GPU use (%)": "0", "VRAM Total Memory (B)": "536870912", "VRAM Total Used Memory (B)": "445550592", "Card Series": "Rembrandt [Radeon 680M]", "Card Model": "0x1681", "Card Vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", "Card SKU": "REMBRANDT", "Subsystem ID": "0x8a22", "Device Rev": "0xc8", "Node ID": "1", "GUID": "34756", "GFX Version": "gfx1035"}}`+"\n") }, validate: func(t *testing.T, gm *GPUManager) { gpu, exists := gm.GpuDataMap["34756"] @@ -1328,14 +1302,9 @@ echo '{"card0": {"Temperature (Sensor edge) (C)": "49.0", "Current Socket Graphi { name: "tegrastats collector", command: "tegrastats", - setup: func(t *testing.T) error { - path := filepath.Join(dir, "tegrastats") - script := `#!/bin/sh -echo "11-14-2024 22:54:33 RAM 1024/4096MB GR3D_FREQ 80% tj@70C VDD_GPU_SOC 1000mW"` - if err := os.WriteFile(path, []byte(script), 0755); err != nil { - return err - } - return nil + gpuID: "0", + setup: func(t *testing.T) { + gpuCommandFixture(t, dir, "tegrastats", `11-14-2024 22:54:33 RAM 1024/4096MB GR3D_FREQ 80% tj@70C VDD_GPU_SOC 1000mW`+"\n") }, validate: func(t *testing.T, gm *GPUManager) { gpu, exists := gm.GpuDataMap["0"] @@ -1353,14 +1322,9 @@ echo "11-14-2024 22:54:33 RAM 1024/4096MB GR3D_FREQ 80% tj@70C VDD_GPU_SOC 1000m { name: "nvtop collector", command: "nvtop", - setup: func(t *testing.T) error { - path := filepath.Join(dir, "nvtop") - script := `#!/bin/sh -echo '[{"device_name":"NVIDIA Test GPU","temp":"52C","power_draw":"31W","gpu_util":"37%","mem_total":"4294967296","mem_used":"536870912","processes":[]}]'` - if err := os.WriteFile(path, []byte(script), 0755); err != nil { - return err - } - return nil + gpuID: "n0", + setup: func(t *testing.T) { + gpuCommandFixture(t, dir, "nvtop", `[{"device_name":"NVIDIA Test GPU","temp":"52C","power_draw":"31W","gpu_util":"37%","mem_total":"4294967296","mem_used":"536870912","processes":[]}]`+"\n") }, validate: func(t *testing.T, gm *GPUManager) { gpu, exists := gm.GpuDataMap["n0"] @@ -1375,9 +1339,7 @@ echo '[{"device_name":"NVIDIA Test GPU","temp":"52C","power_draw":"31W","gpu_uti for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if err := tt.setup(t); err != nil { - t.Fatal(err) - } + tt.setup(t) if tt.gm == nil { tt.gm = &GPUManager{ GpuDataMap: make(map[string]*system.GPUData), @@ -1395,7 +1357,9 @@ echo '[{"device_name":"NVIDIA Test GPU","temp":"52C","power_draw":"31W","gpu_uti default: t.Fatalf("unknown test command %q", tt.command) } - time.Sleep(50 * time.Millisecond) // Give collector time to run + waitGPUs(t, tt.gm, tt.gpuID) + tt.gm.Lock() + defer tt.gm.Unlock() tt.validate(t, tt.gm) }) } @@ -1406,21 +1370,17 @@ func TestNewGPUManagerPriorityNvtopFallback(t *testing.T) { t.Setenv("PATH", dir) t.Setenv("BESZEL_AGENT_GPU_COLLECTOR", "nvtop,nvidia-smi") - nvtopPath := filepath.Join(dir, "nvtop") - nvtopScript := `#!/bin/sh -echo 'not-json'` - require.NoError(t, os.WriteFile(nvtopPath, []byte(nvtopScript), 0755)) + gpuCommandFixture(t, dir, "nvtop", `not-json`+"\n") - nvidiaPath := filepath.Join(dir, "nvidia-smi") - nvidiaScript := `#!/bin/sh -echo "0, NVIDIA Priority GPU, 45, 512, 2048, 12, 25"` - require.NoError(t, os.WriteFile(nvidiaPath, []byte(nvidiaScript), 0755)) + gpuCommandFixture(t, dir, "nvidia-smi", `0, NVIDIA Priority GPU, 45, 512, 2048, 12, 25`+"\n") gm, err := NewGPUManager() require.NoError(t, err) require.NotNil(t, gm) - time.Sleep(150 * time.Millisecond) + waitGPUs(t, gm, "0") + gm.Lock() + defer gm.Unlock() gpu, ok := gm.GpuDataMap["0"] require.True(t, ok) assert.Equal(t, "Priority GPU", gpu.Name) @@ -1432,50 +1392,27 @@ func TestNewGPUManagerPriorityMixedCollectors(t *testing.T) { t.Setenv("PATH", dir) t.Setenv("BESZEL_AGENT_GPU_COLLECTOR", "intel_gpu_top,rocm-smi") - intelPath := filepath.Join(dir, "intel_gpu_top") - intelScript := "#!/bin/sh\necho '" + intelJSONStream(true, + intelOutput := intelJSONStream(true, intelJSONSample(2, 2.69, map[string]float64{"Render/3D": 0, "Video": 0}), intelJSONSample(1.8, 2.45, map[string]float64{"Render/3D": 8.5, "Video": 15}), - ) + "'\n" - require.NoError(t, os.WriteFile(intelPath, []byte(intelScript), 0755)) + ) + gpuCommandFixture(t, dir, intelGpuStatsCmd, intelOutput+"\n") - rocmPath := filepath.Join(dir, "rocm-smi") - rocmScript := `#!/bin/sh -echo '{"card0": {"Temperature (Sensor edge) (C)": "49.0", "Current Socket Graphics Package Power (W)": "28.159", "GPU use (%)": "0", "VRAM Total Memory (B)": "536870912", "VRAM Total Used Memory (B)": "445550592", "Card Series": "Rembrandt [Radeon 680M]", "GUID": "34756"}}' -` - require.NoError(t, os.WriteFile(rocmPath, []byte(rocmScript), 0755)) + gpuCommandFixture(t, dir, "rocm-smi", `{"card0": {"Temperature (Sensor edge) (C)": "49.0", "Current Socket Graphics Package Power (W)": "28.159", "GPU use (%)": "0", "VRAM Total Memory (B)": "536870912", "VRAM Total Used Memory (B)": "445550592", "Card Series": "Rembrandt [Radeon 680M]", "GUID": "34756"}}`+"\n") gm, err := NewGPUManager() require.NoError(t, err) require.NotNil(t, gm) - time.Sleep(150 * time.Millisecond) + waitGPUs(t, gm, "i0", "34756") + gm.Lock() + defer gm.Unlock() _, intelOk := gm.GpuDataMap["i0"] _, amdOk := gm.GpuDataMap["34756"] assert.True(t, intelOk) assert.True(t, amdOk) } -func TestNewGPUManagerPriorityNvmlFallbackToNvidiaSmi(t *testing.T) { - dir := t.TempDir() - t.Setenv("PATH", dir) - t.Setenv("BESZEL_AGENT_GPU_COLLECTOR", "nvml,nvidia-smi") - - nvidiaPath := filepath.Join(dir, "nvidia-smi") - nvidiaScript := `#!/bin/sh -echo "0, NVIDIA Fallback GPU, 41, 256, 1024, 8, 14"` - require.NoError(t, os.WriteFile(nvidiaPath, []byte(nvidiaScript), 0755)) - - gm, err := NewGPUManager() - require.NoError(t, err) - require.NotNil(t, gm) - - time.Sleep(150 * time.Millisecond) - gpu, ok := gm.GpuDataMap["0"] - require.True(t, ok) - assert.Equal(t, "Fallback GPU", gpu.Name) -} - func TestNewGPUManagerConfiguredCollectorsMustStart(t *testing.T) { dir := t.TempDir() t.Setenv("PATH", dir) @@ -1510,8 +1447,12 @@ func TestNewGPUManagerConfiguredNvmlBypassesCapabilityGate(t *testing.T) { t.Setenv("BESZEL_AGENT_GPU_COLLECTOR", "nvml") gm, err := NewGPUManager() + if err == nil { + // Native NVML can be available even with no tools on PATH. + require.NotNil(t, gm) + return + } require.Nil(t, gm) - require.Error(t, err) assert.Contains(t, err.Error(), "no configured GPU collectors are available") assert.NotContains(t, err.Error(), noGPUFoundMsg) } @@ -1521,16 +1462,15 @@ func TestNewGPUManagerJetsonIgnoresCollectorConfig(t *testing.T) { t.Setenv("PATH", dir) t.Setenv("BESZEL_AGENT_GPU_COLLECTOR", "nvidia-smi") - tegraPath := filepath.Join(dir, "tegrastats") - tegraScript := `#!/bin/sh -echo "11-14-2024 22:54:33 RAM 1024/4096MB GR3D_FREQ 80% tj@70C VDD_GPU_SOC 1000mW"` - require.NoError(t, os.WriteFile(tegraPath, []byte(tegraScript), 0755)) + gpuCommandFixture(t, dir, "tegrastats", `11-14-2024 22:54:33 RAM 1024/4096MB GR3D_FREQ 80% tj@70C VDD_GPU_SOC 1000mW`+"\n") gm, err := NewGPUManager() require.NoError(t, err) require.NotNil(t, gm) - time.Sleep(100 * time.Millisecond) + waitGPUs(t, gm, "0") + gm.Lock() + defer gm.Unlock() gpu, ok := gm.GpuDataMap["0"] require.True(t, ok) assert.Equal(t, "GPU", gpu.Name) @@ -1805,11 +1745,7 @@ func TestIntelCollectorStreaming(t *testing.T) { ) + "\n]" // Create a fake intel_gpu_top that prints -J output with four samples (first will be skipped) and exits - scriptPath := filepath.Join(dir, "intel_gpu_top") - script := "#!/bin/sh\necho '" + output + "'\n" - if err := os.WriteFile(scriptPath, []byte(script), 0755); err != nil { - t.Fatal(err) - } + gpuCommandFixture(t, dir, intelGpuStatsCmd, output+"\n") gm := &GPUManager{ GpuDataMap: make(map[string]*system.GPUData), @@ -1992,19 +1928,12 @@ func TestIntelCollectorDeviceEnv(t *testing.T) { dir := t.TempDir() t.Setenv("PATH", dir) - // Prepare a file to capture args - argsFile := filepath.Join(dir, "args.txt") - // Create a fake intel_gpu_top that records its arguments and prints minimal valid output - scriptPath := filepath.Join(dir, "intel_gpu_top") output := intelJSONStream(true, intelJSONSample(2, 2.69, map[string]float64{"Render/3D": 0, "Video": 0}), intelJSONSample(1.8, 2.45, map[string]float64{"Render/3D": 8.5, "Video": 15}), ) - script := fmt.Sprintf("#!/bin/sh\necho \"$@\" > %s\necho '%s'\n", argsFile, output) - if err := os.WriteFile(scriptPath, []byte(script), 0755); err != nil { - t.Fatal(err) - } + argsFile := gpuCommandFixture(t, dir, intelGpuStatsCmd, output) // Set device selector via prefixed env var t.Setenv("BESZEL_AGENT_INTEL_GPU_DEVICE", "sriov") diff --git a/agent/sensors_test.go b/agent/sensors_test.go index 0523bc6f9..0a3c91572 100644 --- a/agent/sensors_test.go +++ b/agent/sensors_test.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "testing" "time" @@ -695,6 +696,11 @@ func TestNewSensorConfigSkipGpuWiresShadow(t *testing.T) { config := agent.newSensorConfig() assert.True(t, config.skipGPU) + if runtime.GOOS != "linux" { + assert.Empty(t, config.sensorShadow) + assert.Nil(t, config.context.Value(common.EnvKey)) + return + } envMap, ok := config.context.Value(common.EnvKey).(common.EnvMap) require.True(t, ok, "SKIP_GPU should point the sensor context at a sysfs shadow") shadow, ok := envMap[common.HostSysEnvKey] @@ -721,6 +727,11 @@ func TestSkipGpuShadowUsesSysSensorsRoot(t *testing.T) { require.True(t, ok, "SKIP_GPU should point the sensor context at a sysfs shadow") shadow, ok := envMap[common.HostSysEnvKey] require.True(t, ok) + if runtime.GOOS != "linux" { + assert.Equal(t, sysRoot, shadow) + assert.Empty(t, config.sensorShadow) + return + } require.NotEqual(t, sysRoot, shadow, "shadow must not be the SYS_SENSORS tree itself") target, err := os.Readlink(filepath.Join(shadow, "class", "hwmon", "hwmon0")) diff --git a/internal/entities/systemd/systemd_test.go b/internal/entities/systemd/systemd_test.go index 8e5ceb104..987c82572 100644 --- a/internal/entities/systemd/systemd_test.go +++ b/internal/entities/systemd/systemd_test.go @@ -3,7 +3,9 @@ package systemd_test import ( + "runtime" "testing" + "testing/synctest" "time" "github.com/henrygd/beszel/internal/entities/systemd" @@ -65,30 +67,36 @@ func TestServiceUpdateCPUPercent(t *testing.T) { }) t.Run("subsequent call calculates CPU percentage", func(t *testing.T) { - service := &systemd.Service{} - service.PrevCpuUsage = 1000 - service.PrevReadTime = time.Now().Add(-time.Second) + synctest.Test(t, func(t *testing.T) { + service := &systemd.Service{} + service.PrevCpuUsage = 1000 + service.PrevReadTime = time.Now().Add(-time.Second) - service.UpdateCPUPercent(8000000000) // 8 seconds of CPU time + // Half of one second's CPU capacity across all cores. + cpuUsage := service.PrevCpuUsage + uint64(time.Second/2)*uint64(runtime.NumCPU()) + service.UpdateCPUPercent(cpuUsage) - // CPU usage should be positive and reasonable - assert.Greater(t, service.Cpu, 0.0, "CPU usage should be positive") - assert.LessOrEqual(t, service.Cpu, 100.0, "CPU usage should not exceed 100%") - assert.Equal(t, uint64(8000000000), service.PrevCpuUsage) - assert.Greater(t, service.CpuPeak, 0.0, "CPU peak should be set") + assert.Equal(t, 50.0, service.Cpu) + assert.Equal(t, cpuUsage, service.PrevCpuUsage) + assert.Equal(t, time.Now(), service.PrevReadTime) + assert.Equal(t, 50.0, service.CpuPeak) + }) }) t.Run("CPU peak updates only when higher", func(t *testing.T) { - service := &systemd.Service{} - service.PrevCpuUsage = 1000 - service.PrevReadTime = time.Now().Add(-time.Second) - service.UpdateCPUPercent(8000000000) // Set initial peak to ~50% - initialPeak := service.CpuPeak + synctest.Test(t, func(t *testing.T) { + service := &systemd.Service{} + service.PrevCpuUsage = 1000 + service.PrevReadTime = time.Now().Add(-time.Second) + service.UpdateCPUPercent(service.PrevCpuUsage + uint64(time.Second/2)*uint64(runtime.NumCPU())) + assert.Equal(t, 50.0, service.CpuPeak) - // Now try with much lower CPU usage - should not update peak - service.PrevReadTime = time.Now().Add(-time.Second) - service.UpdateCPUPercent(1000000) // Much lower usage - assert.Equal(t, initialPeak, service.CpuPeak, "Peak should not update for lower CPU usage") + // A smaller increase in the cumulative counter gives 25% usage. + service.PrevReadTime = time.Now().Add(-time.Second) + service.UpdateCPUPercent(service.PrevCpuUsage + uint64(time.Second/4)*uint64(runtime.NumCPU())) + assert.Equal(t, 25.0, service.Cpu) + assert.Equal(t, 50.0, service.CpuPeak, "Peak should not update for lower CPU usage") + }) }) t.Run("handles zero duration", func(t *testing.T) {