fix(agent): SKIP_GPU excludes GPU hwmon from temperatures and fans (#2313)

agent/agent.go

@@ -252,7 +252,11 @@ func (a *Agent) gatherStats(options common.DataRequestOptions) *system.CombinedD
 // Start initializes and starts the agent with optional WebSocket connection
 func (a *Agent) Start(serverOptions ServerOptions) error {
 	a.keys = serverOptions.Keys
-	return a.connectionManager.Start(serverOptions)
+	err := a.connectionManager.Start(serverOptions)
+	if err != nil {
+		a.cleanupSensorShadow()
+	}
+	return err
 }
 
 func (a *Agent) getFingerprint() string {

agent/connection_manager.go

@@ -155,6 +155,7 @@ func (c *ConnectionManager) stop() error {
 	_ = c.agent.StopServer()
 	c.agent.monitorManager.Stop()
 	c.closeWebSocket()
+	c.agent.cleanupSensorShadow()
 	return health.CleanUp()
 }
 

agent/fans.go

@@ -12,7 +12,7 @@ import (
 )
 
 type fanSensor struct {
-	key, path string
+	key, path, chip string
 }
 
 var getFanSensors = newFanSensorCache(hwmonRoot)
@@ -34,6 +34,10 @@ func (a *Agent) updateFans(systemStats *system.Stats) {
 		slog.Debug("Error reading fans", "err", err)
 		return
 	}
+	// Filter before reading fan*_input: each read can wake an idle GPU.
+	if a.sensorConfig != nil && a.sensorConfig.skipGPU {
+		sensors = filterGpuFans(sensors)
+	}
 	fans := readFanSensors(sensors)
 	if len(fans) == 0 {
 		return
@@ -100,7 +104,7 @@ func discoverHwmonFans(root string) ([]fanSensor, error) {
 			if label != "" {
 				key = chipName + "_" + label
 			}
-			sensors = append(sensors, fanSensor{key, inputPath})
+			sensors = append(sensors, fanSensor{key, inputPath, chipName})
 		}
 	}
 	return sensors, nil
@@ -115,3 +119,15 @@ func readFanSensors(sensors []fanSensor) map[string]uint16 {
 	}
 	return fans
 }
+
+// filterGpuFans drops GPU chips without touching the shared cache backing array.
+func filterGpuFans(sensors []fanSensor) []fanSensor {
+	kept := make([]fanSensor, 0, len(sensors))
+	for _, sensor := range sensors {
+		if isGpuChipName(sensor.chip) {
+			continue
+		}
+		kept = append(kept, sensor)
+	}
+	return kept
+}

agent/fans_test.go

@@ -103,3 +103,20 @@ func TestFanDiscoveryCache(t *testing.T) {
 	fans = readFanSensors(sensors)
 	assert.Equal(t, map[string]uint16{"chip_fan1": 1200}, fans)
 }
+
+func TestFilterGpuFans(t *testing.T) {
+	root := t.TempDir()
+	writeFile(t, filepath.Join(root, "hwmon0", "name"), "xe\n")
+	writeFile(t, filepath.Join(root, "hwmon0", "fan1_input"), "1200\n")
+	writeFile(t, filepath.Join(root, "hwmon1", "name"), "nct6798\n")
+	writeFile(t, filepath.Join(root, "hwmon1", "fan1_input"), "800\n")
+
+	discovered, err := discoverHwmonFans(root)
+	require.NoError(t, err)
+	require.Len(t, discovered, 2)
+
+	filtered := filterGpuFans(discovered)
+	require.Len(t, filtered, 1)
+	assert.Equal(t, "nct6798_fan1", filtered[0].key)
+	assert.Len(t, discovered, 2)
+}

agent/gpu.go

@@ -454,8 +454,8 @@ func (gm *GPUManager) storeSnapshot(id string, gpu *system.GPUData, cacheKey uin
 // It only reports capability presence and does not apply policy decisions.
 func (gm *GPUManager) discoverGpuCapabilities() gpuCapabilities {
 	caps := gpuCapabilities{
-		hasAmdSysfs: gm.hasAmdSysfs(),
-		hasXe:       gm.hasXe(),
+		hasAmdSysfs:   gm.hasAmdSysfs(),
+		hasXe:         gm.hasXe(),
 		hasIntelSysfs: gm.hasIntelSysfs(),
 	}
 	if _, err := exec.LookPath(nvidiaSmiCmd); err == nil {
@@ -750,9 +750,36 @@ func (gm *GPUManager) resolveLegacyCollectorPriority(caps gpuCapabilities) []col
 	return priorities
 }
 
+// gpuHwmonChips are hwmon chip names belonging to GPUs. Sensor reads on some
+// of these drivers (notably Intel Xe, where each read is a runtime PM resume)
+// wake the card, so SKIP_GPU must avoid touching them, not just hide them.
+var gpuHwmonChips = []string{"xe", "i915", "amdgpu", "radeon", "nvidia", "nouveau"}
+
+func isGpuChipName(name string) bool {
+	name = strings.ToLower(strings.TrimSpace(name))
+	for _, chip := range gpuHwmonChips {
+		if name == chip {
+			return true
+		}
+	}
+	return false
+}
+
+// SensorKeys are "<chip>" or "<chip>_<label>".
+func isGpuSensorKey(key string) bool {
+	key = strings.ToLower(strings.TrimSpace(key))
+	for _, chip := range gpuHwmonChips {
+		if key == chip || strings.HasPrefix(key, chip+"_") {
+			return true
+		}
+	}
+	return false
+}
+
 // NewGPUManager creates and initializes a new GPUManager
 func NewGPUManager() (*GPUManager, error) {
 	if skipGPU, _ := utils.GetEnv("SKIP_GPU"); skipGPU == "true" {
+		slog.Info("SKIP_GPU enabled, skipping GPU monitoring (collectors, temperatures, and fans)")
 		return nil, nil
 	}
 	var gm GPUManager

agent/sensors.go

@@ -5,7 +5,9 @@ import (
 	"errors"
 	"fmt"
 	"log/slog"
+	"os"
 	"path"
+	"path/filepath"
 	"runtime"
 	"strconv"
 	"strings"
@@ -32,6 +34,8 @@ type SensorConfig struct {
 	isBlacklist    bool
 	hasWildcards   bool
 	skipCollection bool
+	skipGPU        bool
+	sensorShadow   string
 	firstRun       bool
 }
 
@@ -41,13 +45,14 @@ func (a *Agent) newSensorConfig() *SensorConfig {
 	sensorsEnvVal, sensorsSet := utils.GetEnv("SENSORS")
 	skipCollection := sensorsSet && sensorsEnvVal == ""
 	sensorsTimeout, _ := utils.GetEnv("SENSORS_TIMEOUT")
+	skipGPU, _ := utils.GetEnv("SKIP_GPU")
 
-	return a.newSensorConfigWithEnv(primarySensor, sysSensors, sensorsEnvVal, sensorsTimeout, skipCollection)
+	return a.newSensorConfigWithEnv(primarySensor, sysSensors, sensorsEnvVal, sensorsTimeout, skipCollection, skipGPU == "true")
 }
 
 // newSensorConfigWithEnv creates a SensorConfig with the provided environment variables
 // sensorsSet indicates if the SENSORS environment variable was explicitly set (even to empty string)
-func (a *Agent) newSensorConfigWithEnv(primarySensor, sysSensors, sensorsEnvVal, sensorsTimeout string, skipCollection bool) *SensorConfig {
+func (a *Agent) newSensorConfigWithEnv(primarySensor, sysSensors, sensorsEnvVal, sensorsTimeout string, skipCollection, skipGPU bool) *SensorConfig {
 	timeout := 2 * time.Second
 	if sensorsTimeout != "" {
 		if d, err := time.ParseDuration(sensorsTimeout); err == nil {
@@ -62,6 +67,7 @@ func (a *Agent) newSensorConfigWithEnv(primarySensor, sysSensors, sensorsEnvVal,
 		primarySensor:  primarySensor,
 		timeout:        timeout,
 		skipCollection: skipCollection,
+		skipGPU:        skipGPU,
 		firstRun:       true,
 		sensors:        make(map[string]struct{}),
 	}
@@ -73,6 +79,19 @@ func (a *Agent) newSensorConfigWithEnv(primarySensor, sysSensors, sensorsEnvVal,
 			common.EnvKey, common.EnvMap{common.HostSysEnvKey: sysSensors},
 		)
 	}
+	if skipGPU && runtime.GOOS == "linux" {
+		// gopsutil reads every temp*_input before results can be filtered, so
+		// point it at a shadow tree built from the effective sysfs root instead.
+		if shadow, err := buildNonGpuSysShadow(effectiveSysRoot(config.context)); err == nil {
+			slog.Info("SKIP_GPU enabled, using non-GPU sensor sysfs shadow", "path", shadow)
+			config.sensorShadow = shadow
+			config.context = context.WithValue(config.context,
+				common.EnvKey, common.EnvMap{common.HostSysEnvKey: shadow},
+			)
+		} else {
+			slog.Warn("SKIP_GPU sensor shadow unavailable, falling back to post-read filtering", "err", err)
+		}
+	}
 
 	// handle blacklist
 	if strings.HasPrefix(sensorsEnvVal, "-") {
@@ -149,6 +168,9 @@ func (a *Agent) updateTemperatures(systemStats *system.Stats) {
 		if !isValidSensor(sensorName, a.sensorConfig) {
 			continue
 		}
+		if a.sensorConfig.skipGPU && isGpuSensorKey(sensorName) {
+			continue
+		}
 		// set dashboard temperature
 		switch a.sensorConfig.primarySensor {
 		case "":
@@ -245,3 +267,102 @@ func scaleTemperature(temp float64) float64 {
 	}
 	return scaled100
 }
+
+// effectiveSysRoot mirrors gopsutil's HostSys lookup, which lives in its
+// internal package: context override, then HOST_SYS env, then /sys.
+func effectiveSysRoot(ctx context.Context) string {
+	if envMap, ok := ctx.Value(common.EnvKey).(common.EnvMap); ok {
+		if v := envMap[common.HostSysEnvKey]; v != "" {
+			return v
+		}
+	}
+	if v := os.Getenv("HOST_SYS"); v != "" {
+		return v
+	}
+	return "/sys"
+}
+
+func (config *SensorConfig) cleanupSensorShadow() {
+	if config.sensorShadow == "" {
+		return
+	}
+	if err := os.RemoveAll(config.sensorShadow); err != nil {
+		slog.Warn("Error removing sensor sysfs shadow", "path", config.sensorShadow, "err", err)
+		return
+	}
+	config.sensorShadow = ""
+}
+
+func (a *Agent) cleanupSensorShadow() {
+	if a.sensorConfig != nil {
+		a.sensorConfig.cleanupSensorShadow()
+	}
+}
+
+func isGpuThermalZone(zoneType string) bool {
+	zoneType = strings.ToLower(strings.TrimSpace(zoneType))
+	return isGpuChipName(zoneType) || strings.Contains(zoneType, "gpu")
+}
+
+// buildNonGpuSysShadow links non-GPU sensor directories into a temp dir. Only
+// static chip names and thermal-zone types are read; no sensor values are touched.
+func buildNonGpuSysShadow(sysRoot string) (string, error) {
+	shadow, err := os.MkdirTemp("", "beszel-sensors-*")
+	if err != nil {
+		return "", err
+	}
+	shadowHwmon := filepath.Join(shadow, "class", "hwmon")
+	if err := os.MkdirAll(shadowHwmon, 0o755); err != nil {
+		os.RemoveAll(shadow)
+		return "", err
+	}
+	entries, err := os.ReadDir(filepath.Join(sysRoot, "class", "hwmon"))
+	if err != nil && !os.IsNotExist(err) {
+		os.RemoveAll(shadow)
+		return "", err
+	}
+	for _, entry := range entries {
+		chipDir := filepath.Join(sysRoot, "class", "hwmon", entry.Name())
+		// Some hwmon devices expose name under device/ (gopsutil's CentOS fallback).
+		name, ok := utils.ReadStringFileOK(filepath.Join(chipDir, "name"))
+		if !ok {
+			name, ok = utils.ReadStringFileOK(filepath.Join(chipDir, "device", "name"))
+		}
+		if !ok || isGpuChipName(name) {
+			continue
+		}
+		if err := os.Symlink(chipDir, filepath.Join(shadowHwmon, entry.Name())); err != nil {
+			os.RemoveAll(shadow)
+			return "", err
+		}
+	}
+
+	thermalEntries, err := os.ReadDir(filepath.Join(sysRoot, "class", "thermal"))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return shadow, nil
+		}
+		os.RemoveAll(shadow)
+		return "", err
+	}
+	shadowThermal := filepath.Join(shadow, "class", "thermal")
+	if err := os.MkdirAll(shadowThermal, 0o755); err != nil {
+		os.RemoveAll(shadow)
+		return "", err
+	}
+	for _, entry := range thermalEntries {
+		if !strings.HasPrefix(entry.Name(), "thermal_zone") {
+			continue
+		}
+		zoneDir := filepath.Join(sysRoot, "class", "thermal", entry.Name())
+		zoneType, ok := utils.ReadStringFileOK(filepath.Join(zoneDir, "type"))
+		if !ok || isGpuThermalZone(zoneType) {
+			continue
+		}
+		if err := os.Symlink(zoneDir, filepath.Join(shadowThermal, entry.Name())); err != nil {
+			os.RemoveAll(shadow)
+			return "", err
+		}
+	}
+	return shadow, nil
+}

agent/sensors_test.go

@@ -5,6 +5,8 @@ package agent
 import (
 	"context"
 	"fmt"
+	"os"
+	"path/filepath"
 	"testing"
 	"time"
 
@@ -328,7 +330,7 @@ func TestNewSensorConfigWithEnv(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			result := agent.newSensorConfigWithEnv(tt.primarySensor, tt.sysSensors, tt.sensors, tt.sensorsTimeout, tt.skipCollection)
+			result := agent.newSensorConfigWithEnv(tt.primarySensor, tt.sysSensors, tt.sensors, tt.sensorsTimeout, tt.skipCollection, false)
 
 			// Check primary sensor
 			assert.Equal(t, tt.expectedConfig.primarySensor, result.primarySensor)
@@ -620,3 +622,143 @@ func TestUpdateTemperaturesSkipsOnTimeout(t *testing.T) {
 	assert.Equal(t, 0.0, agent.systemInfo.DashboardTemp)
 	assert.Equal(t, map[string]float64{}, stats.Temperatures)
 }
+
+func TestIsGpuSensorKey(t *testing.T) {
+	for _, key := range []string{"xe", "XE_temp1", "amdgpu_edge", "NVIDIA"} {
+		assert.True(t, isGpuSensorKey(key), key)
+	}
+	for _, key := range []string{"coretemp_core_0", "acpitz", "xen_temp", "myxe", ""} {
+		assert.False(t, isGpuSensorKey(key), key)
+	}
+}
+
+func TestSkipGpuSensorShadow(t *testing.T) {
+	sysRoot := t.TempDir()
+	writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0", "name"), "coretemp\n")
+	writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0", "temp1_input"), "55000\n")
+	writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon1", "name"), "xe\n")
+	writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon1", "temp1_input"), "48000\n")
+	writeFile(t, filepath.Join(sysRoot, "class", "thermal", "thermal_zone0", "type"), "cpu-thermal\n")
+	writeFile(t, filepath.Join(sysRoot, "class", "thermal", "thermal_zone0", "temp"), "55000\n")
+	writeFile(t, filepath.Join(sysRoot, "class", "thermal", "thermal_zone1", "type"), "gpu\n")
+	writeFile(t, filepath.Join(sysRoot, "class", "thermal", "thermal_zone1", "temp"), "48000\n")
+
+	shadow, err := buildNonGpuSysShadow(sysRoot)
+	require.NoError(t, err)
+	t.Cleanup(func() { os.RemoveAll(shadow) })
+
+	assert.FileExists(t, filepath.Join(shadow, "class", "hwmon", "hwmon0", "temp1_input"))
+	assert.NoFileExists(t, filepath.Join(shadow, "class", "hwmon", "hwmon1"))
+	assert.FileExists(t, filepath.Join(shadow, "class", "thermal", "thermal_zone0", "temp"))
+	assert.NoFileExists(t, filepath.Join(shadow, "class", "thermal", "thermal_zone1"))
+}
+
+func TestSkipGpuSensorShadowDeviceName(t *testing.T) {
+	sysRoot := t.TempDir()
+	writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0", "device", "name"), "coretemp\n")
+	writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0", "device", "temp1_input"), "55000\n")
+	writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon1", "device", "name"), "xe\n")
+	writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon1", "device", "temp1_input"), "48000\n")
+
+	shadow, err := buildNonGpuSysShadow(sysRoot)
+	require.NoError(t, err)
+	t.Cleanup(func() { os.RemoveAll(shadow) })
+
+	assert.FileExists(t, filepath.Join(shadow, "class", "hwmon", "hwmon0", "device", "temp1_input"))
+	assert.NoFileExists(t, filepath.Join(shadow, "class", "hwmon", "hwmon1"))
+}
+
+func TestSkipGpuSensorShadowKeepsThermalZonesWithoutNonGpuHwmon(t *testing.T) {
+	sysRoot := t.TempDir()
+	writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0", "name"), "xe\n")
+	writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0", "temp1_input"), "48000\n")
+	writeFile(t, filepath.Join(sysRoot, "class", "thermal", "thermal_zone0", "type"), "cpu-thermal\n")
+	writeFile(t, filepath.Join(sysRoot, "class", "thermal", "thermal_zone0", "temp"), "55000\n")
+	writeFile(t, filepath.Join(sysRoot, "class", "thermal", "thermal_zone1", "type"), "gpu\n")
+	writeFile(t, filepath.Join(sysRoot, "class", "thermal", "thermal_zone1", "temp"), "48000\n")
+
+	shadow, err := buildNonGpuSysShadow(sysRoot)
+	require.NoError(t, err)
+	t.Cleanup(func() { os.RemoveAll(shadow) })
+
+	hwmonTemps, err := filepath.Glob(filepath.Join(shadow, "class", "hwmon", "hwmon*", "temp*_input"))
+	require.NoError(t, err)
+	assert.Empty(t, hwmonTemps)
+	assert.FileExists(t, filepath.Join(shadow, "class", "thermal", "thermal_zone0", "temp"))
+	assert.NoFileExists(t, filepath.Join(shadow, "class", "thermal", "thermal_zone1"))
+}
+
+func TestNewSensorConfigSkipGpuWiresShadow(t *testing.T) {
+	t.Setenv("SKIP_GPU", "true")
+
+	agent := &Agent{}
+	config := agent.newSensorConfig()
+
+	assert.True(t, config.skipGPU)
+	envMap, ok := config.context.Value(common.EnvKey).(common.EnvMap)
+	require.True(t, ok, "SKIP_GPU should point the sensor context at a sysfs shadow")
+	shadow, ok := envMap[common.HostSysEnvKey]
+	require.True(t, ok)
+	assert.DirExists(t, filepath.Join(shadow, "class", "hwmon"))
+	assert.Equal(t, shadow, config.sensorShadow)
+	config.cleanupSensorShadow()
+	assert.NoDirExists(t, shadow)
+	assert.Empty(t, config.sensorShadow)
+}
+
+func TestSkipGpuShadowUsesSysSensorsRoot(t *testing.T) {
+	sysRoot := t.TempDir()
+	writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0", "name"), "coretemp\n")
+	writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0", "temp1_input"), "55000\n")
+	writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon1", "name"), "xe\n")
+	writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon1", "temp1_input"), "48000\n")
+
+	agent := &Agent{}
+	config := agent.newSensorConfigWithEnv("", sysRoot, "", "", false, true)
+	t.Cleanup(config.cleanupSensorShadow)
+
+	envMap, ok := config.context.Value(common.EnvKey).(common.EnvMap)
+	require.True(t, ok, "SKIP_GPU should point the sensor context at a sysfs shadow")
+	shadow, ok := envMap[common.HostSysEnvKey]
+	require.True(t, ok)
+	require.NotEqual(t, sysRoot, shadow, "shadow must not be the SYS_SENSORS tree itself")
+
+	target, err := os.Readlink(filepath.Join(shadow, "class", "hwmon", "hwmon0"))
+	require.NoError(t, err)
+	assert.Equal(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0"), target)
+	assert.NoFileExists(t, filepath.Join(shadow, "class", "hwmon", "hwmon1"))
+}
+
+func TestUpdateTemperaturesSkipGpu(t *testing.T) {
+	originalGetSensorTemps := getSensorTemps
+	t.Cleanup(func() {
+		getSensorTemps = originalGetSensorTemps
+	})
+	getSensorTemps = func(ctx context.Context) ([]sensors.TemperatureStat, error) {
+		return []sensors.TemperatureStat{
+			{SensorKey: "coretemp_core_0", Temperature: 55},
+			{SensorKey: "XE", Temperature: 48},
+		}, nil
+	}
+
+	newAgent := func(skipGPU bool) *Agent {
+		agent := &Agent{
+			systemInfo: system.Info{},
+			sensorConfig: &SensorConfig{
+				context: context.Background(),
+				timeout: 2 * time.Second,
+				sensors: map[string]struct{}{},
+				skipGPU: skipGPU,
+			},
+		}
+		return agent
+	}
+
+	stats := &system.Stats{}
+	newAgent(true).updateTemperatures(stats)
+	assert.Equal(t, map[string]float64{"coretemp_core_0": 55}, stats.Temperatures)
+
+	stats = &system.Stats{}
+	newAgent(false).updateTemperatures(stats)
+	assert.Len(t, stats.Temperatures, 2)
+}