mirror of
https://github.com/henrygd/beszel.git
synced 2026-08-19 16:57:47 +02:00
Add Intel sysfs GPU power collector (Xe/i915 hwmon energy counters) (#2020)
This commit is contained in:
21
agent/gpu.go
21
agent/gpu.go
@@ -48,6 +48,8 @@ type GPUManager struct {
|
|||||||
// Per-cache-key tracking for delta calculations
|
// Per-cache-key tracking for delta calculations
|
||||||
// cacheKey -> gpuId -> snapshot of last count/usage/power values
|
// cacheKey -> gpuId -> snapshot of last count/usage/power values
|
||||||
lastSnapshots map[uint16]map[string]*gpuSnapshot
|
lastSnapshots map[uint16]map[string]*gpuSnapshot
|
||||||
|
// Per-card energy snapshots for Intel sysfs power calculation.
|
||||||
|
intelSysfsEnergySnapshots map[string]intelSysfsEnergySnapshot
|
||||||
}
|
}
|
||||||
|
|
||||||
// gpuSnapshot stores the last observed incremental values for delta tracking
|
// gpuSnapshot stores the last observed incremental values for delta tracking
|
||||||
@@ -90,6 +92,7 @@ const (
|
|||||||
collectorSourceNVML collectorSource = "nvml"
|
collectorSourceNVML collectorSource = "nvml"
|
||||||
collectorSourceNvidiaSMI collectorSource = collectorSource(nvidiaSmiCmd)
|
collectorSourceNvidiaSMI collectorSource = collectorSource(nvidiaSmiCmd)
|
||||||
collectorSourceIntelGpuTop collectorSource = collectorSource(intelGpuStatsCmd)
|
collectorSourceIntelGpuTop collectorSource = collectorSource(intelGpuStatsCmd)
|
||||||
|
collectorSourceIntelSysfs collectorSource = "intel_sysfs"
|
||||||
collectorSourceAmdSysfs collectorSource = "amd_sysfs"
|
collectorSourceAmdSysfs collectorSource = "amd_sysfs"
|
||||||
collectorSourceRocmSMI collectorSource = collectorSource(rocmSmiCmd)
|
collectorSourceRocmSMI collectorSource = collectorSource(rocmSmiCmd)
|
||||||
collectorSourceMacmon collectorSource = collectorSource(macmonCmd)
|
collectorSourceMacmon collectorSource = collectorSource(macmonCmd)
|
||||||
@@ -106,6 +109,7 @@ func isValidCollectorSource(source collectorSource) bool {
|
|||||||
collectorSourceNVML,
|
collectorSourceNVML,
|
||||||
collectorSourceNvidiaSMI,
|
collectorSourceNvidiaSMI,
|
||||||
collectorSourceIntelGpuTop,
|
collectorSourceIntelGpuTop,
|
||||||
|
collectorSourceIntelSysfs,
|
||||||
collectorSourceAmdSysfs,
|
collectorSourceAmdSysfs,
|
||||||
collectorSourceRocmSMI,
|
collectorSourceRocmSMI,
|
||||||
collectorSourceMacmon,
|
collectorSourceMacmon,
|
||||||
@@ -122,6 +126,7 @@ type gpuCapabilities struct {
|
|||||||
hasAmdSysfs bool
|
hasAmdSysfs bool
|
||||||
hasTegrastats bool
|
hasTegrastats bool
|
||||||
hasIntelGpuTop bool
|
hasIntelGpuTop bool
|
||||||
|
hasIntelSysfs bool
|
||||||
hasNvtop bool
|
hasNvtop bool
|
||||||
hasMacmon bool
|
hasMacmon bool
|
||||||
hasPowermetrics bool
|
hasPowermetrics bool
|
||||||
@@ -369,12 +374,13 @@ func (gm *GPUManager) calculateGPUAverage(id string, gpu *system.GPUData, cacheK
|
|||||||
|
|
||||||
gpuAvg.Power = utils.TwoDecimals(deltaPower / float64(deltaCount))
|
gpuAvg.Power = utils.TwoDecimals(deltaPower / float64(deltaCount))
|
||||||
|
|
||||||
|
gpuAvg.PowerPkg = utils.TwoDecimals(deltaPowerPkg / float64(deltaCount))
|
||||||
|
|
||||||
if gpu.Engines != nil {
|
if gpu.Engines != nil {
|
||||||
// make fresh map for averaged engine metrics to avoid mutating
|
// make fresh map for averaged engine metrics to avoid mutating
|
||||||
// the accumulator map stored in gm.GpuDataMap
|
// the accumulator map stored in gm.GpuDataMap
|
||||||
gpuAvg.Engines = make(map[string]float64, len(gpu.Engines))
|
gpuAvg.Engines = make(map[string]float64, len(gpu.Engines))
|
||||||
gpuAvg.Usage = gm.calculateIntelGPUUsage(&gpuAvg, gpu, lastSnapshot, deltaCount)
|
gpuAvg.Usage = gm.calculateIntelGPUUsage(&gpuAvg, gpu, lastSnapshot, deltaCount)
|
||||||
gpuAvg.PowerPkg = utils.TwoDecimals(deltaPowerPkg / float64(deltaCount))
|
|
||||||
} else {
|
} else {
|
||||||
gpuAvg.Usage = utils.TwoDecimals(deltaUsage / float64(deltaCount))
|
gpuAvg.Usage = utils.TwoDecimals(deltaUsage / float64(deltaCount))
|
||||||
}
|
}
|
||||||
@@ -444,6 +450,7 @@ func (gm *GPUManager) storeSnapshot(id string, gpu *system.GPUData, cacheKey uin
|
|||||||
func (gm *GPUManager) discoverGpuCapabilities() gpuCapabilities {
|
func (gm *GPUManager) discoverGpuCapabilities() gpuCapabilities {
|
||||||
caps := gpuCapabilities{
|
caps := gpuCapabilities{
|
||||||
hasAmdSysfs: gm.hasAmdSysfs(),
|
hasAmdSysfs: gm.hasAmdSysfs(),
|
||||||
|
hasIntelSysfs: gm.hasIntelSysfs(),
|
||||||
}
|
}
|
||||||
if _, err := exec.LookPath(nvidiaSmiCmd); err == nil {
|
if _, err := exec.LookPath(nvidiaSmiCmd); err == nil {
|
||||||
caps.hasNvidiaSmi = true
|
caps.hasNvidiaSmi = true
|
||||||
@@ -472,7 +479,7 @@ func (gm *GPUManager) discoverGpuCapabilities() gpuCapabilities {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func hasAnyGpuCollector(caps gpuCapabilities) bool {
|
func hasAnyGpuCollector(caps gpuCapabilities) bool {
|
||||||
return caps.hasNvidiaSmi || caps.hasRocmSmi || caps.hasAmdSysfs || caps.hasTegrastats || caps.hasIntelGpuTop || caps.hasNvtop || caps.hasMacmon || caps.hasPowermetrics
|
return caps.hasNvidiaSmi || caps.hasRocmSmi || caps.hasAmdSysfs || caps.hasTegrastats || caps.hasIntelGpuTop || caps.hasIntelSysfs || caps.hasNvtop || caps.hasMacmon || caps.hasPowermetrics
|
||||||
}
|
}
|
||||||
|
|
||||||
func (gm *GPUManager) startIntelCollector() {
|
func (gm *GPUManager) startIntelCollector() {
|
||||||
@@ -563,6 +570,13 @@ func (gm *GPUManager) collectorDefinitions(caps gpuCapabilities) map[collectorSo
|
|||||||
return true
|
return true
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
collectorSourceIntelSysfs: {
|
||||||
|
group: collectorGroupIntel,
|
||||||
|
available: caps.hasIntelSysfs,
|
||||||
|
start: func(_ func()) bool {
|
||||||
|
return gm.startIntelSysfsCollector()
|
||||||
|
},
|
||||||
|
},
|
||||||
collectorSourceAmdSysfs: {
|
collectorSourceAmdSysfs: {
|
||||||
group: collectorGroupAmd,
|
group: collectorGroupAmd,
|
||||||
available: caps.hasAmdSysfs,
|
available: caps.hasAmdSysfs,
|
||||||
@@ -708,6 +722,9 @@ func (gm *GPUManager) resolveLegacyCollectorPriority(caps gpuCapabilities) []col
|
|||||||
if caps.hasIntelGpuTop {
|
if caps.hasIntelGpuTop {
|
||||||
priorities = append(priorities, collectorSourceIntelGpuTop)
|
priorities = append(priorities, collectorSourceIntelGpuTop)
|
||||||
}
|
}
|
||||||
|
if caps.hasIntelSysfs {
|
||||||
|
priorities = append(priorities, collectorSourceIntelSysfs)
|
||||||
|
}
|
||||||
|
|
||||||
// Apple collectors are currently opt-in only for testing.
|
// Apple collectors are currently opt-in only for testing.
|
||||||
// Enable them with GPU_COLLECTOR=macmon or GPU_COLLECTOR=powermetrics.
|
// Enable them with GPU_COLLECTOR=macmon or GPU_COLLECTOR=powermetrics.
|
||||||
|
|||||||
280
agent/gpu_intel_sysfs_linux.go
Normal file
280
agent/gpu_intel_sysfs_linux.go
Normal file
@@ -0,0 +1,280 @@
|
|||||||
|
//go:build linux
|
||||||
|
|
||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/henrygd/beszel/agent/utils"
|
||||||
|
"github.com/henrygd/beszel/internal/entities/system"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
drmSysfsRoot = "/sys/class/drm"
|
||||||
|
intelSysfsNow = time.Now
|
||||||
|
)
|
||||||
|
|
||||||
|
type intelSysfsEnergySnapshot struct {
|
||||||
|
microjoules uint64
|
||||||
|
timestamp time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
type intelSysfsCard struct {
|
||||||
|
cardPath string
|
||||||
|
hwmonDir string
|
||||||
|
}
|
||||||
|
|
||||||
|
// hasIntelSysfs returns true if any Intel DRM card exposes an hwmon energy counter.
|
||||||
|
func (gm *GPUManager) hasIntelSysfs() bool {
|
||||||
|
cards, err := discoverIntelSysfsCards()
|
||||||
|
return err == nil && len(cards) > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// startIntelSysfsCollector starts Intel GPU collection via sysfs.
|
||||||
|
func (gm *GPUManager) startIntelSysfsCollector() bool {
|
||||||
|
go func() {
|
||||||
|
if err := gm.collectIntelSysfsStats(); err != nil {
|
||||||
|
slog.Warn("Error collecting Intel GPU data via sysfs", "err", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// collectIntelSysfsStats collects Intel GPU metrics directly from DRM sysfs / hwmon.
|
||||||
|
func (gm *GPUManager) collectIntelSysfsStats() error {
|
||||||
|
sysfsPollInterval := 3000 * time.Millisecond
|
||||||
|
cards, err := discoverIntelSysfsCards()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(cards) == 0 {
|
||||||
|
return errNoValidData
|
||||||
|
}
|
||||||
|
|
||||||
|
slog.Debug("Using sysfs for Intel GPU data collection", "cards", len(cards))
|
||||||
|
for _, card := range cards {
|
||||||
|
slog.Debug("Intel sysfs card detected", "card", filepath.Base(card.cardPath), "hwmon", card.hwmonDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
failures := 0
|
||||||
|
for {
|
||||||
|
hasData := false
|
||||||
|
for _, card := range cards {
|
||||||
|
if gm.updateIntelSysfsGpuData(card.cardPath, card.hwmonDir) {
|
||||||
|
hasData = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !hasData {
|
||||||
|
failures++
|
||||||
|
if failures > maxFailureRetries {
|
||||||
|
return errNoValidData
|
||||||
|
}
|
||||||
|
slog.Warn("No Intel GPU data from sysfs", "failures", failures)
|
||||||
|
time.Sleep(retryWaitTime)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
failures = 0
|
||||||
|
time.Sleep(sysfsPollInterval)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func discoverIntelSysfsCards() ([]intelSysfsCard, error) {
|
||||||
|
paths, err := filepath.Glob(filepath.Join(drmSysfsRoot, "card*"))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
var cards []intelSysfsCard
|
||||||
|
for _, cardPath := range paths {
|
||||||
|
if strings.Contains(filepath.Base(cardPath), "-") || !isIntelGpu(cardPath) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
hwmonDir := findIntelEnergyHwmon(filepath.Join(cardPath, "device"))
|
||||||
|
if hwmonDir == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
cards = append(cards, intelSysfsCard{cardPath: cardPath, hwmonDir: hwmonDir})
|
||||||
|
}
|
||||||
|
return cards, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func isIntelGpu(cardPath string) bool {
|
||||||
|
vendor, err := utils.ReadStringFileLimited(filepath.Join(cardPath, "device/vendor"), 64)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return strings.EqualFold(strings.TrimSpace(vendor), "0x8086")
|
||||||
|
}
|
||||||
|
|
||||||
|
func findIntelEnergyHwmon(devicePath string) string {
|
||||||
|
hwmons, _ := filepath.Glob(filepath.Join(devicePath, "hwmon/hwmon*"))
|
||||||
|
var fallback string
|
||||||
|
for _, hwmonDir := range hwmons {
|
||||||
|
if !sysfsFileExists(filepath.Join(hwmonDir, "energy1_input")) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if name, err := utils.ReadStringFileLimited(filepath.Join(hwmonDir, "name"), 64); err == nil && strings.EqualFold(strings.TrimSpace(name), "xe") {
|
||||||
|
return hwmonDir
|
||||||
|
}
|
||||||
|
if fallback == "" {
|
||||||
|
fallback = hwmonDir
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
|
||||||
|
func sysfsFileExists(path string) bool {
|
||||||
|
_, err := utils.ReadStringFileLimited(path, 1)
|
||||||
|
return err == nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// updateIntelSysfsGpuData reads GPU metrics from sysfs and updates the GPU data map.
|
||||||
|
// Returns true if the required energy counter was read successfully.
|
||||||
|
func (gm *GPUManager) updateIntelSysfsGpuData(cardPath, hwmonDir string) bool {
|
||||||
|
devicePath := filepath.Join(cardPath, "device")
|
||||||
|
id := filepath.Base(cardPath)
|
||||||
|
|
||||||
|
energy, err := readSysfsUint(filepath.Join(hwmonDir, "energy1_input"))
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
now := intelSysfsNow()
|
||||||
|
power, hasPower := gm.calculateIntelSysfsPower(id, energy, now)
|
||||||
|
powerPkg, hasPowerPkg := gm.readIntelSysfsPowerPkg(id, hwmonDir, now)
|
||||||
|
temp := readIntelSysfsTemperature(hwmonDir)
|
||||||
|
usage, usageErr := readOptionalSysfsFloat(filepath.Join(devicePath, "gpu_busy_percent"))
|
||||||
|
memUsed, memUsedErr := readFirstOptionalSysfsFloat(
|
||||||
|
filepath.Join(devicePath, "mem_info_vram_used"),
|
||||||
|
filepath.Join(devicePath, "mem_info_lmem_used"),
|
||||||
|
filepath.Join(devicePath, "mem_info_local_mem_used"),
|
||||||
|
)
|
||||||
|
memTotal, memTotalErr := readFirstOptionalSysfsFloat(
|
||||||
|
filepath.Join(devicePath, "mem_info_vram_total"),
|
||||||
|
filepath.Join(devicePath, "mem_info_lmem_total"),
|
||||||
|
filepath.Join(devicePath, "mem_info_local_mem_total"),
|
||||||
|
)
|
||||||
|
|
||||||
|
gm.Lock()
|
||||||
|
defer gm.Unlock()
|
||||||
|
|
||||||
|
gpu, ok := gm.GpuDataMap[id]
|
||||||
|
if !ok {
|
||||||
|
gpu = &system.GPUData{Name: getIntelSysfsGpuName(cardPath)}
|
||||||
|
gm.GpuDataMap[id] = gpu
|
||||||
|
}
|
||||||
|
|
||||||
|
if usageErr == nil {
|
||||||
|
gpu.Usage += usage
|
||||||
|
}
|
||||||
|
if memUsedErr == nil {
|
||||||
|
gpu.MemoryUsed = utils.BytesToMegabytes(memUsed)
|
||||||
|
}
|
||||||
|
if memTotalErr == nil {
|
||||||
|
gpu.MemoryTotal = utils.BytesToMegabytes(memTotal)
|
||||||
|
}
|
||||||
|
if temp > 0 {
|
||||||
|
gpu.Temperature = temp
|
||||||
|
}
|
||||||
|
if hasPower {
|
||||||
|
gpu.Power += power
|
||||||
|
slog.Debug("Computed Intel sysfs GPU power", "card", id, "watts", power)
|
||||||
|
}
|
||||||
|
if hasPowerPkg {
|
||||||
|
gpu.PowerPkg += powerPkg
|
||||||
|
}
|
||||||
|
gpu.Count++
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (gm *GPUManager) calculateIntelSysfsPower(cardID string, microjoules uint64, timestamp time.Time) (float64, bool) {
|
||||||
|
if gm.intelSysfsEnergySnapshots == nil {
|
||||||
|
gm.intelSysfsEnergySnapshots = make(map[string]intelSysfsEnergySnapshot)
|
||||||
|
}
|
||||||
|
|
||||||
|
last, ok := gm.intelSysfsEnergySnapshots[cardID]
|
||||||
|
gm.intelSysfsEnergySnapshots[cardID] = intelSysfsEnergySnapshot{microjoules: microjoules, timestamp: timestamp}
|
||||||
|
if !ok {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
if microjoules < last.microjoules {
|
||||||
|
slog.Debug("Intel sysfs energy counter reset", "card", cardID)
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
elapsed := timestamp.Sub(last.timestamp).Seconds()
|
||||||
|
if elapsed <= 0 {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
delta := microjoules - last.microjoules
|
||||||
|
return float64(delta) / 1_000_000.0 / elapsed, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (gm *GPUManager) readIntelSysfsPowerPkg(cardID, hwmonDir string, timestamp time.Time) (float64, bool) {
|
||||||
|
energyPaths, _ := filepath.Glob(filepath.Join(hwmonDir, "energy*_input"))
|
||||||
|
for _, path := range energyPaths {
|
||||||
|
if filepath.Base(path) == "energy1_input" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
energy, err := readSysfsUint(path)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return gm.calculateIntelSysfsPower(cardID+":"+filepath.Base(path), energy, timestamp)
|
||||||
|
}
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
|
||||||
|
func readIntelSysfsTemperature(hwmonDir string) float64 {
|
||||||
|
tempPaths, _ := filepath.Glob(filepath.Join(hwmonDir, "temp*_input"))
|
||||||
|
for _, path := range tempPaths {
|
||||||
|
temp, err := readSysfsFloat(path)
|
||||||
|
if err == nil && temp > 0 {
|
||||||
|
return temp / 1000.0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func readSysfsUint(path string) (uint64, error) {
|
||||||
|
val, err := utils.ReadStringFileLimited(path, 64)
|
||||||
|
if err != nil {
|
||||||
|
slog.Debug("Failed to read sysfs value", "path", path, "error", err)
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return strconv.ParseUint(strings.TrimSpace(val), 10, 64)
|
||||||
|
}
|
||||||
|
|
||||||
|
func readOptionalSysfsFloat(path string) (float64, error) {
|
||||||
|
val, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return strconv.ParseFloat(strings.TrimSpace(string(val)), 64)
|
||||||
|
}
|
||||||
|
|
||||||
|
func readFirstOptionalSysfsFloat(paths ...string) (float64, error) {
|
||||||
|
for _, path := range paths {
|
||||||
|
val, err := readOptionalSysfsFloat(path)
|
||||||
|
if err == nil {
|
||||||
|
return val, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0, fmt.Errorf("no sysfs values found")
|
||||||
|
}
|
||||||
|
|
||||||
|
func getIntelSysfsGpuName(cardPath string) string {
|
||||||
|
devicePath := filepath.Join(cardPath, "device")
|
||||||
|
if product, err := utils.ReadStringFileLimited(filepath.Join(devicePath, "product_name"), 128); err == nil && strings.TrimSpace(product) != "" {
|
||||||
|
return strings.TrimSpace(product)
|
||||||
|
}
|
||||||
|
if name, err := utils.ReadStringFileLimited(filepath.Join(devicePath, "name"), 128); err == nil && strings.TrimSpace(name) != "" {
|
||||||
|
return strings.TrimSpace(name)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("Intel GPU %s", filepath.Base(cardPath))
|
||||||
|
}
|
||||||
217
agent/gpu_intel_sysfs_linux_test.go
Normal file
217
agent/gpu_intel_sysfs_linux_test.go
Normal file
@@ -0,0 +1,217 @@
|
|||||||
|
//go:build linux
|
||||||
|
|
||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/henrygd/beszel/agent/utils"
|
||||||
|
"github.com/henrygd/beszel/internal/entities/system"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func setupIntelSysfsTest(t *testing.T) (root, cardPath, hwmonPath string) {
|
||||||
|
t.Helper()
|
||||||
|
root = t.TempDir()
|
||||||
|
oldRoot := drmSysfsRoot
|
||||||
|
drmSysfsRoot = root
|
||||||
|
t.Cleanup(func() {
|
||||||
|
drmSysfsRoot = oldRoot
|
||||||
|
})
|
||||||
|
|
||||||
|
cardPath = filepath.Join(root, "card0")
|
||||||
|
devicePath := filepath.Join(cardPath, "device")
|
||||||
|
hwmonPath = filepath.Join(devicePath, "hwmon", "hwmon0")
|
||||||
|
require.NoError(t, os.MkdirAll(hwmonPath, 0o755))
|
||||||
|
return root, cardPath, hwmonPath
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeIntelSysfsFile(t *testing.T, basePath, name, content string) {
|
||||||
|
t.Helper()
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(basePath, name), []byte(content), 0o644))
|
||||||
|
}
|
||||||
|
|
||||||
|
func setIntelSysfsTime(t *testing.T, now time.Time) {
|
||||||
|
t.Helper()
|
||||||
|
oldNow := intelSysfsNow
|
||||||
|
intelSysfsNow = func() time.Time { return now }
|
||||||
|
t.Cleanup(func() {
|
||||||
|
intelSysfsNow = oldNow
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIntelSysfsDetectsIntelCardWithEnergy(t *testing.T) {
|
||||||
|
_, cardPath, hwmonPath := setupIntelSysfsTest(t)
|
||||||
|
devicePath := filepath.Join(cardPath, "device")
|
||||||
|
writeIntelSysfsFile(t, devicePath, "vendor", "0x8086\n")
|
||||||
|
writeIntelSysfsFile(t, hwmonPath, "name", "xe\n")
|
||||||
|
writeIntelSysfsFile(t, hwmonPath, "energy1_input", "1000000\n")
|
||||||
|
|
||||||
|
gm := &GPUManager{}
|
||||||
|
assert.True(t, gm.hasIntelSysfs())
|
||||||
|
|
||||||
|
cards, err := discoverIntelSysfsCards()
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, cards, 1)
|
||||||
|
assert.Equal(t, cardPath, cards[0].cardPath)
|
||||||
|
assert.Equal(t, hwmonPath, cards[0].hwmonDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIntelSysfsRejectsNonIntelCard(t *testing.T) {
|
||||||
|
_, cardPath, hwmonPath := setupIntelSysfsTest(t)
|
||||||
|
devicePath := filepath.Join(cardPath, "device")
|
||||||
|
writeIntelSysfsFile(t, devicePath, "vendor", "0x1002\n")
|
||||||
|
writeIntelSysfsFile(t, hwmonPath, "name", "xe\n")
|
||||||
|
writeIntelSysfsFile(t, hwmonPath, "energy1_input", "1000000\n")
|
||||||
|
|
||||||
|
gm := &GPUManager{}
|
||||||
|
assert.False(t, gm.hasIntelSysfs())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIntelSysfsRequiresEnergyInput(t *testing.T) {
|
||||||
|
_, cardPath, hwmonPath := setupIntelSysfsTest(t)
|
||||||
|
devicePath := filepath.Join(cardPath, "device")
|
||||||
|
writeIntelSysfsFile(t, devicePath, "vendor", "0x8086\n")
|
||||||
|
writeIntelSysfsFile(t, hwmonPath, "name", "xe\n")
|
||||||
|
|
||||||
|
gm := &GPUManager{}
|
||||||
|
assert.False(t, gm.hasIntelSysfs())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIntelSysfsFirstSampleInitializesWithoutBogusPower(t *testing.T) {
|
||||||
|
_, cardPath, hwmonPath := setupIntelSysfsTest(t)
|
||||||
|
devicePath := filepath.Join(cardPath, "device")
|
||||||
|
writeIntelSysfsFile(t, devicePath, "vendor", "0x8086\n")
|
||||||
|
writeIntelSysfsFile(t, hwmonPath, "name", "xe\n")
|
||||||
|
writeIntelSysfsFile(t, hwmonPath, "energy1_input", "1000000\n")
|
||||||
|
setIntelSysfsTime(t, time.Unix(100, 0))
|
||||||
|
|
||||||
|
gm := &GPUManager{GpuDataMap: make(map[string]*system.GPUData)}
|
||||||
|
ok := gm.updateIntelSysfsGpuData(cardPath, hwmonPath)
|
||||||
|
require.True(t, ok)
|
||||||
|
|
||||||
|
gpu := gm.GpuDataMap["card0"]
|
||||||
|
require.NotNil(t, gpu)
|
||||||
|
assert.Equal(t, "Intel GPU card0", gpu.Name)
|
||||||
|
assert.Equal(t, 0.0, gpu.Power)
|
||||||
|
assert.Equal(t, 1.0, gpu.Count)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIntelSysfsSecondSampleComputesWatts(t *testing.T) {
|
||||||
|
_, cardPath, hwmonPath := setupIntelSysfsTest(t)
|
||||||
|
devicePath := filepath.Join(cardPath, "device")
|
||||||
|
writeIntelSysfsFile(t, devicePath, "vendor", "0x8086\n")
|
||||||
|
writeIntelSysfsFile(t, hwmonPath, "energy1_input", "1000000\n")
|
||||||
|
|
||||||
|
gm := &GPUManager{GpuDataMap: make(map[string]*system.GPUData)}
|
||||||
|
oldNow := intelSysfsNow
|
||||||
|
intelSysfsNow = func() time.Time { return time.Unix(100, 0) }
|
||||||
|
t.Cleanup(func() { intelSysfsNow = oldNow })
|
||||||
|
require.True(t, gm.updateIntelSysfsGpuData(cardPath, hwmonPath))
|
||||||
|
|
||||||
|
writeIntelSysfsFile(t, hwmonPath, "energy1_input", "6000000\n")
|
||||||
|
intelSysfsNow = func() time.Time { return time.Unix(102, 0) }
|
||||||
|
require.True(t, gm.updateIntelSysfsGpuData(cardPath, hwmonPath))
|
||||||
|
|
||||||
|
gpu := gm.GpuDataMap["card0"]
|
||||||
|
require.NotNil(t, gpu)
|
||||||
|
assert.Equal(t, 2.5, gpu.Power)
|
||||||
|
assert.Equal(t, 2.0, gpu.Count)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIntelSysfsSecondEnergyCounterMapsToPowerPkg(t *testing.T) {
|
||||||
|
_, cardPath, hwmonPath := setupIntelSysfsTest(t)
|
||||||
|
devicePath := filepath.Join(cardPath, "device")
|
||||||
|
writeIntelSysfsFile(t, devicePath, "vendor", "0x8086\n")
|
||||||
|
writeIntelSysfsFile(t, hwmonPath, "energy1_input", "1000000\n")
|
||||||
|
writeIntelSysfsFile(t, hwmonPath, "energy2_input", "2000000\n")
|
||||||
|
|
||||||
|
oldNow := intelSysfsNow
|
||||||
|
t.Cleanup(func() { intelSysfsNow = oldNow })
|
||||||
|
gm := &GPUManager{GpuDataMap: make(map[string]*system.GPUData)}
|
||||||
|
intelSysfsNow = func() time.Time { return time.Unix(100, 0) }
|
||||||
|
require.True(t, gm.updateIntelSysfsGpuData(cardPath, hwmonPath))
|
||||||
|
|
||||||
|
writeIntelSysfsFile(t, hwmonPath, "energy1_input", "2000000\n")
|
||||||
|
writeIntelSysfsFile(t, hwmonPath, "energy2_input", "8000000\n")
|
||||||
|
intelSysfsNow = func() time.Time { return time.Unix(102, 0) }
|
||||||
|
require.True(t, gm.updateIntelSysfsGpuData(cardPath, hwmonPath))
|
||||||
|
|
||||||
|
gpu := gm.GpuDataMap["card0"]
|
||||||
|
require.NotNil(t, gpu)
|
||||||
|
assert.Equal(t, 0.5, gpu.Power)
|
||||||
|
assert.Equal(t, 3.0, gpu.PowerPkg)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIntelSysfsCounterResetSkipsOneSample(t *testing.T) {
|
||||||
|
gm := &GPUManager{}
|
||||||
|
power, ok := gm.calculateIntelSysfsPower("card0", 5000000, time.Unix(100, 0))
|
||||||
|
assert.False(t, ok)
|
||||||
|
assert.Equal(t, 0.0, power)
|
||||||
|
|
||||||
|
power, ok = gm.calculateIntelSysfsPower("card0", 1000000, time.Unix(101, 0))
|
||||||
|
assert.False(t, ok)
|
||||||
|
assert.Equal(t, 0.0, power)
|
||||||
|
|
||||||
|
power, ok = gm.calculateIntelSysfsPower("card0", 3000000, time.Unix(103, 0))
|
||||||
|
assert.True(t, ok)
|
||||||
|
assert.Equal(t, 1.0, power)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIntelSysfsTempInputMapsToCelsius(t *testing.T) {
|
||||||
|
_, cardPath, hwmonPath := setupIntelSysfsTest(t)
|
||||||
|
devicePath := filepath.Join(cardPath, "device")
|
||||||
|
writeIntelSysfsFile(t, devicePath, "vendor", "0x8086\n")
|
||||||
|
writeIntelSysfsFile(t, hwmonPath, "energy1_input", "1000000\n")
|
||||||
|
writeIntelSysfsFile(t, hwmonPath, "temp1_input", "43500\n")
|
||||||
|
setIntelSysfsTime(t, time.Unix(100, 0))
|
||||||
|
|
||||||
|
gm := &GPUManager{GpuDataMap: make(map[string]*system.GPUData)}
|
||||||
|
require.True(t, gm.updateIntelSysfsGpuData(cardPath, hwmonPath))
|
||||||
|
|
||||||
|
gpu := gm.GpuDataMap["card0"]
|
||||||
|
require.NotNil(t, gpu)
|
||||||
|
assert.Equal(t, 43.5, gpu.Temperature)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIntelSysfsMissingOptionalFilesDoNotFail(t *testing.T) {
|
||||||
|
_, cardPath, hwmonPath := setupIntelSysfsTest(t)
|
||||||
|
devicePath := filepath.Join(cardPath, "device")
|
||||||
|
writeIntelSysfsFile(t, devicePath, "vendor", "0x8086\n")
|
||||||
|
writeIntelSysfsFile(t, hwmonPath, "energy1_input", "1000000\n")
|
||||||
|
setIntelSysfsTime(t, time.Unix(100, 0))
|
||||||
|
|
||||||
|
gm := &GPUManager{GpuDataMap: make(map[string]*system.GPUData)}
|
||||||
|
require.True(t, gm.updateIntelSysfsGpuData(cardPath, hwmonPath))
|
||||||
|
|
||||||
|
gpu := gm.GpuDataMap["card0"]
|
||||||
|
require.NotNil(t, gpu)
|
||||||
|
assert.Equal(t, 0.0, gpu.Usage)
|
||||||
|
assert.Equal(t, 0.0, gpu.MemoryUsed)
|
||||||
|
assert.Equal(t, 0.0, gpu.MemoryTotal)
|
||||||
|
assert.Equal(t, 0.0, gpu.Temperature)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIntelSysfsMapsOpportunisticMemoryAndUsage(t *testing.T) {
|
||||||
|
_, cardPath, hwmonPath := setupIntelSysfsTest(t)
|
||||||
|
devicePath := filepath.Join(cardPath, "device")
|
||||||
|
writeIntelSysfsFile(t, devicePath, "vendor", "0x8086\n")
|
||||||
|
writeIntelSysfsFile(t, devicePath, "gpu_busy_percent", "37\n")
|
||||||
|
writeIntelSysfsFile(t, devicePath, "mem_info_lmem_used", "1073741824\n")
|
||||||
|
writeIntelSysfsFile(t, devicePath, "mem_info_lmem_total", "2147483648\n")
|
||||||
|
writeIntelSysfsFile(t, hwmonPath, "energy1_input", "1000000\n")
|
||||||
|
setIntelSysfsTime(t, time.Unix(100, 0))
|
||||||
|
|
||||||
|
gm := &GPUManager{GpuDataMap: make(map[string]*system.GPUData)}
|
||||||
|
require.True(t, gm.updateIntelSysfsGpuData(cardPath, hwmonPath))
|
||||||
|
|
||||||
|
gpu := gm.GpuDataMap["card0"]
|
||||||
|
require.NotNil(t, gpu)
|
||||||
|
assert.Equal(t, 37.0, gpu.Usage)
|
||||||
|
assert.Equal(t, utils.BytesToMegabytes(1073741824), gpu.MemoryUsed)
|
||||||
|
assert.Equal(t, utils.BytesToMegabytes(2147483648), gpu.MemoryTotal)
|
||||||
|
}
|
||||||
13
agent/gpu_intel_sysfs_unsupported.go
Normal file
13
agent/gpu_intel_sysfs_unsupported.go
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
//go:build !linux
|
||||||
|
|
||||||
|
package agent
|
||||||
|
|
||||||
|
type intelSysfsEnergySnapshot struct{}
|
||||||
|
|
||||||
|
func (gm *GPUManager) hasIntelSysfs() bool {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (gm *GPUManager) startIntelSysfsCollector() bool {
|
||||||
|
return false
|
||||||
|
}
|
||||||
@@ -332,11 +332,12 @@ func TestUpdateNvtopSnapshotsKeepsDeviceAssociationWhenOrderChanges(t *testing.T
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestParseCollectorPriority(t *testing.T) {
|
func TestParseCollectorPriority(t *testing.T) {
|
||||||
got := parseCollectorPriority(" nvml, nvidia-smi, intel_gpu_top, amd_sysfs, nvtop, rocm-smi, bad ")
|
got := parseCollectorPriority(" nvml, nvidia-smi, intel_gpu_top, intel_sysfs, amd_sysfs, nvtop, rocm-smi, bad ")
|
||||||
want := []collectorSource{
|
want := []collectorSource{
|
||||||
collectorSourceNVML,
|
collectorSourceNVML,
|
||||||
collectorSourceNvidiaSMI,
|
collectorSourceNvidiaSMI,
|
||||||
collectorSourceIntelGpuTop,
|
collectorSourceIntelGpuTop,
|
||||||
|
collectorSourceIntelSysfs,
|
||||||
collectorSourceAmdSysfs,
|
collectorSourceAmdSysfs,
|
||||||
collectorSourceNVTop,
|
collectorSourceNVTop,
|
||||||
collectorSourceRocmSMI,
|
collectorSourceRocmSMI,
|
||||||
|
|||||||
Reference in New Issue
Block a user