mirror of
https://github.com/henrygd/beszel.git
synced 2026-03-24 06:26:17 +01:00
fix(agent): avoid mismatched root disk I/O mapping in docker (#1737)
- Stop using max-read fallback when mapping root filesystem to diskstats. - Keep root usage reporting even when root I/O cannot be mapped. - Expand docker fallback mount detection to include /etc/resolv.conf and /etc/hostname (in addition to /etc/hosts). - Add clearer warnings when root block device detection is uncertain.
This commit is contained in:
@@ -26,6 +26,15 @@ func parseFilesystemEntry(entry string) (device, customName string) {
|
|||||||
return device, customName
|
return device, customName
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func isDockerSpecialMountpoint(mountpoint string) bool {
|
||||||
|
switch mountpoint {
|
||||||
|
case "/etc/hosts", "/etc/resolv.conf", "/etc/hostname":
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Sets up the filesystems to monitor for disk usage and I/O.
|
// Sets up the filesystems to monitor for disk usage and I/O.
|
||||||
func (a *Agent) initializeDiskInfo() {
|
func (a *Agent) initializeDiskInfo() {
|
||||||
filesystem, _ := GetEnv("FILESYSTEM")
|
filesystem, _ := GetEnv("FILESYSTEM")
|
||||||
@@ -69,11 +78,15 @@ func (a *Agent) initializeDiskInfo() {
|
|||||||
if _, exists := a.fsStats[key]; !exists {
|
if _, exists := a.fsStats[key]; !exists {
|
||||||
if root {
|
if root {
|
||||||
slog.Info("Detected root device", "name", key)
|
slog.Info("Detected root device", "name", key)
|
||||||
// Check if root device is in /proc/diskstats, use fallback if not
|
// Check if root device is in /proc/diskstats. Do not guess a
|
||||||
|
// fallback device for root: that can misattribute root I/O to a
|
||||||
|
// different disk while usage remains tied to root mountpoint.
|
||||||
if _, ioMatch = diskIoCounters[key]; !ioMatch {
|
if _, ioMatch = diskIoCounters[key]; !ioMatch {
|
||||||
key, ioMatch = findIoDevice(filesystem, diskIoCounters, a.fsStats)
|
if matchedKey, match := findIoDevice(filesystem, diskIoCounters); match {
|
||||||
if !ioMatch {
|
key = matchedKey
|
||||||
slog.Info("Using I/O fallback", "device", device, "mountpoint", mountpoint, "fallback", key)
|
ioMatch = true
|
||||||
|
} else {
|
||||||
|
slog.Warn("Root I/O unmapped; set FILESYSTEM", "device", device, "mountpoint", mountpoint)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
@@ -141,8 +154,8 @@ func (a *Agent) initializeDiskInfo() {
|
|||||||
for _, p := range partitions {
|
for _, p := range partitions {
|
||||||
// fmt.Println(p.Device, p.Mountpoint)
|
// fmt.Println(p.Device, p.Mountpoint)
|
||||||
// Binary root fallback or docker root fallback
|
// Binary root fallback or docker root fallback
|
||||||
if !hasRoot && (p.Mountpoint == rootMountPoint || (p.Mountpoint == "/etc/hosts" && strings.HasPrefix(p.Device, "/dev"))) {
|
if !hasRoot && (p.Mountpoint == rootMountPoint || (isDockerSpecialMountpoint(p.Mountpoint) && strings.HasPrefix(p.Device, "/dev"))) {
|
||||||
fs, match := findIoDevice(filepath.Base(p.Device), diskIoCounters, a.fsStats)
|
fs, match := findIoDevice(filepath.Base(p.Device), diskIoCounters)
|
||||||
if match {
|
if match {
|
||||||
addFsStat(fs, p.Mountpoint, true)
|
addFsStat(fs, p.Mountpoint, true)
|
||||||
hasRoot = true
|
hasRoot = true
|
||||||
@@ -176,33 +189,26 @@ func (a *Agent) initializeDiskInfo() {
|
|||||||
|
|
||||||
// If no root filesystem set, use fallback
|
// If no root filesystem set, use fallback
|
||||||
if !hasRoot {
|
if !hasRoot {
|
||||||
rootDevice, _ := findIoDevice(filepath.Base(filesystem), diskIoCounters, a.fsStats)
|
rootKey := filepath.Base(rootMountPoint)
|
||||||
slog.Info("Root disk", "mountpoint", rootMountPoint, "io", rootDevice)
|
if _, exists := a.fsStats[rootKey]; exists {
|
||||||
a.fsStats[rootDevice] = &system.FsStats{Root: true, Mountpoint: rootMountPoint}
|
rootKey = "root"
|
||||||
|
}
|
||||||
|
slog.Warn("Root device not detected; root I/O disabled", "mountpoint", rootMountPoint)
|
||||||
|
a.fsStats[rootKey] = &system.FsStats{Root: true, Mountpoint: rootMountPoint}
|
||||||
}
|
}
|
||||||
|
|
||||||
a.initializeDiskIoStats(diskIoCounters)
|
a.initializeDiskIoStats(diskIoCounters)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Returns matching device from /proc/diskstats,
|
// Returns matching device from /proc/diskstats.
|
||||||
// or the device with the most reads if no match is found.
|
|
||||||
// bool is true if a match was found.
|
// bool is true if a match was found.
|
||||||
func findIoDevice(filesystem string, diskIoCounters map[string]disk.IOCountersStat, fsStats map[string]*system.FsStats) (string, bool) {
|
func findIoDevice(filesystem string, diskIoCounters map[string]disk.IOCountersStat) (string, bool) {
|
||||||
var maxReadBytes uint64
|
|
||||||
maxReadDevice := "/"
|
|
||||||
for _, d := range diskIoCounters {
|
for _, d := range diskIoCounters {
|
||||||
if d.Name == filesystem || (d.Label != "" && d.Label == filesystem) {
|
if d.Name == filesystem || (d.Label != "" && d.Label == filesystem) {
|
||||||
return d.Name, true
|
return d.Name, true
|
||||||
}
|
}
|
||||||
if d.ReadBytes > maxReadBytes {
|
|
||||||
// don't use if device already exists in fsStats
|
|
||||||
if _, exists := fsStats[d.Name]; !exists {
|
|
||||||
maxReadBytes = d.ReadBytes
|
|
||||||
maxReadDevice = d.Name
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return maxReadDevice, false
|
return "", false
|
||||||
}
|
}
|
||||||
|
|
||||||
// Sets start values for disk I/O stats.
|
// Sets start values for disk I/O stats.
|
||||||
|
|||||||
@@ -94,6 +94,62 @@ func TestParseFilesystemEntry(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFindIoDevice(t *testing.T) {
|
||||||
|
t.Run("matches by device name", func(t *testing.T) {
|
||||||
|
ioCounters := map[string]disk.IOCountersStat{
|
||||||
|
"sda": {Name: "sda"},
|
||||||
|
"sdb": {Name: "sdb"},
|
||||||
|
}
|
||||||
|
|
||||||
|
device, ok := findIoDevice("sdb", ioCounters)
|
||||||
|
assert.True(t, ok)
|
||||||
|
assert.Equal(t, "sdb", device)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("matches by device label", func(t *testing.T) {
|
||||||
|
ioCounters := map[string]disk.IOCountersStat{
|
||||||
|
"sda": {Name: "sda", Label: "rootfs"},
|
||||||
|
"sdb": {Name: "sdb"},
|
||||||
|
}
|
||||||
|
|
||||||
|
device, ok := findIoDevice("rootfs", ioCounters)
|
||||||
|
assert.True(t, ok)
|
||||||
|
assert.Equal(t, "sda", device)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("returns no fallback when not found", func(t *testing.T) {
|
||||||
|
ioCounters := map[string]disk.IOCountersStat{
|
||||||
|
"sda": {Name: "sda"},
|
||||||
|
"sdb": {Name: "sdb"},
|
||||||
|
}
|
||||||
|
|
||||||
|
device, ok := findIoDevice("nvme0n1p1", ioCounters)
|
||||||
|
assert.False(t, ok)
|
||||||
|
assert.Equal(t, "", device)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIsDockerSpecialMountpoint(t *testing.T) {
|
||||||
|
testCases := []struct {
|
||||||
|
name string
|
||||||
|
mountpoint string
|
||||||
|
expected bool
|
||||||
|
}{
|
||||||
|
{name: "hosts", mountpoint: "/etc/hosts", expected: true},
|
||||||
|
{name: "resolv", mountpoint: "/etc/resolv.conf", expected: true},
|
||||||
|
{name: "hostname", mountpoint: "/etc/hostname", expected: true},
|
||||||
|
{name: "root", mountpoint: "/", expected: false},
|
||||||
|
{name: "passwd", mountpoint: "/etc/passwd", expected: false},
|
||||||
|
{name: "extra-filesystem", mountpoint: "/extra-filesystems/sda1", expected: false},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range testCases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
assert.Equal(t, tc.expected, isDockerSpecialMountpoint(tc.mountpoint))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestInitializeDiskInfoWithCustomNames(t *testing.T) {
|
func TestInitializeDiskInfoWithCustomNames(t *testing.T) {
|
||||||
// Set up environment variables
|
// Set up environment variables
|
||||||
oldEnv := os.Getenv("EXTRA_FILESYSTEMS")
|
oldEnv := os.Getenv("EXTRA_FILESYSTEMS")
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -18,8 +19,6 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/smart"
|
"github.com/henrygd/beszel/internal/entities/smart"
|
||||||
|
|
||||||
"log/slog"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// SmartManager manages data collection for SMART devices
|
// SmartManager manages data collection for SMART devices
|
||||||
@@ -1125,7 +1124,6 @@ func NewSmartManager() (*SmartManager, error) {
|
|||||||
sm.refreshExcludedDevices()
|
sm.refreshExcludedDevices()
|
||||||
path, err := sm.detectSmartctl()
|
path, err := sm.detectSmartctl()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
slog.Debug(err.Error())
|
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
slog.Debug("smartctl", "path", path)
|
slog.Debug("smartctl", "path", path)
|
||||||
|
|||||||
Reference in New Issue
Block a user