mirror of
https://github.com/henrygd/beszel.git
synced 2026-09-21 17:07:47 +02:00
feat: add ZFS monitoring (#2209)
- track pool capacity, health, I/O, scrub status, and vdev errors - report dataset usage and correct ZFS filesystem metrics - add pool charts, detail views, refresh controls, and health alerts - persist pool details and include ZFS usage in disk alerts - support configurable detail intervals and legacy agent compatibility --------- Co-authored-by: hank <hank@henrygd.me>
This commit is contained in:
@@ -48,6 +48,7 @@ type Agent struct {
|
|||||||
keys []gossh.PublicKey // SSH public keys
|
keys []gossh.PublicKey // SSH public keys
|
||||||
smartManager *SmartManager // Manages SMART data
|
smartManager *SmartManager // Manages SMART data
|
||||||
systemdManager *systemdManager // Manages systemd services
|
systemdManager *systemdManager // Manages systemd services
|
||||||
|
zfsManager *ZfsManager // Manages ZFS pool and dataset data
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewAgent creates a new agent with the given data directory for persisting data.
|
// NewAgent creates a new agent with the given data directory for persisting data.
|
||||||
@@ -121,6 +122,19 @@ func NewAgent(dataDir ...string) (agent *Agent, err error) {
|
|||||||
// initialize handler registry
|
// initialize handler registry
|
||||||
agent.handlerRegistry = NewHandlerRegistry()
|
agent.handlerRegistry = NewHandlerRegistry()
|
||||||
|
|
||||||
|
agent.zfsManager = newZfsManager()
|
||||||
|
|
||||||
|
// ZFS_INTERVAL env var to update ZFS detail data at this interval
|
||||||
|
if zfsIntervalEnv, exists := utils.GetEnv("ZFS_INTERVAL"); exists {
|
||||||
|
if duration, err := time.ParseDuration(zfsIntervalEnv); err == nil && duration > 0 {
|
||||||
|
agent.zfsManager.detailInterval = duration
|
||||||
|
agent.systemDetails.ZfsInterval = duration
|
||||||
|
slog.Info("ZFS_INTERVAL", "duration", duration)
|
||||||
|
} else {
|
||||||
|
slog.Warn("Invalid ZFS_INTERVAL", "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// initialize disk info
|
// initialize disk info
|
||||||
agent.initializeDiskInfo()
|
agent.initializeDiskInfo()
|
||||||
|
|
||||||
|
|||||||
@@ -537,7 +537,16 @@ func normalizeDeviceName(value string) string {
|
|||||||
func (a *Agent) initializeDiskIoStats(diskIoCounters map[string]disk.IOCountersStat) {
|
func (a *Agent) initializeDiskIoStats(diskIoCounters map[string]disk.IOCountersStat) {
|
||||||
a.fsNames = a.fsNames[:0]
|
a.fsNames = a.fsNames[:0]
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
|
// ZFS datasets have no /proc/diskstats entry, so they are excluded from
|
||||||
|
// I/O tracking instead of warning about a missing device (#1541).
|
||||||
|
var zfsMountpoints map[string]bool
|
||||||
|
if a.zfsManager != nil {
|
||||||
|
zfsMountpoints = a.zfsManager.ZfsMountpoints()
|
||||||
|
}
|
||||||
for device, stats := range a.fsStats {
|
for device, stats := range a.fsStats {
|
||||||
|
if zfsMountpoints[stats.Mountpoint] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
// skip if not in diskIoCounters
|
// skip if not in diskIoCounters
|
||||||
d, exists := diskIoCounters[device]
|
d, exists := diskIoCounters[device]
|
||||||
if !exists {
|
if !exists {
|
||||||
@@ -562,20 +571,31 @@ func (a *Agent) updateDiskUsage(systemStats *system.Stats) {
|
|||||||
!a.lastDiskUsageUpdate.IsZero() &&
|
!a.lastDiskUsageUpdate.IsZero() &&
|
||||||
time.Since(a.lastDiskUsageUpdate) < a.diskUsageCacheDuration
|
time.Since(a.lastDiskUsageUpdate) < a.diskUsageCacheDuration
|
||||||
|
|
||||||
|
// ZFS dataset mountpoints use `zfs list` values because statfs(2) reports
|
||||||
|
// dataset-level usage that excludes child datasets (#1541).
|
||||||
|
var zfsUsage map[string]zfsDatasetUsage
|
||||||
|
if a.zfsManager != nil {
|
||||||
|
zfsUsage = a.zfsManager.DatasetUsage()
|
||||||
|
}
|
||||||
|
|
||||||
// disk usage
|
// disk usage
|
||||||
for _, stats := range a.fsStats {
|
for _, stats := range a.fsStats {
|
||||||
// Skip non-root filesystems if caching is active
|
// Skip non-root filesystems if caching is active
|
||||||
if cacheExtraFs && !stats.Root {
|
if cacheExtraFs && !stats.Root {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if d, err := disk.Usage(stats.Mountpoint); err == nil {
|
var total, used uint64
|
||||||
stats.DiskTotal = utils.BytesToGigabytes(d.Total)
|
var usedPct float64
|
||||||
stats.DiskUsed = utils.BytesToGigabytes(d.Used)
|
if u, ok := zfsUsage[stats.Mountpoint]; ok {
|
||||||
if stats.Root {
|
total = u.used + u.avail
|
||||||
systemStats.DiskTotal = utils.BytesToGigabytes(d.Total)
|
used = u.used
|
||||||
systemStats.DiskUsed = utils.BytesToGigabytes(d.Used)
|
if total > 0 {
|
||||||
systemStats.DiskPct = utils.TwoDecimals(d.UsedPercent)
|
usedPct = float64(used) / float64(total) * 100
|
||||||
}
|
}
|
||||||
|
} else if d, err := disk.Usage(stats.Mountpoint); err == nil {
|
||||||
|
total = d.Total
|
||||||
|
used = d.Used
|
||||||
|
usedPct = d.UsedPercent
|
||||||
} else {
|
} else {
|
||||||
// reset stats if error (likely unmounted)
|
// reset stats if error (likely unmounted)
|
||||||
slog.Error("Error getting disk stats", "name", stats.Mountpoint, "err", err)
|
slog.Error("Error getting disk stats", "name", stats.Mountpoint, "err", err)
|
||||||
@@ -583,6 +603,14 @@ func (a *Agent) updateDiskUsage(systemStats *system.Stats) {
|
|||||||
stats.DiskUsed = 0
|
stats.DiskUsed = 0
|
||||||
stats.TotalRead = 0
|
stats.TotalRead = 0
|
||||||
stats.TotalWrite = 0
|
stats.TotalWrite = 0
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
stats.DiskTotal = utils.BytesToGigabytes(total)
|
||||||
|
stats.DiskUsed = utils.BytesToGigabytes(used)
|
||||||
|
if stats.Root {
|
||||||
|
systemStats.DiskTotal = stats.DiskTotal
|
||||||
|
systemStats.DiskUsed = stats.DiskUsed
|
||||||
|
systemStats.DiskPct = utils.TwoDecimals(usedPct)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
109
agent/disk_zfs_test.go
Normal file
109
agent/disk_zfs_test.go
Normal file
@@ -0,0 +1,109 @@
|
|||||||
|
//go:build testing
|
||||||
|
|
||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/henrygd/beszel/agent/zfs"
|
||||||
|
"github.com/henrygd/beszel/internal/entities/system"
|
||||||
|
"github.com/shirou/gopsutil/v4/disk"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestUpdateDiskUsageZfsMountpoint verifies that a filesystem whose mountpoint
|
||||||
|
// is a ZFS dataset reports `zfs list` usage (which includes child datasets)
|
||||||
|
// instead of the dataset-scoped statfs values (#1541).
|
||||||
|
func TestUpdateDiskUsageZfsMountpoint(t *testing.T) {
|
||||||
|
zm := &ZfsManager{}
|
||||||
|
zm.datasetsFn = func() ([]zfs.Dataset, error) {
|
||||||
|
return []zfs.Dataset{
|
||||||
|
{Name: "tank", Used: 12000000000000, Avail: 11999000000000, Mountpoint: "/tank"},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
agent := &Agent{
|
||||||
|
fsStats: map[string]*system.FsStats{
|
||||||
|
"tank": {Root: false, Mountpoint: "/tank"},
|
||||||
|
},
|
||||||
|
zfsManager: zm,
|
||||||
|
}
|
||||||
|
|
||||||
|
var stats system.Stats
|
||||||
|
agent.updateDiskUsage(&stats)
|
||||||
|
|
||||||
|
fs := agent.fsStats["tank"]
|
||||||
|
require.NotNil(t, fs)
|
||||||
|
assert.Equal(t, 22350.81, fs.DiskTotal) // (used + avail) in GiB
|
||||||
|
assert.Equal(t, 11175.87, fs.DiskUsed)
|
||||||
|
// Non-root filesystems do not populate system-level stats.
|
||||||
|
assert.Equal(t, float64(0), stats.DiskTotal)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestUpdateDiskUsageZfsRootPopulatesSystemStats verifies the root disk values
|
||||||
|
// are derived from ZFS usage when the root mountpoint is a ZFS dataset.
|
||||||
|
func TestUpdateDiskUsageZfsRootPopulatesSystemStats(t *testing.T) {
|
||||||
|
zm := &ZfsManager{}
|
||||||
|
zm.datasetsFn = func() ([]zfs.Dataset, error) {
|
||||||
|
return []zfs.Dataset{
|
||||||
|
{Name: "rpool/ROOT/pve-1", Used: 900000000000, Avail: 300000000000, Mountpoint: "/"},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
agent := &Agent{
|
||||||
|
fsStats: map[string]*system.FsStats{
|
||||||
|
"rpool/ROOT/pve-1": {Root: true, Mountpoint: "/"},
|
||||||
|
},
|
||||||
|
zfsManager: zm,
|
||||||
|
}
|
||||||
|
|
||||||
|
var stats system.Stats
|
||||||
|
agent.updateDiskUsage(&stats)
|
||||||
|
|
||||||
|
assert.Equal(t, 1117.59, agent.fsStats["rpool/ROOT/pve-1"].DiskTotal)
|
||||||
|
assert.Equal(t, 838.19, agent.fsStats["rpool/ROOT/pve-1"].DiskUsed)
|
||||||
|
assert.Equal(t, 75.0, stats.DiskPct)
|
||||||
|
assert.Equal(t, 1117.59, stats.DiskTotal)
|
||||||
|
assert.Equal(t, 838.19, stats.DiskUsed)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestUpdateDiskUsageWithoutZfsManager falls back to statfs when no manager is
|
||||||
|
// present (e.g. tests constructing bare Agent values).
|
||||||
|
func TestUpdateDiskUsageWithoutZfsManager(t *testing.T) {
|
||||||
|
agent := &Agent{
|
||||||
|
fsStats: map[string]*system.FsStats{
|
||||||
|
"root": {Root: true, Mountpoint: "/"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
var stats system.Stats
|
||||||
|
agent.updateDiskUsage(&stats)
|
||||||
|
|
||||||
|
assert.True(t, agent.fsStats["root"].DiskTotal > 0, "root usage should come from statfs")
|
||||||
|
assert.True(t, stats.DiskTotal > 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestInitializeDiskIoStatsSkipsZfsMountpoints verifies ZFS filesystems are
|
||||||
|
// excluded from diskstats I/O tracking instead of warning about a missing device.
|
||||||
|
func TestInitializeDiskIoStatsSkipsZfsMountpoints(t *testing.T) {
|
||||||
|
zm := &ZfsManager{}
|
||||||
|
zm.datasetsFn = func() ([]zfs.Dataset, error) {
|
||||||
|
return []zfs.Dataset{{Name: "tank", Mountpoint: "/tank"}}, nil
|
||||||
|
}
|
||||||
|
agent := &Agent{
|
||||||
|
fsStats: map[string]*system.FsStats{
|
||||||
|
"tank": {Root: false, Mountpoint: "/tank"},
|
||||||
|
"sda1": {Root: false, Mountpoint: "/mnt/data"},
|
||||||
|
},
|
||||||
|
zfsManager: zm,
|
||||||
|
diskPrev: make(map[uint16]map[string]prevDisk),
|
||||||
|
}
|
||||||
|
|
||||||
|
agent.initializeDiskIoStats(map[string]disk.IOCountersStat{
|
||||||
|
"sda1": {Name: "sda1", ReadBytes: 100, WriteBytes: 100},
|
||||||
|
})
|
||||||
|
|
||||||
|
assert.Equal(t, []string{"sda1"}, agent.fsNames)
|
||||||
|
assert.Equal(t, uint64(100), agent.fsStats["sda1"].TotalRead)
|
||||||
|
// ZFS entry is present but untouched by diskstats initialization.
|
||||||
|
assert.Equal(t, uint64(0), agent.fsStats["tank"].TotalRead)
|
||||||
|
}
|
||||||
@@ -51,6 +51,7 @@ func NewHandlerRegistry() *HandlerRegistry {
|
|||||||
registry.Register(common.GetContainerInfo, &GetContainerInfoHandler{})
|
registry.Register(common.GetContainerInfo, &GetContainerInfoHandler{})
|
||||||
registry.Register(common.GetSmartData, &GetSmartDataHandler{})
|
registry.Register(common.GetSmartData, &GetSmartDataHandler{})
|
||||||
registry.Register(common.GetSystemdInfo, &GetSystemdInfoHandler{})
|
registry.Register(common.GetSystemdInfo, &GetSystemdInfoHandler{})
|
||||||
|
registry.Register(common.GetZfsData, &GetZfsDataHandler{})
|
||||||
|
|
||||||
return registry
|
return registry
|
||||||
}
|
}
|
||||||
@@ -178,6 +179,23 @@ func (h *GetSmartDataHandler) Handle(hctx *HandlerContext) error {
|
|||||||
}, hctx.RequestID)
|
}, hctx.RequestID)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
////////////////////////////////////////////////////////////////////////////
|
||||||
|
////////////////////////////////////////////////////////////////////////////
|
||||||
|
|
||||||
|
// GetZfsDataHandler handles ZFS detail data requests
|
||||||
|
type GetZfsDataHandler struct{}
|
||||||
|
|
||||||
|
func (h *GetZfsDataHandler) Handle(hctx *HandlerContext) error {
|
||||||
|
if hctx.Agent.zfsManager == nil {
|
||||||
|
return hctx.SendResponse(nil, hctx.RequestID)
|
||||||
|
}
|
||||||
|
var req common.ZfsDataRequest
|
||||||
|
if err := cbor.Unmarshal(hctx.Request.Data, &req); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return hctx.SendResponse(hctx.Agent.zfsManager.GetDetail(req.Force), hctx.RequestID)
|
||||||
|
}
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////
|
////////////////////////////////////////////////////////////////////////////
|
||||||
////////////////////////////////////////////////////////////////////////////
|
////////////////////////////////////////////////////////////////////////////
|
||||||
////////////////////////////////////////////////////////////////////////////
|
////////////////////////////////////////////////////////////////////////////
|
||||||
|
|||||||
@@ -4,8 +4,10 @@ package agent
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/fxamacker/cbor/v2"
|
"github.com/fxamacker/cbor/v2"
|
||||||
|
"github.com/henrygd/beszel/agent/zfs"
|
||||||
"github.com/henrygd/beszel/internal/common"
|
"github.com/henrygd/beszel/internal/common"
|
||||||
"github.com/henrygd/beszel/internal/entities/smart"
|
"github.com/henrygd/beszel/internal/entities/smart"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
@@ -30,6 +32,32 @@ func TestNewAgentResponseSmartData(t *testing.T) {
|
|||||||
assert.True(t, response.SmartComplete)
|
assert.True(t, response.SmartComplete)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestGetZfsDataHandlerForceRefresh(t *testing.T) {
|
||||||
|
poolCalls := 0
|
||||||
|
zm := &ZfsManager{detailInterval: time.Hour}
|
||||||
|
zm.poolStatsFn = func() ([]zfs.PoolStat, error) {
|
||||||
|
poolCalls++
|
||||||
|
return []zfs.PoolStat{{Name: "tank", Alloc: uint64(poolCalls)}}, nil
|
||||||
|
}
|
||||||
|
zm.poolStatusesFn = func() ([]zfs.PoolStatus, error) { return nil, nil }
|
||||||
|
zm.datasetsFn = func() ([]zfs.Dataset, error) { return nil, nil }
|
||||||
|
zm.GetDetail(false)
|
||||||
|
|
||||||
|
requestData, err := cbor.Marshal(common.ZfsDataRequest{Force: true})
|
||||||
|
assert.NoError(t, err)
|
||||||
|
ctx := &HandlerContext{
|
||||||
|
Agent: &Agent{zfsManager: zm},
|
||||||
|
Request: &common.HubRequest[cbor.RawMessage]{
|
||||||
|
Action: common.GetZfsData,
|
||||||
|
Data: requestData,
|
||||||
|
},
|
||||||
|
SendResponse: func(any, *uint32) error { return nil },
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.NoError(t, (&GetZfsDataHandler{}).Handle(ctx))
|
||||||
|
assert.Equal(t, 2, poolCalls)
|
||||||
|
}
|
||||||
|
|
||||||
func (m *MockHandler) Handle(ctx *HandlerContext) error {
|
func (m *MockHandler) Handle(ctx *HandlerContext) error {
|
||||||
if m.handleFunc != nil {
|
if m.handleFunc != nil {
|
||||||
return m.handleFunc(ctx)
|
return m.handleFunc(ctx)
|
||||||
|
|||||||
@@ -219,6 +219,9 @@ func (a *Agent) getSystemStats(cacheTimeMs uint16) system.Stats {
|
|||||||
// disk i/o (cache-aware per interval)
|
// disk i/o (cache-aware per interval)
|
||||||
a.updateDiskIo(cacheTimeMs, &systemStats)
|
a.updateDiskIo(cacheTimeMs, &systemStats)
|
||||||
|
|
||||||
|
// zfs pool stats
|
||||||
|
a.zfsManager.Update(&systemStats)
|
||||||
|
|
||||||
// network stats (per cache interval)
|
// network stats (per cache interval)
|
||||||
a.updateNetworkStats(cacheTimeMs, &systemStats)
|
a.updateNetworkStats(cacheTimeMs, &systemStats)
|
||||||
|
|
||||||
|
|||||||
9
agent/test-data/zfs/zfs_list.txt
Normal file
9
agent/test-data/zfs/zfs_list.txt
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
tank 12000000000000 11999000000000 /tank
|
||||||
|
tank/apps 1000000000000 11999000000000 /tank/apps
|
||||||
|
tank/backup 2000000000000 11999000000000 /tank/backup
|
||||||
|
tank/media 1000000000000 11999000000000 /tank/my media
|
||||||
|
rpool 900000000000 300000000000 -
|
||||||
|
rpool/ROOT 1000000000 300000000000 -
|
||||||
|
rpool/ROOT/pve-1 890000000000 300000000000 /
|
||||||
|
rpool/data 9000000000 300000000000 -
|
||||||
|
rpool/data/subvol-100-disk-0 400000000000 300000000000 /subvol-100-disk-0
|
||||||
2
agent/test-data/zfs/zpool_list.txt
Normal file
2
agent/test-data/zfs/zpool_list.txt
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
tank 23999000000000 12000000000000 11999000000000 ONLINE
|
||||||
|
rpool 1200000000000 900000000000 300000000000 DEGRADED
|
||||||
29
agent/test-data/zfs/zpool_status.txt
Normal file
29
agent/test-data/zfs/zpool_status.txt
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
pool: tank
|
||||||
|
state: ONLINE
|
||||||
|
scan: scrub repaired 0B in 00:05:12 with 0 errors on Sun Jun 1 02:00:12 2025
|
||||||
|
config:
|
||||||
|
|
||||||
|
NAME STATE READ WRITE CKSUM
|
||||||
|
tank ONLINE 0 0 0
|
||||||
|
mirror-0 ONLINE 0 0 0
|
||||||
|
sda ONLINE 0 0 0
|
||||||
|
sdb ONLINE 0 0 0
|
||||||
|
|
||||||
|
errors: No known data errors
|
||||||
|
|
||||||
|
pool: rpool
|
||||||
|
state: DEGRADED
|
||||||
|
status: One or more devices could not be used because the label is missing or
|
||||||
|
invalid. Sufficient replicas exist for the pool to continue functioning in a
|
||||||
|
degraded state.
|
||||||
|
scan: scrub in progress since Sun Jun 8 01:00:00 2025
|
||||||
|
10.00% done, 01:30:00 to go, 0.00/s
|
||||||
|
config:
|
||||||
|
|
||||||
|
NAME STATE READ WRITE CKSUM
|
||||||
|
rpool DEGRADED 0 0 0
|
||||||
|
mirror-0 DEGRADED 0 0 0
|
||||||
|
sda ONLINE 0 0 0
|
||||||
|
sdb FAULTED 1 2 3
|
||||||
|
|
||||||
|
errors: 1 data errors, use '-v' for a list
|
||||||
160
agent/zfs/zfs.go
Normal file
160
agent/zfs/zfs.go
Normal file
@@ -0,0 +1,160 @@
|
|||||||
|
// Package zfs provides functions to read ZFS statistics.
|
||||||
|
package zfs
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
var commandTimeout = 10 * time.Second
|
||||||
|
|
||||||
|
var commandOutput = func(name string, args ...string) ([]byte, error) {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), commandTimeout)
|
||||||
|
defer cancel()
|
||||||
|
cmd := exec.CommandContext(ctx, name, args...)
|
||||||
|
cmd.Env = append(os.Environ(), "LC_ALL=C", "LANG=C")
|
||||||
|
out, err := cmd.Output()
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil, fmt.Errorf("%s timed out after %s: %w", name, commandTimeout, ctx.Err())
|
||||||
|
}
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// ErrNoZfs is returned when the ZFS utilities or kernel interfaces are unavailable.
|
||||||
|
var ErrNoZfs = errors.New("zfs utilities unavailable")
|
||||||
|
|
||||||
|
// PoolStat is a snapshot of a ZFS pool's capacity and health.
|
||||||
|
type PoolStat struct {
|
||||||
|
Name string
|
||||||
|
Size uint64 // total capacity in bytes
|
||||||
|
Alloc uint64 // allocated bytes
|
||||||
|
Free uint64 // free bytes
|
||||||
|
Health string // ONLINE, DEGRADED, FAULTED, ...
|
||||||
|
}
|
||||||
|
|
||||||
|
// PoolKernelStat is the inexpensive pool telemetry exposed by the ZFS kernel.
|
||||||
|
// NRead and NWrite are cumulative byte counters since the pool was imported.
|
||||||
|
type PoolKernelStat struct {
|
||||||
|
Name string
|
||||||
|
Health string
|
||||||
|
NRead uint64
|
||||||
|
NWrite uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
// PoolIoStats holds calculated per-second I/O rates for a pool.
|
||||||
|
type PoolIoStats struct {
|
||||||
|
NRead uint64
|
||||||
|
NWrite uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
// Dataset is a single ZFS dataset with usage information.
|
||||||
|
type Dataset struct {
|
||||||
|
Name string
|
||||||
|
Used uint64
|
||||||
|
Avail uint64
|
||||||
|
Mountpoint string
|
||||||
|
}
|
||||||
|
|
||||||
|
// PoolStats returns capacity and health for all pools on the system using
|
||||||
|
// `zpool list`. Frequent health and I/O sampling uses PoolKernelStats instead.
|
||||||
|
func PoolStats() ([]PoolStat, error) {
|
||||||
|
out, err := commandOutput("zpool", "list", "-Hp", "-o", "name,size,alloc,free,health")
|
||||||
|
if err != nil {
|
||||||
|
var exitErr *exec.ExitError
|
||||||
|
if errors.As(err, &exitErr) && strings.Contains(string(exitErr.Stderr), "no pools available") {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("zpool list: %w", err)
|
||||||
|
}
|
||||||
|
return parseZpoolListOutput(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Datasets returns all datasets on the system with usage and mountpoint
|
||||||
|
// information using `zfs list` (recursive by default).
|
||||||
|
func Datasets() ([]Dataset, error) {
|
||||||
|
out, err := commandOutput("zfs", "list", "-Hp", "-o", "name,used,avail,mountpoint")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("zfs list: %w", err)
|
||||||
|
}
|
||||||
|
return parseZfsListOutput(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseZpoolListOutput parses `zpool list -Hp -o name,size,alloc,free,health` output.
|
||||||
|
// Columns are tab-separated; numeric columns are raw bytes.
|
||||||
|
func parseZpoolListOutput(out []byte) ([]PoolStat, error) {
|
||||||
|
var pools []PoolStat
|
||||||
|
scanner := bufio.NewScanner(bytes.NewReader(out))
|
||||||
|
for scanner.Scan() {
|
||||||
|
line := strings.TrimSpace(scanner.Text())
|
||||||
|
if line == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if line == "no pools available" && len(pools) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
fields := strings.Split(line, "\t")
|
||||||
|
if len(fields) < 5 {
|
||||||
|
return nil, fmt.Errorf("unexpected zpool list line: %q", line)
|
||||||
|
}
|
||||||
|
size, err := strconv.ParseUint(fields[1], 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("parsing size for pool %q: %w", fields[0], err)
|
||||||
|
}
|
||||||
|
alloc, err := strconv.ParseUint(fields[2], 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("parsing alloc for pool %q: %w", fields[0], err)
|
||||||
|
}
|
||||||
|
free, err := strconv.ParseUint(fields[3], 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("parsing free for pool %q: %w", fields[0], err)
|
||||||
|
}
|
||||||
|
pools = append(pools, PoolStat{
|
||||||
|
Name: fields[0],
|
||||||
|
Size: size,
|
||||||
|
Alloc: alloc,
|
||||||
|
Free: free,
|
||||||
|
Health: fields[4],
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return pools, scanner.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseZfsListOutput parses `zfs list -Hp -o name,used,avail,mountpoint` output.
|
||||||
|
// The mountpoint column may contain spaces, so it is split on tabs only.
|
||||||
|
func parseZfsListOutput(out []byte) ([]Dataset, error) {
|
||||||
|
var datasets []Dataset
|
||||||
|
scanner := bufio.NewScanner(bytes.NewReader(out))
|
||||||
|
for scanner.Scan() {
|
||||||
|
line := strings.TrimSpace(scanner.Text())
|
||||||
|
if line == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fields := strings.SplitN(line, "\t", 4)
|
||||||
|
if len(fields) < 4 {
|
||||||
|
return nil, fmt.Errorf("unexpected zfs list line: %q", line)
|
||||||
|
}
|
||||||
|
used, err := strconv.ParseUint(fields[1], 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("parsing used for dataset %q: %w", fields[0], err)
|
||||||
|
}
|
||||||
|
avail, err := strconv.ParseUint(fields[2], 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("parsing avail for dataset %q: %w", fields[0], err)
|
||||||
|
}
|
||||||
|
datasets = append(datasets, Dataset{
|
||||||
|
Name: fields[0],
|
||||||
|
Used: used,
|
||||||
|
Avail: avail,
|
||||||
|
Mountpoint: fields[3],
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return datasets, scanner.Err()
|
||||||
|
}
|
||||||
@@ -3,9 +3,17 @@
|
|||||||
package zfs
|
package zfs
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
|
|
||||||
"golang.org/x/sys/unix"
|
"golang.org/x/sys/unix"
|
||||||
)
|
)
|
||||||
|
|
||||||
func ARCSize() (uint64, error) {
|
func ARCSize() (uint64, error) {
|
||||||
return unix.SysctlUint64("kstat.zfs.misc.arcstats.size")
|
return unix.SysctlUint64("kstat.zfs.misc.arcstats.size")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// FreeBSD does not expose Linux's per-pool procfs kstats. Capacity, health,
|
||||||
|
// and detail collection still work through the cached utilities.
|
||||||
|
func PoolKernelStats() ([]PoolKernelStat, error) {
|
||||||
|
return nil, errors.ErrUnsupported
|
||||||
|
}
|
||||||
|
|||||||
@@ -5,14 +5,18 @@ package zfs
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bufio"
|
"bufio"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var procZfsPath = "/proc/spl/kstat/zfs"
|
||||||
|
|
||||||
func ARCSize() (uint64, error) {
|
func ARCSize() (uint64, error) {
|
||||||
file, err := os.Open("/proc/spl/kstat/zfs/arcstats")
|
file, err := os.Open(filepath.Join(procZfsPath, "arcstats"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
@@ -29,6 +33,167 @@ func ARCSize() (uint64, error) {
|
|||||||
return strconv.ParseUint(fields[2], 10, 64)
|
return strconv.ParseUint(fields[2], 10, 64)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if err := scanner.Err(); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
return 0, fmt.Errorf("size field not found in arcstats")
|
return 0, fmt.Errorf("size field not found in arcstats")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PoolKernelStats reads pool state and cumulative I/O counters directly from
|
||||||
|
// procfs. These kstats are the same interfaces used by node_exporter's Linux
|
||||||
|
// ZFS collector and avoid keeping a `zpool iostat` subprocess alive.
|
||||||
|
func PoolKernelStats() ([]PoolKernelStat, error) {
|
||||||
|
poolDirs := make(map[string]struct{})
|
||||||
|
for _, filename := range []string{"state", "io", "objset-*"} {
|
||||||
|
paths, err := filepath.Glob(filepath.Join(procZfsPath, "*", filename))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for _, path := range paths {
|
||||||
|
poolDirs[filepath.Dir(path)] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(poolDirs) == 0 {
|
||||||
|
return nil, ErrNoZfs
|
||||||
|
}
|
||||||
|
pools := make([]PoolKernelStat, 0, len(poolDirs))
|
||||||
|
for poolDir := range poolDirs {
|
||||||
|
nread, nwrite, err := readPoolCounters(poolDir)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
continue // pool may have been exported after the glob
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
state, err := os.ReadFile(filepath.Join(poolDir, "state"))
|
||||||
|
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
pools = append(pools, PoolKernelStat{
|
||||||
|
Name: filepath.Base(poolDir), Health: strings.ToUpper(strings.TrimSpace(string(state))),
|
||||||
|
NRead: nread, NWrite: nwrite,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if len(pools) == 0 {
|
||||||
|
return nil, ErrNoZfs
|
||||||
|
}
|
||||||
|
return pools, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// readPoolCounters supports both ZFS kernel interfaces. OpenZFS through 2.3
|
||||||
|
// exposes aggregate vdev counters in "io". When that file is unavailable, sum
|
||||||
|
// the logical I/O counters exposed for each dataset in the pool.
|
||||||
|
func readPoolCounters(poolDir string) (uint64, uint64, error) {
|
||||||
|
nread, nwrite, err := readPoolIO(filepath.Join(poolDir, "io"))
|
||||||
|
if err == nil || !errors.Is(err, os.ErrNotExist) {
|
||||||
|
return nread, nwrite, err
|
||||||
|
}
|
||||||
|
return readPoolObjsets(poolDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
func readPoolIO(path string) (uint64, uint64, error) {
|
||||||
|
file, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, err
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
scanner := bufio.NewScanner(file)
|
||||||
|
for scanner.Scan() {
|
||||||
|
fields := strings.Fields(scanner.Text())
|
||||||
|
if len(fields) < 2 || fields[0] != "nread" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !scanner.Scan() {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
values := strings.Fields(scanner.Text())
|
||||||
|
if len(values) < 2 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
nread, err := strconv.ParseUint(values[0], 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, fmt.Errorf("parsing nread in %s: %w", path, err)
|
||||||
|
}
|
||||||
|
nwrite, err := strconv.ParseUint(values[1], 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, fmt.Errorf("parsing nwritten in %s: %w", path, err)
|
||||||
|
}
|
||||||
|
return nread, nwrite, nil
|
||||||
|
}
|
||||||
|
if err := scanner.Err(); err != nil {
|
||||||
|
return 0, 0, err
|
||||||
|
}
|
||||||
|
return 0, 0, fmt.Errorf("I/O counters not found in %s", path)
|
||||||
|
}
|
||||||
|
|
||||||
|
func readPoolObjsets(poolDir string) (uint64, uint64, error) {
|
||||||
|
paths, err := filepath.Glob(filepath.Join(poolDir, "objset-*"))
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, err
|
||||||
|
}
|
||||||
|
if len(paths) == 0 {
|
||||||
|
return 0, 0, fmt.Errorf("dataset I/O counters not found in %s", poolDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
var totalRead, totalWrite uint64
|
||||||
|
objsetsRead := 0
|
||||||
|
for _, path := range paths {
|
||||||
|
nread, nwrite, err := readObjsetIO(path)
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
continue // dataset may have been destroyed after the glob
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, err
|
||||||
|
}
|
||||||
|
totalRead += nread
|
||||||
|
totalWrite += nwrite
|
||||||
|
objsetsRead++
|
||||||
|
}
|
||||||
|
if objsetsRead == 0 {
|
||||||
|
return 0, 0, fmt.Errorf("dataset I/O counters not found in %s", poolDir)
|
||||||
|
}
|
||||||
|
return totalRead, totalWrite, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func readObjsetIO(path string) (uint64, uint64, error) {
|
||||||
|
file, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, err
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
var nread, nwrite uint64
|
||||||
|
var foundRead, foundWrite bool
|
||||||
|
scanner := bufio.NewScanner(file)
|
||||||
|
for scanner.Scan() {
|
||||||
|
fields := strings.Fields(scanner.Text())
|
||||||
|
if len(fields) < 3 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var target *uint64
|
||||||
|
switch fields[0] {
|
||||||
|
case "nread":
|
||||||
|
target = &nread
|
||||||
|
foundRead = true
|
||||||
|
case "nwritten":
|
||||||
|
target = &nwrite
|
||||||
|
foundWrite = true
|
||||||
|
default:
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
value, err := strconv.ParseUint(fields[2], 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, fmt.Errorf("parsing %s in %s: %w", fields[0], path, err)
|
||||||
|
}
|
||||||
|
*target = value
|
||||||
|
}
|
||||||
|
if err := scanner.Err(); err != nil {
|
||||||
|
return 0, 0, err
|
||||||
|
}
|
||||||
|
if !foundRead || !foundWrite {
|
||||||
|
return 0, 0, fmt.Errorf("incomplete I/O counters in %s", path)
|
||||||
|
}
|
||||||
|
return nread, nwrite, nil
|
||||||
|
}
|
||||||
|
|||||||
90
agent/zfs/zfs_linux_test.go
Normal file
90
agent/zfs/zfs_linux_test.go
Normal file
@@ -0,0 +1,90 @@
|
|||||||
|
//go:build testing && linux
|
||||||
|
|
||||||
|
package zfs
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPoolKernelStats(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
oldPath := procZfsPath
|
||||||
|
procZfsPath = root
|
||||||
|
t.Cleanup(func() { procZfsPath = oldPath })
|
||||||
|
|
||||||
|
poolDir := filepath.Join(root, "tank")
|
||||||
|
require.NoError(t, os.MkdirAll(poolDir, 0o755))
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(poolDir, "io"), []byte(
|
||||||
|
"11 3 0x00 1 80 0 0\n"+
|
||||||
|
"nread nwritten reads writes wtime wlentime wupdate rtime rlentime rupdate wcnt rcnt\n"+
|
||||||
|
"1884160 6450688 22 978 0 0 0 0 0 0 0 0\n",
|
||||||
|
), 0o644))
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(poolDir, "state"), []byte("DEGRADED\n"), 0o644))
|
||||||
|
|
||||||
|
stats, err := PoolKernelStats()
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, stats, 1)
|
||||||
|
assert.Equal(t, PoolKernelStat{
|
||||||
|
Name: "tank", Health: "DEGRADED", NRead: 1884160, NWrite: 6450688,
|
||||||
|
}, stats[0])
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPoolKernelStatsOpenZfs24(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
oldPath := procZfsPath
|
||||||
|
procZfsPath = root
|
||||||
|
t.Cleanup(func() { procZfsPath = oldPath })
|
||||||
|
|
||||||
|
poolDir := filepath.Join(root, "tank")
|
||||||
|
require.NoError(t, os.MkdirAll(poolDir, 0o755))
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(poolDir, "state"), []byte("ONLINE\n"), 0o644))
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(poolDir, "objset-0x1"), []byte(
|
||||||
|
"34 1 0x01 28 7872 0 0\n"+
|
||||||
|
"name type data\n"+
|
||||||
|
"dataset_name 7 tank\n"+
|
||||||
|
"nwritten 4 2000\n"+
|
||||||
|
"nread 4 1000\n",
|
||||||
|
), 0o644))
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(poolDir, "objset-0x2"), []byte(
|
||||||
|
"34 1 0x01 28 7872 0 0\n"+
|
||||||
|
"name type data\n"+
|
||||||
|
"dataset_name 7 tank/videos\n"+
|
||||||
|
"nwritten 4 400\n"+
|
||||||
|
"nread 4 300\n",
|
||||||
|
), 0o644))
|
||||||
|
|
||||||
|
stats, err := PoolKernelStats()
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, stats, 1)
|
||||||
|
assert.Equal(t, PoolKernelStat{
|
||||||
|
Name: "tank", Health: "ONLINE", NRead: 1300, NWrite: 2400,
|
||||||
|
}, stats[0])
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPoolKernelStatsNoZfs(t *testing.T) {
|
||||||
|
oldPath := procZfsPath
|
||||||
|
procZfsPath = t.TempDir()
|
||||||
|
t.Cleanup(func() { procZfsPath = oldPath })
|
||||||
|
|
||||||
|
_, err := PoolKernelStats()
|
||||||
|
assert.ErrorIs(t, err, ErrNoZfs)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReadPoolIORejectsMalformedCounters(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "io")
|
||||||
|
require.NoError(t, os.WriteFile(path, []byte("nread nwritten\nnope 10\n"), 0o644))
|
||||||
|
_, _, err := readPoolIO(path)
|
||||||
|
require.Error(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReadObjsetIORequiresAllCounters(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "objset-0x1")
|
||||||
|
require.NoError(t, os.WriteFile(path, []byte("nread 4 10\n"), 0o644))
|
||||||
|
_, _, err := readObjsetIO(path)
|
||||||
|
require.Error(t, err)
|
||||||
|
}
|
||||||
150
agent/zfs/zfs_status.go
Normal file
150
agent/zfs/zfs_status.go
Normal file
@@ -0,0 +1,150 @@
|
|||||||
|
package zfs
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"bytes"
|
||||||
|
"fmt"
|
||||||
|
"regexp"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// PoolStatus holds parsed `zpool status` information for one pool.
|
||||||
|
type PoolStatus struct {
|
||||||
|
Name string
|
||||||
|
State string // ONLINE, DEGRADED, FAULTED, ...
|
||||||
|
Scrub ScrubStatus
|
||||||
|
Vdevs []VdevStatus
|
||||||
|
}
|
||||||
|
|
||||||
|
// ScrubStatus holds the scrub (or resilver) status parsed from the scan line.
|
||||||
|
type ScrubStatus struct {
|
||||||
|
State string // NONE, SCANNING, FINISHED, CANCELED
|
||||||
|
Progress string // e.g. "10.00%" while scanning
|
||||||
|
Errors uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
// VdevStatus is a single vdev row (mirror, raidz, or leaf disk).
|
||||||
|
type VdevStatus struct {
|
||||||
|
Name string
|
||||||
|
State string
|
||||||
|
ReadErrs uint64
|
||||||
|
WriteErrs uint64
|
||||||
|
ChecksumErrs uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
progressRe = regexp.MustCompile(`(\d+\.\d+)%\s+done`)
|
||||||
|
errorsRe = regexp.MustCompile(`with\s+(\d+)\s+errors`)
|
||||||
|
)
|
||||||
|
|
||||||
|
// PoolStatuses runs `zpool status` and parses per-pool state, scrub, and vdev
|
||||||
|
// information. The human-readable format has been stable across OpenZFS
|
||||||
|
// releases; rows are matched by their tabular shape rather than position.
|
||||||
|
func PoolStatuses() ([]PoolStatus, error) {
|
||||||
|
out, err := commandOutput("zpool", "status")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("zpool status: %w", err)
|
||||||
|
}
|
||||||
|
return parseZpoolStatusOutput(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseZpoolStatusOutput parses the output of `zpool status`.
|
||||||
|
func parseZpoolStatusOutput(out []byte) ([]PoolStatus, error) {
|
||||||
|
var pools []PoolStatus
|
||||||
|
var current *PoolStatus
|
||||||
|
inConfig := false
|
||||||
|
scanContinuation := false // next non-blank line continues the scan line (progress)
|
||||||
|
|
||||||
|
scanner := bufio.NewScanner(bytes.NewReader(out))
|
||||||
|
for scanner.Scan() {
|
||||||
|
line := scanner.Text()
|
||||||
|
trimmed := strings.TrimSpace(line)
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case strings.HasPrefix(trimmed, "pool:"):
|
||||||
|
pools = append(pools, PoolStatus{Name: strings.TrimSpace(strings.TrimPrefix(trimmed, "pool:"))})
|
||||||
|
current = &pools[len(pools)-1]
|
||||||
|
inConfig = false
|
||||||
|
scanContinuation = false
|
||||||
|
case current == nil:
|
||||||
|
continue
|
||||||
|
case strings.HasPrefix(trimmed, "state:"):
|
||||||
|
current.State = strings.TrimSpace(strings.TrimPrefix(trimmed, "state:"))
|
||||||
|
case strings.HasPrefix(trimmed, "scan:"):
|
||||||
|
current.Scrub = parseScanLine(trimmed)
|
||||||
|
// zpool status prints the progress percentage on the line after scan.
|
||||||
|
scanContinuation = true
|
||||||
|
case trimmed == "config:":
|
||||||
|
inConfig = true
|
||||||
|
case scanContinuation:
|
||||||
|
// The line after scan: may be an indented progress continuation.
|
||||||
|
if m := progressRe.FindStringSubmatch(trimmed); m != nil {
|
||||||
|
current.Scrub.Progress = m[1] + "%"
|
||||||
|
}
|
||||||
|
scanContinuation = false
|
||||||
|
case inConfig && (line == "" || strings.HasPrefix(line, " ") || strings.HasPrefix(line, "\t")):
|
||||||
|
// Table rows are indented; blank lines separate sections. The
|
||||||
|
// column header and the pool's own row are skipped.
|
||||||
|
if trimmed != "" && !strings.HasPrefix(trimmed, "NAME") {
|
||||||
|
if vdev, ok := parseVdevLine(trimmed, current.Name); ok {
|
||||||
|
current.Vdevs = append(current.Vdevs, vdev)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case inConfig:
|
||||||
|
// unindented line (errors:, status:, next pool:) ends the table
|
||||||
|
inConfig = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return pools, scanner.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseScanLine maps a `scan:` line to a ScrubStatus.
|
||||||
|
func parseScanLine(line string) ScrubStatus {
|
||||||
|
var scrub ScrubStatus
|
||||||
|
switch {
|
||||||
|
case strings.Contains(line, "in progress"):
|
||||||
|
scrub.State = "SCANNING"
|
||||||
|
case strings.Contains(line, "canceled"):
|
||||||
|
scrub.State = "CANCELED"
|
||||||
|
case strings.Contains(line, "repaired"), strings.Contains(line, "resilvered"):
|
||||||
|
scrub.State = "FINISHED"
|
||||||
|
default:
|
||||||
|
scrub.State = "NONE"
|
||||||
|
}
|
||||||
|
if m := progressRe.FindStringSubmatch(line); m != nil {
|
||||||
|
scrub.Progress = m[1] + "%"
|
||||||
|
}
|
||||||
|
if m := errorsRe.FindStringSubmatch(line); m != nil {
|
||||||
|
if n, err := strconv.ParseUint(m[1], 10, 64); err == nil {
|
||||||
|
scrub.Errors = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return scrub
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseVdevLine parses one row of the config table. Rows have the shape
|
||||||
|
// "NAME STATE READ WRITE CKSUM [extra...]". The first data row is the pool
|
||||||
|
// itself and is skipped since it duplicates pool-level info.
|
||||||
|
func parseVdevLine(line, poolName string) (VdevStatus, bool) {
|
||||||
|
fields := strings.Fields(line)
|
||||||
|
if len(fields) < 5 {
|
||||||
|
return VdevStatus{}, false
|
||||||
|
}
|
||||||
|
if fields[0] == poolName {
|
||||||
|
return VdevStatus{}, false
|
||||||
|
}
|
||||||
|
read, err1 := strconv.ParseUint(fields[2], 10, 64)
|
||||||
|
write, err2 := strconv.ParseUint(fields[3], 10, 64)
|
||||||
|
cksum, err3 := strconv.ParseUint(fields[4], 10, 64)
|
||||||
|
if err1 != nil || err2 != nil || err3 != nil {
|
||||||
|
return VdevStatus{}, false
|
||||||
|
}
|
||||||
|
return VdevStatus{
|
||||||
|
Name: fields[0],
|
||||||
|
State: fields[1],
|
||||||
|
ReadErrs: read,
|
||||||
|
WriteErrs: write,
|
||||||
|
ChecksumErrs: cksum,
|
||||||
|
}, true
|
||||||
|
}
|
||||||
143
agent/zfs/zfs_test.go
Normal file
143
agent/zfs/zfs_test.go
Normal file
@@ -0,0 +1,143 @@
|
|||||||
|
//go:build testing
|
||||||
|
|
||||||
|
package zfs
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func fixturePath(name string) string {
|
||||||
|
return filepath.Join("..", "test-data", "zfs", name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseZpoolListOutput(t *testing.T) {
|
||||||
|
data, err := os.ReadFile(fixturePath("zpool_list.txt"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
pools, err := parseZpoolListOutput(data)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, pools, 2)
|
||||||
|
|
||||||
|
assert.Equal(t, PoolStat{Name: "tank", Size: 23999000000000, Alloc: 12000000000000, Free: 11999000000000, Health: "ONLINE"}, pools[0])
|
||||||
|
assert.Equal(t, PoolStat{Name: "rpool", Size: 1200000000000, Alloc: 900000000000, Free: 300000000000, Health: "DEGRADED"}, pools[1])
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseZpoolListOutputIgnoresEmptyLines(t *testing.T) {
|
||||||
|
pools, err := parseZpoolListOutput([]byte("tank\t100\t50\t50\tONLINE\n\n"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, pools, 1)
|
||||||
|
assert.Equal(t, "tank", pools[0].Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseZpoolListOutputNoPools(t *testing.T) {
|
||||||
|
pools, err := parseZpoolListOutput([]byte("no pools available\n"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Empty(t, pools)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseZpoolListOutputRejectsMalformedLine(t *testing.T) {
|
||||||
|
_, err := parseZpoolListOutput([]byte("tank\t100\t50\n"))
|
||||||
|
require.Error(t, err)
|
||||||
|
|
||||||
|
_, err = parseZpoolListOutput([]byte("tank\tnotanumber\t50\t50\tONLINE\n"))
|
||||||
|
require.Error(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseZfsListOutput(t *testing.T) {
|
||||||
|
data, err := os.ReadFile(fixturePath("zfs_list.txt"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
datasets, err := parseZfsListOutput(data)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, datasets, 9)
|
||||||
|
|
||||||
|
// Mountpoint with a space must be kept intact (tab-split only).
|
||||||
|
assert.Equal(t, "/tank/my media", datasets[3].Mountpoint)
|
||||||
|
// Unmounted datasets/zvols report "-".
|
||||||
|
assert.Equal(t, "-", datasets[4].Mountpoint)
|
||||||
|
assert.Equal(t, uint64(12000000000000), datasets[0].Used)
|
||||||
|
assert.Equal(t, uint64(11999000000000), datasets[0].Avail)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseZpoolStatusOutput(t *testing.T) {
|
||||||
|
data, err := os.ReadFile(fixturePath("zpool_status.txt"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
pools, err := parseZpoolStatusOutput(data)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, pools, 2)
|
||||||
|
|
||||||
|
tank := pools[0]
|
||||||
|
assert.Equal(t, "tank", tank.Name)
|
||||||
|
assert.Equal(t, "ONLINE", tank.State)
|
||||||
|
assert.Equal(t, "FINISHED", tank.Scrub.State)
|
||||||
|
assert.Equal(t, "", tank.Scrub.Progress)
|
||||||
|
assert.Equal(t, uint64(0), tank.Scrub.Errors)
|
||||||
|
// Pool row itself is skipped; mirror + 2 disks remain.
|
||||||
|
require.Len(t, tank.Vdevs, 3)
|
||||||
|
assert.Equal(t, "mirror-0", tank.Vdevs[0].Name)
|
||||||
|
assert.Equal(t, "sda", tank.Vdevs[1].Name)
|
||||||
|
assert.Equal(t, "sdb", tank.Vdevs[2].Name)
|
||||||
|
|
||||||
|
rpool := pools[1]
|
||||||
|
assert.Equal(t, "rpool", rpool.Name)
|
||||||
|
assert.Equal(t, "DEGRADED", rpool.State)
|
||||||
|
assert.Equal(t, "SCANNING", rpool.Scrub.State)
|
||||||
|
assert.Equal(t, "10.00%", rpool.Scrub.Progress)
|
||||||
|
require.Len(t, rpool.Vdevs, 3)
|
||||||
|
assert.Equal(t, "FAULTED", rpool.Vdevs[2].State)
|
||||||
|
assert.Equal(t, uint64(1), rpool.Vdevs[2].ReadErrs)
|
||||||
|
assert.Equal(t, uint64(2), rpool.Vdevs[2].WriteErrs)
|
||||||
|
assert.Equal(t, uint64(3), rpool.Vdevs[2].ChecksumErrs)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseScanLine(t *testing.T) {
|
||||||
|
assert.Equal(t, "FINISHED", parseScanLine("scan: scrub repaired 0B in 00:05:12 with 0 errors on Sun Jun 1 02:00:12 2025").State)
|
||||||
|
assert.Equal(t, uint64(3), parseScanLine("scan: scrub repaired 10G in 01:00:00 with 3 errors on Sun Jun 1 02:00:12 2025").Errors)
|
||||||
|
assert.Equal(t, "SCANNING", parseScanLine("scan: scrub in progress since Sun Jun 8 01:00:00 2025").State)
|
||||||
|
assert.Equal(t, "CANCELED", parseScanLine("scan: scrub canceled on Sun Jun 1 02:00:12 2025").State)
|
||||||
|
assert.Equal(t, "FINISHED", parseScanLine("scan: resilvered 1.23G in 00:01:00 with 0 errors on Sun Jun 1 02:00:12 2025").State)
|
||||||
|
assert.Equal(t, "NONE", parseScanLine("scan: none requested").State)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCommandOutputForcesLocaleAndTimesOut(t *testing.T) {
|
||||||
|
t.Setenv("BESZEL_ZFS_COMMAND_HELPER", "1")
|
||||||
|
out, err := commandOutput(os.Args[0], "-test.run=TestZfsCommandHelperProcess", "--", "locale")
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "C/C", string(out))
|
||||||
|
|
||||||
|
oldTimeout := commandTimeout
|
||||||
|
commandTimeout = 20 * time.Millisecond
|
||||||
|
t.Cleanup(func() { commandTimeout = oldTimeout })
|
||||||
|
_, err = commandOutput(os.Args[0], "-test.run=TestZfsCommandHelperProcess", "--", "sleep")
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Contains(t, err.Error(), "timed out")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestZfsCommandHelperProcess(t *testing.T) {
|
||||||
|
if os.Getenv("BESZEL_ZFS_COMMAND_HELPER") != "1" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
mode := ""
|
||||||
|
for i, arg := range os.Args {
|
||||||
|
if arg == "--" && i+1 < len(os.Args) {
|
||||||
|
mode = os.Args[i+1]
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
switch strings.TrimSpace(mode) {
|
||||||
|
case "locale":
|
||||||
|
_, _ = fmt.Printf("%s/%s", os.Getenv("LC_ALL"), os.Getenv("LANG"))
|
||||||
|
case "sleep":
|
||||||
|
time.Sleep(time.Second)
|
||||||
|
}
|
||||||
|
os.Exit(0)
|
||||||
|
}
|
||||||
@@ -7,3 +7,7 @@ import "errors"
|
|||||||
func ARCSize() (uint64, error) {
|
func ARCSize() (uint64, error) {
|
||||||
return 0, errors.ErrUnsupported
|
return 0, errors.ErrUnsupported
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func PoolKernelStats() ([]PoolKernelStat, error) {
|
||||||
|
return nil, errors.ErrUnsupported
|
||||||
|
}
|
||||||
|
|||||||
320
agent/zfs_pool.go
Normal file
320
agent/zfs_pool.go
Normal file
@@ -0,0 +1,320 @@
|
|||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log/slog"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/henrygd/beszel/agent/zfs"
|
||||||
|
"github.com/henrygd/beszel/internal/entities/system"
|
||||||
|
zfsentity "github.com/henrygd/beszel/internal/entities/zfs"
|
||||||
|
)
|
||||||
|
|
||||||
|
// zfsDatasetUsage holds usage values for a ZFS dataset mountpoint.
|
||||||
|
type zfsDatasetUsage struct {
|
||||||
|
used uint64
|
||||||
|
avail uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
// datasetUsageRefreshInterval controls how often `zfs list` is re-run for the
|
||||||
|
// mountpoint usage map. Dataset inventory changes rarely.
|
||||||
|
const datasetUsageRefreshInterval = 5 * time.Minute
|
||||||
|
|
||||||
|
// poolStatsRefreshInterval controls how often `zpool list` is re-run for pool
|
||||||
|
// capacity. Health and I/O are read from procfs on Linux, so the utility only
|
||||||
|
// needs to refresh slow-moving space accounting.
|
||||||
|
const poolStatsRefreshInterval = time.Minute
|
||||||
|
|
||||||
|
type poolKernelSample struct {
|
||||||
|
nread uint64
|
||||||
|
nwrite uint64
|
||||||
|
at time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// ZfsManager collects ZFS pool and dataset statistics. Collection functions
|
||||||
|
// are fields so unit tests can substitute them (same pattern as
|
||||||
|
// diskDiscovery.usageFn). It is safe for concurrent use by a single goroutine
|
||||||
|
// only; callers must hold the agent lock like updateDiskUsage does.
|
||||||
|
type ZfsManager struct {
|
||||||
|
poolStatsFn func() ([]zfs.PoolStat, error) // capacity/health source
|
||||||
|
datasetsFn func() ([]zfs.Dataset, error) // dataset inventory source
|
||||||
|
kernelStatsFn func() ([]zfs.PoolKernelStat, error) // procfs pool state/I/O source
|
||||||
|
poolStatusesFn func() ([]zfs.PoolStatus, error) // scrub/vdev detail source
|
||||||
|
|
||||||
|
poolData []zfs.PoolStat // cached pool inventory (TTL below)
|
||||||
|
lastPoolStats time.Time
|
||||||
|
kernelSamples map[string]poolKernelSample
|
||||||
|
|
||||||
|
datasetUsage map[string]zfsDatasetUsage // mountpoint -> usage
|
||||||
|
lastUsageRefresh time.Time
|
||||||
|
|
||||||
|
// Detail data (pools, vdevs, scrub, datasets) is cached and refreshed on
|
||||||
|
// an interval. Accessed from handler goroutines, so it is mutex-protected.
|
||||||
|
detailMu sync.Mutex
|
||||||
|
detail *zfsentity.ZfsData
|
||||||
|
lastDetailRefresh time.Time
|
||||||
|
detailInterval time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// newZfsManager creates a ZfsManager wired to the system's ZFS utilities.
|
||||||
|
func newZfsManager() *ZfsManager {
|
||||||
|
return &ZfsManager{
|
||||||
|
poolStatsFn: zfs.PoolStats,
|
||||||
|
datasetsFn: zfs.Datasets,
|
||||||
|
kernelStatsFn: zfs.PoolKernelStats,
|
||||||
|
poolStatusesFn: zfs.PoolStatuses,
|
||||||
|
detailInterval: time.Hour,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update refreshes systemStats.ZfsPools with the latest pool data. I/O
|
||||||
|
// throughput and health come from inexpensive kernel kstats on Linux. Pool
|
||||||
|
// capacity and dataset usage come from separately cached utility calls. It is
|
||||||
|
// a no-op when ZFS is absent.
|
||||||
|
func (zm *ZfsManager) Update(systemStats *system.Stats) {
|
||||||
|
pools := zm.poolStats()
|
||||||
|
if len(pools) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
kernelStats, ioRates := zm.kernelStats()
|
||||||
|
|
||||||
|
if systemStats.ZfsPools == nil {
|
||||||
|
systemStats.ZfsPools = make(map[string]*system.ZfsPool, len(pools))
|
||||||
|
}
|
||||||
|
for i := range pools {
|
||||||
|
pool := &pools[i]
|
||||||
|
// Full precision, matching the dataset values below; the frontend
|
||||||
|
// formats any magnitude.
|
||||||
|
stats := &system.ZfsPool{
|
||||||
|
Total: float64(pool.Size) / (1024 * 1024 * 1024),
|
||||||
|
Used: float64(pool.Alloc) / (1024 * 1024 * 1024),
|
||||||
|
Health: pool.Health,
|
||||||
|
}
|
||||||
|
if kernel, exists := kernelStats[pool.Name]; exists && kernel.Health != "" {
|
||||||
|
stats.Health = kernel.Health
|
||||||
|
}
|
||||||
|
if io, exists := ioRates[pool.Name]; exists {
|
||||||
|
stats.ReadBytes = io.NRead
|
||||||
|
stats.WriteBytes = io.NWrite
|
||||||
|
}
|
||||||
|
slog.Debug("ZFS pool sample", "pool", pool.Name, "health", stats.Health, "used_gb", stats.Used, "read_bps", stats.ReadBytes, "write_bps", stats.WriteBytes)
|
||||||
|
systemStats.ZfsPools[pool.Name] = stats
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// poolStats returns the cached pool inventory, re-running `zpool list` at most
|
||||||
|
// every poolStatsRefreshInterval. On failure the previous inventory is
|
||||||
|
// retained and the refresh is retried on the next cadence.
|
||||||
|
func (zm *ZfsManager) poolStats() []zfs.PoolStat {
|
||||||
|
if zm.lastPoolStats.IsZero() || time.Since(zm.lastPoolStats) >= poolStatsRefreshInterval {
|
||||||
|
pools, err := zm.poolStatsFn()
|
||||||
|
if err != nil {
|
||||||
|
slog.Debug("ZFS pool stats unavailable", "err", err)
|
||||||
|
} else {
|
||||||
|
zm.poolData = pools
|
||||||
|
}
|
||||||
|
zm.lastPoolStats = time.Now()
|
||||||
|
}
|
||||||
|
return zm.poolData
|
||||||
|
}
|
||||||
|
|
||||||
|
// kernelStats reads cumulative pool counters and converts them to per-second
|
||||||
|
// rates. Counter decreases indicate a pool export/import and reset the
|
||||||
|
// baseline instead of producing an underflow spike.
|
||||||
|
func (zm *ZfsManager) kernelStats() (map[string]zfs.PoolKernelStat, map[string]zfs.PoolIoStats) {
|
||||||
|
if zm.kernelStatsFn == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
stats, err := zm.kernelStatsFn()
|
||||||
|
if err != nil {
|
||||||
|
slog.Debug("ZFS kernel stats unavailable", "err", err)
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
byName := make(map[string]zfs.PoolKernelStat, len(stats))
|
||||||
|
rates := make(map[string]zfs.PoolIoStats, len(stats))
|
||||||
|
nextSamples := make(map[string]poolKernelSample, len(stats))
|
||||||
|
for _, stat := range stats {
|
||||||
|
byName[stat.Name] = stat
|
||||||
|
if previous, ok := zm.kernelSamples[stat.Name]; ok && now.After(previous.at) &&
|
||||||
|
stat.NRead >= previous.nread && stat.NWrite >= previous.nwrite {
|
||||||
|
seconds := now.Sub(previous.at).Seconds()
|
||||||
|
rates[stat.Name] = zfs.PoolIoStats{
|
||||||
|
NRead: uint64(float64(stat.NRead-previous.nread) / seconds),
|
||||||
|
NWrite: uint64(float64(stat.NWrite-previous.nwrite) / seconds),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
nextSamples[stat.Name] = poolKernelSample{nread: stat.NRead, nwrite: stat.NWrite, at: now}
|
||||||
|
}
|
||||||
|
zm.kernelSamples = nextSamples
|
||||||
|
return byName, rates
|
||||||
|
}
|
||||||
|
|
||||||
|
// refreshDatasetUsage re-runs `zfs list` when the refresh window has elapsed
|
||||||
|
// and rebuilds the mountpoint-keyed usage map.
|
||||||
|
func (zm *ZfsManager) refreshDatasetUsage() {
|
||||||
|
if !zm.lastUsageRefresh.IsZero() && time.Since(zm.lastUsageRefresh) < datasetUsageRefreshInterval {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
datasets, err := zm.datasetsFn()
|
||||||
|
if err != nil {
|
||||||
|
slog.Debug("ZFS dataset usage unavailable", "err", err)
|
||||||
|
} else {
|
||||||
|
usage := make(map[string]zfsDatasetUsage, len(datasets))
|
||||||
|
for _, ds := range datasets {
|
||||||
|
if ds.Mountpoint != "" && ds.Mountpoint != "-" {
|
||||||
|
usage[ds.Mountpoint] = zfsDatasetUsage{used: ds.Used, avail: ds.Avail}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
zm.datasetUsage = usage
|
||||||
|
}
|
||||||
|
zm.lastUsageRefresh = time.Now()
|
||||||
|
}
|
||||||
|
|
||||||
|
// DatasetUsage returns ZFS dataset usage keyed by mountpoint, refreshed at
|
||||||
|
// most every datasetUsageRefreshInterval. On failure the previous map is
|
||||||
|
// retained and a debug log is emitted.
|
||||||
|
func (zm *ZfsManager) DatasetUsage() map[string]zfsDatasetUsage {
|
||||||
|
zm.refreshDatasetUsage()
|
||||||
|
return zm.datasetUsage
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetDetail returns ZFS detail data (pool health, scrub, vdevs, datasets).
|
||||||
|
// Scheduled requests use the cached snapshot until stale; manual requests can
|
||||||
|
// force collection. On failure the previous snapshot is retained.
|
||||||
|
func (zm *ZfsManager) GetDetail(force bool) *zfsentity.ZfsData {
|
||||||
|
zm.detailMu.Lock()
|
||||||
|
defer zm.detailMu.Unlock()
|
||||||
|
|
||||||
|
if force || zm.detail == nil || time.Since(zm.lastDetailRefresh) >= zm.detailInterval {
|
||||||
|
if data, err := zm.collectDetail(zm.detail); err != nil {
|
||||||
|
slog.Debug("ZFS detail collection failed", "err", err)
|
||||||
|
if zm.detail == nil {
|
||||||
|
return &zfsentity.ZfsData{}
|
||||||
|
}
|
||||||
|
return &zfsentity.ZfsData{Pools: zm.detail.Pools}
|
||||||
|
} else {
|
||||||
|
zm.detail = data
|
||||||
|
zm.lastDetailRefresh = time.Now()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if zm.detail == nil {
|
||||||
|
return &zfsentity.ZfsData{}
|
||||||
|
}
|
||||||
|
return zm.detail
|
||||||
|
}
|
||||||
|
|
||||||
|
// collectDetail builds a ZfsData payload from the current system state.
|
||||||
|
func (zm *ZfsManager) collectDetail(previous *zfsentity.ZfsData) (*zfsentity.ZfsData, error) {
|
||||||
|
pools, err := zm.poolStatsFn()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(pools) == 0 {
|
||||||
|
return &zfsentity.ZfsData{Pools: []*zfsentity.PoolDetail{}, Complete: true}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
statuses, statusErr := zm.poolStatusesFn()
|
||||||
|
if statusErr != nil {
|
||||||
|
slog.Debug("ZFS pool status unavailable", "err", statusErr)
|
||||||
|
}
|
||||||
|
datasets, datasetsErr := zm.datasetsFn()
|
||||||
|
if datasetsErr != nil {
|
||||||
|
slog.Debug("ZFS datasets unavailable", "err", datasetsErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
statusByPool := make(map[string]zfs.PoolStatus, len(statuses))
|
||||||
|
for _, st := range statuses {
|
||||||
|
statusByPool[st.Name] = st
|
||||||
|
}
|
||||||
|
|
||||||
|
previousByPool := make(map[string]*zfsentity.PoolDetail)
|
||||||
|
if previous != nil {
|
||||||
|
for _, pool := range previous.Pools {
|
||||||
|
if pool != nil {
|
||||||
|
previousByPool[pool.Name] = pool
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
data := &zfsentity.ZfsData{Pools: make([]*zfsentity.PoolDetail, 0, len(pools)), Complete: true}
|
||||||
|
for i := range pools {
|
||||||
|
p := &pools[i]
|
||||||
|
detail := &zfsentity.PoolDetail{
|
||||||
|
Name: p.Name,
|
||||||
|
Health: p.Health,
|
||||||
|
Size: p.Size,
|
||||||
|
Alloc: p.Alloc,
|
||||||
|
Free: p.Free,
|
||||||
|
}
|
||||||
|
if st, ok := statusByPool[p.Name]; statusErr == nil && ok {
|
||||||
|
if st.Scrub.State != "" && st.Scrub.State != "NONE" {
|
||||||
|
detail.Scrub = &zfsentity.Scrub{
|
||||||
|
State: st.Scrub.State,
|
||||||
|
Progress: st.Scrub.Progress,
|
||||||
|
Errors: st.Scrub.Errors,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, v := range st.Vdevs {
|
||||||
|
detail.Vdevs = append(detail.Vdevs, &zfsentity.Vdev{
|
||||||
|
Name: v.Name,
|
||||||
|
State: v.State,
|
||||||
|
ReadErrs: v.ReadErrs,
|
||||||
|
WriteErrs: v.WriteErrs,
|
||||||
|
ChecksumErrs: v.ChecksumErrs,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if cached := previousByPool[p.Name]; cached != nil {
|
||||||
|
detail.Scrub = cached.Scrub
|
||||||
|
detail.Vdevs = cached.Vdevs
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if datasetsErr == nil {
|
||||||
|
foundDataset := false
|
||||||
|
for _, ds := range datasets {
|
||||||
|
if poolOfDataset(ds.Name) == p.Name {
|
||||||
|
foundDataset = true
|
||||||
|
detail.Datasets = append(detail.Datasets, &zfsentity.Dataset{
|
||||||
|
Name: ds.Name,
|
||||||
|
Used: ds.Used,
|
||||||
|
Avail: ds.Avail,
|
||||||
|
Mountpoint: ds.Mountpoint,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !foundDataset {
|
||||||
|
if cached := previousByPool[p.Name]; cached != nil {
|
||||||
|
detail.Datasets = cached.Datasets
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if cached := previousByPool[p.Name]; cached != nil {
|
||||||
|
detail.Datasets = cached.Datasets
|
||||||
|
}
|
||||||
|
data.Pools = append(data.Pools, detail)
|
||||||
|
}
|
||||||
|
return data, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// poolOfDataset returns the pool name for a dataset name (everything before
|
||||||
|
// the first '/'). Datasets without a separator belong to a pool of the same
|
||||||
|
// name.
|
||||||
|
func poolOfDataset(name string) string {
|
||||||
|
if idx := strings.IndexByte(name, '/'); idx >= 0 {
|
||||||
|
return name[:idx]
|
||||||
|
}
|
||||||
|
return name
|
||||||
|
}
|
||||||
|
|
||||||
|
// ZfsMountpoints returns the set of mountpoints backed by ZFS datasets.
|
||||||
|
func (zm *ZfsManager) ZfsMountpoints() map[string]bool {
|
||||||
|
usage := zm.DatasetUsage()
|
||||||
|
mountpoints := make(map[string]bool, len(usage))
|
||||||
|
for mountpoint := range usage {
|
||||||
|
mountpoints[mountpoint] = true
|
||||||
|
}
|
||||||
|
return mountpoints
|
||||||
|
}
|
||||||
245
agent/zfs_pool_test.go
Normal file
245
agent/zfs_pool_test.go
Normal file
@@ -0,0 +1,245 @@
|
|||||||
|
//go:build testing
|
||||||
|
|
||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/henrygd/beszel/agent/zfs"
|
||||||
|
"github.com/henrygd/beszel/internal/entities/system"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestUpdatePopulatesZfsPools(t *testing.T) {
|
||||||
|
zm := &ZfsManager{}
|
||||||
|
zm.poolStatsFn = func() ([]zfs.PoolStat, error) {
|
||||||
|
return []zfs.PoolStat{{Name: "tank", Size: 23999000000000, Alloc: 12000000000000, Free: 11999000000000, Health: "DEGRADED"}}, nil
|
||||||
|
}
|
||||||
|
zm.datasetsFn = func() ([]zfs.Dataset, error) {
|
||||||
|
return []zfs.Dataset{
|
||||||
|
{Name: "tank/apps", Used: 5000000000000, Avail: 11999000000000, Mountpoint: "/tank/apps"},
|
||||||
|
{Name: "tank/backup", Used: 6000000000000, Avail: 11999000000000, Mountpoint: "/tank/backup"},
|
||||||
|
// Small zvol (Proxmox VM EFI disk): must not round to zero.
|
||||||
|
{Name: "rpool/vm-100-disk-2", Used: 4194304, Avail: 0, Mountpoint: "-"},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
var kernelCalls int
|
||||||
|
zm.kernelStatsFn = func() ([]zfs.PoolKernelStat, error) {
|
||||||
|
kernelCalls++
|
||||||
|
return []zfs.PoolKernelStat{{
|
||||||
|
Name: "tank", Health: "ONLINE",
|
||||||
|
NRead: uint64(kernelCalls-1) * 1250, NWrite: uint64(kernelCalls-1) * 5120,
|
||||||
|
}}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var stats system.Stats
|
||||||
|
// The first kernel sample establishes the cumulative-counter baseline.
|
||||||
|
zm.Update(&stats)
|
||||||
|
zm.kernelSamples["tank"] = poolKernelSample{at: time.Now().Add(-time.Second)}
|
||||||
|
zm.Update(&stats)
|
||||||
|
require.NotNil(t, stats.ZfsPools)
|
||||||
|
require.Contains(t, stats.ZfsPools, "tank")
|
||||||
|
assert.InDelta(t, 22350.8105, stats.ZfsPools["tank"].Total, 0.0001) // Size in GiB
|
||||||
|
assert.InDelta(t, 11175.8709, stats.ZfsPools["tank"].Used, 0.0001) // Alloc in GiB
|
||||||
|
assert.Equal(t, "ONLINE", stats.ZfsPools["tank"].Health)
|
||||||
|
assert.InDelta(t, 1250, stats.ZfsPools["tank"].ReadBytes, 5)
|
||||||
|
assert.InDelta(t, 5120, stats.ZfsPools["tank"].WriteBytes, 5)
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestUpdateKernelStatsMissing verifies pools without a kernel sample report zero
|
||||||
|
// I/O instead of erroring.
|
||||||
|
func TestUpdateKernelStatsMissing(t *testing.T) {
|
||||||
|
zm := &ZfsManager{}
|
||||||
|
zm.poolStatsFn = func() ([]zfs.PoolStat, error) {
|
||||||
|
return []zfs.PoolStat{{Name: "tank", Size: 1, Alloc: 1, Health: "ONLINE"}}, nil
|
||||||
|
}
|
||||||
|
zm.datasetsFn = func() ([]zfs.Dataset, error) { return nil, nil }
|
||||||
|
zm.kernelStatsFn = func() ([]zfs.PoolKernelStat, error) {
|
||||||
|
return nil, zfs.ErrNoZfs
|
||||||
|
}
|
||||||
|
|
||||||
|
var stats system.Stats
|
||||||
|
zm.Update(&stats)
|
||||||
|
require.NotNil(t, stats.ZfsPools)
|
||||||
|
assert.Equal(t, uint64(0), stats.ZfsPools["tank"].ReadBytes)
|
||||||
|
assert.Equal(t, uint64(0), stats.ZfsPools["tank"].WriteBytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUpdateKernelCounterReset(t *testing.T) {
|
||||||
|
zm := &ZfsManager{}
|
||||||
|
zm.poolStatsFn = func() ([]zfs.PoolStat, error) {
|
||||||
|
return []zfs.PoolStat{{Name: "tank", Health: "ONLINE"}}, nil
|
||||||
|
}
|
||||||
|
zm.datasetsFn = func() ([]zfs.Dataset, error) { return nil, nil }
|
||||||
|
zm.kernelSamples = map[string]poolKernelSample{
|
||||||
|
"tank": {nread: 100, nwrite: 200, at: time.Now().Add(-time.Second)},
|
||||||
|
}
|
||||||
|
zm.kernelStatsFn = func() ([]zfs.PoolKernelStat, error) {
|
||||||
|
return []zfs.PoolKernelStat{{Name: "tank", Health: "ONLINE", NRead: 10, NWrite: 20}}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var stats system.Stats
|
||||||
|
zm.Update(&stats)
|
||||||
|
assert.Equal(t, uint64(0), stats.ZfsPools["tank"].ReadBytes)
|
||||||
|
assert.Equal(t, uint64(0), stats.ZfsPools["tank"].WriteBytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUpdateNoZfs(t *testing.T) {
|
||||||
|
zm := &ZfsManager{}
|
||||||
|
calls := 0
|
||||||
|
zm.poolStatsFn = func() ([]zfs.PoolStat, error) {
|
||||||
|
calls++
|
||||||
|
return nil, zfs.ErrNoZfs
|
||||||
|
}
|
||||||
|
|
||||||
|
var stats system.Stats
|
||||||
|
zm.Update(&stats)
|
||||||
|
zm.Update(&stats)
|
||||||
|
assert.Nil(t, stats.ZfsPools)
|
||||||
|
assert.Equal(t, 1, calls, "failed pool discovery should be cached until the next refresh interval")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUpdateEmptyPools(t *testing.T) {
|
||||||
|
zm := &ZfsManager{}
|
||||||
|
calls := 0
|
||||||
|
zm.poolStatsFn = func() ([]zfs.PoolStat, error) {
|
||||||
|
calls++
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var stats system.Stats
|
||||||
|
zm.Update(&stats)
|
||||||
|
zm.Update(&stats)
|
||||||
|
assert.Nil(t, stats.ZfsPools)
|
||||||
|
assert.Equal(t, 1, calls, "an empty pool inventory should be cached until the next refresh interval")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDatasetUsage(t *testing.T) {
|
||||||
|
zm := &ZfsManager{}
|
||||||
|
calls := 0
|
||||||
|
zm.datasetsFn = func() ([]zfs.Dataset, error) {
|
||||||
|
calls++
|
||||||
|
return []zfs.Dataset{
|
||||||
|
{Name: "tank", Used: 12000000000000, Avail: 11999000000000, Mountpoint: "/tank"},
|
||||||
|
{Name: "tank/apps", Used: 1000000000000, Avail: 11999000000000, Mountpoint: "/tank/apps"},
|
||||||
|
{Name: "rpool", Used: 900000000000, Avail: 300000000000, Mountpoint: "-"}, // zvol/unmounted: excluded
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
usage := zm.DatasetUsage()
|
||||||
|
require.Len(t, usage, 2)
|
||||||
|
assert.Equal(t, zfsDatasetUsage{used: 12000000000000, avail: 11999000000000}, usage["/tank"])
|
||||||
|
assert.Equal(t, zfsDatasetUsage{used: 1000000000000, avail: 11999000000000}, usage["/tank/apps"])
|
||||||
|
assert.Equal(t, 1, calls)
|
||||||
|
|
||||||
|
// Second call within the refresh window must not re-run the collector.
|
||||||
|
zm.DatasetUsage()
|
||||||
|
assert.Equal(t, 1, calls)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDatasetUsageRefreshOnErrorKeepsPrevious(t *testing.T) {
|
||||||
|
zm := &ZfsManager{}
|
||||||
|
zm.datasetsFn = func() ([]zfs.Dataset, error) {
|
||||||
|
return []zfs.Dataset{{Name: "tank", Used: 1, Avail: 1, Mountpoint: "/tank"}}, nil
|
||||||
|
}
|
||||||
|
assert.Len(t, zm.DatasetUsage(), 1)
|
||||||
|
|
||||||
|
// Force refresh window expiry, then a failing collector.
|
||||||
|
zm.lastUsageRefresh = time.Now().Add(-10 * time.Minute)
|
||||||
|
zm.datasetsFn = func() ([]zfs.Dataset, error) {
|
||||||
|
return nil, zfs.ErrNoZfs
|
||||||
|
}
|
||||||
|
usage := zm.DatasetUsage()
|
||||||
|
assert.Len(t, usage, 1, "previous usage should be retained on error")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetDetailForceRefresh(t *testing.T) {
|
||||||
|
zm := &ZfsManager{detailInterval: time.Hour}
|
||||||
|
poolCalls := 0
|
||||||
|
zm.poolStatsFn = func() ([]zfs.PoolStat, error) {
|
||||||
|
poolCalls++
|
||||||
|
return []zfs.PoolStat{{Name: "tank", Alloc: uint64(poolCalls)}}, nil
|
||||||
|
}
|
||||||
|
zm.poolStatusesFn = func() ([]zfs.PoolStatus, error) { return nil, nil }
|
||||||
|
zm.datasetsFn = func() ([]zfs.Dataset, error) { return nil, nil }
|
||||||
|
|
||||||
|
first := zm.GetDetail(false)
|
||||||
|
assert.True(t, first.Complete)
|
||||||
|
require.Len(t, first.Pools, 1)
|
||||||
|
assert.Equal(t, uint64(1), first.Pools[0].Alloc)
|
||||||
|
|
||||||
|
cached := zm.GetDetail(false)
|
||||||
|
require.Len(t, cached.Pools, 1)
|
||||||
|
assert.Equal(t, uint64(1), cached.Pools[0].Alloc)
|
||||||
|
assert.Equal(t, 1, poolCalls)
|
||||||
|
|
||||||
|
refreshed := zm.GetDetail(true)
|
||||||
|
assert.True(t, refreshed.Complete)
|
||||||
|
require.Len(t, refreshed.Pools, 1)
|
||||||
|
assert.Equal(t, uint64(2), refreshed.Pools[0].Alloc)
|
||||||
|
assert.Equal(t, 2, poolCalls)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetDetailSuccessfulEmptyInventoryClearsCache(t *testing.T) {
|
||||||
|
zm := &ZfsManager{detailInterval: time.Hour}
|
||||||
|
zm.poolStatsFn = func() ([]zfs.PoolStat, error) {
|
||||||
|
return []zfs.PoolStat{{Name: "tank"}}, nil
|
||||||
|
}
|
||||||
|
zm.poolStatusesFn = func() ([]zfs.PoolStatus, error) { return nil, nil }
|
||||||
|
zm.datasetsFn = func() ([]zfs.Dataset, error) { return nil, nil }
|
||||||
|
|
||||||
|
require.Len(t, zm.GetDetail(false).Pools, 1)
|
||||||
|
zm.poolStatsFn = func() ([]zfs.PoolStat, error) { return nil, nil }
|
||||||
|
empty := zm.GetDetail(true)
|
||||||
|
assert.True(t, empty.Complete)
|
||||||
|
assert.Empty(t, empty.Pools)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetDetailFailureReturnsIncompleteCachedInventory(t *testing.T) {
|
||||||
|
zm := &ZfsManager{detailInterval: time.Hour}
|
||||||
|
zm.poolStatsFn = func() ([]zfs.PoolStat, error) {
|
||||||
|
return []zfs.PoolStat{{Name: "tank"}}, nil
|
||||||
|
}
|
||||||
|
zm.poolStatusesFn = func() ([]zfs.PoolStatus, error) {
|
||||||
|
return []zfs.PoolStatus{{Name: "tank", Vdevs: []zfs.VdevStatus{{Name: "mirror-0"}}}}, nil
|
||||||
|
}
|
||||||
|
zm.datasetsFn = func() ([]zfs.Dataset, error) {
|
||||||
|
return []zfs.Dataset{{Name: "tank/data"}}, nil
|
||||||
|
}
|
||||||
|
first := zm.GetDetail(false)
|
||||||
|
require.True(t, first.Complete)
|
||||||
|
require.Len(t, first.Pools[0].Vdevs, 1)
|
||||||
|
require.Len(t, first.Pools[0].Datasets, 1)
|
||||||
|
|
||||||
|
zm.poolStatusesFn = func() ([]zfs.PoolStatus, error) { return nil, zfs.ErrNoZfs }
|
||||||
|
zm.datasetsFn = func() ([]zfs.Dataset, error) { return nil, zfs.ErrNoZfs }
|
||||||
|
partial := zm.GetDetail(true)
|
||||||
|
require.True(t, partial.Complete)
|
||||||
|
require.Len(t, partial.Pools[0].Vdevs, 1)
|
||||||
|
require.Len(t, partial.Pools[0].Datasets, 1)
|
||||||
|
|
||||||
|
zm.poolStatsFn = func() ([]zfs.PoolStat, error) { return nil, zfs.ErrNoZfs }
|
||||||
|
lastSuccessfulRefresh := zm.lastDetailRefresh
|
||||||
|
failed := zm.GetDetail(true)
|
||||||
|
assert.False(t, failed.Complete)
|
||||||
|
require.Len(t, failed.Pools, 1)
|
||||||
|
assert.Equal(t, "tank", failed.Pools[0].Name)
|
||||||
|
assert.Equal(t, lastSuccessfulRefresh, zm.lastDetailRefresh)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestZfsMountpoints(t *testing.T) {
|
||||||
|
zm := &ZfsManager{}
|
||||||
|
zm.datasetsFn = func() ([]zfs.Dataset, error) {
|
||||||
|
return []zfs.Dataset{
|
||||||
|
{Name: "tank", Mountpoint: "/tank"},
|
||||||
|
{Name: "rpool/ROOT/pve-1", Mountpoint: "/"},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
mountpoints := zm.ZfsMountpoints()
|
||||||
|
assert.Len(t, mountpoints, 2)
|
||||||
|
assert.True(t, mountpoints["/tank"])
|
||||||
|
assert.True(t, mountpoints["/"])
|
||||||
|
}
|
||||||
@@ -6,7 +6,7 @@ import "github.com/blang/semver"
|
|||||||
|
|
||||||
const (
|
const (
|
||||||
// Version is the current version of the application.
|
// Version is the current version of the application.
|
||||||
Version = "0.18.8"
|
Version = "0.18.9"
|
||||||
// AppName is the name of the application.
|
// AppName is the name of the application.
|
||||||
AppName = "beszel"
|
AppName = "beszel"
|
||||||
)
|
)
|
||||||
@@ -16,3 +16,6 @@ var MinVersionCbor = semver.MustParse("0.12.0")
|
|||||||
|
|
||||||
// MinVersionAgentResponse is the minimum supported version for AgentResponse compatibility.
|
// MinVersionAgentResponse is the minimum supported version for AgentResponse compatibility.
|
||||||
var MinVersionAgentResponse = semver.MustParse("0.13.0")
|
var MinVersionAgentResponse = semver.MustParse("0.13.0")
|
||||||
|
|
||||||
|
// MinVersionZfsData is the minimum agent version that supports ZFS detail requests.
|
||||||
|
var MinVersionZfsData = semver.MustParse("0.18.9")
|
||||||
|
|||||||
@@ -57,12 +57,18 @@ type SystemAlertStats struct {
|
|||||||
Battery [2]uint8 `json:"bat"`
|
Battery [2]uint8 `json:"bat"`
|
||||||
Batteries map[string]uint8 `json:"bats"`
|
Batteries map[string]uint8 `json:"bats"`
|
||||||
ExtraFs map[string]SystemAlertFsStats `json:"efs"`
|
ExtraFs map[string]SystemAlertFsStats `json:"efs"`
|
||||||
|
ZfsPools map[string]SystemAlertZfsPool `json:"z"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type SystemAlertGPUData struct {
|
type SystemAlertGPUData struct {
|
||||||
Usage float64 `json:"u"`
|
Usage float64 `json:"u"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type SystemAlertZfsPool struct {
|
||||||
|
Total float64 `json:"d"`
|
||||||
|
Used float64 `json:"du"`
|
||||||
|
}
|
||||||
|
|
||||||
type SystemAlertData struct {
|
type SystemAlertData struct {
|
||||||
systemRecord *core.Record
|
systemRecord *core.Record
|
||||||
alertData CachedAlertData
|
alertData CachedAlertData
|
||||||
@@ -111,6 +117,9 @@ func (am *AlertManager) bindEvents() {
|
|||||||
am.hub.OnRecordAfterUpdateSuccess("alerts").BindFunc(updateHistoryOnAlertUpdate)
|
am.hub.OnRecordAfterUpdateSuccess("alerts").BindFunc(updateHistoryOnAlertUpdate)
|
||||||
am.hub.OnRecordAfterDeleteSuccess("alerts").BindFunc(resolveHistoryOnAlertDelete)
|
am.hub.OnRecordAfterDeleteSuccess("alerts").BindFunc(resolveHistoryOnAlertDelete)
|
||||||
am.hub.OnRecordAfterUpdateSuccess("smart_devices").BindFunc(am.handleSmartDeviceAlert)
|
am.hub.OnRecordAfterUpdateSuccess("smart_devices").BindFunc(am.handleSmartDeviceAlert)
|
||||||
|
am.hub.OnRecordAfterCreateSuccess("zfs_pools").BindFunc(am.handleZfsPoolCreateAlert)
|
||||||
|
am.hub.OnRecordAfterUpdateSuccess("zfs_pools").BindFunc(am.handleZfsPoolAlert)
|
||||||
|
am.hub.OnRecordAfterDeleteSuccess("zfs_pools").BindFunc(resolveZfsPoolHistoryOnDelete)
|
||||||
|
|
||||||
am.hub.OnServe().BindFunc(func(e *core.ServeEvent) error {
|
am.hub.OnServe().BindFunc(func(e *core.ServeEvent) error {
|
||||||
// Populate all alerts into cache on startup
|
// Populate all alerts into cache on startup
|
||||||
|
|||||||
@@ -44,6 +44,14 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
|
|||||||
maxUsedPct = usedPct
|
maxUsedPct = usedPct
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
for _, pool := range data.Stats.ZfsPools {
|
||||||
|
if pool != nil && pool.Total > 0 {
|
||||||
|
usedPct := pool.Used / pool.Total * 100
|
||||||
|
if usedPct > maxUsedPct {
|
||||||
|
maxUsedPct = usedPct
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
val = maxUsedPct
|
val = maxUsedPct
|
||||||
case "Temperature":
|
case "Temperature":
|
||||||
if data.Info.DashboardTemp < 1 {
|
if data.Info.DashboardTemp < 1 {
|
||||||
@@ -208,6 +216,16 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
|
|||||||
alert.mapSums[key] += float32(fs.DiskUsed / fs.DiskTotal * 100)
|
alert.mapSums[key] += float32(fs.DiskUsed / fs.DiskTotal * 100)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// add zfs pool usage from historical record
|
||||||
|
for key, pool := range stats.ZfsPools {
|
||||||
|
if pool.Total > 0 {
|
||||||
|
zfsKey := zfsDiskAlertKey(key)
|
||||||
|
if _, ok := alert.mapSums[zfsKey]; !ok {
|
||||||
|
alert.mapSums[zfsKey] = 0.0
|
||||||
|
}
|
||||||
|
alert.mapSums[zfsKey] += float32(pool.Used / pool.Total * 100)
|
||||||
|
}
|
||||||
|
}
|
||||||
case "Temperature":
|
case "Temperature":
|
||||||
if alert.mapSums == nil {
|
if alert.mapSums == nil {
|
||||||
alert.mapSums = make(map[string]float32, len(stats.Temperatures))
|
alert.mapSums = make(map[string]float32, len(stats.Temperatures))
|
||||||
@@ -255,7 +273,7 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
|
|||||||
sumPct := float32(value)
|
sumPct := float32(value)
|
||||||
if sumPct > maxPct {
|
if sumPct > maxPct {
|
||||||
maxPct = sumPct
|
maxPct = sumPct
|
||||||
alert.descriptor = fmt.Sprintf("Usage of %s", key)
|
alert.descriptor = diskAlertDescriptor(key)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
alert.val = float64(maxPct / float32(alert.count))
|
alert.val = float64(maxPct / float32(alert.count))
|
||||||
@@ -301,6 +319,17 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func zfsDiskAlertKey(poolName string) string {
|
||||||
|
return "zfs:" + poolName
|
||||||
|
}
|
||||||
|
|
||||||
|
func diskAlertDescriptor(key string) string {
|
||||||
|
if poolName, ok := strings.CutPrefix(key, "zfs:"); ok {
|
||||||
|
return fmt.Sprintf("Usage of ZFS pool %s", poolName)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("Usage of %s", key)
|
||||||
|
}
|
||||||
|
|
||||||
func hasRepresentativeBattery(legacy [2]uint8, batteries map[string]uint8) bool {
|
func hasRepresentativeBattery(legacy [2]uint8, batteries map[string]uint8) bool {
|
||||||
return legacy != [2]uint8{} || len(batteries) > 0
|
return legacy != [2]uint8{} || len(batteries) > 0
|
||||||
}
|
}
|
||||||
|
|||||||
142
internal/alerts/alerts_zfs.go
Normal file
142
internal/alerts/alerts_zfs.go
Normal file
@@ -0,0 +1,142 @@
|
|||||||
|
package alerts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/pocketbase/dbx"
|
||||||
|
"github.com/pocketbase/pocketbase/core"
|
||||||
|
)
|
||||||
|
|
||||||
|
// handleZfsPoolAlert sends alerts when a ZFS pool health state worsens and
|
||||||
|
// resolves the alert history entry when the pool recovers. Like the SMART
|
||||||
|
// hook, this is automatic and does not require user opt-in.
|
||||||
|
func (am *AlertManager) handleZfsPoolAlert(e *core.RecordEvent) error {
|
||||||
|
return am.handleZfsPoolHealthAlert(e, e.Record.Original().GetString("health"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (am *AlertManager) handleZfsPoolCreateAlert(e *core.RecordEvent) error {
|
||||||
|
return am.handleZfsPoolHealthAlert(e, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (am *AlertManager) handleZfsPoolHealthAlert(e *core.RecordEvent, oldHealth string) error {
|
||||||
|
newHealth := e.Record.GetString("health")
|
||||||
|
oldSeverity := zfsPoolSeverity(oldHealth)
|
||||||
|
newSeverity := zfsPoolSeverity(newHealth)
|
||||||
|
|
||||||
|
systemID := e.Record.GetString("system")
|
||||||
|
if systemID == "" {
|
||||||
|
return e.Next()
|
||||||
|
}
|
||||||
|
|
||||||
|
systemRecord, err := e.App.FindRecordById("systems", systemID)
|
||||||
|
if err != nil {
|
||||||
|
e.App.Logger().Error("Failed to find system for ZFS alert", "err", err, "systemID", systemID)
|
||||||
|
return e.Next()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Pool recovered to a healthy state: resolve any open history entries.
|
||||||
|
if newSeverity == 1 && oldSeverity > 1 {
|
||||||
|
resolveAllAlertHistoryRecords(e.App, e.Record.Id)
|
||||||
|
return e.Next()
|
||||||
|
}
|
||||||
|
|
||||||
|
if !shouldSendZfsPoolAlert(oldSeverity, newSeverity) {
|
||||||
|
return e.Next()
|
||||||
|
}
|
||||||
|
|
||||||
|
systemName := systemRecord.GetString("name")
|
||||||
|
poolName := e.Record.GetString("name")
|
||||||
|
|
||||||
|
title := fmt.Sprintf("ZFS pool %s on %s: %s", newHealth, systemName, poolName)
|
||||||
|
message := fmt.Sprintf("ZFS pool %s (%s) was first observed as %s", poolName, systemName, newHealth)
|
||||||
|
if oldSeverity > 0 {
|
||||||
|
message = fmt.Sprintf("ZFS pool %s (%s) health changed from %s to %s", poolName, systemName, oldHealth, newHealth)
|
||||||
|
}
|
||||||
|
|
||||||
|
userIDs := systemRecord.GetStringSlice("users")
|
||||||
|
if len(userIDs) == 0 {
|
||||||
|
return e.Next()
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, userID := range userIDs {
|
||||||
|
if err := am.SendAlert(AlertMessageData{
|
||||||
|
UserID: userID,
|
||||||
|
SystemID: systemID,
|
||||||
|
Title: title,
|
||||||
|
Message: message,
|
||||||
|
Link: am.hub.MakeLink("system", systemID),
|
||||||
|
LinkText: "View " + systemName,
|
||||||
|
}); err != nil {
|
||||||
|
e.App.Logger().Error("Failed to send ZFS alert", "err", err, "userID", userID)
|
||||||
|
}
|
||||||
|
_ = createZfsPoolHistoryRecord(e.App, userID, systemID, e.Record.Id, poolName)
|
||||||
|
}
|
||||||
|
|
||||||
|
return e.Next()
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolveZfsPoolHistoryOnDelete resolves open alert history entries when a
|
||||||
|
// pool record is deleted (manually or because the pool disappeared), so the
|
||||||
|
// UI does not keep showing an ongoing alert for a pool that no longer exists.
|
||||||
|
func resolveZfsPoolHistoryOnDelete(e *core.RecordEvent) error {
|
||||||
|
resolveAllAlertHistoryRecords(e.App, e.Record.Id)
|
||||||
|
return e.Next()
|
||||||
|
}
|
||||||
|
|
||||||
|
// shouldSendZfsPoolAlert reports whether a health transition warrants an alert.
|
||||||
|
// First observations of unhealthy pools and worsening transitions are reported.
|
||||||
|
func shouldSendZfsPoolAlert(oldSeverity, newSeverity int) bool {
|
||||||
|
return newSeverity > 1 && (oldSeverity == 0 || newSeverity > oldSeverity)
|
||||||
|
}
|
||||||
|
|
||||||
|
// zfsPoolSeverity ranks pool health states: healthy (1), degraded (2),
|
||||||
|
// failed/unavailable (3), unknown (0).
|
||||||
|
func zfsPoolSeverity(health string) int {
|
||||||
|
switch health {
|
||||||
|
case "ONLINE":
|
||||||
|
return 1
|
||||||
|
case "DEGRADED":
|
||||||
|
return 2
|
||||||
|
case "FAULTED", "OFFLINE", "UNAVAIL", "REMOVED", "SUSPENDED":
|
||||||
|
return 3
|
||||||
|
default:
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// createZfsPoolHistoryRecord logs a pool health alert in the alerts history so
|
||||||
|
// it is visible in the UI without creating an editable alert configuration.
|
||||||
|
func createZfsPoolHistoryRecord(app core.App, userID, systemID, alertID, poolName string) error {
|
||||||
|
collection, err := app.FindCachedCollectionByNameOrId("alerts_history")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
record := core.NewRecord(collection)
|
||||||
|
record.Set("user", userID)
|
||||||
|
record.Set("system", systemID)
|
||||||
|
record.Set("alert_id", alertID)
|
||||||
|
record.Set("name", "ZFS Pool: "+poolName)
|
||||||
|
return app.Save(record)
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolveAllAlertHistoryRecords resolves every open history entry for an alert
|
||||||
|
// record id (one per system user).
|
||||||
|
func resolveAllAlertHistoryRecords(app core.App, alertID string) {
|
||||||
|
records, err := app.FindRecordsByFilter(
|
||||||
|
"alerts_history",
|
||||||
|
"alert_id={:alert_id} && resolved=null",
|
||||||
|
"", 0, 0,
|
||||||
|
dbx.Params{"alert_id": alertID},
|
||||||
|
)
|
||||||
|
if err != nil || len(records) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
now := time.Now().UTC()
|
||||||
|
for _, record := range records {
|
||||||
|
record.Set("resolved", now)
|
||||||
|
if err := app.Save(record); err != nil {
|
||||||
|
app.Logger().Error("Failed to resolve ZFS alert history", "err", err, "recordId", record.Id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
145
internal/alerts/alerts_zfs_disk_test.go
Normal file
145
internal/alerts/alerts_zfs_disk_test.go
Normal file
@@ -0,0 +1,145 @@
|
|||||||
|
//go:build testing
|
||||||
|
|
||||||
|
package alerts_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/henrygd/beszel/internal/entities/system"
|
||||||
|
beszelTests "github.com/henrygd/beszel/internal/tests"
|
||||||
|
|
||||||
|
"github.com/pocketbase/dbx"
|
||||||
|
"github.com/pocketbase/pocketbase/tools/types"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestDiskAlertZfsPoolMultiMinute verifies that ZFS pool usage participates in
|
||||||
|
// the Disk threshold alert using historical per-minute values, mirroring the
|
||||||
|
// extra-filesystem behavior.
|
||||||
|
func TestDiskAlertZfsPoolMultiMinute(t *testing.T) {
|
||||||
|
hub, user := beszelTests.GetHubWithUser(t)
|
||||||
|
defer hub.Cleanup()
|
||||||
|
|
||||||
|
systems, err := beszelTests.CreateSystems(hub, 1, user.Id, "up")
|
||||||
|
require.NoError(t, err)
|
||||||
|
systemRecord := systems[0]
|
||||||
|
|
||||||
|
diskAlert, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{
|
||||||
|
"name": "Disk",
|
||||||
|
"system": systemRecord.Id,
|
||||||
|
"user": user.Id,
|
||||||
|
"value": 80, // threshold: 80%
|
||||||
|
"min": 2, // requires historical averaging
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
am := hub.GetAlertManager()
|
||||||
|
now := time.Now().UTC()
|
||||||
|
|
||||||
|
poolHigh := map[string]*system.ZfsPool{
|
||||||
|
"tank": {Total: 1000, Used: 920}, // 92% - above threshold
|
||||||
|
}
|
||||||
|
|
||||||
|
recordTimes := []time.Duration{
|
||||||
|
-180 * time.Second,
|
||||||
|
-90 * time.Second,
|
||||||
|
-60 * time.Second,
|
||||||
|
-30 * time.Second,
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, offset := range recordTimes {
|
||||||
|
stats := system.Stats{
|
||||||
|
DiskPct: 30, // root disk at 30% - below threshold
|
||||||
|
ZfsPools: poolHigh,
|
||||||
|
}
|
||||||
|
statsJSON, _ := json.Marshal(stats)
|
||||||
|
|
||||||
|
recordTime := now.Add(offset)
|
||||||
|
record, err := beszelTests.CreateRecord(hub, "system_stats", map[string]any{
|
||||||
|
"system": systemRecord.Id,
|
||||||
|
"type": "1m",
|
||||||
|
"stats": string(statsJSON),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
record.SetRaw("created", recordTime.Format(types.DefaultDateLayout))
|
||||||
|
err = hub.SaveNoValidate(record)
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
combinedDataHigh := &system.CombinedData{
|
||||||
|
Stats: system.Stats{
|
||||||
|
DiskPct: 30,
|
||||||
|
ZfsPools: poolHigh,
|
||||||
|
},
|
||||||
|
Info: system.Info{
|
||||||
|
DiskPct: 30,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
systemRecord.Set("updated", now)
|
||||||
|
err = hub.SaveNoValidate(systemRecord)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
err = am.HandleSystemAlerts(systemRecord, combinedDataHigh)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
time.Sleep(20 * time.Millisecond)
|
||||||
|
|
||||||
|
diskAlert, err = hub.FindFirstRecordByFilter("alerts", "id={:id}", dbx.Params{"id": diskAlert.Id})
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.True(t, diskAlert.GetBool("triggered"),
|
||||||
|
"Alert should be triggered when ZFS pool average (92%%) exceeds threshold (80%%)")
|
||||||
|
|
||||||
|
// --- Resolution: pool drops to 50%, alert should resolve ---
|
||||||
|
|
||||||
|
poolLow := map[string]*system.ZfsPool{
|
||||||
|
"tank": {Total: 1000, Used: 500}, // 50% - below threshold
|
||||||
|
}
|
||||||
|
|
||||||
|
newNow := now.Add(2 * time.Minute)
|
||||||
|
for _, offset := range recordTimes {
|
||||||
|
stats := system.Stats{
|
||||||
|
DiskPct: 30,
|
||||||
|
ZfsPools: poolLow,
|
||||||
|
}
|
||||||
|
statsJSON, _ := json.Marshal(stats)
|
||||||
|
|
||||||
|
recordTime := newNow.Add(offset)
|
||||||
|
record, err := beszelTests.CreateRecord(hub, "system_stats", map[string]any{
|
||||||
|
"system": systemRecord.Id,
|
||||||
|
"type": "1m",
|
||||||
|
"stats": string(statsJSON),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
record.SetRaw("created", recordTime.Format(types.DefaultDateLayout))
|
||||||
|
err = hub.SaveNoValidate(record)
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
combinedDataLow := &system.CombinedData{
|
||||||
|
Stats: system.Stats{
|
||||||
|
DiskPct: 30,
|
||||||
|
ZfsPools: poolLow,
|
||||||
|
},
|
||||||
|
Info: system.Info{
|
||||||
|
DiskPct: 30,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
systemRecord.Set("updated", newNow)
|
||||||
|
err = hub.SaveNoValidate(systemRecord)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
err = am.HandleSystemAlerts(systemRecord, combinedDataLow)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
time.Sleep(20 * time.Millisecond)
|
||||||
|
|
||||||
|
diskAlert, err = hub.FindFirstRecordByFilter("alerts", "id={:id}", dbx.Params{"id": diskAlert.Id})
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.False(t, diskAlert.GetBool("triggered"),
|
||||||
|
"Alert should be resolved when ZFS pool average (50%%) drops below threshold (80%%)")
|
||||||
|
}
|
||||||
15
internal/alerts/alerts_zfs_key_test.go
Normal file
15
internal/alerts/alerts_zfs_key_test.go
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
//go:build testing
|
||||||
|
|
||||||
|
package alerts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestZfsDiskAlertKeyIsNamespaced(t *testing.T) {
|
||||||
|
assert.Equal(t, "zfs:tank", zfsDiskAlertKey("tank"))
|
||||||
|
assert.Equal(t, "Usage of ZFS pool tank", diskAlertDescriptor(zfsDiskAlertKey("tank")))
|
||||||
|
assert.Equal(t, "Usage of tank", diskAlertDescriptor("tank"))
|
||||||
|
}
|
||||||
292
internal/alerts/alerts_zfs_test.go
Normal file
292
internal/alerts/alerts_zfs_test.go
Normal file
@@ -0,0 +1,292 @@
|
|||||||
|
//go:build testing
|
||||||
|
|
||||||
|
package alerts_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
beszelTests "github.com/henrygd/beszel/internal/tests"
|
||||||
|
"github.com/pocketbase/pocketbase/core"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestZfsPoolAlertOnlineToDegraded(t *testing.T) {
|
||||||
|
hub, user := beszelTests.GetHubWithUser(t)
|
||||||
|
defer hub.Cleanup()
|
||||||
|
|
||||||
|
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
|
||||||
|
"name": "test-system",
|
||||||
|
"users": []string{user.Id},
|
||||||
|
"host": "127.0.0.1",
|
||||||
|
})
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
|
||||||
|
"system": system.Id,
|
||||||
|
"name": "tank",
|
||||||
|
"health": "ONLINE",
|
||||||
|
})
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
// Re-fetch so PocketBase tracks original values
|
||||||
|
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
pool.Set("health", "DEGRADED")
|
||||||
|
err = hub.Save(pool)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
|
||||||
|
assert.EqualValues(t, 1, hub.TestMailer.TotalSend(), "should have 1 email sent after pool became DEGRADED")
|
||||||
|
lastMessage := hub.TestMailer.LastMessage()
|
||||||
|
assert.Contains(t, lastMessage.Subject, "ZFS pool DEGRADED on test-system")
|
||||||
|
assert.Contains(t, lastMessage.Subject, "tank")
|
||||||
|
assert.Contains(t, lastMessage.Text, "ONLINE to DEGRADED")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestZfsPoolAlertDegradedToFaulted(t *testing.T) {
|
||||||
|
hub, user := beszelTests.GetHubWithUser(t)
|
||||||
|
defer hub.Cleanup()
|
||||||
|
|
||||||
|
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
|
||||||
|
"name": "test-system",
|
||||||
|
"users": []string{user.Id},
|
||||||
|
"host": "127.0.0.1",
|
||||||
|
})
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
|
||||||
|
"system": system.Id,
|
||||||
|
"name": "rpool",
|
||||||
|
"health": "DEGRADED",
|
||||||
|
})
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
pool.Set("health", "FAULTED")
|
||||||
|
err = hub.Save(pool)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
|
||||||
|
assert.EqualValues(t, 2, hub.TestMailer.TotalSend(), "should alert on initial DEGRADED state and later FAULTED transition")
|
||||||
|
lastMessage := hub.TestMailer.LastMessage()
|
||||||
|
assert.Contains(t, lastMessage.Subject, "ZFS pool FAULTED on test-system")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestZfsPoolAlertNoAlertOnRecovery(t *testing.T) {
|
||||||
|
hub, user := beszelTests.GetHubWithUser(t)
|
||||||
|
defer hub.Cleanup()
|
||||||
|
|
||||||
|
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
|
||||||
|
"name": "test-system",
|
||||||
|
"users": []string{user.Id},
|
||||||
|
"host": "127.0.0.1",
|
||||||
|
})
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
|
||||||
|
"system": system.Id,
|
||||||
|
"name": "tank",
|
||||||
|
"health": "DEGRADED",
|
||||||
|
})
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
// Trigger a worsening alert first
|
||||||
|
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
pool.Set("health", "FAULTED")
|
||||||
|
err = hub.Save(pool)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
assert.EqualValues(t, 2, hub.TestMailer.TotalSend(), "expected alerts for initial DEGRADED state and DEGRADED -> FAULTED")
|
||||||
|
|
||||||
|
// Recovery back to ONLINE must not send a new alert
|
||||||
|
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
pool.Set("health", "ONLINE")
|
||||||
|
err = hub.Save(pool)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
assert.EqualValues(t, 2, hub.TestMailer.TotalSend(), "recovery should not send a new alert")
|
||||||
|
|
||||||
|
// And the open history entry should have been resolved
|
||||||
|
history, err := hub.FindRecordsByFilter("alerts_history", "alert_id={:alert_id}", "", 0, 0, map[string]any{"alert_id": pool.Id})
|
||||||
|
assert.NoError(t, err)
|
||||||
|
requireHistoryResolved(t, history)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestZfsPoolAlertUnknownHealthDoesNotResolve(t *testing.T) {
|
||||||
|
hub, user := beszelTests.GetHubWithUser(t)
|
||||||
|
defer hub.Cleanup()
|
||||||
|
|
||||||
|
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
|
||||||
|
"name": "test-system",
|
||||||
|
"users": []string{user.Id},
|
||||||
|
"host": "127.0.0.1",
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
|
||||||
|
"system": system.Id,
|
||||||
|
"name": "tank",
|
||||||
|
"health": "DEGRADED",
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
|
||||||
|
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
|
||||||
|
require.NoError(t, err)
|
||||||
|
pool.Set("health", "")
|
||||||
|
require.NoError(t, hub.Save(pool))
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
|
||||||
|
history, err := hub.FindRecordsByFilter("alerts_history", "alert_id={:alert_id} && resolved=null", "", 0, 0, map[string]any{"alert_id": pool.Id})
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, history, 1, "unknown health must not resolve an active alert")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestZfsPoolAlertUnknownToFaulted(t *testing.T) {
|
||||||
|
hub, user := beszelTests.GetHubWithUser(t)
|
||||||
|
defer hub.Cleanup()
|
||||||
|
|
||||||
|
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
|
||||||
|
"name": "test-system",
|
||||||
|
"users": []string{user.Id},
|
||||||
|
"host": "127.0.0.1",
|
||||||
|
})
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
|
||||||
|
"system": system.Id,
|
||||||
|
"name": "tank",
|
||||||
|
"health": "",
|
||||||
|
})
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
pool.Set("health", "FAULTED")
|
||||||
|
err = hub.Save(pool)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
|
||||||
|
assert.EqualValues(t, 1, hub.TestMailer.TotalSend(), "should alert when a previously unknown pool becomes FAULTED")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestZfsPoolAlertOnInitialUnhealthyState(t *testing.T) {
|
||||||
|
hub, user := beszelTests.GetHubWithUser(t)
|
||||||
|
defer hub.Cleanup()
|
||||||
|
|
||||||
|
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
|
||||||
|
"name": "test-system",
|
||||||
|
"users": []string{user.Id},
|
||||||
|
"host": "127.0.0.1",
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
|
||||||
|
"system": system.Id,
|
||||||
|
"name": "tank",
|
||||||
|
"health": "DEGRADED",
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
|
||||||
|
require.EqualValues(t, 1, hub.TestMailer.TotalSend())
|
||||||
|
assert.Contains(t, hub.TestMailer.LastMessage().Text, "first observed as DEGRADED")
|
||||||
|
|
||||||
|
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, hub.Save(pool))
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
assert.EqualValues(t, 1, hub.TestMailer.TotalSend(), "unchanged unhealthy health must not duplicate alerts")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestZfsPoolAlertWritesHistory(t *testing.T) {
|
||||||
|
hub, user := beszelTests.GetHubWithUser(t)
|
||||||
|
defer hub.Cleanup()
|
||||||
|
|
||||||
|
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
|
||||||
|
"name": "test-system",
|
||||||
|
"users": []string{user.Id},
|
||||||
|
"host": "127.0.0.1",
|
||||||
|
})
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
|
||||||
|
"system": system.Id,
|
||||||
|
"name": "tank",
|
||||||
|
"health": "ONLINE",
|
||||||
|
})
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
pool.Set("health", "FAULTED")
|
||||||
|
err = hub.Save(pool)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
|
||||||
|
history, err := hub.FindRecordsByFilter("alerts_history", "alert_id={:alert_id}", "", 0, 0, map[string]any{"alert_id": pool.Id})
|
||||||
|
assert.NoError(t, err)
|
||||||
|
require.Len(t, history, 1, "expected one history entry per user")
|
||||||
|
assert.Equal(t, "ZFS Pool: tank", history[0].GetString("name"))
|
||||||
|
assert.Equal(t, system.Id, history[0].GetString("system"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestZfsPoolAlertResolvedOnRecordDelete(t *testing.T) {
|
||||||
|
hub, user := beszelTests.GetHubWithUser(t)
|
||||||
|
defer hub.Cleanup()
|
||||||
|
|
||||||
|
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
|
||||||
|
"name": "test-system",
|
||||||
|
"users": []string{user.Id},
|
||||||
|
"host": "127.0.0.1",
|
||||||
|
})
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
|
||||||
|
"system": system.Id,
|
||||||
|
"name": "tank",
|
||||||
|
"health": "ONLINE",
|
||||||
|
})
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
// Trigger an alert so an open history entry exists.
|
||||||
|
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
pool.Set("health", "FAULTED")
|
||||||
|
err = hub.Save(pool)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
|
||||||
|
history, err := hub.FindRecordsByFilter("alerts_history", "alert_id={:alert_id} && resolved=null", "", 0, 0, map[string]any{"alert_id": pool.Id})
|
||||||
|
assert.NoError(t, err)
|
||||||
|
require.Len(t, history, 1, "expected one open history entry")
|
||||||
|
|
||||||
|
// Deleting the pool record must resolve the open entry.
|
||||||
|
err = hub.Delete(pool)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
|
||||||
|
history, err = hub.FindRecordsByFilter("alerts_history", "alert_id={:alert_id}", "", 0, 0, map[string]any{"alert_id": pool.Id})
|
||||||
|
assert.NoError(t, err)
|
||||||
|
require.Len(t, history, 1)
|
||||||
|
requireHistoryResolved(t, history)
|
||||||
|
}
|
||||||
|
|
||||||
|
func requireHistoryResolved(t *testing.T, history []*core.Record) {
|
||||||
|
t.Helper()
|
||||||
|
for _, record := range history {
|
||||||
|
assert.False(t, record.GetDateTime("resolved").Time().IsZero(), "expected history entry to be resolved")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -22,6 +22,8 @@ const (
|
|||||||
GetSmartData
|
GetSmartData
|
||||||
// Request detailed systemd service info from agent
|
// Request detailed systemd service info from agent
|
||||||
GetSystemdInfo
|
GetSystemdInfo
|
||||||
|
// Request ZFS detail data from agent
|
||||||
|
GetZfsData
|
||||||
// Add new actions here...
|
// Add new actions here...
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -64,6 +66,10 @@ type DataRequestOptions struct {
|
|||||||
IncludeDetails bool `cbor:"1,keyasint"`
|
IncludeDetails bool `cbor:"1,keyasint"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ZfsDataRequest struct {
|
||||||
|
Force bool `cbor:"0,keyasint,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
type ContainerLogsRequest struct {
|
type ContainerLogsRequest struct {
|
||||||
ContainerID string `cbor:"0,keyasint"`
|
ContainerID string `cbor:"0,keyasint"`
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ COPY --from=builder /agent /agent
|
|||||||
# AMD GPU name lookup (used by agent on Linux when /usr/share/libdrm/amdgpu.ids is read)
|
# AMD GPU name lookup (used by agent on Linux when /usr/share/libdrm/amdgpu.ids is read)
|
||||||
COPY --from=builder /app/agent/test-data/amdgpu.ids /usr/share/libdrm/amdgpu.ids
|
COPY --from=builder /app/agent/test-data/amdgpu.ids /usr/share/libdrm/amdgpu.ids
|
||||||
|
|
||||||
RUN apk add --no-cache smartmontools
|
RUN apk add --no-cache smartmontools zfs
|
||||||
|
|
||||||
# Ensure data persistence across container recreations
|
# Ensure data persistence across container recreations
|
||||||
VOLUME ["/var/lib/beszel-agent"]
|
VOLUME ["/var/lib/beszel-agent"]
|
||||||
|
|||||||
@@ -33,8 +33,6 @@ type Stats struct {
|
|||||||
MaxNetworkSent float64 `json:"nsm,omitempty" cbor:"-"`
|
MaxNetworkSent float64 `json:"nsm,omitempty" cbor:"-"`
|
||||||
MaxNetworkRecv float64 `json:"nrm,omitempty" cbor:"-"`
|
MaxNetworkRecv float64 `json:"nrm,omitempty" cbor:"-"`
|
||||||
Temperatures map[string]float64 `json:"t,omitempty" cbor:"20,keyasint,omitempty"`
|
Temperatures map[string]float64 `json:"t,omitempty" cbor:"20,keyasint,omitempty"`
|
||||||
Fans map[string]uint16 `json:"f,omitempty" cbor:"36,keyasint,omitempty"`
|
|
||||||
Batteries map[string]uint8 `json:"bats,omitempty" cbor:"37,keyasint,omitempty"`
|
|
||||||
ExtraFs map[string]*FsStats `json:"efs,omitempty" cbor:"21,keyasint,omitempty"`
|
ExtraFs map[string]*FsStats `json:"efs,omitempty" cbor:"21,keyasint,omitempty"`
|
||||||
GPUData map[string]GPUData `json:"g,omitempty" cbor:"22,keyasint,omitempty"`
|
GPUData map[string]GPUData `json:"g,omitempty" cbor:"22,keyasint,omitempty"`
|
||||||
// LoadAvg1 float64 `json:"l1,omitempty" cbor:"23,keyasint,omitempty"`
|
// LoadAvg1 float64 `json:"l1,omitempty" cbor:"23,keyasint,omitempty"`
|
||||||
@@ -52,7 +50,20 @@ type Stats struct {
|
|||||||
CpuCoresUsage Uint8Slice `json:"cpus,omitempty" cbor:"34,keyasint,omitempty"` // per-core busy usage [CPU0..]
|
CpuCoresUsage Uint8Slice `json:"cpus,omitempty" cbor:"34,keyasint,omitempty"` // per-core busy usage [CPU0..]
|
||||||
DiskIoStats [6]float64 `json:"dios,omitzero" cbor:"35,keyasint,omitzero"` // [read time %, write time %, io utilization %, r_await ms, w_await ms, weighted io %]
|
DiskIoStats [6]float64 `json:"dios,omitzero" cbor:"35,keyasint,omitzero"` // [read time %, write time %, io utilization %, r_await ms, w_await ms, weighted io %]
|
||||||
MaxDiskIoStats [6]float64 `json:"diosm,omitzero" cbor:"-"` // max values for DiskIoStats
|
MaxDiskIoStats [6]float64 `json:"diosm,omitzero" cbor:"-"` // max values for DiskIoStats
|
||||||
|
Fans map[string]uint16 `json:"f,omitempty" cbor:"36,keyasint,omitempty"`
|
||||||
|
Batteries map[string]uint8 `json:"bats,omitempty" cbor:"37,keyasint,omitempty"`
|
||||||
|
ZfsPools map[string]*ZfsPool `json:"z,omitempty" cbor:"39,keyasint,omitempty"` // ZFS pool metrics, keyed by pool name
|
||||||
DiskIOTotal [2]uint64 `json:"diot,omitzero" cbor:"38,keyasint,omitzero"` // [total read bytes, total write bytes] cumulative device counters
|
DiskIOTotal [2]uint64 `json:"diot,omitzero" cbor:"38,keyasint,omitzero"` // [total read bytes, total write bytes] cumulative device counters
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// ZfsPool holds per-pool ZFS metrics for a single collection interval.
|
||||||
|
type ZfsPool struct {
|
||||||
|
Total float64 `json:"d" cbor:"0,keyasint"` // total capacity in GiB
|
||||||
|
Used float64 `json:"du" cbor:"1,keyasint"` // allocated in GiB
|
||||||
|
ReadBytes uint64 `json:"rb,omitzero" cbor:"2,keyasint,omitzero"` // read throughput in bytes/s
|
||||||
|
WriteBytes uint64 `json:"wb,omitzero" cbor:"3,keyasint,omitzero"` // write throughput in bytes/s
|
||||||
|
Health string `json:"h,omitempty" cbor:"4,keyasint,omitempty"` // ONLINE, DEGRADED, FAULTED, ...
|
||||||
}
|
}
|
||||||
|
|
||||||
// Uint8Slice wraps []uint8 to customize JSON encoding while keeping CBOR efficient.
|
// Uint8Slice wraps []uint8 to customize JSON encoding while keeping CBOR efficient.
|
||||||
@@ -183,6 +194,7 @@ type Details struct {
|
|||||||
Podman bool `cbor:"8,keyasint,omitempty"`
|
Podman bool `cbor:"8,keyasint,omitempty"`
|
||||||
MemoryTotal uint64 `cbor:"9,keyasint"`
|
MemoryTotal uint64 `cbor:"9,keyasint"`
|
||||||
SmartInterval time.Duration `cbor:"10,keyasint,omitempty"`
|
SmartInterval time.Duration `cbor:"10,keyasint,omitempty"`
|
||||||
|
ZfsInterval time.Duration `cbor:"11,keyasint,omitempty"` // interval for ZFS detail refresh
|
||||||
}
|
}
|
||||||
|
|
||||||
// Final data structure to return to the hub
|
// Final data structure to return to the hub
|
||||||
|
|||||||
45
internal/entities/zfs/zfs.go
Normal file
45
internal/entities/zfs/zfs.go
Normal file
@@ -0,0 +1,45 @@
|
|||||||
|
// Package zfs defines the ZFS detail data exchanged between agent and hub.
|
||||||
|
package zfs
|
||||||
|
|
||||||
|
// ZfsData is the detail payload returned by the agent for the GetZfsData action.
|
||||||
|
type ZfsData struct {
|
||||||
|
Pools []*PoolDetail `json:"pools,omitempty"`
|
||||||
|
Complete bool `json:"complete,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// PoolDetail holds the verbose state of a single pool: capacity, health,
|
||||||
|
// scrub, vdev, and dataset information.
|
||||||
|
type PoolDetail struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Health string `json:"health,omitempty"`
|
||||||
|
Size uint64 `json:"size,omitempty"` // bytes
|
||||||
|
Alloc uint64 `json:"alloc,omitempty"` // bytes
|
||||||
|
Free uint64 `json:"free,omitempty"` // bytes
|
||||||
|
Scrub *Scrub `json:"scrub,omitempty"`
|
||||||
|
Vdevs []*Vdev `json:"vdevs,omitempty"`
|
||||||
|
Datasets []*Dataset `json:"datasets,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Scrub holds the scrub (or resilver) status of a pool.
|
||||||
|
type Scrub struct {
|
||||||
|
State string `json:"state,omitempty"` // NONE, SCANNING, FINISHED, CANCELED
|
||||||
|
Progress string `json:"progress,omitempty"`
|
||||||
|
Errors uint64 `json:"errors,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Vdev is a single vdev (mirror, raidz, or leaf disk) with error counters.
|
||||||
|
type Vdev struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
State string `json:"state,omitempty"`
|
||||||
|
ReadErrs uint64 `json:"readErrs,omitempty"`
|
||||||
|
WriteErrs uint64 `json:"writeErrs,omitempty"`
|
||||||
|
ChecksumErrs uint64 `json:"checksumErrs,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Dataset is a single ZFS dataset with usage information.
|
||||||
|
type Dataset struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Used uint64 `json:"used,omitempty"`
|
||||||
|
Avail uint64 `json:"avail,omitempty"`
|
||||||
|
Mountpoint string `json:"mount,omitempty"`
|
||||||
|
}
|
||||||
@@ -125,6 +125,8 @@ func (h *Hub) registerApiRoutes(se *core.ServeEvent) error {
|
|||||||
apiAuth.DELETE("/user-alerts", alerts.DeleteUserAlerts)
|
apiAuth.DELETE("/user-alerts", alerts.DeleteUserAlerts)
|
||||||
// refresh SMART devices for a system
|
// refresh SMART devices for a system
|
||||||
apiAuth.POST("/smart/refresh", h.refreshSmartData).BindFunc(excludeReadOnlyRole)
|
apiAuth.POST("/smart/refresh", h.refreshSmartData).BindFunc(excludeReadOnlyRole)
|
||||||
|
// refresh ZFS pool details for a system
|
||||||
|
apiAuth.POST("/zfs/refresh", h.refreshZfsData).BindFunc(excludeReadOnlyRole)
|
||||||
// get systemd service details
|
// get systemd service details
|
||||||
apiAuth.GET("/systemd/info", h.getSystemdInfo)
|
apiAuth.GET("/systemd/info", h.getSystemdInfo)
|
||||||
// /containers routes
|
// /containers routes
|
||||||
@@ -389,3 +391,23 @@ func (h *Hub) refreshSmartData(e *core.RequestEvent) error {
|
|||||||
|
|
||||||
return e.JSON(http.StatusOK, map[string]string{"status": "ok"})
|
return e.JSON(http.StatusOK, map[string]string{"status": "ok"})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// refreshZfsData handles POST /api/beszel/zfs/refresh requests
|
||||||
|
// Fetches fresh ZFS detail data from the agent and updates the collection
|
||||||
|
func (h *Hub) refreshZfsData(e *core.RequestEvent) error {
|
||||||
|
systemID := e.Request.URL.Query().Get("system")
|
||||||
|
if systemID == "" {
|
||||||
|
return e.BadRequestError("Invalid system parameter", nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
system, err := h.sm.GetSystem(systemID)
|
||||||
|
if err != nil || !system.HasUser(e.App, e.Auth) {
|
||||||
|
return e.NotFoundError("", nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := system.FetchAndSaveZfsPools(true); err != nil {
|
||||||
|
return e.InternalServerError("", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return e.JSON(http.StatusOK, map[string]string{"status": "ok"})
|
||||||
|
}
|
||||||
|
|||||||
@@ -91,6 +91,12 @@ func setCollectionAuthSettings(app core.App) error {
|
|||||||
}); err != nil {
|
}); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if err := applyCollectionRules(app, []string{"zfs_pools"}, collectionRules{
|
||||||
|
list: &systemScopedReadRule,
|
||||||
|
view: &systemScopedReadRule,
|
||||||
|
}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
if err := applyCollectionRules(app, []string{"fingerprints"}, collectionRules{
|
if err := applyCollectionRules(app, []string{"fingerprints"}, collectionRules{
|
||||||
list: &systemScopedReadRule,
|
list: &systemScopedReadRule,
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ import (
|
|||||||
"github.com/henrygd/beszel/internal/entities/smart"
|
"github.com/henrygd/beszel/internal/entities/smart"
|
||||||
"github.com/henrygd/beszel/internal/entities/system"
|
"github.com/henrygd/beszel/internal/entities/system"
|
||||||
"github.com/henrygd/beszel/internal/entities/systemd"
|
"github.com/henrygd/beszel/internal/entities/systemd"
|
||||||
|
"github.com/henrygd/beszel/internal/entities/zfs"
|
||||||
|
|
||||||
"github.com/henrygd/beszel"
|
"github.com/henrygd/beszel"
|
||||||
|
|
||||||
@@ -49,6 +50,8 @@ type System struct {
|
|||||||
detailsFetched atomic.Bool // True if static system details have been fetched and saved
|
detailsFetched atomic.Bool // True if static system details have been fetched and saved
|
||||||
smartFetching atomic.Bool // True if SMART devices are currently being fetched
|
smartFetching atomic.Bool // True if SMART devices are currently being fetched
|
||||||
smartInterval time.Duration // Interval for periodic SMART data updates
|
smartInterval time.Duration // Interval for periodic SMART data updates
|
||||||
|
zfsFetching atomic.Bool // True if ZFS pools are currently being fetched
|
||||||
|
zfsInterval time.Duration // Interval for periodic ZFS detail data updates
|
||||||
}
|
}
|
||||||
|
|
||||||
func (sm *SystemManager) NewSystem(systemId string) *System {
|
func (sm *SystemManager) NewSystem(systemId string) *System {
|
||||||
@@ -154,6 +157,12 @@ func (sys *System) update() error {
|
|||||||
// to prevent premature expiration leading to new fetch if interval is different.
|
// to prevent premature expiration leading to new fetch if interval is different.
|
||||||
sys.manager.smartFetchMap.UpdateExpiration(sys.Id, sys.smartInterval+time.Minute)
|
sys.manager.smartFetchMap.UpdateExpiration(sys.Id, sys.smartInterval+time.Minute)
|
||||||
}
|
}
|
||||||
|
// update zfs interval if it's set on the agent side
|
||||||
|
if data.Details.ZfsInterval > 0 {
|
||||||
|
sys.zfsInterval = data.Details.ZfsInterval
|
||||||
|
sys.manager.hub.Logger().Info("ZFS interval updated from agent details", "system", sys.Id, "interval", sys.zfsInterval.String())
|
||||||
|
sys.manager.zfsFetchMap.UpdateExpiration(sys.Id, sys.zfsInterval+time.Minute)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fetch and save SMART devices when system first comes online or at intervals
|
// Fetch and save SMART devices when system first comes online or at intervals
|
||||||
@@ -170,6 +179,20 @@ func (sys *System) update() error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Fetch and save ZFS pool details when system first comes online or at intervals
|
||||||
|
if backgroundZfsFetchEnabled() && sys.detailsFetched.Load() && sys.supportsZfsData() {
|
||||||
|
if sys.zfsInterval <= 0 {
|
||||||
|
sys.zfsInterval = time.Hour
|
||||||
|
}
|
||||||
|
if sys.shouldFetchZfs() && sys.zfsFetching.CompareAndSwap(false, true) {
|
||||||
|
sys.manager.hub.Logger().Info("ZFS fetch", "system", sys.Id, "interval", sys.zfsInterval.String())
|
||||||
|
go func() {
|
||||||
|
defer sys.zfsFetching.Store(false)
|
||||||
|
_ = sys.FetchAndSaveZfsPools(false)
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -241,6 +264,10 @@ func (sys *System) createRecords(data *system.CombinedData) (*core.Record, error
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if err := sys.syncZfsPoolHealth(txApp, data.Stats.ZfsPools); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
// update system record (do this last because it triggers alerts and we need above records to be inserted first)
|
// update system record (do this last because it triggers alerts and we need above records to be inserted first)
|
||||||
systemRecord.Set("status", up)
|
systemRecord.Set("status", up)
|
||||||
systemRecord.Set("info", data.Info)
|
systemRecord.Set("info", data.Info)
|
||||||
@@ -558,6 +585,15 @@ func (sys *System) FetchSmartDataFromAgent() (smart.SmartDataResponse, error) {
|
|||||||
return result, err
|
return result, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// FetchZfsDataFromAgent fetches ZFS detail data from the agent.
|
||||||
|
func (sys *System) FetchZfsDataFromAgent(force bool) (*zfs.ZfsData, error) {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
var result zfs.ZfsData
|
||||||
|
err := sys.request(ctx, common.GetZfsData, common.ZfsDataRequest{Force: force}, &result)
|
||||||
|
return &result, err
|
||||||
|
}
|
||||||
|
|
||||||
func makeStableHashId(strings ...string) string {
|
func makeStableHashId(strings ...string) string {
|
||||||
hash := fnv.New32a()
|
hash := fnv.New32a()
|
||||||
for _, str := range strings {
|
for _, str := range strings {
|
||||||
@@ -728,6 +764,7 @@ func (s *System) createSSHClient() error {
|
|||||||
}
|
}
|
||||||
s.agentVersion, _ = extractAgentVersion(string(client.Conn.ServerVersion()))
|
s.agentVersion, _ = extractAgentVersion(string(client.Conn.ServerVersion()))
|
||||||
s.manager.resetFailedSmartFetchState(s.Id)
|
s.manager.resetFailedSmartFetchState(s.Id)
|
||||||
|
s.manager.resetFailedZfsFetchState(s.Id)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -46,6 +46,7 @@ type SystemManager struct {
|
|||||||
systems *store.Store[string, *System] // Thread-safe store of active systems
|
systems *store.Store[string, *System] // Thread-safe store of active systems
|
||||||
sshConfig *ssh.ClientConfig // SSH client configuration for system connections
|
sshConfig *ssh.ClientConfig // SSH client configuration for system connections
|
||||||
smartFetchMap *expirymap.ExpiryMap[smartFetchState] // Stores last SMART fetch time/result; TTL is only for cleanup
|
smartFetchMap *expirymap.ExpiryMap[smartFetchState] // Stores last SMART fetch time/result; TTL is only for cleanup
|
||||||
|
zfsFetchMap *expirymap.ExpiryMap[zfsFetchState] // Stores last ZFS fetch time/result; TTL is only for cleanup
|
||||||
ctx context.Context // Cancelled when the app terminates
|
ctx context.Context // Cancelled when the app terminates
|
||||||
cancel context.CancelFunc // Cancels ctx and all child system contexts
|
cancel context.CancelFunc // Cancels ctx and all child system contexts
|
||||||
}
|
}
|
||||||
@@ -67,6 +68,7 @@ func NewSystemManager(hub hubLike) *SystemManager {
|
|||||||
systems: store.New(map[string]*System{}),
|
systems: store.New(map[string]*System{}),
|
||||||
hub: hub,
|
hub: hub,
|
||||||
smartFetchMap: expirymap.New[smartFetchState](time.Hour),
|
smartFetchMap: expirymap.New[smartFetchState](time.Hour),
|
||||||
|
zfsFetchMap: expirymap.New[zfsFetchState](time.Hour),
|
||||||
}
|
}
|
||||||
sm.ctx, sm.cancel = context.WithCancel(context.Background())
|
sm.ctx, sm.cancel = context.WithCancel(context.Background())
|
||||||
return sm
|
return sm
|
||||||
@@ -343,6 +345,15 @@ func (sm *SystemManager) resetFailedSmartFetchState(systemID string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// resetFailedZfsFetchState clears only failed ZFS cooldown entries so a fresh
|
||||||
|
// agent reconnect retries ZFS discovery immediately after configuration changes.
|
||||||
|
func (sm *SystemManager) resetFailedZfsFetchState(systemID string) {
|
||||||
|
state, ok := sm.zfsFetchMap.GetOk(systemID)
|
||||||
|
if ok && !state.Successful {
|
||||||
|
sm.zfsFetchMap.Remove(systemID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// createSSHClientConfig initializes the SSH client configuration for connecting to an agent's server
|
// createSSHClientConfig initializes the SSH client configuration for connecting to an agent's server
|
||||||
func (sm *SystemManager) createSSHClientConfig() error {
|
func (sm *SystemManager) createSSHClientConfig() error {
|
||||||
privateKey, err := sm.hub.GetSSHKey("")
|
privateKey, err := sm.hub.GetSSHKey("")
|
||||||
|
|||||||
193
internal/hub/systems/system_zfs.go
Normal file
193
internal/hub/systems/system_zfs.go
Normal file
@@ -0,0 +1,193 @@
|
|||||||
|
package systems
|
||||||
|
|
||||||
|
import (
|
||||||
|
"database/sql"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/henrygd/beszel"
|
||||||
|
"github.com/henrygd/beszel/internal/entities/system"
|
||||||
|
"github.com/henrygd/beszel/internal/entities/zfs"
|
||||||
|
"github.com/pocketbase/dbx"
|
||||||
|
"github.com/pocketbase/pocketbase/core"
|
||||||
|
)
|
||||||
|
|
||||||
|
var errIncompleteZfsData = errors.New("incomplete ZFS pool inventory")
|
||||||
|
|
||||||
|
type zfsFetchState struct {
|
||||||
|
LastAttempt int64
|
||||||
|
Successful bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sys *System) supportsZfsData() bool {
|
||||||
|
return sys.agentVersion.GTE(beszel.MinVersionZfsData)
|
||||||
|
}
|
||||||
|
|
||||||
|
// FetchAndSaveZfsPools fetches ZFS detail data from the agent and saves it to
|
||||||
|
// the database. force bypasses the agent's detail cache for manual refreshes.
|
||||||
|
func (sys *System) FetchAndSaveZfsPools(force bool) error {
|
||||||
|
zfsData, err := sys.FetchZfsDataFromAgent(force)
|
||||||
|
if err != nil {
|
||||||
|
sys.recordZfsFetchResult(err, 0)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if zfsData == nil || !zfsData.Complete {
|
||||||
|
err = errIncompleteZfsData
|
||||||
|
sys.recordZfsFetchResult(err, 0)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
err = sys.saveZfsPools(zfsData)
|
||||||
|
sys.recordZfsFetchResult(err, len(zfsData.Pools))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// recordZfsFetchResult stores a cooldown entry for the ZFS interval and marks
|
||||||
|
// whether the last fetch produced any pools, so failed setup can retry on reconnect.
|
||||||
|
func (sys *System) recordZfsFetchResult(err error, poolCount int) {
|
||||||
|
if sys.manager == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
interval := sys.zfsFetchInterval()
|
||||||
|
success := err == nil && poolCount > 0
|
||||||
|
if sys.manager.hub != nil {
|
||||||
|
sys.manager.hub.Logger().Info("ZFS fetch result", "system", sys.Id, "success", success, "pools", poolCount, "interval", interval.String(), "err", err)
|
||||||
|
}
|
||||||
|
sys.manager.zfsFetchMap.Set(sys.Id, zfsFetchState{LastAttempt: time.Now().UnixMilli(), Successful: success}, interval+time.Minute)
|
||||||
|
}
|
||||||
|
|
||||||
|
// shouldFetchZfs returns true when there is no active ZFS cooldown entry for this system.
|
||||||
|
func (sys *System) shouldFetchZfs() bool {
|
||||||
|
if sys.manager == nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
state, ok := sys.manager.zfsFetchMap.GetOk(sys.Id)
|
||||||
|
if !ok {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return !time.UnixMilli(state.LastAttempt).Add(sys.zfsFetchInterval()).After(time.Now())
|
||||||
|
}
|
||||||
|
|
||||||
|
// zfsFetchInterval returns the agent-provided ZFS interval or the default when unset.
|
||||||
|
func (sys *System) zfsFetchInterval() time.Duration {
|
||||||
|
if sys.zfsInterval > 0 {
|
||||||
|
return sys.zfsInterval
|
||||||
|
}
|
||||||
|
return time.Hour
|
||||||
|
}
|
||||||
|
|
||||||
|
// saveZfsPools saves ZFS pool detail data to the zfs_pools collection and
|
||||||
|
// removes records for pools no longer reported by a complete agent inventory.
|
||||||
|
func (sys *System) saveZfsPools(zfsData *zfs.ZfsData) error {
|
||||||
|
if zfsData == nil || !zfsData.Complete {
|
||||||
|
return errIncompleteZfsData
|
||||||
|
}
|
||||||
|
|
||||||
|
hub := sys.manager.hub
|
||||||
|
collection, err := hub.FindCachedCollectionByNameOrId("zfs_pools")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return hub.RunInTransaction(func(txApp core.App) error {
|
||||||
|
alive := make(map[string]bool, len(zfsData.Pools))
|
||||||
|
for _, pool := range zfsData.Pools {
|
||||||
|
if pool == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
alive[pool.Name] = true
|
||||||
|
if err := sys.upsertZfsPoolRecord(txApp, collection, pool); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
existing, err := txApp.FindRecordsByFilter(
|
||||||
|
collection,
|
||||||
|
"system={:system}",
|
||||||
|
"", 0, 0,
|
||||||
|
dbx.Params{"system": sys.Id},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, record := range existing {
|
||||||
|
if !alive[record.GetString("name")] {
|
||||||
|
if err := txApp.Delete(record); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sys *System) upsertZfsPoolRecord(app core.App, collection *core.Collection, pool *zfs.PoolDetail) error {
|
||||||
|
recordID := makeStableHashId(sys.Id, pool.Name)
|
||||||
|
|
||||||
|
record, err := app.FindRecordById(collection, recordID)
|
||||||
|
if err != nil {
|
||||||
|
if !errors.Is(err, sql.ErrNoRows) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
record = core.NewRecord(collection)
|
||||||
|
record.Set("id", recordID)
|
||||||
|
}
|
||||||
|
|
||||||
|
record.Set("system", sys.Id)
|
||||||
|
record.Set("name", pool.Name)
|
||||||
|
record.Set("health", pool.Health)
|
||||||
|
record.Set("size", pool.Size)
|
||||||
|
record.Set("alloc", pool.Alloc)
|
||||||
|
record.Set("free", pool.Free)
|
||||||
|
record.Set("scrub", pool.Scrub)
|
||||||
|
record.Set("vdevs", pool.Vdevs)
|
||||||
|
record.Set("datasets", pool.Datasets)
|
||||||
|
record.Set("details_updated", time.Now().UTC())
|
||||||
|
|
||||||
|
return app.SaveNoValidate(record)
|
||||||
|
}
|
||||||
|
|
||||||
|
// syncZfsPoolHealth persists newly discovered pools and health transitions from
|
||||||
|
// regular system samples. Detailed fields remain owned by the hourly refresh.
|
||||||
|
func (sys *System) syncZfsPoolHealth(app core.App, pools map[string]*system.ZfsPool) error {
|
||||||
|
if len(pools) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
collection, err := app.FindCachedCollectionByNameOrId("zfs_pools")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
const gib = 1024 * 1024 * 1024
|
||||||
|
for name, pool := range pools {
|
||||||
|
if pool == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
recordID := makeStableHashId(sys.Id, name)
|
||||||
|
record, err := app.FindRecordById(collection, recordID)
|
||||||
|
if err != nil {
|
||||||
|
if !errors.Is(err, sql.ErrNoRows) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
record = core.NewRecord(collection)
|
||||||
|
record.Set("id", recordID)
|
||||||
|
record.Set("system", sys.Id)
|
||||||
|
record.Set("name", name)
|
||||||
|
record.Set("health", pool.Health)
|
||||||
|
record.Set("size", uint64(pool.Total*gib))
|
||||||
|
record.Set("alloc", uint64(pool.Used*gib))
|
||||||
|
record.Set("free", uint64(max(pool.Total-pool.Used, 0)*gib))
|
||||||
|
if err := app.SaveNoValidate(record); err != nil {
|
||||||
|
return fmt.Errorf("creating ZFS pool summary %q: %w", name, err)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if record.GetString("health") == pool.Health {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
record.Set("health", pool.Health)
|
||||||
|
if err := app.SaveNoValidate(record); err != nil {
|
||||||
|
return fmt.Errorf("updating ZFS pool health %q: %w", name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
153
internal/hub/systems/system_zfs_test.go
Normal file
153
internal/hub/systems/system_zfs_test.go
Normal file
@@ -0,0 +1,153 @@
|
|||||||
|
//go:build testing
|
||||||
|
|
||||||
|
package systems
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/blang/semver"
|
||||||
|
"github.com/henrygd/beszel/internal/entities/system"
|
||||||
|
"github.com/henrygd/beszel/internal/entities/zfs"
|
||||||
|
"github.com/henrygd/beszel/internal/hub/expirymap"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSupportsZfsData(t *testing.T) {
|
||||||
|
sys := &System{agentVersion: semver.MustParse("0.18.8")}
|
||||||
|
assert.False(t, sys.supportsZfsData())
|
||||||
|
|
||||||
|
sys.agentVersion = semver.MustParse("0.18.9")
|
||||||
|
assert.True(t, sys.supportsZfsData())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRecordZfsFetchResult(t *testing.T) {
|
||||||
|
sm := &SystemManager{zfsFetchMap: expirymap.New[zfsFetchState](time.Hour)}
|
||||||
|
t.Cleanup(sm.zfsFetchMap.StopCleaner)
|
||||||
|
|
||||||
|
sys := &System{
|
||||||
|
Id: "system-1",
|
||||||
|
manager: sm,
|
||||||
|
zfsInterval: time.Hour,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Successful fetch with pools
|
||||||
|
sys.recordZfsFetchResult(nil, 2)
|
||||||
|
state, ok := sm.zfsFetchMap.GetOk(sys.Id)
|
||||||
|
assert.True(t, ok, "expected zfs fetch result to be stored")
|
||||||
|
assert.True(t, state.Successful, "expected successful fetch state to be recorded")
|
||||||
|
|
||||||
|
// Failed fetch
|
||||||
|
sys.recordZfsFetchResult(errors.New("failed"), 0)
|
||||||
|
state, ok = sm.zfsFetchMap.GetOk(sys.Id)
|
||||||
|
assert.True(t, ok, "expected failed zfs fetch state to be stored")
|
||||||
|
assert.False(t, state.Successful, "expected failed zfs fetch state to be marked unsuccessful")
|
||||||
|
|
||||||
|
// Successful fetch but no pools
|
||||||
|
sys.recordZfsFetchResult(nil, 0)
|
||||||
|
state, ok = sm.zfsFetchMap.GetOk(sys.Id)
|
||||||
|
assert.True(t, ok, "expected fetch with zero pools to be stored")
|
||||||
|
assert.False(t, state.Successful, "expected fetch with zero pools to be marked unsuccessful")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestShouldFetchZfs(t *testing.T) {
|
||||||
|
sm := &SystemManager{zfsFetchMap: expirymap.New[zfsFetchState](time.Hour)}
|
||||||
|
t.Cleanup(sm.zfsFetchMap.StopCleaner)
|
||||||
|
|
||||||
|
sys := &System{
|
||||||
|
Id: "system-1",
|
||||||
|
manager: sm,
|
||||||
|
zfsInterval: time.Hour,
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.True(t, sys.shouldFetchZfs(), "expected initial zfs fetch to be allowed")
|
||||||
|
|
||||||
|
sys.recordZfsFetchResult(errors.New("failed"), 0)
|
||||||
|
assert.False(t, sys.shouldFetchZfs(), "expected zfs fetch to be blocked while interval entry exists")
|
||||||
|
|
||||||
|
sm.zfsFetchMap.Remove(sys.Id)
|
||||||
|
assert.True(t, sys.shouldFetchZfs(), "expected zfs fetch to be allowed after interval entry is cleared")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestZfsFetchIntervalDefault(t *testing.T) {
|
||||||
|
sys := &System{}
|
||||||
|
assert.Equal(t, time.Hour, sys.zfsFetchInterval())
|
||||||
|
|
||||||
|
sys.zfsInterval = 5 * time.Minute
|
||||||
|
assert.Equal(t, 5*time.Minute, sys.zfsFetchInterval())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResetFailedZfsFetchState(t *testing.T) {
|
||||||
|
sm := &SystemManager{zfsFetchMap: expirymap.New[zfsFetchState](time.Hour)}
|
||||||
|
t.Cleanup(sm.zfsFetchMap.StopCleaner)
|
||||||
|
|
||||||
|
sm.zfsFetchMap.Set("system-1", zfsFetchState{LastAttempt: time.Now().UnixMilli(), Successful: false}, time.Hour)
|
||||||
|
sm.resetFailedZfsFetchState("system-1")
|
||||||
|
_, ok := sm.zfsFetchMap.GetOk("system-1")
|
||||||
|
assert.False(t, ok, "expected failed zfs fetch state to be cleared on reconnect")
|
||||||
|
|
||||||
|
sm.zfsFetchMap.Set("system-1", zfsFetchState{LastAttempt: time.Now().UnixMilli(), Successful: true}, time.Hour)
|
||||||
|
sm.resetFailedZfsFetchState("system-1")
|
||||||
|
_, ok = sm.zfsFetchMap.GetOk("system-1")
|
||||||
|
assert.True(t, ok, "expected successful zfs fetch state to be preserved")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSaveZfsPoolsCompleteEmptyPrunesFinalPool(t *testing.T) {
|
||||||
|
sys, app := newTestSystemWithHub(t)
|
||||||
|
require.NoError(t, sys.saveZfsPools(&zfs.ZfsData{
|
||||||
|
Complete: true,
|
||||||
|
Pools: []*zfs.PoolDetail{{Name: "tank", Health: "ONLINE"}},
|
||||||
|
}))
|
||||||
|
records, err := app.FindRecordsByFilter("zfs_pools", "system={:system}", "", 0, 0, map[string]any{"system": sys.Id})
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, records, 1)
|
||||||
|
assert.False(t, records[0].GetDateTime("details_updated").Time().IsZero())
|
||||||
|
|
||||||
|
require.NoError(t, sys.saveZfsPools(&zfs.ZfsData{Complete: true}))
|
||||||
|
records, err = app.FindRecordsByFilter("zfs_pools", "system={:system}", "", 0, 0, map[string]any{"system": sys.Id})
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Empty(t, records)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSaveZfsPoolsIncompletePreservesRecords(t *testing.T) {
|
||||||
|
sys, app := newTestSystemWithHub(t)
|
||||||
|
require.NoError(t, sys.saveZfsPools(&zfs.ZfsData{
|
||||||
|
Complete: true,
|
||||||
|
Pools: []*zfs.PoolDetail{{Name: "tank", Health: "ONLINE"}},
|
||||||
|
}))
|
||||||
|
assert.ErrorIs(t, sys.saveZfsPools(&zfs.ZfsData{}), errIncompleteZfsData)
|
||||||
|
records, err := app.FindRecordsByFilter("zfs_pools", "system={:system}", "", 0, 0, map[string]any{"system": sys.Id})
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Len(t, records, 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSyncZfsPoolHealthWritesOnlyTransitions(t *testing.T) {
|
||||||
|
sys, app := newTestSystemWithHub(t)
|
||||||
|
collection, err := app.FindCachedCollectionByNameOrId("zfs_pools")
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{
|
||||||
|
"tank": {Total: 100, Used: 25, Health: "ONLINE"},
|
||||||
|
}))
|
||||||
|
record, err := app.FindRecordById(collection, makeStableHashId(sys.Id, "tank"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
firstUpdated := record.GetDateTime("updated")
|
||||||
|
assert.Equal(t, "ONLINE", record.GetString("health"))
|
||||||
|
assert.EqualValues(t, 100*1024*1024*1024, record.GetInt("size"))
|
||||||
|
|
||||||
|
require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{
|
||||||
|
"tank": {Total: 100, Used: 30, Health: "ONLINE"},
|
||||||
|
}))
|
||||||
|
record, err = app.FindRecordById(collection, record.Id)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, firstUpdated, record.GetDateTime("updated"))
|
||||||
|
|
||||||
|
require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{
|
||||||
|
"tank": {Total: 100, Used: 30, Health: "DEGRADED"},
|
||||||
|
}))
|
||||||
|
record, err = app.FindRecordById(collection, record.Id)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "DEGRADED", record.GetString("health"))
|
||||||
|
}
|
||||||
@@ -7,3 +7,6 @@ package systems
|
|||||||
// The hub integration tests create/replace systems and clean up the test apps quickly.
|
// The hub integration tests create/replace systems and clean up the test apps quickly.
|
||||||
// Background SMART fetching can outlive teardown and crash in PocketBase internals (nil DB).
|
// Background SMART fetching can outlive teardown and crash in PocketBase internals (nil DB).
|
||||||
func backgroundSmartFetchEnabled() bool { return true }
|
func backgroundSmartFetchEnabled() bool { return true }
|
||||||
|
|
||||||
|
// Background ZFS fetching follows the same policy as SMART fetching.
|
||||||
|
func backgroundZfsFetchEnabled() bool { return true }
|
||||||
|
|||||||
@@ -17,6 +17,9 @@ import (
|
|||||||
// the automatic background fetch during tests.
|
// the automatic background fetch during tests.
|
||||||
func backgroundSmartFetchEnabled() bool { return false }
|
func backgroundSmartFetchEnabled() bool { return false }
|
||||||
|
|
||||||
|
// Background ZFS fetching follows the same policy as SMART fetching.
|
||||||
|
func backgroundZfsFetchEnabled() bool { return false }
|
||||||
|
|
||||||
// TESTING ONLY: GetSystemCount returns the number of systems in the store
|
// TESTING ONLY: GetSystemCount returns the number of systems in the store
|
||||||
func (sm *SystemManager) GetSystemCount() int {
|
func (sm *SystemManager) GetSystemCount() int {
|
||||||
return sm.systems.Length()
|
return sm.systems.Length()
|
||||||
@@ -115,6 +118,7 @@ func (sm *SystemManager) RemoveAllSystems() {
|
|||||||
sm.RemoveSystem(system.Id)
|
sm.RemoveSystem(system.Id)
|
||||||
}
|
}
|
||||||
sm.smartFetchMap.StopCleaner()
|
sm.smartFetchMap.StopCleaner()
|
||||||
|
sm.zfsFetchMap.StopCleaner()
|
||||||
}
|
}
|
||||||
|
|
||||||
// ResetContextForTesting replaces the manager context for a new synctest bubble.
|
// ResetContextForTesting replaces the manager context for a new synctest bubble.
|
||||||
|
|||||||
184
internal/migrations/zfs_pools.go
Normal file
184
internal/migrations/zfs_pools.go
Normal file
@@ -0,0 +1,184 @@
|
|||||||
|
package migrations
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/pocketbase/pocketbase/core"
|
||||||
|
m "github.com/pocketbase/pocketbase/migrations"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Creates the zfs_pools collection for per-system ZFS pool detail data
|
||||||
|
// (pool health, capacity, scrub, vdevs, datasets). Upserts rather than
|
||||||
|
// deletes missing collections, so it is safe on fresh and existing installs.
|
||||||
|
func init() {
|
||||||
|
m.Register(func(app core.App) error {
|
||||||
|
// update collections
|
||||||
|
jsonData := `[
|
||||||
|
{
|
||||||
|
"createRule": null,
|
||||||
|
"deleteRule": null,
|
||||||
|
"fields": [
|
||||||
|
{
|
||||||
|
"autogeneratePattern": "[a-z0-9]{15}",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "text3208210256",
|
||||||
|
"max": 15,
|
||||||
|
"min": 15,
|
||||||
|
"name": "id",
|
||||||
|
"pattern": "^[a-z0-9]+$",
|
||||||
|
"presentable": false,
|
||||||
|
"primaryKey": true,
|
||||||
|
"required": true,
|
||||||
|
"system": true,
|
||||||
|
"type": "text"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cascadeDelete": true,
|
||||||
|
"collectionId": "2hz5ncl8tizk5nx",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "relation1204987316",
|
||||||
|
"maxSelect": 1,
|
||||||
|
"minSelect": 0,
|
||||||
|
"name": "system",
|
||||||
|
"presentable": false,
|
||||||
|
"required": true,
|
||||||
|
"system": false,
|
||||||
|
"type": "relation"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"autogeneratePattern": "",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "text7739291048",
|
||||||
|
"max": 0,
|
||||||
|
"min": 0,
|
||||||
|
"name": "name",
|
||||||
|
"pattern": "",
|
||||||
|
"presentable": false,
|
||||||
|
"primaryKey": false,
|
||||||
|
"required": false,
|
||||||
|
"system": false,
|
||||||
|
"type": "text"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"autogeneratePattern": "",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "text5528164482",
|
||||||
|
"max": 0,
|
||||||
|
"min": 0,
|
||||||
|
"name": "health",
|
||||||
|
"pattern": "",
|
||||||
|
"presentable": false,
|
||||||
|
"primaryKey": false,
|
||||||
|
"required": false,
|
||||||
|
"system": false,
|
||||||
|
"type": "text"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"hidden": false,
|
||||||
|
"id": "number8862034195",
|
||||||
|
"max": null,
|
||||||
|
"min": null,
|
||||||
|
"name": "size",
|
||||||
|
"onlyInt": true,
|
||||||
|
"presentable": false,
|
||||||
|
"required": false,
|
||||||
|
"system": false,
|
||||||
|
"type": "number"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"hidden": false,
|
||||||
|
"id": "number4418907321",
|
||||||
|
"max": null,
|
||||||
|
"min": null,
|
||||||
|
"name": "alloc",
|
||||||
|
"onlyInt": true,
|
||||||
|
"presentable": false,
|
||||||
|
"required": false,
|
||||||
|
"system": false,
|
||||||
|
"type": "number"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"hidden": false,
|
||||||
|
"id": "number2904183765",
|
||||||
|
"max": null,
|
||||||
|
"min": null,
|
||||||
|
"name": "free",
|
||||||
|
"onlyInt": true,
|
||||||
|
"presentable": false,
|
||||||
|
"required": false,
|
||||||
|
"system": false,
|
||||||
|
"type": "number"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"hidden": false,
|
||||||
|
"id": "json4466109723",
|
||||||
|
"maxSize": 0,
|
||||||
|
"name": "scrub",
|
||||||
|
"presentable": false,
|
||||||
|
"required": false,
|
||||||
|
"system": false,
|
||||||
|
"type": "json"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"hidden": false,
|
||||||
|
"id": "json9012873456",
|
||||||
|
"maxSize": 0,
|
||||||
|
"name": "vdevs",
|
||||||
|
"presentable": false,
|
||||||
|
"required": false,
|
||||||
|
"system": false,
|
||||||
|
"type": "json"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"hidden": false,
|
||||||
|
"id": "json7182045639",
|
||||||
|
"maxSize": 0,
|
||||||
|
"name": "datasets",
|
||||||
|
"presentable": false,
|
||||||
|
"required": false,
|
||||||
|
"system": false,
|
||||||
|
"type": "json"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"hidden": false,
|
||||||
|
"id": "date9274163058",
|
||||||
|
"max": "",
|
||||||
|
"min": "",
|
||||||
|
"name": "details_updated",
|
||||||
|
"presentable": false,
|
||||||
|
"required": false,
|
||||||
|
"system": false,
|
||||||
|
"type": "date"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"hidden": false,
|
||||||
|
"id": "autodate3332085495",
|
||||||
|
"name": "updated",
|
||||||
|
"onCreate": true,
|
||||||
|
"onUpdate": true,
|
||||||
|
"presentable": false,
|
||||||
|
"system": false,
|
||||||
|
"type": "autodate"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"id": "pbc_8441057391",
|
||||||
|
"indexes": [
|
||||||
|
"CREATE INDEX ` + "`" + `idx_zfsPoolsSystem` + "`" + ` ON ` + "`" + `zfs_pools` + "`" + ` (` + "`" + `system` + "`" + `)"
|
||||||
|
],
|
||||||
|
"listRule": null,
|
||||||
|
"name": "zfs_pools",
|
||||||
|
"system": false,
|
||||||
|
"type": "base",
|
||||||
|
"updateRule": null,
|
||||||
|
"viewRule": null
|
||||||
|
}
|
||||||
|
]`
|
||||||
|
|
||||||
|
err := app.ImportCollectionsByMarshaledJSON([]byte(jsonData), false)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}, func(app core.App) error {
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -127,6 +127,7 @@ func (rm *RecordManager) CreateLongerRecords() {
|
|||||||
"created": shorterRecordPeriod,
|
"created": shorterRecordPeriod,
|
||||||
},
|
},
|
||||||
)).
|
)).
|
||||||
|
OrderBy("created").
|
||||||
All(&recordIds)
|
All(&recordIds)
|
||||||
|
|
||||||
// continue if not enough shorter records
|
// continue if not enough shorter records
|
||||||
@@ -196,6 +197,7 @@ func AverageSystemStatsSlice(records []system.Stats) system.Stats {
|
|||||||
tempCount := float64(0)
|
tempCount := float64(0)
|
||||||
var fanSums map[string]uint64
|
var fanSums map[string]uint64
|
||||||
fanCount := uint64(0)
|
fanCount := uint64(0)
|
||||||
|
zfsPoolCounts := make(map[string]uint64)
|
||||||
|
|
||||||
// Accumulate totals
|
// Accumulate totals
|
||||||
for i := range records {
|
for i := range records {
|
||||||
@@ -336,6 +338,31 @@ func AverageSystemStatsSlice(records []system.Stats) system.Stats {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Accumulate ZFS pool stats. Counts are tracked per entry so a pool
|
||||||
|
// missing from some samples is not averaged as zero.
|
||||||
|
if stats.ZfsPools != nil {
|
||||||
|
if sum.ZfsPools == nil {
|
||||||
|
sum.ZfsPools = make(map[string]*system.ZfsPool, len(stats.ZfsPools))
|
||||||
|
}
|
||||||
|
for name, value := range stats.ZfsPools {
|
||||||
|
if value == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
pool := sum.ZfsPools[name]
|
||||||
|
if pool == nil {
|
||||||
|
pool = &system.ZfsPool{}
|
||||||
|
sum.ZfsPools[name] = pool
|
||||||
|
}
|
||||||
|
pool.Total += value.Total
|
||||||
|
pool.Used += value.Used
|
||||||
|
pool.ReadBytes += value.ReadBytes
|
||||||
|
pool.WriteBytes += value.WriteBytes
|
||||||
|
if value.Health != "" {
|
||||||
|
pool.Health = value.Health
|
||||||
|
}
|
||||||
|
zfsPoolCounts[name]++
|
||||||
|
}
|
||||||
|
}
|
||||||
// Accumulate GPU data
|
// Accumulate GPU data
|
||||||
if stats.GPUData != nil {
|
if stats.GPUData != nil {
|
||||||
if sum.GPUData == nil {
|
if sum.GPUData == nil {
|
||||||
@@ -446,6 +473,14 @@ func AverageSystemStatsSlice(records []system.Stats) system.Stats {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Average ZFS pool stats.
|
||||||
|
for name, pool := range sum.ZfsPools {
|
||||||
|
entryCount := zfsPoolCounts[name]
|
||||||
|
pool.Total = twoDecimals(pool.Total / float64(entryCount))
|
||||||
|
pool.Used = twoDecimals(pool.Used / float64(entryCount))
|
||||||
|
pool.ReadBytes /= entryCount
|
||||||
|
pool.WriteBytes /= entryCount
|
||||||
|
}
|
||||||
// Average GPU data
|
// Average GPU data
|
||||||
if sum.GPUData != nil {
|
if sum.GPUData != nil {
|
||||||
for id := range sum.GPUData {
|
for id := range sum.GPUData {
|
||||||
|
|||||||
@@ -669,6 +669,33 @@ func TestAverageSystemStatsSlice_MixedOptionalFields(t *testing.T) {
|
|||||||
assert.Equal(t, 20.0, result.GPUData["gpu0"].Usage)
|
assert.Equal(t, 20.0, result.GPUData["gpu0"].Usage)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestAverageSystemStatsSlice_Zfs(t *testing.T) {
|
||||||
|
input := []system.Stats{
|
||||||
|
{
|
||||||
|
ZfsPools: map[string]*system.ZfsPool{
|
||||||
|
"tank": {Total: 100, Used: 40, ReadBytes: 100, WriteBytes: 200, Health: "ONLINE"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{},
|
||||||
|
{
|
||||||
|
ZfsPools: map[string]*system.ZfsPool{
|
||||||
|
"tank": {Total: 120, Used: 60, ReadBytes: 300, WriteBytes: 400, Health: "DEGRADED"},
|
||||||
|
"backup": {Total: 50, Used: 10, ReadBytes: 25, WriteBytes: 50, Health: "ONLINE"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
result := records.AverageSystemStatsSlice(input)
|
||||||
|
|
||||||
|
require.Len(t, result.ZfsPools, 2)
|
||||||
|
assert.Equal(t, &system.ZfsPool{
|
||||||
|
Total: 110, Used: 50, ReadBytes: 200, WriteBytes: 300, Health: "DEGRADED",
|
||||||
|
}, result.ZfsPools["tank"])
|
||||||
|
assert.Equal(t, &system.ZfsPool{
|
||||||
|
Total: 50, Used: 10, ReadBytes: 25, WriteBytes: 50, Health: "ONLINE",
|
||||||
|
}, result.ZfsPools["backup"])
|
||||||
|
}
|
||||||
|
|
||||||
// Tests with 10 records matching the common real-world case (10 x 1m -> 1 x 10m).
|
// Tests with 10 records matching the common real-world case (10 x 1m -> 1 x 10m).
|
||||||
func TestAverageSystemStatsSlice_TenRecords(t *testing.T) {
|
func TestAverageSystemStatsSlice_TenRecords(t *testing.T) {
|
||||||
input := make([]system.Stats, 10)
|
input := make([]system.Stats, 10)
|
||||||
|
|||||||
@@ -8,10 +8,11 @@ import { useSystemData } from "./system/use-system-data"
|
|||||||
import { CpuChart, ContainerCpuChart } from "./system/charts/cpu-charts"
|
import { CpuChart, ContainerCpuChart } from "./system/charts/cpu-charts"
|
||||||
import { MemoryChart, ContainerMemoryChart, SwapChart } from "./system/charts/memory-charts"
|
import { MemoryChart, ContainerMemoryChart, SwapChart } from "./system/charts/memory-charts"
|
||||||
import { RootDiskCharts, ExtraFsCharts } from "./system/charts/disk-charts"
|
import { RootDiskCharts, ExtraFsCharts } from "./system/charts/disk-charts"
|
||||||
|
import { ZfsCharts } from "./system/charts/zfs-charts"
|
||||||
import { BandwidthChart, ContainerNetworkChart } from "./system/charts/network-charts"
|
import { BandwidthChart, ContainerNetworkChart } from "./system/charts/network-charts"
|
||||||
import { TemperatureChart, FanChart, BatteryChart } from "./system/charts/sensor-charts"
|
import { TemperatureChart, FanChart, BatteryChart } from "./system/charts/sensor-charts"
|
||||||
import { GpuPowerChart, GpuDetailCharts } from "./system/charts/gpu-charts"
|
import { GpuPowerChart, GpuDetailCharts } from "./system/charts/gpu-charts"
|
||||||
import { LazyContainersTable, LazySmartTable, LazySystemdTable } from "./system/lazy-tables"
|
import { LazyContainersTable, LazySmartTable, LazySystemdTable, LazyZfsTable } from "./system/lazy-tables"
|
||||||
import { LoadAverageChart } from "./system/charts/load-average-chart"
|
import { LoadAverageChart } from "./system/charts/load-average-chart"
|
||||||
import { ContainerIcon, CpuIcon, HardDriveIcon, TerminalSquareIcon } from "lucide-react"
|
import { ContainerIcon, CpuIcon, HardDriveIcon, TerminalSquareIcon } from "lucide-react"
|
||||||
import { GpuIcon } from "../ui/icons"
|
import { GpuIcon } from "../ui/icons"
|
||||||
@@ -63,6 +64,7 @@ export default memo(function SystemDetail({ id }: { id: string }) {
|
|||||||
const hasContainersTable = hasContainers && compareSemVer(chartData.agentVersion, SEMVER_0_14_0) >= 0
|
const hasContainersTable = hasContainers && compareSemVer(chartData.agentVersion, SEMVER_0_14_0) >= 0
|
||||||
const hasSystemd = system.info.sv
|
const hasSystemd = system.info.sv
|
||||||
const hasGpu = hasGpuData || hasGpuPowerData
|
const hasGpu = hasGpuData || hasGpuPowerData
|
||||||
|
const hasZfs = Object.keys(systemStats.at(-1)?.stats?.z ?? {}).length > 0
|
||||||
|
|
||||||
// keep tabsRef in sync for keyboard navigation
|
// keep tabsRef in sync for keyboard navigation
|
||||||
const tabs = ["core", "disk"]
|
const tabs = ["core", "disk"]
|
||||||
@@ -142,6 +144,10 @@ export default memo(function SystemDetail({ id }: { id: string }) {
|
|||||||
|
|
||||||
<ExtraFsCharts systemData={systemData} />
|
<ExtraFsCharts systemData={systemData} />
|
||||||
|
|
||||||
|
{hasZfs && <ZfsCharts systemData={systemData} />}
|
||||||
|
|
||||||
|
{hasZfs && <LazyZfsTable systemId={system.id} />}
|
||||||
|
|
||||||
{maybeHasSmartData && <LazySmartTable systemId={system.id} />}
|
{maybeHasSmartData && <LazySmartTable systemId={system.id} />}
|
||||||
|
|
||||||
{hasContainersTable && <LazyContainersTable systemId={system.id} />}
|
{hasContainersTable && <LazyContainersTable systemId={system.id} />}
|
||||||
@@ -204,6 +210,8 @@ export default memo(function SystemDetail({ id }: { id: string }) {
|
|||||||
<RootDiskCharts systemData={systemData} />
|
<RootDiskCharts systemData={systemData} />
|
||||||
</div>
|
</div>
|
||||||
<ExtraFsCharts systemData={systemData} />
|
<ExtraFsCharts systemData={systemData} />
|
||||||
|
{hasZfs && <ZfsCharts systemData={systemData} />}
|
||||||
|
{hasZfs && <LazyZfsTable systemId={system.id} />}
|
||||||
{maybeHasSmartData && <LazySmartTable systemId={system.id} />}
|
{maybeHasSmartData && <LazySmartTable systemId={system.id} />}
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
|
|||||||
130
internal/site/src/components/routes/system/charts/zfs-charts.tsx
Normal file
130
internal/site/src/components/routes/system/charts/zfs-charts.tsx
Normal file
@@ -0,0 +1,130 @@
|
|||||||
|
import { t } from "@lingui/core/macro"
|
||||||
|
import AreaChartDefault from "@/components/charts/area-chart"
|
||||||
|
import { decimalString, formatBytes, toFixedFloat } from "@/lib/utils"
|
||||||
|
import type { SystemStatsRecord } from "@/types"
|
||||||
|
import { ChartCard } from "../chart-card"
|
||||||
|
import { Unit } from "@/lib/enums"
|
||||||
|
import { useStore } from "@nanostores/react"
|
||||||
|
import { $userSettings } from "@/lib/stores"
|
||||||
|
import type { SystemData } from "../use-system-data"
|
||||||
|
|
||||||
|
// Accessors for ZFS metrics
|
||||||
|
const poolUsage =
|
||||||
|
(name: string) =>
|
||||||
|
({ stats }: SystemStatsRecord) =>
|
||||||
|
stats?.z?.[name]?.du ?? 0
|
||||||
|
const poolRead =
|
||||||
|
(name: string) =>
|
||||||
|
({ stats }: SystemStatsRecord) =>
|
||||||
|
stats?.z?.[name]?.rb ?? 0
|
||||||
|
const poolWrite =
|
||||||
|
(name: string) =>
|
||||||
|
({ stats }: SystemStatsRecord) =>
|
||||||
|
stats?.z?.[name]?.wb ?? 0
|
||||||
|
|
||||||
|
export function ZfsPoolUsageChart({ systemData, poolName }: { systemData: SystemData; poolName: string }) {
|
||||||
|
const { chartData, grid, dataEmpty } = systemData
|
||||||
|
const latest = chartData.systemStats.at(-1)?.stats
|
||||||
|
const pool = latest?.z?.[poolName]
|
||||||
|
if (!pool) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
let poolTotal = pool.d
|
||||||
|
// round to nearest GB
|
||||||
|
if (poolTotal >= 100) {
|
||||||
|
poolTotal = Math.round(poolTotal)
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<ChartCard
|
||||||
|
empty={dataEmpty}
|
||||||
|
grid={grid}
|
||||||
|
title={`${poolName} ${t`Usage`}`}
|
||||||
|
description={t`Usage of ZFS pool ${poolName}`}
|
||||||
|
>
|
||||||
|
<AreaChartDefault
|
||||||
|
chartData={chartData}
|
||||||
|
domain={[0, poolTotal]}
|
||||||
|
showTotal={true}
|
||||||
|
tickFormatter={(val) => {
|
||||||
|
const { value, unit } = formatBytes(val * 1024, false, Unit.Bytes, true)
|
||||||
|
return `${toFixedFloat(value, value >= 10 ? 0 : 1)} ${unit}`
|
||||||
|
}}
|
||||||
|
contentFormatter={({ value }) => {
|
||||||
|
const { value: convertedValue, unit } = formatBytes(value * 1024, false, Unit.Bytes, true)
|
||||||
|
return `${decimalString(convertedValue, convertedValue >= 100 ? 1 : 2)} ${unit}`
|
||||||
|
}}
|
||||||
|
dataPoints={[
|
||||||
|
{
|
||||||
|
label: t`Pool Usage`,
|
||||||
|
dataKey: poolUsage(poolName),
|
||||||
|
color: 4,
|
||||||
|
opacity: 0.4,
|
||||||
|
},
|
||||||
|
]}
|
||||||
|
/>
|
||||||
|
</ChartCard>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
export function ZfsPoolIOChart({ systemData, poolName }: { systemData: SystemData; poolName: string }) {
|
||||||
|
const { chartData, grid, dataEmpty } = systemData
|
||||||
|
const userSettings = useStore($userSettings)
|
||||||
|
if (!chartData.systemStats?.length) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
return (
|
||||||
|
<ChartCard
|
||||||
|
empty={dataEmpty}
|
||||||
|
grid={grid}
|
||||||
|
title={`${poolName} I/O`}
|
||||||
|
description={t`Throughput of ZFS pool ${poolName}`}
|
||||||
|
>
|
||||||
|
<AreaChartDefault
|
||||||
|
chartData={chartData}
|
||||||
|
showTotal={true}
|
||||||
|
dataPoints={[
|
||||||
|
{
|
||||||
|
label: t({ message: "Write", comment: "Disk write" }),
|
||||||
|
dataKey: poolWrite(poolName),
|
||||||
|
color: 3,
|
||||||
|
opacity: 0.3,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: t({ message: "Read", comment: "Disk read" }),
|
||||||
|
dataKey: poolRead(poolName),
|
||||||
|
color: 1,
|
||||||
|
opacity: 0.3,
|
||||||
|
},
|
||||||
|
]}
|
||||||
|
tickFormatter={(val) => {
|
||||||
|
const { value, unit } = formatBytes(val, true, userSettings.unitDisk, false)
|
||||||
|
return `${toFixedFloat(value, value >= 10 ? 0 : 1)} ${unit}`
|
||||||
|
}}
|
||||||
|
contentFormatter={({ value }) => {
|
||||||
|
const { value: convertedValue, unit } = formatBytes(value, true, userSettings.unitDisk, false)
|
||||||
|
return `${decimalString(convertedValue, convertedValue >= 100 ? 1 : 2)} ${unit}`
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</ChartCard>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** ZFS section: one stacked usage card per pool plus per-pool I/O cards. */
|
||||||
|
export function ZfsCharts({ systemData }: { systemData: SystemData }) {
|
||||||
|
const latest = systemData.chartData.systemStats?.at(-1)?.stats
|
||||||
|
const pools = latest?.z ?? {}
|
||||||
|
if (Object.keys(pools).length === 0) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
return (
|
||||||
|
<div className="grid xl:grid-cols-2 gap-4">
|
||||||
|
{Object.keys(pools).map((poolName) => (
|
||||||
|
<div key={poolName} className="contents">
|
||||||
|
<ZfsPoolUsageChart systemData={systemData} poolName={poolName} />
|
||||||
|
<ZfsPoolIOChart systemData={systemData} poolName={poolName} />
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
@@ -24,6 +24,17 @@ export function LazySmartTable({ systemId }: { systemId: string }) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const ZfsTable = lazy(() => import("./zfs-table"))
|
||||||
|
|
||||||
|
export function LazyZfsTable({ systemId }: { systemId: string }) {
|
||||||
|
const { isIntersecting, ref } = useIntersectionObserver({ rootMargin: "90px" })
|
||||||
|
return (
|
||||||
|
<div ref={ref} className={cn(isIntersecting && "contents")}>
|
||||||
|
{isIntersecting && <ZfsTable systemId={systemId} />}
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
const SystemdTable = lazy(() => import("../../systemd-table/systemd-table"))
|
const SystemdTable = lazy(() => import("../../systemd-table/systemd-table"))
|
||||||
|
|
||||||
export function LazySystemdTable({ systemId }: { systemId: string }) {
|
export function LazySystemdTable({ systemId }: { systemId: string }) {
|
||||||
|
|||||||
652
internal/site/src/components/routes/system/zfs-table.tsx
Normal file
652
internal/site/src/components/routes/system/zfs-table.tsx
Normal file
@@ -0,0 +1,652 @@
|
|||||||
|
import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"
|
||||||
|
import { Badge } from "@/components/ui/badge"
|
||||||
|
import { Button } from "@/components/ui/button"
|
||||||
|
import { Card, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"
|
||||||
|
import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuTrigger } from "@/components/ui/dropdown-menu"
|
||||||
|
import { Input } from "@/components/ui/input"
|
||||||
|
import { Separator } from "@/components/ui/separator"
|
||||||
|
import { Sheet, SheetContent, SheetDescription, SheetHeader, SheetTitle } from "@/components/ui/sheet"
|
||||||
|
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"
|
||||||
|
import { isReadOnlyUser, pb } from "@/lib/api"
|
||||||
|
import { cn, formatBytes, formatShortDate, hourWithSeconds, toFixedFloat } from "@/lib/utils"
|
||||||
|
import type { ZfsDataset, ZfsPoolRecord, ZfsVdev } from "@/types"
|
||||||
|
import { t } from "@lingui/core/macro"
|
||||||
|
import { Trans } from "@lingui/react/macro"
|
||||||
|
import type { Column, ColumnDef } from "@tanstack/react-table"
|
||||||
|
import {
|
||||||
|
flexRender,
|
||||||
|
getCoreRowModel,
|
||||||
|
getFilteredRowModel,
|
||||||
|
getSortedRowModel,
|
||||||
|
useReactTable,
|
||||||
|
} from "@tanstack/react-table"
|
||||||
|
import {
|
||||||
|
ActivityIcon,
|
||||||
|
BinaryIcon,
|
||||||
|
CheckCircleIcon,
|
||||||
|
CircleAlertIcon,
|
||||||
|
ClockIcon,
|
||||||
|
HardDriveDownloadIcon,
|
||||||
|
HardDriveIcon,
|
||||||
|
HardDriveUploadIcon,
|
||||||
|
LoaderCircleIcon,
|
||||||
|
MoreHorizontalIcon,
|
||||||
|
RefreshCwIcon,
|
||||||
|
RotateCwIcon,
|
||||||
|
XCircleIcon,
|
||||||
|
XIcon,
|
||||||
|
} from "lucide-react"
|
||||||
|
import { useCallback, useEffect, useMemo, useState } from "react"
|
||||||
|
|
||||||
|
const ZFS_POOL_FIELDS = "id,system,name,health,size,alloc,free,scrub,details_updated,updated"
|
||||||
|
|
||||||
|
/** Maps a zpool health string to a Badge variant. */
|
||||||
|
function healthVariant(health: string): "success" | "warning" | "danger" | "outline" {
|
||||||
|
switch (health) {
|
||||||
|
case "ONLINE":
|
||||||
|
return "success"
|
||||||
|
case "DEGRADED":
|
||||||
|
return "warning"
|
||||||
|
case "FAULTED":
|
||||||
|
case "OFFLINE":
|
||||||
|
case "UNAVAIL":
|
||||||
|
case "REMOVED":
|
||||||
|
case "SUSPENDED":
|
||||||
|
return "danger"
|
||||||
|
default:
|
||||||
|
return "outline"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatCapacity(bytes: number): string {
|
||||||
|
if (!bytes) return "-"
|
||||||
|
const { value, unit } = formatBytes(bytes)
|
||||||
|
return `${toFixedFloat(value, value >= 10 ? 1 : 2)} ${unit}`
|
||||||
|
}
|
||||||
|
|
||||||
|
function HeaderButton<T>({ column, name, Icon }: { column: Column<T>; name: string; Icon: React.ElementType }) {
|
||||||
|
const isSorted = column.getIsSorted()
|
||||||
|
return (
|
||||||
|
<Button
|
||||||
|
className={cn(
|
||||||
|
"h-9 px-3 flex items-center gap-2 duration-50",
|
||||||
|
isSorted && "bg-accent/70 light:bg-accent text-accent-foreground/90"
|
||||||
|
)}
|
||||||
|
variant="ghost"
|
||||||
|
onClick={() => column.toggleSorting(column.getIsSorted() === "asc")}
|
||||||
|
>
|
||||||
|
<Icon className="size-4" />
|
||||||
|
{name}
|
||||||
|
</Button>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
const columns: ColumnDef<ZfsPoolRecord>[] = [
|
||||||
|
{
|
||||||
|
accessorKey: "name",
|
||||||
|
sortingFn: (a, b) => a.original.name.localeCompare(b.original.name),
|
||||||
|
header: ({ column }) => <HeaderButton column={column} name={t`Pool`} Icon={HardDriveIcon} />,
|
||||||
|
cell: ({ getValue }) => <span className="font-medium ms-1.5">{getValue() as string}</span>,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
accessorKey: "health",
|
||||||
|
sortingFn: (a, b) => a.original.health.localeCompare(b.original.health),
|
||||||
|
header: ({ column }) => <HeaderButton column={column} name={t`Health`} Icon={ActivityIcon} />,
|
||||||
|
cell: ({ getValue }) => {
|
||||||
|
const health = (getValue() as string) || ""
|
||||||
|
return <Badge variant={healthVariant(health)}>{health || t`Unknown`}</Badge>
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "size",
|
||||||
|
accessorFn: (record) => record.size,
|
||||||
|
invertSorting: true,
|
||||||
|
header: ({ column }) => <HeaderButton column={column} name={t`Size`} Icon={BinaryIcon} />,
|
||||||
|
cell: ({ getValue }) => <span className="ms-1.5 tabular-nums">{formatCapacity(getValue() as number)}</span>,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "used",
|
||||||
|
accessorFn: (record) => record.alloc,
|
||||||
|
invertSorting: true,
|
||||||
|
header: ({ column }) => <HeaderButton column={column} name={t`Used`} Icon={HardDriveDownloadIcon} />,
|
||||||
|
cell: ({ getValue }) => <span className="ms-1.5 tabular-nums">{formatCapacity(getValue() as number)}</span>,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "free",
|
||||||
|
accessorFn: (record) => record.free,
|
||||||
|
invertSorting: true,
|
||||||
|
header: ({ column }) => <HeaderButton column={column} name={t`Free`} Icon={HardDriveUploadIcon} />,
|
||||||
|
cell: ({ getValue }) => <span className="ms-1.5 tabular-nums">{formatCapacity(getValue() as number)}</span>,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "scrub",
|
||||||
|
accessorFn: (record) => record.scrub?.state ?? "",
|
||||||
|
header: ({ column }) => <HeaderButton column={column} name={t`Scrub`} Icon={RotateCwIcon} />,
|
||||||
|
cell: ({ row }) => {
|
||||||
|
const scrub = row.original.scrub
|
||||||
|
if (!scrub?.state) return <span className="ms-1.5 text-muted-foreground">{t`None`}</span>
|
||||||
|
return (
|
||||||
|
<span className="ms-1.5 tabular-nums">
|
||||||
|
{scrub.state}
|
||||||
|
{scrub.progress ? ` (${scrub.progress})` : ""}
|
||||||
|
</span>
|
||||||
|
)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "updated",
|
||||||
|
invertSorting: true,
|
||||||
|
accessorFn: (record) => record.details_updated || record.updated,
|
||||||
|
header: ({ column }) => <HeaderButton column={column} name={t`Updated`} Icon={ClockIcon} />,
|
||||||
|
cell: ({ getValue }) => {
|
||||||
|
const timestamp = getValue() as string
|
||||||
|
if (!timestamp) return null
|
||||||
|
const formatter =
|
||||||
|
new Date(timestamp).toDateString() === new Date().toDateString() ? hourWithSeconds : formatShortDate
|
||||||
|
return <span className="ms-1 tabular-nums">{formatter(timestamp)}</span>
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
function VdevTable({ vdevs }: { vdevs: ZfsVdev[] }) {
|
||||||
|
if (!vdevs?.length) return null
|
||||||
|
return (
|
||||||
|
<div className="overflow-x-auto rounded-md border">
|
||||||
|
<Table>
|
||||||
|
<TableHeader>
|
||||||
|
<TableRow>
|
||||||
|
<TableHead>{t`Vdev`}</TableHead>
|
||||||
|
<TableHead>{t`State`}</TableHead>
|
||||||
|
<TableHead className="text-right">{t`Read errors`}</TableHead>
|
||||||
|
<TableHead className="text-right">{t`Write errors`}</TableHead>
|
||||||
|
<TableHead className="text-right">{t`Checksum errors`}</TableHead>
|
||||||
|
</TableRow>
|
||||||
|
</TableHeader>
|
||||||
|
<TableBody>
|
||||||
|
{vdevs.map((vdev) => (
|
||||||
|
<TableRow key={vdev.name}>
|
||||||
|
<TableCell className="font-mono text-xs">{vdev.name}</TableCell>
|
||||||
|
<TableCell>
|
||||||
|
<Badge variant={healthVariant(vdev.state ?? "")} className="font-normal">
|
||||||
|
{vdev.state ?? "-"}
|
||||||
|
</Badge>
|
||||||
|
</TableCell>
|
||||||
|
<TableCell
|
||||||
|
className={cn("text-right tabular-nums", (vdev.readErrs ?? 0) > 0 && "text-red-600 dark:text-red-400")}
|
||||||
|
>
|
||||||
|
{vdev.readErrs ?? 0}
|
||||||
|
</TableCell>
|
||||||
|
<TableCell
|
||||||
|
className={cn("text-right tabular-nums", (vdev.writeErrs ?? 0) > 0 && "text-red-600 dark:text-red-400")}
|
||||||
|
>
|
||||||
|
{vdev.writeErrs ?? 0}
|
||||||
|
</TableCell>
|
||||||
|
<TableCell
|
||||||
|
className={cn(
|
||||||
|
"text-right tabular-nums",
|
||||||
|
(vdev.checksumErrs ?? 0) > 0 && "text-red-600 dark:text-red-400"
|
||||||
|
)}
|
||||||
|
>
|
||||||
|
{vdev.checksumErrs ?? 0}
|
||||||
|
</TableCell>
|
||||||
|
</TableRow>
|
||||||
|
))}
|
||||||
|
</TableBody>
|
||||||
|
</Table>
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
const datasetColumns: ColumnDef<ZfsDataset>[] = [
|
||||||
|
{
|
||||||
|
accessorKey: "name",
|
||||||
|
sortingFn: (a, b) => a.original.name.localeCompare(b.original.name),
|
||||||
|
header: ({ column }) => <HeaderButton column={column} name={t`Dataset`} Icon={HardDriveIcon} />,
|
||||||
|
cell: ({ getValue }) => <span className="font-mono text-xs">{getValue() as string}</span>,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "used",
|
||||||
|
accessorFn: (ds) => ds.used ?? 0,
|
||||||
|
invertSorting: true,
|
||||||
|
header: ({ column }) => <HeaderButton column={column} name={t`Used`} Icon={HardDriveDownloadIcon} />,
|
||||||
|
cell: ({ getValue }) => <span className="text-right tabular-nums">{formatCapacity(getValue() as number)}</span>,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "avail",
|
||||||
|
accessorFn: (ds) => ds.avail ?? 0,
|
||||||
|
invertSorting: true,
|
||||||
|
header: ({ column }) => <HeaderButton column={column} name={t`Available`} Icon={HardDriveUploadIcon} />,
|
||||||
|
cell: ({ getValue }) => <span className="text-right tabular-nums">{formatCapacity(getValue() as number)}</span>,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
accessorKey: "mount",
|
||||||
|
sortingFn: (a, b) => (a.original.mount ?? "").localeCompare(b.original.mount ?? ""),
|
||||||
|
header: ({ column }) => <HeaderButton column={column} name={t`Mountpoint`} Icon={HardDriveIcon} />,
|
||||||
|
cell: ({ getValue }) => (
|
||||||
|
<span className="font-mono text-xs text-muted-foreground">{(getValue() as string) || "-"}</span>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
function DatasetTable({ datasets }: { datasets: ZfsDataset[] }) {
|
||||||
|
const [filter, setFilter] = useState("")
|
||||||
|
const filtered = useMemo(() => {
|
||||||
|
if (!datasets) return []
|
||||||
|
if (!filter) return datasets
|
||||||
|
const needle = filter.toLowerCase()
|
||||||
|
return datasets.filter((ds) => ds.name.toLowerCase().includes(needle))
|
||||||
|
}, [datasets, filter])
|
||||||
|
|
||||||
|
const table = useReactTable({
|
||||||
|
data: filtered,
|
||||||
|
columns: datasetColumns,
|
||||||
|
getCoreRowModel: getCoreRowModel(),
|
||||||
|
getSortedRowModel: getSortedRowModel(),
|
||||||
|
})
|
||||||
|
|
||||||
|
if (!datasets?.length) return null
|
||||||
|
return (
|
||||||
|
<div>
|
||||||
|
<div className="mb-2 flex items-center justify-between gap-4">
|
||||||
|
<h3 className="text-base font-semibold">
|
||||||
|
<Trans>Datasets</Trans>
|
||||||
|
</h3>
|
||||||
|
<div className="relative w-64 max-w-full">
|
||||||
|
<Input
|
||||||
|
placeholder={t`Filter datasets...`}
|
||||||
|
value={filter}
|
||||||
|
onChange={(e) => setFilter(e.target.value)}
|
||||||
|
className="px-4 w-full"
|
||||||
|
/>
|
||||||
|
{filter && (
|
||||||
|
<Button
|
||||||
|
type="button"
|
||||||
|
variant="ghost"
|
||||||
|
size="icon"
|
||||||
|
aria-label={t`Clear`}
|
||||||
|
className="absolute right-1 top-1/2 -translate-y-1/2 h-7 w-7 text-muted-foreground"
|
||||||
|
onClick={() => setFilter("")}
|
||||||
|
>
|
||||||
|
<XIcon className="h-4 w-4" />
|
||||||
|
</Button>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div className="max-h-80 overflow-auto rounded-md border">
|
||||||
|
<Table>
|
||||||
|
<TableHeader className="sticky top-0 z-10">
|
||||||
|
{table.getHeaderGroups().map((headerGroup) => (
|
||||||
|
<TableRow key={headerGroup.id}>
|
||||||
|
{headerGroup.headers.map((header) => (
|
||||||
|
<TableHead key={header.id} className="px-2">
|
||||||
|
{header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())}
|
||||||
|
</TableHead>
|
||||||
|
))}
|
||||||
|
</TableRow>
|
||||||
|
))}
|
||||||
|
</TableHeader>
|
||||||
|
<TableBody>
|
||||||
|
{table.getRowModel().rows.map((row) => (
|
||||||
|
<TableRow key={row.id}>
|
||||||
|
{row.getVisibleCells().map((cell) => (
|
||||||
|
<TableCell key={cell.id} className="ps-5 whitespace-pre">
|
||||||
|
{flexRender(cell.column.columnDef.cell, cell.getContext())}
|
||||||
|
</TableCell>
|
||||||
|
))}
|
||||||
|
</TableRow>
|
||||||
|
))}
|
||||||
|
</TableBody>
|
||||||
|
</Table>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
function PoolSheet({
|
||||||
|
poolId,
|
||||||
|
open,
|
||||||
|
onOpenChange,
|
||||||
|
}: {
|
||||||
|
poolId: string | null
|
||||||
|
open: boolean
|
||||||
|
onOpenChange: (open: boolean) => void
|
||||||
|
}) {
|
||||||
|
const [pool, setPool] = useState<ZfsPoolRecord | null>(null)
|
||||||
|
const [isLoading, setIsLoading] = useState(false)
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
let active = true
|
||||||
|
if (!poolId) {
|
||||||
|
setPool(null)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Only fetch when opening, not when closing (keeps data visible during close animation)
|
||||||
|
if (!open) return
|
||||||
|
setIsLoading(true)
|
||||||
|
pb.collection("zfs_pools")
|
||||||
|
.getOne(poolId)
|
||||||
|
.then((record) => active && setPool(record as ZfsPoolRecord))
|
||||||
|
.catch(() => active && setPool(null))
|
||||||
|
.finally(() => active && setIsLoading(false))
|
||||||
|
return () => {
|
||||||
|
active = false
|
||||||
|
}
|
||||||
|
}, [open, poolId])
|
||||||
|
|
||||||
|
const health = pool?.health || ""
|
||||||
|
const healthVariantValue = healthVariant(health)
|
||||||
|
const HealthIcon =
|
||||||
|
healthVariantValue === "success"
|
||||||
|
? CheckCircleIcon
|
||||||
|
: healthVariantValue === "warning"
|
||||||
|
? CircleAlertIcon
|
||||||
|
: XCircleIcon
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Sheet open={open} onOpenChange={onOpenChange}>
|
||||||
|
<SheetContent className="w-full sm:max-w-220 gap-0 overflow-y-auto">
|
||||||
|
<SheetHeader className="mb-0 border-b">
|
||||||
|
<SheetTitle className="flex items-center gap-2">
|
||||||
|
{pool ? pool.name : t`ZFS Pool`}
|
||||||
|
{pool && <Badge variant={healthVariantValue}>{health}</Badge>}
|
||||||
|
</SheetTitle>
|
||||||
|
<SheetDescription className="flex flex-wrap items-center gap-x-2 gap-y-1">
|
||||||
|
{pool?.size ? formatCapacity(pool.size) : null}
|
||||||
|
{pool?.alloc ? (
|
||||||
|
<>
|
||||||
|
<Separator orientation="vertical" className="h-2.5 bg-muted-foreground opacity-70" />
|
||||||
|
<span>
|
||||||
|
<Trans>Used</Trans>: {formatCapacity(pool.alloc)}
|
||||||
|
</span>
|
||||||
|
</>
|
||||||
|
) : null}
|
||||||
|
{pool?.free ? (
|
||||||
|
<>
|
||||||
|
<Separator orientation="vertical" className="h-2.5 bg-muted-foreground opacity-70" />
|
||||||
|
<span>
|
||||||
|
<Trans>Free</Trans>: {formatCapacity(pool.free)}
|
||||||
|
</span>
|
||||||
|
</>
|
||||||
|
) : null}
|
||||||
|
</SheetDescription>
|
||||||
|
</SheetHeader>
|
||||||
|
<div className="flex-1 p-4 flex flex-col gap-4">
|
||||||
|
{isLoading ? (
|
||||||
|
<div className="flex justify-center py-8">
|
||||||
|
<LoaderCircleIcon className="animate-spin size-10 opacity-60" />
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<>
|
||||||
|
{pool && health && (
|
||||||
|
<Alert className="pb-3 shrink-0">
|
||||||
|
<HealthIcon className="size-4" />
|
||||||
|
<AlertTitle>
|
||||||
|
<Trans>Pool Health</Trans>: {health}
|
||||||
|
</AlertTitle>
|
||||||
|
{pool.scrub?.state && (
|
||||||
|
<AlertDescription>
|
||||||
|
<Trans>Scrub</Trans>: {pool.scrub.state}
|
||||||
|
{pool.scrub.progress ? ` (${pool.scrub.progress})` : ""}
|
||||||
|
{pool.scrub.errors ? `, ${pool.scrub.errors} errors` : ""}
|
||||||
|
</AlertDescription>
|
||||||
|
)}
|
||||||
|
</Alert>
|
||||||
|
)}
|
||||||
|
{pool?.vdevs?.length || pool?.datasets?.length ? (
|
||||||
|
<>
|
||||||
|
{pool.vdevs?.length ? <VdevTable vdevs={pool.vdevs} /> : null}
|
||||||
|
<DatasetTable datasets={pool.datasets ?? []} />
|
||||||
|
</>
|
||||||
|
) : (
|
||||||
|
!isLoading && (
|
||||||
|
<div className="py-8 text-center text-sm text-muted-foreground">
|
||||||
|
<Trans>No detail data for this pool.</Trans>
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
)}
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</SheetContent>
|
||||||
|
</Sheet>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
export default function ZfsTable({ systemId }: { systemId?: string }) {
|
||||||
|
const [zfsPools, setZfsPools] = useState<ZfsPoolRecord[]>()
|
||||||
|
const [globalFilter, setGlobalFilter] = useState("")
|
||||||
|
const [activePoolId, setActivePoolId] = useState<string | null>(null)
|
||||||
|
const [sheetOpen, setSheetOpen] = useState(false)
|
||||||
|
const [refreshingId, setRefreshingId] = useState<string | null>(null)
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
let disposed = false
|
||||||
|
let unsubscribe: () => void = () => {}
|
||||||
|
// fetch initial records
|
||||||
|
pb.collection<ZfsPoolRecord>("zfs_pools")
|
||||||
|
.getFullList({
|
||||||
|
filter: systemId ? pb.filter("system={:id}", { id: systemId }) : "",
|
||||||
|
sort: "name",
|
||||||
|
fields: ZFS_POOL_FIELDS,
|
||||||
|
})
|
||||||
|
.then((records) => !disposed && setZfsPools(records))
|
||||||
|
.catch((error) => console.error("Failed to fetch ZFS pools:", error))
|
||||||
|
|
||||||
|
// subscribe to realtime updates
|
||||||
|
const pbOptions = systemId ? { filter: `system="${systemId}"` } : undefined
|
||||||
|
;(async () => {
|
||||||
|
try {
|
||||||
|
const unsubscribeNow = await pb.collection<ZfsPoolRecord>("zfs_pools").subscribe(
|
||||||
|
"*",
|
||||||
|
(event) => {
|
||||||
|
const record = event.record as ZfsPoolRecord
|
||||||
|
setZfsPools((current) => {
|
||||||
|
const pools = current ?? []
|
||||||
|
const matchesSystemScope = !systemId || record.system === systemId
|
||||||
|
if (event.action === "delete") {
|
||||||
|
return pools.filter((pool) => pool.id !== record.id)
|
||||||
|
}
|
||||||
|
if (!matchesSystemScope) {
|
||||||
|
return pools.filter((pool) => pool.id !== record.id)
|
||||||
|
}
|
||||||
|
const existingIndex = pools.findIndex((pool) => pool.id === record.id)
|
||||||
|
if (existingIndex === -1) {
|
||||||
|
return [record, ...pools]
|
||||||
|
}
|
||||||
|
const next = [...pools]
|
||||||
|
next[existingIndex] = record
|
||||||
|
return next
|
||||||
|
})
|
||||||
|
},
|
||||||
|
pbOptions
|
||||||
|
)
|
||||||
|
if (disposed) {
|
||||||
|
unsubscribeNow()
|
||||||
|
} else {
|
||||||
|
unsubscribe = unsubscribeNow
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error("Failed to subscribe to ZFS pool updates:", error)
|
||||||
|
}
|
||||||
|
})()
|
||||||
|
|
||||||
|
return () => {
|
||||||
|
disposed = true
|
||||||
|
unsubscribe?.()
|
||||||
|
}
|
||||||
|
}, [systemId])
|
||||||
|
|
||||||
|
const refreshSystem = useCallback(async (systemId: string) => {
|
||||||
|
try {
|
||||||
|
await pb.send("/api/beszel/zfs/refresh", {
|
||||||
|
method: "POST",
|
||||||
|
query: { system: systemId },
|
||||||
|
})
|
||||||
|
} catch (error) {
|
||||||
|
console.error("Failed to refresh ZFS pools:", error)
|
||||||
|
}
|
||||||
|
}, [])
|
||||||
|
|
||||||
|
const handleRowRefresh = useCallback(
|
||||||
|
async (pool: ZfsPoolRecord) => {
|
||||||
|
if (!pool.system) return
|
||||||
|
setRefreshingId(pool.id)
|
||||||
|
try {
|
||||||
|
await refreshSystem(pool.system)
|
||||||
|
} finally {
|
||||||
|
setRefreshingId((id) => (id === pool.id ? null : id))
|
||||||
|
}
|
||||||
|
},
|
||||||
|
[refreshSystem]
|
||||||
|
)
|
||||||
|
|
||||||
|
const actionColumn = useMemo<ColumnDef<ZfsPoolRecord>>(
|
||||||
|
() => ({
|
||||||
|
id: "actions",
|
||||||
|
enableSorting: false,
|
||||||
|
header: () => (
|
||||||
|
<span className="sr-only">
|
||||||
|
<Trans>Actions</Trans>
|
||||||
|
</span>
|
||||||
|
),
|
||||||
|
cell: ({ row }) => {
|
||||||
|
const pool = row.original
|
||||||
|
const isRowRefreshing = refreshingId === pool.id
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="flex justify-end">
|
||||||
|
<DropdownMenu>
|
||||||
|
<DropdownMenuTrigger asChild>
|
||||||
|
<Button
|
||||||
|
variant="ghost"
|
||||||
|
size="icon"
|
||||||
|
className="size-10"
|
||||||
|
onClick={(event) => event.stopPropagation()}
|
||||||
|
onMouseDown={(event) => event.stopPropagation()}
|
||||||
|
>
|
||||||
|
<span className="sr-only">
|
||||||
|
<Trans>Open menu</Trans>
|
||||||
|
</span>
|
||||||
|
<MoreHorizontalIcon className="w-5" />
|
||||||
|
</Button>
|
||||||
|
</DropdownMenuTrigger>
|
||||||
|
<DropdownMenuContent align="end" onClick={(event) => event.stopPropagation()}>
|
||||||
|
<DropdownMenuItem
|
||||||
|
onClick={(event) => {
|
||||||
|
event.stopPropagation()
|
||||||
|
handleRowRefresh(pool)
|
||||||
|
}}
|
||||||
|
disabled={isRowRefreshing}
|
||||||
|
>
|
||||||
|
<RefreshCwIcon className={cn("me-2.5 size-4", isRowRefreshing && "animate-spin")} />
|
||||||
|
<Trans>Refresh</Trans>
|
||||||
|
</DropdownMenuItem>
|
||||||
|
</DropdownMenuContent>
|
||||||
|
</DropdownMenu>
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
[refreshingId, handleRowRefresh]
|
||||||
|
)
|
||||||
|
|
||||||
|
const tableColumns = useMemo(() => {
|
||||||
|
return isReadOnlyUser() ? columns : [...columns, actionColumn]
|
||||||
|
}, [actionColumn])
|
||||||
|
|
||||||
|
const table = useReactTable({
|
||||||
|
data: zfsPools || ([] as ZfsPoolRecord[]),
|
||||||
|
columns: tableColumns,
|
||||||
|
getCoreRowModel: getCoreRowModel(),
|
||||||
|
getSortedRowModel: getSortedRowModel(),
|
||||||
|
getFilteredRowModel: getFilteredRowModel(),
|
||||||
|
state: { globalFilter },
|
||||||
|
onGlobalFilterChange: setGlobalFilter,
|
||||||
|
globalFilterFn: (row, _columnId, filterValue) => {
|
||||||
|
const pool = row.original
|
||||||
|
const searchString = `${pool.name} ${pool.health ?? ""}`.toLowerCase()
|
||||||
|
return (filterValue as string)
|
||||||
|
.toLowerCase()
|
||||||
|
.split(" ")
|
||||||
|
.every((term) => searchString.includes(term))
|
||||||
|
},
|
||||||
|
})
|
||||||
|
const rows = table.getRowModel().rows
|
||||||
|
|
||||||
|
// Hide the table on system pages if there's no data
|
||||||
|
if (systemId && !zfsPools?.length && !globalFilter) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
const openSheet = (pool: ZfsPoolRecord) => {
|
||||||
|
setActivePoolId(pool.id)
|
||||||
|
setSheetOpen(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div>
|
||||||
|
<Card className="@container w-full px-3 py-5 sm:py-6 sm:px-6">
|
||||||
|
<CardHeader className="p-0 mb-3 sm:mb-4">
|
||||||
|
<div className="grid md:flex gap-x-5 gap-y-3 w-full items-end">
|
||||||
|
<div className="px-2 sm:px-1">
|
||||||
|
<CardTitle className="mb-2">ZFS</CardTitle>
|
||||||
|
<CardDescription className="flex">
|
||||||
|
<Trans>Click on a pool to view vdev and dataset details.</Trans>
|
||||||
|
</CardDescription>
|
||||||
|
</div>
|
||||||
|
<div className="relative ms-auto w-full max-w-full md:w-64">
|
||||||
|
<Input
|
||||||
|
placeholder={t`Filter...`}
|
||||||
|
value={globalFilter}
|
||||||
|
onChange={(event) => setGlobalFilter(event.target.value)}
|
||||||
|
className="px-4 w-full max-w-full md:w-64"
|
||||||
|
/>
|
||||||
|
{globalFilter && (
|
||||||
|
<Button
|
||||||
|
type="button"
|
||||||
|
variant="ghost"
|
||||||
|
size="icon"
|
||||||
|
aria-label={t`Clear`}
|
||||||
|
className="absolute right-1 top-1/2 -translate-y-1/2 h-7 w-7 text-muted-foreground"
|
||||||
|
onClick={() => setGlobalFilter("")}
|
||||||
|
>
|
||||||
|
<XIcon className="h-4 w-4" />
|
||||||
|
</Button>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</CardHeader>
|
||||||
|
<div className="h-min max-h-[calc(100dvh-17rem)] max-w-full relative overflow-auto rounded-md border">
|
||||||
|
<Table>
|
||||||
|
<TableHeader className="sticky top-0 z-50 w-full border-b-2">
|
||||||
|
{table.getHeaderGroups().map((headerGroup) => (
|
||||||
|
<TableRow key={headerGroup.id}>
|
||||||
|
{headerGroup.headers.map((header) => (
|
||||||
|
<TableHead key={header.id} className="px-2">
|
||||||
|
{header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())}
|
||||||
|
</TableHead>
|
||||||
|
))}
|
||||||
|
</TableRow>
|
||||||
|
))}
|
||||||
|
</TableHeader>
|
||||||
|
<TableBody>
|
||||||
|
{rows.map((row) => (
|
||||||
|
<TableRow
|
||||||
|
key={row.id}
|
||||||
|
data-state={row.getIsSelected() && "selected"}
|
||||||
|
className="cursor-pointer"
|
||||||
|
onClick={() => openSheet(row.original)}
|
||||||
|
>
|
||||||
|
{row.getVisibleCells().map((cell) => (
|
||||||
|
<TableCell key={cell.id}>{flexRender(cell.column.columnDef.cell, cell.getContext())}</TableCell>
|
||||||
|
))}
|
||||||
|
</TableRow>
|
||||||
|
))}
|
||||||
|
</TableBody>
|
||||||
|
</Table>
|
||||||
|
</div>
|
||||||
|
</Card>
|
||||||
|
<PoolSheet poolId={activePoolId} open={sheetOpen} onOpenChange={setSheetOpen} />
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
52
internal/site/src/types.d.ts
vendored
52
internal/site/src/types.d.ts
vendored
@@ -151,6 +151,8 @@ export interface SystemStats {
|
|||||||
f?: Record<string, number>
|
f?: Record<string, number>
|
||||||
/** extra filesystems */
|
/** extra filesystems */
|
||||||
efs?: Record<string, ExtraFsStats>
|
efs?: Record<string, ExtraFsStats>
|
||||||
|
/** ZFS pool metrics */
|
||||||
|
z?: Record<string, ZfsPool>
|
||||||
/** GPU data */
|
/** GPU data */
|
||||||
g?: Record<string, GPUData>
|
g?: Record<string, GPUData>
|
||||||
/** battery percent and state */
|
/** battery percent and state */
|
||||||
@@ -178,6 +180,56 @@ export interface GPUData {
|
|||||||
e?: Record<string, number>
|
e?: Record<string, number>
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface ZfsPool {
|
||||||
|
/** total capacity (GiB) */
|
||||||
|
d: number
|
||||||
|
/** allocated (GiB) */
|
||||||
|
du: number
|
||||||
|
/** read throughput (bytes/s) */
|
||||||
|
rb?: number
|
||||||
|
/** write throughput (bytes/s) */
|
||||||
|
wb?: number
|
||||||
|
/** health: ONLINE, DEGRADED, FAULTED, ... */
|
||||||
|
h?: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ZfsScrub {
|
||||||
|
/** NONE, SCANNING, FINISHED, CANCELED */
|
||||||
|
state?: string
|
||||||
|
/** progress while scanning, e.g. "10.00%" */
|
||||||
|
progress?: string
|
||||||
|
errors?: number
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ZfsVdev {
|
||||||
|
name: string
|
||||||
|
state?: string
|
||||||
|
readErrs?: number
|
||||||
|
writeErrs?: number
|
||||||
|
checksumErrs?: number
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ZfsDataset {
|
||||||
|
name: string
|
||||||
|
used?: number
|
||||||
|
avail?: number
|
||||||
|
mount?: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ZfsPoolRecord extends RecordModel {
|
||||||
|
system: string
|
||||||
|
name: string
|
||||||
|
health: string
|
||||||
|
size: number
|
||||||
|
alloc: number
|
||||||
|
free: number
|
||||||
|
scrub: ZfsScrub | null
|
||||||
|
vdevs: ZfsVdev[] | null
|
||||||
|
datasets: ZfsDataset[] | null
|
||||||
|
details_updated: string
|
||||||
|
updated: string
|
||||||
|
}
|
||||||
|
|
||||||
export interface ExtraFsStats {
|
export interface ExtraFsStats {
|
||||||
/** disk size (gb) */
|
/** disk size (gb) */
|
||||||
d: number
|
d: number
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ It has a friendly web interface, simple configuration, and is ready to use out o
|
|||||||
- **Lightweight**: Smaller and less resource-intensive than leading solutions.
|
- **Lightweight**: Smaller and less resource-intensive than leading solutions.
|
||||||
- **Simple**: Easy setup with little manual configuration required.
|
- **Simple**: Easy setup with little manual configuration required.
|
||||||
- **Docker stats**: Tracks CPU, memory, and network usage history for each container.
|
- **Docker stats**: Tracks CPU, memory, and network usage history for each container.
|
||||||
|
- **ZFS**: Tracks pool capacity, health, and I/O, plus per-dataset usage.
|
||||||
- **Alerts**: Configurable alerts for CPU, memory, disk, bandwidth, temperature, fan speed, load average, and status.
|
- **Alerts**: Configurable alerts for CPU, memory, disk, bandwidth, temperature, fan speed, load average, and status.
|
||||||
- **Multi-user**: Users manage their own systems. Admins can share systems across users.
|
- **Multi-user**: Users manage their own systems. Admins can share systems across users.
|
||||||
- **OAuth / OIDC**: Supports many OAuth2 providers. Password auth can be disabled.
|
- **OAuth / OIDC**: Supports many OAuth2 providers. Password auth can be disabled.
|
||||||
@@ -53,6 +54,7 @@ The [quick start guide](https://beszel.dev/guide/getting-started) and other docu
|
|||||||
- **Battery** - Host system battery charge.
|
- **Battery** - Host system battery charge.
|
||||||
- **Containers** - Status and metrics of all running Docker / Podman containers.
|
- **Containers** - Status and metrics of all running Docker / Podman containers.
|
||||||
- **S.M.A.R.T.** - Host system disk health (includes eMMC wear/EOL and Linux mdraid array health via sysfs when available).
|
- **S.M.A.R.T.** - Host system disk health (includes eMMC wear/EOL and Linux mdraid array health via sysfs when available).
|
||||||
|
- **ZFS** - Pool capacity, usage, health, I/O throughput, scrub status, and per-dataset usage.
|
||||||
|
|
||||||
## Help and discussion
|
## Help and discussion
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user