feat: add ZFS monitoring (#2209)

- track pool capacity, health, I/O, scrub status, and vdev errors
- report dataset usage and correct ZFS filesystem metrics
- add pool charts, detail views, refresh controls, and health alerts
- persist pool details and include ZFS usage in disk alerts
- support configurable detail intervals and legacy agent compatibility

---------

Co-authored-by: hank <hank@henrygd.me>
This commit is contained in:
Tamás Vince
2026-09-01 18:19:36 +02:00
committed by GitHub
parent b38fb7dafa
commit 917d069ab3
46 changed files with 3768 additions and 15 deletions

View File

@@ -57,12 +57,18 @@ type SystemAlertStats struct {
Battery [2]uint8 `json:"bat"`
Batteries map[string]uint8 `json:"bats"`
ExtraFs map[string]SystemAlertFsStats `json:"efs"`
ZfsPools map[string]SystemAlertZfsPool `json:"z"`
}
type SystemAlertGPUData struct {
Usage float64 `json:"u"`
}
type SystemAlertZfsPool struct {
Total float64 `json:"d"`
Used float64 `json:"du"`
}
type SystemAlertData struct {
systemRecord *core.Record
alertData CachedAlertData
@@ -111,6 +117,9 @@ func (am *AlertManager) bindEvents() {
am.hub.OnRecordAfterUpdateSuccess("alerts").BindFunc(updateHistoryOnAlertUpdate)
am.hub.OnRecordAfterDeleteSuccess("alerts").BindFunc(resolveHistoryOnAlertDelete)
am.hub.OnRecordAfterUpdateSuccess("smart_devices").BindFunc(am.handleSmartDeviceAlert)
am.hub.OnRecordAfterCreateSuccess("zfs_pools").BindFunc(am.handleZfsPoolCreateAlert)
am.hub.OnRecordAfterUpdateSuccess("zfs_pools").BindFunc(am.handleZfsPoolAlert)
am.hub.OnRecordAfterDeleteSuccess("zfs_pools").BindFunc(resolveZfsPoolHistoryOnDelete)
am.hub.OnServe().BindFunc(func(e *core.ServeEvent) error {
// Populate all alerts into cache on startup

View File

@@ -44,6 +44,14 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
maxUsedPct = usedPct
}
}
for _, pool := range data.Stats.ZfsPools {
if pool != nil && pool.Total > 0 {
usedPct := pool.Used / pool.Total * 100
if usedPct > maxUsedPct {
maxUsedPct = usedPct
}
}
}
val = maxUsedPct
case "Temperature":
if data.Info.DashboardTemp < 1 {
@@ -208,6 +216,16 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
alert.mapSums[key] += float32(fs.DiskUsed / fs.DiskTotal * 100)
}
}
// add zfs pool usage from historical record
for key, pool := range stats.ZfsPools {
if pool.Total > 0 {
zfsKey := zfsDiskAlertKey(key)
if _, ok := alert.mapSums[zfsKey]; !ok {
alert.mapSums[zfsKey] = 0.0
}
alert.mapSums[zfsKey] += float32(pool.Used / pool.Total * 100)
}
}
case "Temperature":
if alert.mapSums == nil {
alert.mapSums = make(map[string]float32, len(stats.Temperatures))
@@ -255,7 +273,7 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
sumPct := float32(value)
if sumPct > maxPct {
maxPct = sumPct
alert.descriptor = fmt.Sprintf("Usage of %s", key)
alert.descriptor = diskAlertDescriptor(key)
}
}
alert.val = float64(maxPct / float32(alert.count))
@@ -301,6 +319,17 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
return nil
}
func zfsDiskAlertKey(poolName string) string {
return "zfs:" + poolName
}
func diskAlertDescriptor(key string) string {
if poolName, ok := strings.CutPrefix(key, "zfs:"); ok {
return fmt.Sprintf("Usage of ZFS pool %s", poolName)
}
return fmt.Sprintf("Usage of %s", key)
}
func hasRepresentativeBattery(legacy [2]uint8, batteries map[string]uint8) bool {
return legacy != [2]uint8{} || len(batteries) > 0
}

View File

@@ -0,0 +1,142 @@
package alerts
import (
"fmt"
"time"
"github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/core"
)
// handleZfsPoolAlert sends alerts when a ZFS pool health state worsens and
// resolves the alert history entry when the pool recovers. Like the SMART
// hook, this is automatic and does not require user opt-in.
func (am *AlertManager) handleZfsPoolAlert(e *core.RecordEvent) error {
return am.handleZfsPoolHealthAlert(e, e.Record.Original().GetString("health"))
}
func (am *AlertManager) handleZfsPoolCreateAlert(e *core.RecordEvent) error {
return am.handleZfsPoolHealthAlert(e, "")
}
func (am *AlertManager) handleZfsPoolHealthAlert(e *core.RecordEvent, oldHealth string) error {
newHealth := e.Record.GetString("health")
oldSeverity := zfsPoolSeverity(oldHealth)
newSeverity := zfsPoolSeverity(newHealth)
systemID := e.Record.GetString("system")
if systemID == "" {
return e.Next()
}
systemRecord, err := e.App.FindRecordById("systems", systemID)
if err != nil {
e.App.Logger().Error("Failed to find system for ZFS alert", "err", err, "systemID", systemID)
return e.Next()
}
// Pool recovered to a healthy state: resolve any open history entries.
if newSeverity == 1 && oldSeverity > 1 {
resolveAllAlertHistoryRecords(e.App, e.Record.Id)
return e.Next()
}
if !shouldSendZfsPoolAlert(oldSeverity, newSeverity) {
return e.Next()
}
systemName := systemRecord.GetString("name")
poolName := e.Record.GetString("name")
title := fmt.Sprintf("ZFS pool %s on %s: %s", newHealth, systemName, poolName)
message := fmt.Sprintf("ZFS pool %s (%s) was first observed as %s", poolName, systemName, newHealth)
if oldSeverity > 0 {
message = fmt.Sprintf("ZFS pool %s (%s) health changed from %s to %s", poolName, systemName, oldHealth, newHealth)
}
userIDs := systemRecord.GetStringSlice("users")
if len(userIDs) == 0 {
return e.Next()
}
for _, userID := range userIDs {
if err := am.SendAlert(AlertMessageData{
UserID: userID,
SystemID: systemID,
Title: title,
Message: message,
Link: am.hub.MakeLink("system", systemID),
LinkText: "View " + systemName,
}); err != nil {
e.App.Logger().Error("Failed to send ZFS alert", "err", err, "userID", userID)
}
_ = createZfsPoolHistoryRecord(e.App, userID, systemID, e.Record.Id, poolName)
}
return e.Next()
}
// resolveZfsPoolHistoryOnDelete resolves open alert history entries when a
// pool record is deleted (manually or because the pool disappeared), so the
// UI does not keep showing an ongoing alert for a pool that no longer exists.
func resolveZfsPoolHistoryOnDelete(e *core.RecordEvent) error {
resolveAllAlertHistoryRecords(e.App, e.Record.Id)
return e.Next()
}
// shouldSendZfsPoolAlert reports whether a health transition warrants an alert.
// First observations of unhealthy pools and worsening transitions are reported.
func shouldSendZfsPoolAlert(oldSeverity, newSeverity int) bool {
return newSeverity > 1 && (oldSeverity == 0 || newSeverity > oldSeverity)
}
// zfsPoolSeverity ranks pool health states: healthy (1), degraded (2),
// failed/unavailable (3), unknown (0).
func zfsPoolSeverity(health string) int {
switch health {
case "ONLINE":
return 1
case "DEGRADED":
return 2
case "FAULTED", "OFFLINE", "UNAVAIL", "REMOVED", "SUSPENDED":
return 3
default:
return 0
}
}
// createZfsPoolHistoryRecord logs a pool health alert in the alerts history so
// it is visible in the UI without creating an editable alert configuration.
func createZfsPoolHistoryRecord(app core.App, userID, systemID, alertID, poolName string) error {
collection, err := app.FindCachedCollectionByNameOrId("alerts_history")
if err != nil {
return err
}
record := core.NewRecord(collection)
record.Set("user", userID)
record.Set("system", systemID)
record.Set("alert_id", alertID)
record.Set("name", "ZFS Pool: "+poolName)
return app.Save(record)
}
// resolveAllAlertHistoryRecords resolves every open history entry for an alert
// record id (one per system user).
func resolveAllAlertHistoryRecords(app core.App, alertID string) {
records, err := app.FindRecordsByFilter(
"alerts_history",
"alert_id={:alert_id} && resolved=null",
"", 0, 0,
dbx.Params{"alert_id": alertID},
)
if err != nil || len(records) == 0 {
return
}
now := time.Now().UTC()
for _, record := range records {
record.Set("resolved", now)
if err := app.Save(record); err != nil {
app.Logger().Error("Failed to resolve ZFS alert history", "err", err, "recordId", record.Id)
}
}
}

View File

@@ -0,0 +1,145 @@
//go:build testing
package alerts_test
import (
"encoding/json"
"testing"
"time"
"github.com/henrygd/beszel/internal/entities/system"
beszelTests "github.com/henrygd/beszel/internal/tests"
"github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/tools/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestDiskAlertZfsPoolMultiMinute verifies that ZFS pool usage participates in
// the Disk threshold alert using historical per-minute values, mirroring the
// extra-filesystem behavior.
func TestDiskAlertZfsPoolMultiMinute(t *testing.T) {
hub, user := beszelTests.GetHubWithUser(t)
defer hub.Cleanup()
systems, err := beszelTests.CreateSystems(hub, 1, user.Id, "up")
require.NoError(t, err)
systemRecord := systems[0]
diskAlert, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{
"name": "Disk",
"system": systemRecord.Id,
"user": user.Id,
"value": 80, // threshold: 80%
"min": 2, // requires historical averaging
})
require.NoError(t, err)
am := hub.GetAlertManager()
now := time.Now().UTC()
poolHigh := map[string]*system.ZfsPool{
"tank": {Total: 1000, Used: 920}, // 92% - above threshold
}
recordTimes := []time.Duration{
-180 * time.Second,
-90 * time.Second,
-60 * time.Second,
-30 * time.Second,
}
for _, offset := range recordTimes {
stats := system.Stats{
DiskPct: 30, // root disk at 30% - below threshold
ZfsPools: poolHigh,
}
statsJSON, _ := json.Marshal(stats)
recordTime := now.Add(offset)
record, err := beszelTests.CreateRecord(hub, "system_stats", map[string]any{
"system": systemRecord.Id,
"type": "1m",
"stats": string(statsJSON),
})
require.NoError(t, err)
record.SetRaw("created", recordTime.Format(types.DefaultDateLayout))
err = hub.SaveNoValidate(record)
require.NoError(t, err)
}
combinedDataHigh := &system.CombinedData{
Stats: system.Stats{
DiskPct: 30,
ZfsPools: poolHigh,
},
Info: system.Info{
DiskPct: 30,
},
}
systemRecord.Set("updated", now)
err = hub.SaveNoValidate(systemRecord)
require.NoError(t, err)
err = am.HandleSystemAlerts(systemRecord, combinedDataHigh)
require.NoError(t, err)
time.Sleep(20 * time.Millisecond)
diskAlert, err = hub.FindFirstRecordByFilter("alerts", "id={:id}", dbx.Params{"id": diskAlert.Id})
require.NoError(t, err)
assert.True(t, diskAlert.GetBool("triggered"),
"Alert should be triggered when ZFS pool average (92%%) exceeds threshold (80%%)")
// --- Resolution: pool drops to 50%, alert should resolve ---
poolLow := map[string]*system.ZfsPool{
"tank": {Total: 1000, Used: 500}, // 50% - below threshold
}
newNow := now.Add(2 * time.Minute)
for _, offset := range recordTimes {
stats := system.Stats{
DiskPct: 30,
ZfsPools: poolLow,
}
statsJSON, _ := json.Marshal(stats)
recordTime := newNow.Add(offset)
record, err := beszelTests.CreateRecord(hub, "system_stats", map[string]any{
"system": systemRecord.Id,
"type": "1m",
"stats": string(statsJSON),
})
require.NoError(t, err)
record.SetRaw("created", recordTime.Format(types.DefaultDateLayout))
err = hub.SaveNoValidate(record)
require.NoError(t, err)
}
combinedDataLow := &system.CombinedData{
Stats: system.Stats{
DiskPct: 30,
ZfsPools: poolLow,
},
Info: system.Info{
DiskPct: 30,
},
}
systemRecord.Set("updated", newNow)
err = hub.SaveNoValidate(systemRecord)
require.NoError(t, err)
err = am.HandleSystemAlerts(systemRecord, combinedDataLow)
require.NoError(t, err)
time.Sleep(20 * time.Millisecond)
diskAlert, err = hub.FindFirstRecordByFilter("alerts", "id={:id}", dbx.Params{"id": diskAlert.Id})
require.NoError(t, err)
assert.False(t, diskAlert.GetBool("triggered"),
"Alert should be resolved when ZFS pool average (50%%) drops below threshold (80%%)")
}

View File

@@ -0,0 +1,15 @@
//go:build testing
package alerts
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestZfsDiskAlertKeyIsNamespaced(t *testing.T) {
assert.Equal(t, "zfs:tank", zfsDiskAlertKey("tank"))
assert.Equal(t, "Usage of ZFS pool tank", diskAlertDescriptor(zfsDiskAlertKey("tank")))
assert.Equal(t, "Usage of tank", diskAlertDescriptor("tank"))
}

View File

@@ -0,0 +1,292 @@
//go:build testing
package alerts_test
import (
"testing"
"time"
beszelTests "github.com/henrygd/beszel/internal/tests"
"github.com/pocketbase/pocketbase/core"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestZfsPoolAlertOnlineToDegraded(t *testing.T) {
hub, user := beszelTests.GetHubWithUser(t)
defer hub.Cleanup()
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
"name": "test-system",
"users": []string{user.Id},
"host": "127.0.0.1",
})
assert.NoError(t, err)
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
"system": system.Id,
"name": "tank",
"health": "ONLINE",
})
assert.NoError(t, err)
// Re-fetch so PocketBase tracks original values
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
assert.NoError(t, err)
pool.Set("health", "DEGRADED")
err = hub.Save(pool)
assert.NoError(t, err)
time.Sleep(50 * time.Millisecond)
assert.EqualValues(t, 1, hub.TestMailer.TotalSend(), "should have 1 email sent after pool became DEGRADED")
lastMessage := hub.TestMailer.LastMessage()
assert.Contains(t, lastMessage.Subject, "ZFS pool DEGRADED on test-system")
assert.Contains(t, lastMessage.Subject, "tank")
assert.Contains(t, lastMessage.Text, "ONLINE to DEGRADED")
}
func TestZfsPoolAlertDegradedToFaulted(t *testing.T) {
hub, user := beszelTests.GetHubWithUser(t)
defer hub.Cleanup()
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
"name": "test-system",
"users": []string{user.Id},
"host": "127.0.0.1",
})
assert.NoError(t, err)
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
"system": system.Id,
"name": "rpool",
"health": "DEGRADED",
})
assert.NoError(t, err)
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
assert.NoError(t, err)
pool.Set("health", "FAULTED")
err = hub.Save(pool)
assert.NoError(t, err)
time.Sleep(50 * time.Millisecond)
assert.EqualValues(t, 2, hub.TestMailer.TotalSend(), "should alert on initial DEGRADED state and later FAULTED transition")
lastMessage := hub.TestMailer.LastMessage()
assert.Contains(t, lastMessage.Subject, "ZFS pool FAULTED on test-system")
}
func TestZfsPoolAlertNoAlertOnRecovery(t *testing.T) {
hub, user := beszelTests.GetHubWithUser(t)
defer hub.Cleanup()
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
"name": "test-system",
"users": []string{user.Id},
"host": "127.0.0.1",
})
assert.NoError(t, err)
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
"system": system.Id,
"name": "tank",
"health": "DEGRADED",
})
assert.NoError(t, err)
// Trigger a worsening alert first
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
assert.NoError(t, err)
pool.Set("health", "FAULTED")
err = hub.Save(pool)
assert.NoError(t, err)
time.Sleep(50 * time.Millisecond)
assert.EqualValues(t, 2, hub.TestMailer.TotalSend(), "expected alerts for initial DEGRADED state and DEGRADED -> FAULTED")
// Recovery back to ONLINE must not send a new alert
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
assert.NoError(t, err)
pool.Set("health", "ONLINE")
err = hub.Save(pool)
assert.NoError(t, err)
time.Sleep(50 * time.Millisecond)
assert.EqualValues(t, 2, hub.TestMailer.TotalSend(), "recovery should not send a new alert")
// And the open history entry should have been resolved
history, err := hub.FindRecordsByFilter("alerts_history", "alert_id={:alert_id}", "", 0, 0, map[string]any{"alert_id": pool.Id})
assert.NoError(t, err)
requireHistoryResolved(t, history)
}
func TestZfsPoolAlertUnknownHealthDoesNotResolve(t *testing.T) {
hub, user := beszelTests.GetHubWithUser(t)
defer hub.Cleanup()
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
"name": "test-system",
"users": []string{user.Id},
"host": "127.0.0.1",
})
require.NoError(t, err)
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
"system": system.Id,
"name": "tank",
"health": "DEGRADED",
})
require.NoError(t, err)
time.Sleep(50 * time.Millisecond)
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
require.NoError(t, err)
pool.Set("health", "")
require.NoError(t, hub.Save(pool))
time.Sleep(50 * time.Millisecond)
history, err := hub.FindRecordsByFilter("alerts_history", "alert_id={:alert_id} && resolved=null", "", 0, 0, map[string]any{"alert_id": pool.Id})
require.NoError(t, err)
require.Len(t, history, 1, "unknown health must not resolve an active alert")
}
func TestZfsPoolAlertUnknownToFaulted(t *testing.T) {
hub, user := beszelTests.GetHubWithUser(t)
defer hub.Cleanup()
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
"name": "test-system",
"users": []string{user.Id},
"host": "127.0.0.1",
})
assert.NoError(t, err)
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
"system": system.Id,
"name": "tank",
"health": "",
})
assert.NoError(t, err)
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
assert.NoError(t, err)
pool.Set("health", "FAULTED")
err = hub.Save(pool)
assert.NoError(t, err)
time.Sleep(50 * time.Millisecond)
assert.EqualValues(t, 1, hub.TestMailer.TotalSend(), "should alert when a previously unknown pool becomes FAULTED")
}
func TestZfsPoolAlertOnInitialUnhealthyState(t *testing.T) {
hub, user := beszelTests.GetHubWithUser(t)
defer hub.Cleanup()
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
"name": "test-system",
"users": []string{user.Id},
"host": "127.0.0.1",
})
require.NoError(t, err)
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
"system": system.Id,
"name": "tank",
"health": "DEGRADED",
})
require.NoError(t, err)
time.Sleep(50 * time.Millisecond)
require.EqualValues(t, 1, hub.TestMailer.TotalSend())
assert.Contains(t, hub.TestMailer.LastMessage().Text, "first observed as DEGRADED")
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
require.NoError(t, err)
require.NoError(t, hub.Save(pool))
time.Sleep(50 * time.Millisecond)
assert.EqualValues(t, 1, hub.TestMailer.TotalSend(), "unchanged unhealthy health must not duplicate alerts")
}
func TestZfsPoolAlertWritesHistory(t *testing.T) {
hub, user := beszelTests.GetHubWithUser(t)
defer hub.Cleanup()
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
"name": "test-system",
"users": []string{user.Id},
"host": "127.0.0.1",
})
assert.NoError(t, err)
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
"system": system.Id,
"name": "tank",
"health": "ONLINE",
})
assert.NoError(t, err)
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
assert.NoError(t, err)
pool.Set("health", "FAULTED")
err = hub.Save(pool)
assert.NoError(t, err)
time.Sleep(50 * time.Millisecond)
history, err := hub.FindRecordsByFilter("alerts_history", "alert_id={:alert_id}", "", 0, 0, map[string]any{"alert_id": pool.Id})
assert.NoError(t, err)
require.Len(t, history, 1, "expected one history entry per user")
assert.Equal(t, "ZFS Pool: tank", history[0].GetString("name"))
assert.Equal(t, system.Id, history[0].GetString("system"))
}
func TestZfsPoolAlertResolvedOnRecordDelete(t *testing.T) {
hub, user := beszelTests.GetHubWithUser(t)
defer hub.Cleanup()
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
"name": "test-system",
"users": []string{user.Id},
"host": "127.0.0.1",
})
assert.NoError(t, err)
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
"system": system.Id,
"name": "tank",
"health": "ONLINE",
})
assert.NoError(t, err)
// Trigger an alert so an open history entry exists.
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
assert.NoError(t, err)
pool.Set("health", "FAULTED")
err = hub.Save(pool)
assert.NoError(t, err)
time.Sleep(50 * time.Millisecond)
history, err := hub.FindRecordsByFilter("alerts_history", "alert_id={:alert_id} && resolved=null", "", 0, 0, map[string]any{"alert_id": pool.Id})
assert.NoError(t, err)
require.Len(t, history, 1, "expected one open history entry")
// Deleting the pool record must resolve the open entry.
err = hub.Delete(pool)
assert.NoError(t, err)
time.Sleep(50 * time.Millisecond)
history, err = hub.FindRecordsByFilter("alerts_history", "alert_id={:alert_id}", "", 0, 0, map[string]any{"alert_id": pool.Id})
assert.NoError(t, err)
require.Len(t, history, 1)
requireHistoryResolved(t, history)
}
func requireHistoryResolved(t *testing.T, history []*core.Record) {
t.Helper()
for _, record := range history {
assert.False(t, record.GetDateTime("resolved").Time().IsZero(), "expected history entry to be resolved")
}
}