mirror of
https://github.com/henrygd/beszel.git
synced 2026-09-21 08:57:48 +02:00
feat: add ZFS monitoring (#2209)
- track pool capacity, health, I/O, scrub status, and vdev errors - report dataset usage and correct ZFS filesystem metrics - add pool charts, detail views, refresh controls, and health alerts - persist pool details and include ZFS usage in disk alerts - support configurable detail intervals and legacy agent compatibility --------- Co-authored-by: hank <hank@henrygd.me>
This commit is contained in:
@@ -57,12 +57,18 @@ type SystemAlertStats struct {
|
||||
Battery [2]uint8 `json:"bat"`
|
||||
Batteries map[string]uint8 `json:"bats"`
|
||||
ExtraFs map[string]SystemAlertFsStats `json:"efs"`
|
||||
ZfsPools map[string]SystemAlertZfsPool `json:"z"`
|
||||
}
|
||||
|
||||
type SystemAlertGPUData struct {
|
||||
Usage float64 `json:"u"`
|
||||
}
|
||||
|
||||
type SystemAlertZfsPool struct {
|
||||
Total float64 `json:"d"`
|
||||
Used float64 `json:"du"`
|
||||
}
|
||||
|
||||
type SystemAlertData struct {
|
||||
systemRecord *core.Record
|
||||
alertData CachedAlertData
|
||||
@@ -111,6 +117,9 @@ func (am *AlertManager) bindEvents() {
|
||||
am.hub.OnRecordAfterUpdateSuccess("alerts").BindFunc(updateHistoryOnAlertUpdate)
|
||||
am.hub.OnRecordAfterDeleteSuccess("alerts").BindFunc(resolveHistoryOnAlertDelete)
|
||||
am.hub.OnRecordAfterUpdateSuccess("smart_devices").BindFunc(am.handleSmartDeviceAlert)
|
||||
am.hub.OnRecordAfterCreateSuccess("zfs_pools").BindFunc(am.handleZfsPoolCreateAlert)
|
||||
am.hub.OnRecordAfterUpdateSuccess("zfs_pools").BindFunc(am.handleZfsPoolAlert)
|
||||
am.hub.OnRecordAfterDeleteSuccess("zfs_pools").BindFunc(resolveZfsPoolHistoryOnDelete)
|
||||
|
||||
am.hub.OnServe().BindFunc(func(e *core.ServeEvent) error {
|
||||
// Populate all alerts into cache on startup
|
||||
|
||||
@@ -44,6 +44,14 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
|
||||
maxUsedPct = usedPct
|
||||
}
|
||||
}
|
||||
for _, pool := range data.Stats.ZfsPools {
|
||||
if pool != nil && pool.Total > 0 {
|
||||
usedPct := pool.Used / pool.Total * 100
|
||||
if usedPct > maxUsedPct {
|
||||
maxUsedPct = usedPct
|
||||
}
|
||||
}
|
||||
}
|
||||
val = maxUsedPct
|
||||
case "Temperature":
|
||||
if data.Info.DashboardTemp < 1 {
|
||||
@@ -208,6 +216,16 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
|
||||
alert.mapSums[key] += float32(fs.DiskUsed / fs.DiskTotal * 100)
|
||||
}
|
||||
}
|
||||
// add zfs pool usage from historical record
|
||||
for key, pool := range stats.ZfsPools {
|
||||
if pool.Total > 0 {
|
||||
zfsKey := zfsDiskAlertKey(key)
|
||||
if _, ok := alert.mapSums[zfsKey]; !ok {
|
||||
alert.mapSums[zfsKey] = 0.0
|
||||
}
|
||||
alert.mapSums[zfsKey] += float32(pool.Used / pool.Total * 100)
|
||||
}
|
||||
}
|
||||
case "Temperature":
|
||||
if alert.mapSums == nil {
|
||||
alert.mapSums = make(map[string]float32, len(stats.Temperatures))
|
||||
@@ -255,7 +273,7 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
|
||||
sumPct := float32(value)
|
||||
if sumPct > maxPct {
|
||||
maxPct = sumPct
|
||||
alert.descriptor = fmt.Sprintf("Usage of %s", key)
|
||||
alert.descriptor = diskAlertDescriptor(key)
|
||||
}
|
||||
}
|
||||
alert.val = float64(maxPct / float32(alert.count))
|
||||
@@ -301,6 +319,17 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
|
||||
return nil
|
||||
}
|
||||
|
||||
func zfsDiskAlertKey(poolName string) string {
|
||||
return "zfs:" + poolName
|
||||
}
|
||||
|
||||
func diskAlertDescriptor(key string) string {
|
||||
if poolName, ok := strings.CutPrefix(key, "zfs:"); ok {
|
||||
return fmt.Sprintf("Usage of ZFS pool %s", poolName)
|
||||
}
|
||||
return fmt.Sprintf("Usage of %s", key)
|
||||
}
|
||||
|
||||
func hasRepresentativeBattery(legacy [2]uint8, batteries map[string]uint8) bool {
|
||||
return legacy != [2]uint8{} || len(batteries) > 0
|
||||
}
|
||||
|
||||
142
internal/alerts/alerts_zfs.go
Normal file
142
internal/alerts/alerts_zfs.go
Normal file
@@ -0,0 +1,142 @@
|
||||
package alerts
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/pocketbase/dbx"
|
||||
"github.com/pocketbase/pocketbase/core"
|
||||
)
|
||||
|
||||
// handleZfsPoolAlert sends alerts when a ZFS pool health state worsens and
|
||||
// resolves the alert history entry when the pool recovers. Like the SMART
|
||||
// hook, this is automatic and does not require user opt-in.
|
||||
func (am *AlertManager) handleZfsPoolAlert(e *core.RecordEvent) error {
|
||||
return am.handleZfsPoolHealthAlert(e, e.Record.Original().GetString("health"))
|
||||
}
|
||||
|
||||
func (am *AlertManager) handleZfsPoolCreateAlert(e *core.RecordEvent) error {
|
||||
return am.handleZfsPoolHealthAlert(e, "")
|
||||
}
|
||||
|
||||
func (am *AlertManager) handleZfsPoolHealthAlert(e *core.RecordEvent, oldHealth string) error {
|
||||
newHealth := e.Record.GetString("health")
|
||||
oldSeverity := zfsPoolSeverity(oldHealth)
|
||||
newSeverity := zfsPoolSeverity(newHealth)
|
||||
|
||||
systemID := e.Record.GetString("system")
|
||||
if systemID == "" {
|
||||
return e.Next()
|
||||
}
|
||||
|
||||
systemRecord, err := e.App.FindRecordById("systems", systemID)
|
||||
if err != nil {
|
||||
e.App.Logger().Error("Failed to find system for ZFS alert", "err", err, "systemID", systemID)
|
||||
return e.Next()
|
||||
}
|
||||
|
||||
// Pool recovered to a healthy state: resolve any open history entries.
|
||||
if newSeverity == 1 && oldSeverity > 1 {
|
||||
resolveAllAlertHistoryRecords(e.App, e.Record.Id)
|
||||
return e.Next()
|
||||
}
|
||||
|
||||
if !shouldSendZfsPoolAlert(oldSeverity, newSeverity) {
|
||||
return e.Next()
|
||||
}
|
||||
|
||||
systemName := systemRecord.GetString("name")
|
||||
poolName := e.Record.GetString("name")
|
||||
|
||||
title := fmt.Sprintf("ZFS pool %s on %s: %s", newHealth, systemName, poolName)
|
||||
message := fmt.Sprintf("ZFS pool %s (%s) was first observed as %s", poolName, systemName, newHealth)
|
||||
if oldSeverity > 0 {
|
||||
message = fmt.Sprintf("ZFS pool %s (%s) health changed from %s to %s", poolName, systemName, oldHealth, newHealth)
|
||||
}
|
||||
|
||||
userIDs := systemRecord.GetStringSlice("users")
|
||||
if len(userIDs) == 0 {
|
||||
return e.Next()
|
||||
}
|
||||
|
||||
for _, userID := range userIDs {
|
||||
if err := am.SendAlert(AlertMessageData{
|
||||
UserID: userID,
|
||||
SystemID: systemID,
|
||||
Title: title,
|
||||
Message: message,
|
||||
Link: am.hub.MakeLink("system", systemID),
|
||||
LinkText: "View " + systemName,
|
||||
}); err != nil {
|
||||
e.App.Logger().Error("Failed to send ZFS alert", "err", err, "userID", userID)
|
||||
}
|
||||
_ = createZfsPoolHistoryRecord(e.App, userID, systemID, e.Record.Id, poolName)
|
||||
}
|
||||
|
||||
return e.Next()
|
||||
}
|
||||
|
||||
// resolveZfsPoolHistoryOnDelete resolves open alert history entries when a
|
||||
// pool record is deleted (manually or because the pool disappeared), so the
|
||||
// UI does not keep showing an ongoing alert for a pool that no longer exists.
|
||||
func resolveZfsPoolHistoryOnDelete(e *core.RecordEvent) error {
|
||||
resolveAllAlertHistoryRecords(e.App, e.Record.Id)
|
||||
return e.Next()
|
||||
}
|
||||
|
||||
// shouldSendZfsPoolAlert reports whether a health transition warrants an alert.
|
||||
// First observations of unhealthy pools and worsening transitions are reported.
|
||||
func shouldSendZfsPoolAlert(oldSeverity, newSeverity int) bool {
|
||||
return newSeverity > 1 && (oldSeverity == 0 || newSeverity > oldSeverity)
|
||||
}
|
||||
|
||||
// zfsPoolSeverity ranks pool health states: healthy (1), degraded (2),
|
||||
// failed/unavailable (3), unknown (0).
|
||||
func zfsPoolSeverity(health string) int {
|
||||
switch health {
|
||||
case "ONLINE":
|
||||
return 1
|
||||
case "DEGRADED":
|
||||
return 2
|
||||
case "FAULTED", "OFFLINE", "UNAVAIL", "REMOVED", "SUSPENDED":
|
||||
return 3
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
// createZfsPoolHistoryRecord logs a pool health alert in the alerts history so
|
||||
// it is visible in the UI without creating an editable alert configuration.
|
||||
func createZfsPoolHistoryRecord(app core.App, userID, systemID, alertID, poolName string) error {
|
||||
collection, err := app.FindCachedCollectionByNameOrId("alerts_history")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
record := core.NewRecord(collection)
|
||||
record.Set("user", userID)
|
||||
record.Set("system", systemID)
|
||||
record.Set("alert_id", alertID)
|
||||
record.Set("name", "ZFS Pool: "+poolName)
|
||||
return app.Save(record)
|
||||
}
|
||||
|
||||
// resolveAllAlertHistoryRecords resolves every open history entry for an alert
|
||||
// record id (one per system user).
|
||||
func resolveAllAlertHistoryRecords(app core.App, alertID string) {
|
||||
records, err := app.FindRecordsByFilter(
|
||||
"alerts_history",
|
||||
"alert_id={:alert_id} && resolved=null",
|
||||
"", 0, 0,
|
||||
dbx.Params{"alert_id": alertID},
|
||||
)
|
||||
if err != nil || len(records) == 0 {
|
||||
return
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
for _, record := range records {
|
||||
record.Set("resolved", now)
|
||||
if err := app.Save(record); err != nil {
|
||||
app.Logger().Error("Failed to resolve ZFS alert history", "err", err, "recordId", record.Id)
|
||||
}
|
||||
}
|
||||
}
|
||||
145
internal/alerts/alerts_zfs_disk_test.go
Normal file
145
internal/alerts/alerts_zfs_disk_test.go
Normal file
@@ -0,0 +1,145 @@
|
||||
//go:build testing
|
||||
|
||||
package alerts_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/henrygd/beszel/internal/entities/system"
|
||||
beszelTests "github.com/henrygd/beszel/internal/tests"
|
||||
|
||||
"github.com/pocketbase/dbx"
|
||||
"github.com/pocketbase/pocketbase/tools/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestDiskAlertZfsPoolMultiMinute verifies that ZFS pool usage participates in
|
||||
// the Disk threshold alert using historical per-minute values, mirroring the
|
||||
// extra-filesystem behavior.
|
||||
func TestDiskAlertZfsPoolMultiMinute(t *testing.T) {
|
||||
hub, user := beszelTests.GetHubWithUser(t)
|
||||
defer hub.Cleanup()
|
||||
|
||||
systems, err := beszelTests.CreateSystems(hub, 1, user.Id, "up")
|
||||
require.NoError(t, err)
|
||||
systemRecord := systems[0]
|
||||
|
||||
diskAlert, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{
|
||||
"name": "Disk",
|
||||
"system": systemRecord.Id,
|
||||
"user": user.Id,
|
||||
"value": 80, // threshold: 80%
|
||||
"min": 2, // requires historical averaging
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
am := hub.GetAlertManager()
|
||||
now := time.Now().UTC()
|
||||
|
||||
poolHigh := map[string]*system.ZfsPool{
|
||||
"tank": {Total: 1000, Used: 920}, // 92% - above threshold
|
||||
}
|
||||
|
||||
recordTimes := []time.Duration{
|
||||
-180 * time.Second,
|
||||
-90 * time.Second,
|
||||
-60 * time.Second,
|
||||
-30 * time.Second,
|
||||
}
|
||||
|
||||
for _, offset := range recordTimes {
|
||||
stats := system.Stats{
|
||||
DiskPct: 30, // root disk at 30% - below threshold
|
||||
ZfsPools: poolHigh,
|
||||
}
|
||||
statsJSON, _ := json.Marshal(stats)
|
||||
|
||||
recordTime := now.Add(offset)
|
||||
record, err := beszelTests.CreateRecord(hub, "system_stats", map[string]any{
|
||||
"system": systemRecord.Id,
|
||||
"type": "1m",
|
||||
"stats": string(statsJSON),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
record.SetRaw("created", recordTime.Format(types.DefaultDateLayout))
|
||||
err = hub.SaveNoValidate(record)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
combinedDataHigh := &system.CombinedData{
|
||||
Stats: system.Stats{
|
||||
DiskPct: 30,
|
||||
ZfsPools: poolHigh,
|
||||
},
|
||||
Info: system.Info{
|
||||
DiskPct: 30,
|
||||
},
|
||||
}
|
||||
|
||||
systemRecord.Set("updated", now)
|
||||
err = hub.SaveNoValidate(systemRecord)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = am.HandleSystemAlerts(systemRecord, combinedDataHigh)
|
||||
require.NoError(t, err)
|
||||
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
|
||||
diskAlert, err = hub.FindFirstRecordByFilter("alerts", "id={:id}", dbx.Params{"id": diskAlert.Id})
|
||||
require.NoError(t, err)
|
||||
assert.True(t, diskAlert.GetBool("triggered"),
|
||||
"Alert should be triggered when ZFS pool average (92%%) exceeds threshold (80%%)")
|
||||
|
||||
// --- Resolution: pool drops to 50%, alert should resolve ---
|
||||
|
||||
poolLow := map[string]*system.ZfsPool{
|
||||
"tank": {Total: 1000, Used: 500}, // 50% - below threshold
|
||||
}
|
||||
|
||||
newNow := now.Add(2 * time.Minute)
|
||||
for _, offset := range recordTimes {
|
||||
stats := system.Stats{
|
||||
DiskPct: 30,
|
||||
ZfsPools: poolLow,
|
||||
}
|
||||
statsJSON, _ := json.Marshal(stats)
|
||||
|
||||
recordTime := newNow.Add(offset)
|
||||
record, err := beszelTests.CreateRecord(hub, "system_stats", map[string]any{
|
||||
"system": systemRecord.Id,
|
||||
"type": "1m",
|
||||
"stats": string(statsJSON),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
record.SetRaw("created", recordTime.Format(types.DefaultDateLayout))
|
||||
err = hub.SaveNoValidate(record)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
combinedDataLow := &system.CombinedData{
|
||||
Stats: system.Stats{
|
||||
DiskPct: 30,
|
||||
ZfsPools: poolLow,
|
||||
},
|
||||
Info: system.Info{
|
||||
DiskPct: 30,
|
||||
},
|
||||
}
|
||||
|
||||
systemRecord.Set("updated", newNow)
|
||||
err = hub.SaveNoValidate(systemRecord)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = am.HandleSystemAlerts(systemRecord, combinedDataLow)
|
||||
require.NoError(t, err)
|
||||
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
|
||||
diskAlert, err = hub.FindFirstRecordByFilter("alerts", "id={:id}", dbx.Params{"id": diskAlert.Id})
|
||||
require.NoError(t, err)
|
||||
assert.False(t, diskAlert.GetBool("triggered"),
|
||||
"Alert should be resolved when ZFS pool average (50%%) drops below threshold (80%%)")
|
||||
}
|
||||
15
internal/alerts/alerts_zfs_key_test.go
Normal file
15
internal/alerts/alerts_zfs_key_test.go
Normal file
@@ -0,0 +1,15 @@
|
||||
//go:build testing
|
||||
|
||||
package alerts
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestZfsDiskAlertKeyIsNamespaced(t *testing.T) {
|
||||
assert.Equal(t, "zfs:tank", zfsDiskAlertKey("tank"))
|
||||
assert.Equal(t, "Usage of ZFS pool tank", diskAlertDescriptor(zfsDiskAlertKey("tank")))
|
||||
assert.Equal(t, "Usage of tank", diskAlertDescriptor("tank"))
|
||||
}
|
||||
292
internal/alerts/alerts_zfs_test.go
Normal file
292
internal/alerts/alerts_zfs_test.go
Normal file
@@ -0,0 +1,292 @@
|
||||
//go:build testing
|
||||
|
||||
package alerts_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
beszelTests "github.com/henrygd/beszel/internal/tests"
|
||||
"github.com/pocketbase/pocketbase/core"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestZfsPoolAlertOnlineToDegraded(t *testing.T) {
|
||||
hub, user := beszelTests.GetHubWithUser(t)
|
||||
defer hub.Cleanup()
|
||||
|
||||
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
|
||||
"name": "test-system",
|
||||
"users": []string{user.Id},
|
||||
"host": "127.0.0.1",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
|
||||
"system": system.Id,
|
||||
"name": "tank",
|
||||
"health": "ONLINE",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Re-fetch so PocketBase tracks original values
|
||||
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
|
||||
assert.NoError(t, err)
|
||||
|
||||
pool.Set("health", "DEGRADED")
|
||||
err = hub.Save(pool)
|
||||
assert.NoError(t, err)
|
||||
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
assert.EqualValues(t, 1, hub.TestMailer.TotalSend(), "should have 1 email sent after pool became DEGRADED")
|
||||
lastMessage := hub.TestMailer.LastMessage()
|
||||
assert.Contains(t, lastMessage.Subject, "ZFS pool DEGRADED on test-system")
|
||||
assert.Contains(t, lastMessage.Subject, "tank")
|
||||
assert.Contains(t, lastMessage.Text, "ONLINE to DEGRADED")
|
||||
}
|
||||
|
||||
func TestZfsPoolAlertDegradedToFaulted(t *testing.T) {
|
||||
hub, user := beszelTests.GetHubWithUser(t)
|
||||
defer hub.Cleanup()
|
||||
|
||||
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
|
||||
"name": "test-system",
|
||||
"users": []string{user.Id},
|
||||
"host": "127.0.0.1",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
|
||||
"system": system.Id,
|
||||
"name": "rpool",
|
||||
"health": "DEGRADED",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
|
||||
assert.NoError(t, err)
|
||||
|
||||
pool.Set("health", "FAULTED")
|
||||
err = hub.Save(pool)
|
||||
assert.NoError(t, err)
|
||||
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
assert.EqualValues(t, 2, hub.TestMailer.TotalSend(), "should alert on initial DEGRADED state and later FAULTED transition")
|
||||
lastMessage := hub.TestMailer.LastMessage()
|
||||
assert.Contains(t, lastMessage.Subject, "ZFS pool FAULTED on test-system")
|
||||
}
|
||||
|
||||
func TestZfsPoolAlertNoAlertOnRecovery(t *testing.T) {
|
||||
hub, user := beszelTests.GetHubWithUser(t)
|
||||
defer hub.Cleanup()
|
||||
|
||||
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
|
||||
"name": "test-system",
|
||||
"users": []string{user.Id},
|
||||
"host": "127.0.0.1",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
|
||||
"system": system.Id,
|
||||
"name": "tank",
|
||||
"health": "DEGRADED",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Trigger a worsening alert first
|
||||
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
|
||||
assert.NoError(t, err)
|
||||
pool.Set("health", "FAULTED")
|
||||
err = hub.Save(pool)
|
||||
assert.NoError(t, err)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
assert.EqualValues(t, 2, hub.TestMailer.TotalSend(), "expected alerts for initial DEGRADED state and DEGRADED -> FAULTED")
|
||||
|
||||
// Recovery back to ONLINE must not send a new alert
|
||||
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
|
||||
assert.NoError(t, err)
|
||||
pool.Set("health", "ONLINE")
|
||||
err = hub.Save(pool)
|
||||
assert.NoError(t, err)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
assert.EqualValues(t, 2, hub.TestMailer.TotalSend(), "recovery should not send a new alert")
|
||||
|
||||
// And the open history entry should have been resolved
|
||||
history, err := hub.FindRecordsByFilter("alerts_history", "alert_id={:alert_id}", "", 0, 0, map[string]any{"alert_id": pool.Id})
|
||||
assert.NoError(t, err)
|
||||
requireHistoryResolved(t, history)
|
||||
}
|
||||
|
||||
func TestZfsPoolAlertUnknownHealthDoesNotResolve(t *testing.T) {
|
||||
hub, user := beszelTests.GetHubWithUser(t)
|
||||
defer hub.Cleanup()
|
||||
|
||||
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
|
||||
"name": "test-system",
|
||||
"users": []string{user.Id},
|
||||
"host": "127.0.0.1",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
|
||||
"system": system.Id,
|
||||
"name": "tank",
|
||||
"health": "DEGRADED",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
|
||||
require.NoError(t, err)
|
||||
pool.Set("health", "")
|
||||
require.NoError(t, hub.Save(pool))
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
history, err := hub.FindRecordsByFilter("alerts_history", "alert_id={:alert_id} && resolved=null", "", 0, 0, map[string]any{"alert_id": pool.Id})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, history, 1, "unknown health must not resolve an active alert")
|
||||
}
|
||||
|
||||
func TestZfsPoolAlertUnknownToFaulted(t *testing.T) {
|
||||
hub, user := beszelTests.GetHubWithUser(t)
|
||||
defer hub.Cleanup()
|
||||
|
||||
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
|
||||
"name": "test-system",
|
||||
"users": []string{user.Id},
|
||||
"host": "127.0.0.1",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
|
||||
"system": system.Id,
|
||||
"name": "tank",
|
||||
"health": "",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
|
||||
assert.NoError(t, err)
|
||||
|
||||
pool.Set("health", "FAULTED")
|
||||
err = hub.Save(pool)
|
||||
assert.NoError(t, err)
|
||||
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
assert.EqualValues(t, 1, hub.TestMailer.TotalSend(), "should alert when a previously unknown pool becomes FAULTED")
|
||||
}
|
||||
|
||||
func TestZfsPoolAlertOnInitialUnhealthyState(t *testing.T) {
|
||||
hub, user := beszelTests.GetHubWithUser(t)
|
||||
defer hub.Cleanup()
|
||||
|
||||
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
|
||||
"name": "test-system",
|
||||
"users": []string{user.Id},
|
||||
"host": "127.0.0.1",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
|
||||
"system": system.Id,
|
||||
"name": "tank",
|
||||
"health": "DEGRADED",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
require.EqualValues(t, 1, hub.TestMailer.TotalSend())
|
||||
assert.Contains(t, hub.TestMailer.LastMessage().Text, "first observed as DEGRADED")
|
||||
|
||||
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, hub.Save(pool))
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
assert.EqualValues(t, 1, hub.TestMailer.TotalSend(), "unchanged unhealthy health must not duplicate alerts")
|
||||
}
|
||||
|
||||
func TestZfsPoolAlertWritesHistory(t *testing.T) {
|
||||
hub, user := beszelTests.GetHubWithUser(t)
|
||||
defer hub.Cleanup()
|
||||
|
||||
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
|
||||
"name": "test-system",
|
||||
"users": []string{user.Id},
|
||||
"host": "127.0.0.1",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
|
||||
"system": system.Id,
|
||||
"name": "tank",
|
||||
"health": "ONLINE",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
|
||||
assert.NoError(t, err)
|
||||
pool.Set("health", "FAULTED")
|
||||
err = hub.Save(pool)
|
||||
assert.NoError(t, err)
|
||||
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
history, err := hub.FindRecordsByFilter("alerts_history", "alert_id={:alert_id}", "", 0, 0, map[string]any{"alert_id": pool.Id})
|
||||
assert.NoError(t, err)
|
||||
require.Len(t, history, 1, "expected one history entry per user")
|
||||
assert.Equal(t, "ZFS Pool: tank", history[0].GetString("name"))
|
||||
assert.Equal(t, system.Id, history[0].GetString("system"))
|
||||
}
|
||||
|
||||
func TestZfsPoolAlertResolvedOnRecordDelete(t *testing.T) {
|
||||
hub, user := beszelTests.GetHubWithUser(t)
|
||||
defer hub.Cleanup()
|
||||
|
||||
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
|
||||
"name": "test-system",
|
||||
"users": []string{user.Id},
|
||||
"host": "127.0.0.1",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
|
||||
"system": system.Id,
|
||||
"name": "tank",
|
||||
"health": "ONLINE",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Trigger an alert so an open history entry exists.
|
||||
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
|
||||
assert.NoError(t, err)
|
||||
pool.Set("health", "FAULTED")
|
||||
err = hub.Save(pool)
|
||||
assert.NoError(t, err)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
history, err := hub.FindRecordsByFilter("alerts_history", "alert_id={:alert_id} && resolved=null", "", 0, 0, map[string]any{"alert_id": pool.Id})
|
||||
assert.NoError(t, err)
|
||||
require.Len(t, history, 1, "expected one open history entry")
|
||||
|
||||
// Deleting the pool record must resolve the open entry.
|
||||
err = hub.Delete(pool)
|
||||
assert.NoError(t, err)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
history, err = hub.FindRecordsByFilter("alerts_history", "alert_id={:alert_id}", "", 0, 0, map[string]any{"alert_id": pool.Id})
|
||||
assert.NoError(t, err)
|
||||
require.Len(t, history, 1)
|
||||
requireHistoryResolved(t, history)
|
||||
}
|
||||
|
||||
func requireHistoryResolved(t *testing.T, history []*core.Record) {
|
||||
t.Helper()
|
||||
for _, record := range history {
|
||||
assert.False(t, record.GetDateTime("resolved").Time().IsZero(), "expected history entry to be resolved")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user