feat(alerts): add CPU state notifications (#2249)

internal/alerts/alerts.go

@@ -48,6 +48,7 @@ type SystemAlertFsStats struct {
 // Values pulled from system_stats.stats that are relevant to alerts.
 type SystemAlertStats struct {
 	Cpu          float64                       `json:"cpu"`
+	CpuBreakdown []float64                     `json:"cpub"`
 	Mem          float64                       `json:"mp"`
 	Disk         float64                       `json:"dp"`
 	Bandwidth    [2]uint64                     `json:"b"`

internal/alerts/alerts_system.go

@@ -13,6 +13,29 @@ import (
 	"github.com/pocketbase/pocketbase/tools/types"
 )
 
+var cpuStateAlerts = map[string]struct {
+	index int
+	label string
+}{
+	"CPUIOWait": {2, "CPU I/O Wait"},
+	"CPUSteal":  {3, "CPU Steal Time"},
+}
+
+func cpuStateAlertValue(name string, breakdown []float64) (float64, bool) {
+	state, ok := cpuStateAlerts[name]
+	if !ok || len(breakdown) < 5 {
+		return 0, false
+	}
+	var total float64
+	for _, value := range breakdown {
+		total += value
+	}
+	if total <= 0 {
+		return 0, false
+	}
+	return breakdown[state.index], true
+}
+
 func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *system.CombinedData) error {
 	alerts := am.alertsCache.GetAlertsExcludingNames(systemRecord.Id, "Status")
 	if len(alerts) == 0 {
@@ -75,6 +98,11 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
 				continue
 			}
 			val = float64(data.Stats.Battery[0])
+		default:
+			var ok bool
+			if val, ok = cpuStateAlertValue(name, data.Stats.CpuBreakdown); !ok {
+				continue
+			}
 		}
 
 		triggered := alertData.Triggered
@@ -259,13 +287,20 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
 				}
 				alert.val += float64(stats.Battery[0])
 			default:
-				continue
+				value, ok := cpuStateAlertValue(alert.name, stats.CpuBreakdown)
+				if !ok {
+					continue
+				}
+				alert.val += value
 			}
 			alert.count++
 		}
 	}
 	// sum up vals for each alert
 	for _, alert := range validAlerts {
+		if alert.count == 0 {
+			continue
+		}
 		switch alert.name {
 		case "Disk":
 			maxPct := float32(0)
@@ -338,6 +373,9 @@ func (am *AlertManager) sendSystemAlert(alert SystemAlertData) {
 	// log.Printf("Sending alert %s: val %f | count %d | threshold %f\n", alert.name, alert.val, alert.count, alert.threshold)
 	systemName := alert.systemRecord.GetString("name")
 
+	if state, ok := cpuStateAlerts[alert.name]; ok {
+		alert.name = state.label
+	}
 	// change Disk to Disk usage
 	if alert.name == "Disk" {
 		alert.name += " usage"
@@ -349,7 +387,7 @@ func (am *AlertManager) sendSystemAlert(alert SystemAlertData) {
 
 	// make title alert name lowercase if not CPU or GPU
 	titleAlertName := alert.name
-	if titleAlertName != "CPU" && titleAlertName != "GPU" {
+	if titleAlertName != "CPU" && titleAlertName != "GPU" && !strings.HasPrefix(titleAlertName, "CPU") {
 		titleAlertName = strings.ToLower(titleAlertName)
 	}
 

internal/alerts/alerts_system_test.go

@@ -146,6 +146,20 @@ func setCPUAlertValue(info *system.Info, stats *system.Stats, value float64) {
 	stats.Cpu = value
 }
 
+func setCPUStateAlertValue(_ *system.Info, stats *system.Stats, value []float64) {
+	stats.CpuBreakdown = value
+}
+
+var cpuStateAlertTests = []struct {
+	name     string
+	trigger  []float64
+	resolve  []float64
+	baseline []float64
+}{
+	{"CPUIOWait", []float64{0, 0, 51, 0, 49}, []float64{0, 0, 48, 0, 52}, []float64{0, 0, 10, 0, 90}},
+	{"CPUSteal", []float64{0, 0, 0, 51, 49}, []float64{0, 0, 0, 48, 52}, []float64{0, 0, 0, 10, 90}},
+}
+
 func setMemoryAlertValue(info *system.Info, stats *system.Stats, value float64) {
 	info.MemPct = value
 	stats.MemPct = value
@@ -191,6 +205,11 @@ func setBatteryAlertValue(info *system.Info, stats *system.Stats, value [2]uint8
 
 func TestSystemAlertsOneMin(t *testing.T) {
 	testOneMinuteSystemAlert(t, "CPU", 50, setCPUAlertValue, 51, 49)
+	for _, test := range cpuStateAlertTests {
+		t.Run(test.name, func(t *testing.T) {
+			testOneMinuteSystemAlert(t, test.name, 50, setCPUStateAlertValue, test.trigger, test.resolve)
+		})
+	}
 	testOneMinuteSystemAlert(t, "Memory", 50, setMemoryAlertValue, 51, 49)
 	testOneMinuteSystemAlert(t, "Disk", 50, setDiskAlertValue, 51, 49)
 	testOneMinuteSystemAlert(t, "Bandwidth", 50, setBandwidthAlertValue, [2]uint64{megabytesToBytes(26), megabytesToBytes(25)}, [2]uint64{megabytesToBytes(25), megabytesToBytes(24)})
@@ -204,6 +223,11 @@ func TestSystemAlertsOneMin(t *testing.T) {
 
 func TestSystemAlertsTwoMin(t *testing.T) {
 	testMultiMinuteSystemAlert(t, "CPU", 50, 2, setCPUAlertValue, 10, 51, 48)
+	for _, test := range cpuStateAlertTests {
+		t.Run(test.name, func(t *testing.T) {
+			testMultiMinuteSystemAlert(t, test.name, 50, 2, setCPUStateAlertValue, test.baseline, test.trigger, test.resolve)
+		})
+	}
 	testMultiMinuteSystemAlert(t, "Memory", 50, 2, setMemoryAlertValue, 10, 51, 48)
 	testMultiMinuteSystemAlert(t, "Disk", 50, 2, setDiskAlertValue, 10, 51, 48)
 	testMultiMinuteSystemAlert(t, "Bandwidth", 50, 2, setBandwidthAlertValue, [2]uint64{megabytesToBytes(10), megabytesToBytes(10)}, [2]uint64{megabytesToBytes(26), megabytesToBytes(25)}, [2]uint64{megabytesToBytes(10), megabytesToBytes(10)})
@@ -214,3 +238,17 @@ func TestSystemAlertsTwoMin(t *testing.T) {
 	testMultiMinuteSystemAlert(t, "LoadAvg15", 4, 2, setLoadAvgAlertValue, [3]float64{0, 0, 2}, [3]float64{0, 0, 4.1}, [3]float64{0, 0, 3.5})
 	testMultiMinuteSystemAlert(t, "Battery", 20, 2, setBatteryAlertValue, [2]uint8{21, 0}, [2]uint8{19, 0}, [2]uint8{25, 1})
 }
+
+func TestCPUStateAlertWithoutBreakdown(t *testing.T) {
+	fixture := newSystemAlertTestFixture(t, "CPUSteal", 1, 1)
+	defer fixture.cleanup()
+
+	synctest.Test(t, func(t *testing.T) {
+		submitValue(fixture, t, []float64(nil), setCPUStateAlertValue)
+		submitValue(fixture, t, []float64{0, 0, 0, 0, 0}, setCPUStateAlertValue)
+		waitForSystemAlert(time.Second)
+
+		fixture.assertTriggered(t, false, "Alert should ignore missing CPU breakdown data")
+		assert.Zero(t, fixture.hub.TestMailer.TotalSend(), "No email should be sent without CPU breakdown data")
+	})
+}