feat: network monitoring from agents (#2266, #1911)

Co-authored-by: Sven van Ginkel <svenvanginkel@icloud.com>
Co-authored-by: xiaomiku01 <xiaomiku01@outlook.com>

internal/alerts/alerts.go

@@ -20,10 +20,11 @@ type hubLike interface {
 }
 
 type AlertManager struct {
-	hub           hubLike
-	stopOnce      sync.Once
-	pendingAlerts sync.Map
-	alertsCache   *AlertsCache
+	hub             hubLike
+	stopOnce        sync.Once
+	pendingAlerts   sync.Map
+	alertsCache     *AlertsCache
+	networkMonitors *networkMonitorCache
 }
 
 type AlertMessageData struct {
@@ -107,8 +108,9 @@ var supportsTitle = map[string]struct{}{
 // NewAlertManager creates a new AlertManager instance.
 func NewAlertManager(app hubLike) *AlertManager {
 	am := &AlertManager{
-		hub:         app,
-		alertsCache: NewAlertsCache(app),
+		hub:             app,
+		alertsCache:     NewAlertsCache(app),
+		networkMonitors: newNetworkMonitorCache(app),
 	}
 	am.bindEvents()
 	return am
@@ -116,6 +118,7 @@ func NewAlertManager(app hubLike) *AlertManager {
 
 // Bind events to the alerts collection lifecycle
 func (am *AlertManager) bindEvents() {
+	am.bindNetworkMonitorAlertEvents()
 	am.hub.OnRecordAfterUpdateSuccess("alerts").BindFunc(updateHistoryOnAlertUpdate)
 	am.hub.OnRecordAfterDeleteSuccess("alerts").BindFunc(resolveHistoryOnAlertDelete)
 	am.hub.OnRecordAfterUpdateSuccess("smart_devices").BindFunc(am.handleSmartDeviceAlert)

internal/alerts/alerts_api.go

@@ -29,6 +29,13 @@ func UpsertUserAlerts(e *core.RequestEvent) error {
 		return e.BadRequestError("Bad data", err)
 	}
 
+	if reqData.Name == alertNameNetworkMonitorLoss {
+		if reqData.Value < 0 || reqData.Value >= 100 {
+			return e.BadRequestError("Monitor loss threshold must be at least 0 and below 100", nil)
+		}
+		reqData.Min = 0
+	}
+
 	alertsCollection, err := e.App.FindCachedCollectionByNameOrId("alerts")
 	if err != nil {
 		return err

internal/alerts/alerts_cache.go

@@ -1,6 +1,7 @@
 package alerts
 
 import (
+	"sync"
 	"time"
 
 	"github.com/pocketbase/dbx"
@@ -18,6 +19,9 @@ type CachedAlertData struct {
 	Triggered    bool
 	Min          uint8
 	PendingSince time.Time
+	// Immutable after publication; decoded only when the alert record changes.
+	MonitorStates      map[string]string
+	MonitorStatesValid bool
 	// Created   types.DateTime
 }
 
@@ -30,11 +34,18 @@ func (a *CachedAlertData) PopulateFromRecord(record *core.Record) {
 	a.Triggered = record.GetBool("triggered")
 	a.Min = uint8(record.GetInt("min"))
 	a.PendingSince = record.GetDateTime("pending_since").Time()
+	if a.Name == alertNameNetworkMonitorLoss {
+		var state networkMonitorAlertState
+		a.MonitorStatesValid = record.UnmarshalJSONField("state", &state) == nil
+		a.MonitorStates = state.Monitors
+	}
 	// a.Created = record.GetDateTime("created")
 }
 
 // AlertsCache provides an in-memory cache for system alerts.
 type AlertsCache struct {
+	// Serialize lazy loads with updates so a late load cannot replace newer state.
+	loadMu    sync.Mutex
 	app       core.App
 	store     *store.Store[string, *store.Store[string, CachedAlertData]]
 	populated bool
@@ -69,6 +80,8 @@ func (c *AlertsCache) bindEvents() *AlertsCache {
 
 // PopulateFromDB clears current entries and loads all alerts from the database into the cache.
 func (c *AlertsCache) PopulateFromDB(force bool) error {
+	c.loadMu.Lock()
+	defer c.loadMu.Unlock()
 	if !force && c.populated {
 		return nil
 	}
@@ -78,7 +91,7 @@ func (c *AlertsCache) PopulateFromDB(force bool) error {
 	}
 	c.store.RemoveAll()
 	for _, record := range records {
-		c.Update(record)
+		c.update(record)
 	}
 	c.populated = true
 	return nil
@@ -86,6 +99,12 @@ func (c *AlertsCache) PopulateFromDB(force bool) error {
 
 // Update adds or updates an alert record in the cache.
 func (c *AlertsCache) Update(record *core.Record) {
+	c.loadMu.Lock()
+	defer c.loadMu.Unlock()
+	c.update(record)
+}
+
+func (c *AlertsCache) update(record *core.Record) {
 	systemID := record.GetString("system")
 	if systemID == "" {
 		return
@@ -102,6 +121,8 @@ func (c *AlertsCache) Update(record *core.Record) {
 
 // Delete removes an alert record from the cache.
 func (c *AlertsCache) Delete(record *core.Record) {
+	c.loadMu.Lock()
+	defer c.loadMu.Unlock()
 	systemID := record.GetString("system")
 	if systemID == "" {
 		return
@@ -115,18 +136,23 @@ func (c *AlertsCache) Delete(record *core.Record) {
 func (c *AlertsCache) GetSystemAlerts(systemID string) []CachedAlertData {
 	systemStore, ok := c.store.GetOk(systemID)
 	if !ok {
-		// Populate cache for this system
-		records, err := c.app.FindAllRecords("alerts", dbx.NewExp("system={:system}", dbx.Params{"system": systemID}))
-		if err != nil {
-			return nil
+		c.loadMu.Lock()
+		defer c.loadMu.Unlock()
+		systemStore, ok = c.store.GetOk(systemID)
+		if !ok {
+			// Populate cache for this system
+			records, err := c.app.FindAllRecords("alerts", dbx.NewExp("system={:system}", dbx.Params{"system": systemID}))
+			if err != nil {
+				return nil
+			}
+			systemStore = store.New(map[string]CachedAlertData{})
+			for _, record := range records {
+				var ca CachedAlertData
+				ca.PopulateFromRecord(record)
+				systemStore.Set(record.Id, ca)
+			}
+			c.store.Set(systemID, systemStore)
 		}
-		systemStore = store.New(map[string]CachedAlertData{})
-		for _, record := range records {
-			var ca CachedAlertData
-			ca.PopulateFromRecord(record)
-			systemStore.Set(record.Id, ca)
-		}
-		c.store.Set(systemID, systemStore)
 	}
 	all := systemStore.GetAll()
 	alerts := make([]CachedAlertData, 0, len(all))

internal/alerts/alerts_history.go

@@ -9,6 +9,12 @@ import (
 
 // On triggered alert record delete, set matching alert history record to resolved
 func resolveHistoryOnAlertDelete(e *core.RecordEvent) error {
+	if e.Record.GetString("name") == alertNameNetworkMonitorLoss {
+		if err := resolveNetworkMonitorHistory(e.App, e.Record.Id); err != nil {
+			return err
+		}
+		return e.Next()
+	}
 	if !e.Record.GetBool("triggered") {
 		return e.Next()
 	}
@@ -18,6 +24,10 @@ func resolveHistoryOnAlertDelete(e *core.RecordEvent) error {
 
 // On alert record update, update alert history record
 func updateHistoryOnAlertUpdate(e *core.RecordEvent) error {
+	// Network monitor incidents have separate history entries per monitor.
+	if e.Record.GetString("name") == alertNameNetworkMonitorLoss {
+		return e.Next()
+	}
 	original := e.Record.Original()
 	new := e.Record
 

internal/alerts/alerts_network_monitors.go

@@ -0,0 +1,269 @@
+package alerts
+
+import (
+	"database/sql"
+	"errors"
+	"fmt"
+	"math"
+	"net"
+	"strconv"
+	"time"
+
+	"github.com/henrygd/beszel/internal/entities/monitor"
+	"github.com/pocketbase/dbx"
+	"github.com/pocketbase/pocketbase/core"
+)
+
+const alertNameNetworkMonitorLoss = "NetworkMonitorLoss"
+
+// networkMonitorAlertState is this alert type's persisted runtime state.
+// Monitor IDs map to their open history entries independently of history retention.
+type networkMonitorAlertState struct {
+	Monitors map[string]string `json:"monitors"`
+}
+
+func (am *AlertManager) bindNetworkMonitorAlertEvents() {
+	// Hidden fields are still writable through the record API unless protected.
+	protectState := func(e *core.RecordRequestEvent) error {
+		e.Record.Set("state", e.Record.Original().Get("state"))
+		oldName, newName := e.Record.Original().GetString("name"), e.Record.GetString("name")
+		if oldName != "" && (oldName == alertNameNetworkMonitorLoss || newName == alertNameNetworkMonitorLoss) &&
+			(oldName != newName || e.Record.GetString("system") != e.Record.Original().GetString("system")) {
+			return e.BadRequestError("Delete and recreate the alert to change its type or system", nil)
+		}
+		if e.Record.GetString("name") == alertNameNetworkMonitorLoss {
+			if !e.HasSuperuserAuth() && (e.Auth == nil || !userHasSystem(e.App, e.Auth.Id, e.Record.GetString("system"))) {
+				return e.ForbiddenError("You do not have access to this system", nil)
+			}
+			e.Record.Set("triggered", e.Record.Original().GetBool("triggered"))
+			value := e.Record.GetFloat("value")
+			if math.IsNaN(value) || math.IsInf(value, 0) || value < 0 || value >= 100 {
+				return e.BadRequestError("Monitor loss threshold must be at least 0 and below 100", nil)
+			}
+			e.Record.Set("min", 0)
+		}
+		return e.Next()
+	}
+	am.hub.OnRecordCreateRequest("alerts").BindFunc(protectState)
+	am.hub.OnRecordUpdateRequest("alerts").BindFunc(protectState)
+	cleanup := func(e *core.RecordEvent) error {
+		if err := e.Next(); err != nil {
+			return err
+		}
+		return am.evaluateNetworkMonitorAlerts(e.App, e.Record.GetString("system"), nil)
+	}
+	am.hub.OnRecordAfterDeleteSuccess("network_monitors").BindFunc(cleanup)
+	am.hub.OnRecordAfterUpdateSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error {
+		if e.Record.GetBool("enabled") || !e.Record.Original().GetBool("enabled") {
+			return e.Next()
+		}
+		return cleanup(e)
+	})
+}
+
+// HandleNetworkMonitorAlerts runs after the full monitoring transaction commits,
+// using its exact payload (dashboard requests can replace the cached payload).
+// Omitted results and disconnected systems never imply recovery.
+func (am *AlertManager) HandleNetworkMonitorAlerts(systemRecord *core.Record, results map[string]monitor.Result) error {
+	if systemRecord.GetString("status") != "up" {
+		return nil
+	}
+	alerts := am.alertsCache.GetAlertsByName(systemRecord.Id, alertNameNetworkMonitorLoss)
+	if len(alerts) == 0 {
+		return nil
+	}
+	monitors, err := am.networkMonitors.get(systemRecord.Id)
+	if err != nil {
+		return err
+	}
+	if !networkMonitorTransitionPending(alerts, monitors, results, time.Now()) {
+		return nil
+	}
+	// The cache only predicts a transition. Reload and recheck under the DB
+	// transaction before persisting, including current system/monitor status.
+	return am.evaluateNetworkMonitorAlerts(am.hub, systemRecord.Id, results)
+}
+
+// networkMonitorTransitionPending does no IO and never mutates cached maps.
+func networkMonitorTransitionPending(alerts []CachedAlertData, monitors map[string]int, results map[string]monitor.Result, now time.Time) bool {
+	for _, alert := range alerts {
+		if !alert.MonitorStatesValid || alert.Triggered != (len(alert.MonitorStates) > 0) {
+			return true
+		}
+		for id := range alert.MonitorStates {
+			if _, enabled := monitors[id]; !enabled {
+				return true
+			}
+		}
+		for id, result := range results {
+			interval, enabled := monitors[id]
+			if !enabled || !monitorResultReady(result, interval, now) {
+				continue
+			}
+			_, active := alert.MonitorStates[id]
+			if (result.PacketLoss1h > alert.Value) != active {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func (am *AlertManager) evaluateNetworkMonitorAlerts(app core.App, systemID string, results map[string]monitor.Result) error {
+	var messages []AlertMessageData
+	err := app.RunInTransaction(func(tx core.App) error {
+		// Read configuration inside the transaction so concurrent threshold changes,
+		// disabling, and evaluations cannot overwrite each other's incident state.
+		alerts, err := tx.FindAllRecords("alerts", dbx.HashExp{"system": systemID, "name": alertNameNetworkMonitorLoss})
+		if err != nil || len(alerts) == 0 {
+			return err
+		}
+		system, err := tx.FindRecordById("systems", systemID)
+		if errors.Is(err, sql.ErrNoRows) {
+			// System deletion cascades to its alerts.
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+		monitors, err := tx.FindAllRecords("network_monitors", dbx.HashExp{"system": systemID, "enabled": true})
+		if err != nil {
+			return err
+		}
+		enabled := make(map[string]*core.Record, len(monitors))
+		for _, m := range monitors {
+			enabled[m.Id] = m
+		}
+		now := time.Now()
+		for _, alert := range alerts {
+			var state networkMonitorAlertState
+			if err := alert.UnmarshalJSONField("state", &state); err != nil {
+				return err
+			}
+			states := state.Monitors
+			if states == nil {
+				states = map[string]string{}
+			}
+			changed := false
+			// Removing or disabling a monitor closes its incident silently.
+			for id, historyID := range states {
+				if _, ok := enabled[id]; !ok {
+					if err := resolveMonitorIncident(tx, historyID, now); err != nil {
+						return err
+					}
+					delete(states, id)
+					changed = true
+				}
+			}
+			if system.GetString("status") == "up" {
+				for _, m := range monitors {
+					result, ok := results[m.Id]
+					if !ok || !monitorResultReady(result, m.GetInt("interval"), now) {
+						continue
+					}
+					historyID, active := states[m.Id]
+					triggered := result.PacketLoss1h > alert.GetFloat("value")
+					if triggered == active {
+						continue
+					}
+					label := m.GetString("target")
+					if m.GetString("protocol") == "tcp" {
+						label = net.JoinHostPort(label, strconv.Itoa(m.GetInt("port")))
+					}
+					if triggered {
+						collection, err := tx.FindCachedCollectionByNameOrId("alerts_history")
+						if err != nil {
+							return err
+						}
+						history := core.NewRecord(collection)
+						history.Load(map[string]any{
+							"alert_id": alert.Id, "user": alert.GetString("user"), "system": systemID,
+							"name": alertNameNetworkMonitorLoss, "monitor_name": label, "value": result.PacketLoss1h,
+						})
+						if err := tx.Save(history); err != nil {
+							return err
+						}
+						states[m.Id] = history.Id
+					} else {
+						if err := resolveMonitorIncident(tx, historyID, now); err != nil {
+							return err
+						}
+						delete(states, m.Id)
+					}
+					changed = true
+					state, comparison := "loss", "exceeds"
+					if !triggered {
+						state, comparison = "recovered", "is at or below"
+					}
+					messages = append(messages, AlertMessageData{
+						UserID: alert.GetString("user"), SystemID: systemID,
+						Title:   fmt.Sprintf("Network monitor %s on %s: %s", state, system.GetString("name"), label),
+						Message: fmt.Sprintf("%s on %s: loss over the past hour is %.2f%%, which %s the %.2f%% threshold.", label, system.GetString("name"), result.PacketLoss1h, comparison, alert.GetFloat("value")),
+						Link:    am.hub.MakeLink("system", systemID), LinkText: "View " + system.GetString("name"),
+					})
+				}
+			}
+			if changed || alert.GetBool("triggered") != (len(states) > 0) {
+				alert.Set("state", networkMonitorAlertState{Monitors: states})
+				alert.Set("triggered", len(states) > 0)
+				if err := tx.Save(alert); err != nil {
+					return err
+				}
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	// Match other alert types: persist transitions before delivery, and respect
+	// the user's existing notification destinations and quiet hours.
+	for _, message := range messages {
+		if err := am.SendAlert(message); err != nil {
+			app.Logger().Error("Failed to send network monitor alert", "err", err)
+		}
+	}
+	return nil
+}
+
+func monitorResultReady(result monitor.Result, interval int, now time.Time) bool {
+	// Three completed attempts provide a short warm-up, including after an agent
+	// restart.
+	if result.SampleCount < 3 || result.LastProbeAt <= 0 || math.IsNaN(result.PacketLoss1h) || math.IsInf(result.PacketLoss1h, 0) || result.PacketLoss1h < 0 || result.PacketLoss1h > 100 {
+		return false
+	}
+	// Never interpret an empty one-hour window as zero loss.
+	maxAge := min(time.Hour, max(3*time.Duration(interval)*time.Second, 3*time.Minute))
+	age := now.Sub(time.UnixMilli(result.LastProbeAt))
+	return age >= -time.Minute && age <= maxAge
+}
+
+func resolveMonitorIncident(app core.App, id string, now time.Time) error {
+	record, err := app.FindRecordById("alerts_history", id)
+	if errors.Is(err, sql.ErrNoRows) {
+		// History can be purged independently.
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	if !record.GetDateTime("resolved").IsZero() {
+		return nil
+	}
+	record.Set("resolved", now.UTC())
+	return app.Save(record)
+}
+
+func resolveNetworkMonitorHistory(app core.App, alertID string) error {
+	records, err := app.FindAllRecords("alerts_history", dbx.HashExp{"alert_id": alertID, "resolved": ""})
+	if err != nil {
+		return err
+	}
+	for _, record := range records {
+		record.Set("resolved", time.Now().UTC())
+		if err := app.Save(record); err != nil {
+			return err
+		}
+	}
+	return nil
+}

internal/alerts/alerts_network_monitors_test.go

@@ -0,0 +1,513 @@
+//go:build testing
+
+package alerts_test
+
+import (
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/henrygd/beszel/internal/alerts"
+	"github.com/henrygd/beszel/internal/entities/monitor"
+	beszelTests "github.com/henrygd/beszel/internal/tests"
+	"github.com/pocketbase/dbx"
+	"github.com/pocketbase/pocketbase/core"
+	pbTests "github.com/pocketbase/pocketbase/tests"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func networkAlertSetup(t *testing.T) (*beszelTests.TestHub, *core.Record, *core.Record, []*core.Record) {
+	t.Helper()
+	hub, system, alert := systemdTestSetup(t, false)
+	t.Cleanup(hub.Cleanup)
+	alert.Set("name", "NetworkMonitorLoss")
+	alert.Set("value", 5)
+	require.NoError(t, hub.Save(alert))
+	var monitors []*core.Record
+	for _, name := range []string{"gateway", "website"} {
+		record, err := beszelTests.CreateRecord(hub, "network_monitors", map[string]any{
+			"system": system.Id, "target": name + ".example.com", "protocol": "icmp", "interval": 60, "enabled": true,
+		})
+		require.NoError(t, err)
+		monitors = append(monitors, record)
+	}
+	// Avoid starting a system update worker in tests.
+	_, err := hub.DB().Update("systems", dbx.Params{"status": "up"}, dbx.HashExp{"id": system.Id}).Execute()
+	require.NoError(t, err)
+	system.Set("status", "up")
+	return hub, system, alert, monitors
+}
+
+func monitorResult(loss float64) monitor.Result {
+	return monitor.Result{LastProbeAt: time.Now().UnixMilli(), SampleCount: 60, PacketLoss1h: loss}
+}
+
+func TestNetworkMonitorAlertIndependentIncidents(t *testing.T) {
+	hub, system, alert, monitors := networkAlertSetup(t)
+	am := alerts.NewTestAlertManagerWithoutWorker(hub)
+	count := hub.TestMailer.TotalSend()
+	results := map[string]monitor.Result{monitors[0].Id: monitorResult(10), monitors[1].Id: monitorResult(0)}
+	check := func(active bool, open, sent int) {
+		t.Helper()
+		record, err := hub.FindRecordById("alerts", alert.Id)
+		require.NoError(t, err)
+		assert.Equal(t, active, record.GetBool("triggered"))
+		total, err := hub.CountRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id, "resolved": ""})
+		require.NoError(t, err)
+		assert.EqualValues(t, open, total)
+		assert.Equal(t, count+sent, hub.TestMailer.TotalSend())
+	}
+	require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+	check(true, 1, 1)
+	message := hub.TestMailer.Messages()[count]
+	assert.Contains(t, message.Text, "gateway.example.com")
+	assert.Contains(t, message.Text, "10.00%")
+	assert.Contains(t, message.Text, "5.00%")
+	require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+	check(true, 1, 1)
+	// Persisted monitor state prevents duplicate notifications after a hub restart.
+	am = alerts.NewTestAlertManagerWithoutWorker(hub)
+	results[monitors[1].Id] = monitorResult(20)
+	require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+	check(true, 2, 2)
+	results[monitors[0].Id] = monitorResult(5) // Equality is a recovery.
+	require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+	check(true, 1, 3)
+	results[monitors[1].Id] = monitorResult(0)
+	require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+	check(false, 0, 4)
+	histories, err := hub.FindAllRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id})
+	require.NoError(t, err)
+	require.Len(t, histories, 2)
+	for _, history := range histories {
+		assert.NotEmpty(t, history.GetString("monitor_name"))
+	}
+}
+
+func TestNetworkMonitorAlertTargetLabel(t *testing.T) {
+	for _, tc := range []struct {
+		protocol, target, label string
+		port                    int
+	}{
+		{"icmp", "gateway.example.com", "gateway.example.com", 0},
+		{"http", "https://example.com/health", "https://example.com/health", 0},
+		{"tcp", "example.com", "example.com:8443", 8443},
+		{"tcp", "2001:db8::1", "[2001:db8::1]:443", 443},
+	} {
+		t.Run(tc.label, func(t *testing.T) {
+			hub, system, alert, monitors := networkAlertSetup(t)
+			m := monitors[0]
+			m.Set("protocol", tc.protocol)
+			m.Set("target", tc.target)
+			m.Set("port", tc.port)
+			require.NoError(t, hub.Save(m))
+			am := alerts.NewTestAlertManagerWithoutWorker(hub)
+			require.NoError(t, am.HandleNetworkMonitorAlerts(system, map[string]monitor.Result{m.Id: monitorResult(10)}))
+			histories, err := hub.FindAllRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id})
+			require.NoError(t, err)
+			require.Len(t, histories, 1)
+			assert.Equal(t, tc.label, histories[0].GetString("monitor_name"))
+			assert.Contains(t, hub.TestMailer.Messages()[hub.TestMailer.TotalSend()-1].Text, tc.label)
+		})
+	}
+}
+
+func TestNetworkMonitorAlertIgnoresUnknownResults(t *testing.T) {
+	for _, scenario := range []string{"missing", "stale", "warmup", "no probes", "down", "paused", "future", "expired hourly window"} {
+		t.Run(scenario, func(t *testing.T) {
+			hub, system, alert, monitors := networkAlertSetup(t)
+			am := alerts.NewTestAlertManagerWithoutWorker(hub)
+			apply := func(loss float64) {
+				result := monitorResult(loss)
+				results := map[string]monitor.Result{monitors[0].Id: result}
+				switch scenario {
+				case "missing":
+					results = nil
+				case "stale":
+					result.LastProbeAt = time.Now().Add(-10 * time.Minute).UnixMilli()
+					results[monitors[0].Id] = result
+				case "expired hourly window":
+					monitors[0].Set("interval", 3600)
+					require.NoError(t, hub.Save(monitors[0]))
+					result.LastProbeAt = time.Now().Add(-2 * time.Hour).UnixMilli()
+					results[monitors[0].Id] = result
+				case "future":
+					result.LastProbeAt = time.Now().Add(time.Hour).UnixMilli()
+					results[monitors[0].Id] = result
+				case "warmup":
+					result.SampleCount = 2
+					results[monitors[0].Id] = result
+				case "no probes":
+					result.SampleCount = 0
+					results[monitors[0].Id] = result
+				case "down":
+					_, err := hub.DB().Update("systems", dbx.Params{"status": scenario}, dbx.HashExp{"id": system.Id}).Execute()
+					require.NoError(t, err)
+				case "paused":
+					record, err := hub.FindRecordById("systems", system.Id)
+					require.NoError(t, err)
+					record.Set("status", "paused")
+					require.NoError(t, hub.Save(record))
+				}
+				require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+			}
+			count := hub.TestMailer.TotalSend()
+			apply(100)
+			assert.Equal(t, count, hub.TestMailer.TotalSend())
+			_, err := hub.DB().Update("systems", dbx.Params{"status": "up"}, dbx.HashExp{"id": system.Id}).Execute()
+			require.NoError(t, err)
+			require.NoError(t, am.HandleNetworkMonitorAlerts(system, map[string]monitor.Result{monitors[0].Id: monitorResult(10)}))
+			apply(0)
+			assert.Equal(t, count+1, hub.TestMailer.TotalSend(), "unknown data must not recover an incident")
+			record, err := hub.FindRecordById("alerts", alert.Id)
+			require.NoError(t, err)
+			assert.True(t, record.GetBool("triggered"))
+		})
+	}
+}
+
+func TestNetworkMonitorAlertCleanup(t *testing.T) {
+	for _, scenario := range []string{"disable monitor", "delete monitor", "disable alert", "purge history", "delete system"} {
+		t.Run(scenario, func(t *testing.T) {
+			hub, system, alert, monitors := networkAlertSetup(t)
+			am := alerts.NewTestAlertManagerWithoutWorker(hub)
+			results := map[string]monitor.Result{monitors[0].Id: monitorResult(10), monitors[1].Id: monitorResult(20)}
+			require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+			count := hub.TestMailer.TotalSend()
+			switch scenario {
+			case "disable monitor":
+				monitors[0].Set("enabled", false)
+				require.NoError(t, hub.Save(monitors[0]))
+			case "delete monitor":
+				require.NoError(t, hub.Delete(monitors[0]))
+			case "disable alert":
+				require.NoError(t, hub.Delete(alert))
+			case "delete system":
+				require.NoError(t, hub.Delete(system))
+			case "purge history":
+				history, err := hub.FindAllRecords("alerts_history")
+				require.NoError(t, err)
+				for _, record := range history {
+					require.NoError(t, hub.Delete(record))
+				}
+			}
+			require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+			assert.Equal(t, count, hub.TestMailer.TotalSend())
+			open, err := hub.CountRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id, "resolved": ""})
+			require.NoError(t, err)
+			if scenario == "disable monitor" || scenario == "delete monitor" {
+				assert.EqualValues(t, 1, open)
+				record, err := hub.FindRecordById("alerts", alert.Id)
+				require.NoError(t, err)
+				assert.True(t, record.GetBool("triggered"))
+				require.NoError(t, hub.Delete(monitors[1]))
+				record, err = hub.FindRecordById("alerts", alert.Id)
+				require.NoError(t, err)
+				assert.False(t, record.GetBool("triggered"))
+			} else {
+				assert.Zero(t, open)
+			}
+		})
+	}
+}
+
+func TestNetworkMonitorAlertPerUserThresholds(t *testing.T) {
+	hub, system, alert, monitors := networkAlertSetup(t)
+	user, err := beszelTests.CreateUser(hub, "monitor2@example.com", "password")
+	require.NoError(t, err)
+	other, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{"name": "NetworkMonitorLoss", "system": system.Id, "user": user.Id, "value": 20})
+	require.NoError(t, err)
+	am := alerts.NewTestAlertManagerWithoutWorker(hub)
+	results := map[string]monitor.Result{monitors[0].Id: monitorResult(10)}
+	require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+	other, err = hub.FindRecordById("alerts", other.Id)
+	require.NoError(t, err)
+	assert.False(t, other.GetBool("triggered"))
+	// Editing the threshold re-evaluates on the next batch, without losing state.
+	alert, err = hub.FindRecordById("alerts", alert.Id)
+	require.NoError(t, err)
+	alert.Set("value", 15)
+	require.NoError(t, hub.Save(alert))
+	require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+	alert, err = hub.FindRecordById("alerts", alert.Id)
+	require.NoError(t, err)
+	assert.False(t, alert.GetBool("triggered"))
+}
+
+func TestNetworkMonitorAlertAPI(t *testing.T) {
+	for _, tc := range []struct {
+		name                  string
+		value                 float64
+		direct, denied, patch bool
+		status                int
+	}{
+		{name: "zero threshold", value: 0, status: 200},
+		{name: "fractional threshold", value: 5.5, status: 200},
+		{name: "negative threshold", value: -1, status: 400},
+		{name: "unreachable threshold", value: 100, status: 400},
+		{name: "bulk inaccessible system", value: 5, denied: true, status: 200},
+		{name: "direct inaccessible system", value: 5, direct: true, denied: true, status: 403},
+		{name: "direct invalid threshold", value: -1, direct: true, status: 400},
+		{name: "direct private state", value: 5, direct: true, status: 200},
+		{name: "patch preserves state", value: 10, direct: true, patch: true, status: 200},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			hub, user := beszelTests.GetHubWithUser(t)
+			defer hub.Cleanup()
+			owner := user.Id
+			if tc.denied {
+				other, err := beszelTests.CreateUser(hub, "other@example.com", "password")
+				require.NoError(t, err)
+				owner = other.Id
+			}
+			systems, err := beszelTests.CreateSystems(hub, 1, owner, "paused")
+			require.NoError(t, err)
+			token, err := user.NewAuthToken()
+			require.NoError(t, err)
+			body := map[string]any{"name": "NetworkMonitorLoss", "value": tc.value, "min": 60, "systems": []string{systems[0].Id}, "overwrite": true}
+			url, method := "/api/beszel/user-alerts", "POST"
+			if tc.direct {
+				url = "/api/collections/alerts/records"
+				body["system"], body["user"] = systems[0].Id, user.Id
+				body["state"], body["triggered"] = map[string]any{"monitors": map[string]string{"fake": "fake"}}, true
+			}
+			if tc.patch {
+				alert, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{"name": "NetworkMonitorLoss", "system": systems[0].Id, "user": user.Id, "value": 5, "triggered": true, "state": map[string]any{"monitors": map[string]string{"real": "history"}}})
+				require.NoError(t, err)
+				url += "/" + alert.Id
+				method = "PATCH"
+				body["triggered"] = false
+			}
+			content := `"success":true`
+			if tc.direct {
+				content = `"name":"NetworkMonitorLoss"`
+			}
+			if tc.status == 400 {
+				content = `"status":400`
+			}
+			if tc.status == 403 {
+				content = `"status":403`
+			}
+			scenario := beszelTests.ApiScenario{
+				Name: tc.name, Method: method, URL: url, Body: jsonReader(body),
+				Headers: map[string]string{"Authorization": token}, ExpectedStatus: tc.status, ExpectedContent: []string{content},
+				TestAppFactory: func(testing.TB) *pbTests.TestApp { return hub.TestApp },
+			}
+			scenario.Test(t)
+			records, err := hub.FindAllRecords("alerts")
+			require.NoError(t, err)
+			if tc.status != 200 || tc.denied {
+				assert.Empty(t, records)
+				return
+			}
+			require.Len(t, records, 1)
+			assert.Equal(t, tc.value, records[0].GetFloat("value"))
+			assert.Zero(t, records[0].GetInt("min"))
+			state := struct {
+				Monitors map[string]string `json:"monitors"`
+			}{}
+			require.NoError(t, records[0].UnmarshalJSONField("state", &state))
+			states := state.Monitors
+			if tc.patch {
+				assert.Equal(t, map[string]string{"real": "history"}, states)
+				assert.True(t, records[0].GetBool("triggered"))
+			} else {
+				assert.Empty(t, states)
+				assert.False(t, records[0].GetBool("triggered"))
+			}
+		})
+	}
+}
+
+type monitorCountingHub struct {
+	*beszelTests.TestHub
+	transactions      atomic.Int64
+	beforeTransaction func()
+}
+
+func (h *monitorCountingHub) RunInTransaction(fn func(core.App) error) error {
+	h.transactions.Add(1)
+	if h.beforeTransaction != nil {
+		h.beforeTransaction()
+	}
+	return h.App.RunInTransaction(fn)
+}
+
+// Count actual SQL on both DB connections, including queries through record APIs.
+func monitorSQLCounter(t *testing.T, app core.App) *atomic.Int64 {
+	t.Helper()
+	count := &atomic.Int64{}
+	for _, builder := range []dbx.Builder{app.ConcurrentDB(), app.NonconcurrentDB()} {
+		db := builder.(*dbx.DB)
+		old := db.LogFunc
+		db.LogFunc = func(string, ...any) { count.Add(1) }
+		t.Cleanup(func() { db.LogFunc = old })
+	}
+	return count
+}
+
+func TestNetworkMonitorAlertSteadyStateNoDatabaseWork(t *testing.T) {
+	hub, system, alert, monitors := networkAlertSetup(t)
+	counted := &monitorCountingHub{TestHub: hub}
+	am := alerts.NewTestAlertManagerWithoutWorker(counted)
+	sql := monitorSQLCounter(t, hub)
+	results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}
+	evaluate := func() { t.Helper(); require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) }
+	noWork := func() {
+		t.Helper()
+		sql.Store(0)
+		counted.transactions.Store(0)
+		for range 100 {
+			evaluate()
+		}
+		assert.Zero(t, sql.Load(), "steady state must not issue SQL")
+		assert.Zero(t, counted.transactions.Load(), "steady state must not open transactions")
+	}
+	// One-time lazy loads are permitted, including on hub restart.
+	evaluate()
+	assert.Positive(t, sql.Load())
+	noWork()
+	// Realtime metric saves invoke record hooks but must not invalidate config.
+	fresh, err := hub.FindRecordById("network_monitors", monitors[0].Id)
+	require.NoError(t, err)
+	monitors[0] = fresh
+	monitors[0].Set("loss1h", 0)
+	monitors[0].Set("res", 100)
+	require.NoError(t, hub.Save(monitors[0]))
+	noWork()
+	results[monitors[0].Id] = monitorResult(10)
+	evaluate()
+	assert.Positive(t, sql.Load(), "transitions must still be persisted")
+	assert.EqualValues(t, 1, counted.transactions.Load())
+	noWork()
+	// Missing and stale observations must not enter the transaction either.
+	results = nil
+	noWork()
+	results = map[string]monitor.Result{monitors[0].Id: {SampleCount: 60, LastProbeAt: time.Now().Add(-10 * time.Minute).UnixMilli()}}
+	noWork()
+	results[monitors[0].Id] = monitorResult(0)
+	evaluate()
+	noWork()
+	require.NoError(t, hub.Delete(alert))
+	noWork()
+}
+
+func TestNetworkMonitorAlertConfigCacheInvalidation(t *testing.T) {
+	hub, system, alert, monitors := networkAlertSetup(t)
+	am := alerts.NewTestAlertManagerWithoutWorker(hub)
+	results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}
+	require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+	count := hub.TestMailer.TotalSend()
+	// Widening the interval makes this observation fresh. A stale interval cache
+	// would miss the failure indefinitely, even though results keep arriving.
+	result := monitorResult(10)
+	result.LastProbeAt = time.Now().Add(-4 * time.Minute).UnixMilli()
+	results[monitors[0].Id] = result
+	require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+	assert.Equal(t, count, hub.TestMailer.TotalSend())
+	monitors[0].Set("interval", 120)
+	require.NoError(t, hub.Save(monitors[0]))
+	require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+	assert.Equal(t, count+1, hub.TestMailer.TotalSend())
+	// Disable, then re-enable the same ID: its new failure must be detected.
+	monitors[0].Set("enabled", false)
+	require.NoError(t, hub.Save(monitors[0]))
+	require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+	monitors[0].Set("enabled", true)
+	require.NoError(t, hub.Save(monitors[0]))
+	require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+	assert.Equal(t, count+2, hub.TestMailer.TotalSend())
+	// A new monitor must also become eligible without restarting the hub.
+	created, err := beszelTests.CreateRecord(hub, "network_monitors", map[string]any{
+		"system": system.Id, "name": "new", "target": "new.example.com", "protocol": "icmp", "interval": 60, "enabled": true,
+	})
+	require.NoError(t, err)
+	results[created.Id] = monitorResult(10)
+	require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+	assert.Equal(t, count+3, hub.TestMailer.TotalSend())
+	// Threshold changes refresh cached config and preserve the active incidents.
+	alert, err = hub.FindRecordById("alerts", alert.Id)
+	require.NoError(t, err)
+	alert.Set("value", 15)
+	require.NoError(t, hub.Save(alert))
+	require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+	assert.Equal(t, count+5, hub.TestMailer.TotalSend())
+}
+
+func TestNetworkMonitorAlertRevalidatesCandidate(t *testing.T) {
+	for _, change := range []string{"threshold", "disable alert", "disable monitor", "down"} {
+		t.Run(change, func(t *testing.T) {
+			hub, system, alert, monitors := networkAlertSetup(t)
+			counted := &monitorCountingHub{TestHub: hub}
+			am := alerts.NewTestAlertManagerWithoutWorker(counted)
+			results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}
+			require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+			count := hub.TestMailer.TotalSend()
+			// Change the DB after the cache predicts a transition, before its transaction.
+			counted.beforeTransaction = func() {
+				counted.beforeTransaction = nil
+				switch change {
+				case "threshold":
+					alert.Set("value", 20)
+					require.NoError(t, hub.Save(alert))
+				case "disable alert":
+					require.NoError(t, hub.Delete(alert))
+				case "disable monitor":
+					monitors[0].Set("enabled", false)
+					require.NoError(t, hub.Save(monitors[0]))
+				case "down":
+					_, err := hub.DB().Update("systems", dbx.Params{"status": "down"}, dbx.HashExp{"id": system.Id}).Execute()
+					require.NoError(t, err)
+				}
+			}
+			results[monitors[0].Id] = monitorResult(10)
+			require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+			assert.EqualValues(t, 1, counted.transactions.Load())
+			assert.Equal(t, count, hub.TestMailer.TotalSend())
+			histories, err := hub.CountRecords("alerts_history")
+			require.NoError(t, err)
+			assert.Zero(t, histories)
+		})
+	}
+}
+
+func TestNetworkMonitorAlertConcurrentEvaluations(t *testing.T) {
+	hub, system, _, monitors := networkAlertSetup(t)
+	am := alerts.NewTestAlertManagerWithoutWorker(hub)
+	results := map[string]monitor.Result{monitors[0].Id: monitorResult(10)}
+	count := hub.TestMailer.TotalSend()
+	var wg sync.WaitGroup
+	errs := make(chan error, 8)
+	for range 8 {
+		wg.Go(func() { errs <- am.HandleNetworkMonitorAlerts(system, results) })
+	}
+	wg.Wait()
+	close(errs)
+	for err := range errs {
+		require.NoError(t, err)
+	}
+	assert.Equal(t, count+1, hub.TestMailer.TotalSend())
+}
+
+func TestNetworkMonitorAlertCacheAfterRollback(t *testing.T) {
+	hub, system, _, monitors := networkAlertSetup(t)
+	am := alerts.NewTestAlertManagerWithoutWorker(hub)
+	results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}
+	require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+	_, err := hub.DB().NewQuery(`CREATE TRIGGER fail_alert BEFORE UPDATE ON alerts BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute()
+	require.NoError(t, err)
+	count := hub.TestMailer.TotalSend()
+	results[monitors[0].Id] = monitorResult(10)
+	require.Error(t, am.HandleNetworkMonitorAlerts(system, results))
+	assert.Equal(t, count, hub.TestMailer.TotalSend())
+	histories, err := hub.CountRecords("alerts_history")
+	require.NoError(t, err)
+	assert.Zero(t, histories)
+	_, err = hub.DB().NewQuery("DROP TRIGGER fail_alert").Execute()
+	require.NoError(t, err)
+	// A failed transition must not be published to the cache and mask the retry.
+	require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+	assert.Equal(t, count+1, hub.TestMailer.TotalSend())
+}

internal/alerts/alerts_system.go

@@ -47,7 +47,7 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
 		return nil
 	}
 
-	alerts := am.alertsCache.GetAlertsExcludingNames(systemRecord.Id, "Status", alertNameSystemdFailed, containerAlertName)
+	alerts := am.alertsCache.GetAlertsExcludingNames(systemRecord.Id, "Status", alertNameSystemdFailed, containerAlertName, alertNameNetworkMonitorLoss)
 	if len(alerts) == 0 {
 		return nil
 	}

internal/alerts/alerts_test_helpers.go

@@ -11,8 +11,9 @@ import (
 
 func NewTestAlertManagerWithoutWorker(app hubLike) *AlertManager {
 	return &AlertManager{
-		hub:         app,
-		alertsCache: NewAlertsCache(app),
+		hub:             app,
+		alertsCache:     NewAlertsCache(app),
+		networkMonitors: newNetworkMonitorCache(app),
 	}
 }
 

internal/alerts/network_monitor_cache.go

@@ -0,0 +1,79 @@
+package alerts
+
+import (
+	"sync"
+
+	"github.com/pocketbase/dbx"
+	"github.com/pocketbase/pocketbase/core"
+)
+
+// networkMonitorCache keeps just the enabled monitor IDs and probe intervals
+// needed for the alert fast path. Names and targets are read only on transitions.
+// Returned maps are immutable; configuration changes invalidate the whole entry.
+type networkMonitorCache struct {
+	app     core.App
+	mu      sync.RWMutex
+	systems map[string]map[string]int
+}
+
+func newNetworkMonitorCache(app core.App) *networkMonitorCache {
+	c := &networkMonitorCache{app: app, systems: make(map[string]map[string]int)}
+	invalidate := func(e *core.RecordEvent) error {
+		c.invalidate(e.Record.GetString("system"))
+		return e.Next()
+	}
+	app.OnRecordAfterCreateSuccess("network_monitors").BindFunc(invalidate)
+	app.OnRecordAfterDeleteSuccess("network_monitors").BindFunc(invalidate)
+	app.OnRecordAfterUpdateSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error {
+		old := e.Record.Original()
+		// Realtime metric saves also invoke this hook. They must not evict config.
+		if old.GetString("system") != e.Record.GetString("system") ||
+			old.GetBool("enabled") != e.Record.GetBool("enabled") ||
+			old.GetInt("interval") != e.Record.GetInt("interval") {
+			c.invalidate(old.GetString("system"))
+			c.invalidate(e.Record.GetString("system"))
+		}
+		return e.Next()
+	})
+	app.OnRecordAfterDeleteSuccess("systems").BindFunc(func(e *core.RecordEvent) error {
+		c.invalidate(e.Record.Id)
+		return e.Next()
+	})
+	return c
+}
+
+func (c *networkMonitorCache) invalidate(systemID string) {
+	c.mu.Lock()
+	delete(c.systems, systemID)
+	c.mu.Unlock()
+}
+
+func (c *networkMonitorCache) get(systemID string) (map[string]int, error) {
+	c.mu.RLock()
+	monitors, ok := c.systems[systemID]
+	c.mu.RUnlock()
+	if ok {
+		return monitors, nil
+	}
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if monitors, ok := c.systems[systemID]; ok {
+		return monitors, nil
+	}
+	// Keep the lock through the load so a concurrent config change cannot be
+	// invalidated first and then overwritten by the older query result.
+	var rows []struct {
+		ID       string `db:"id"`
+		Interval int    `db:"interval"`
+	}
+	if err := c.app.DB().Select("id", "interval").From("network_monitors").
+		Where(dbx.HashExp{"system": systemID, "enabled": true}).All(&rows); err != nil {
+		return nil, err
+	}
+	monitors = make(map[string]int, len(rows))
+	for _, row := range rows {
+		monitors[row.ID] = row.Interval
+	}
+	c.systems[systemID] = monitors
+	return monitors, nil
+}