feat: network monitoring from agents (#2266, #1911)

Co-authored-by: Sven van Ginkel <svenvanginkel@icloud.com>
Co-authored-by: xiaomiku01 <xiaomiku01@outlook.com>

internal/hub/systems/network_monitor_stats_test.go

@@ -0,0 +1,224 @@
+//go:build testing
+
+package systems
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/fxamacker/cbor/v2"
+	"github.com/henrygd/beszel/internal/entities/monitor"
+	"github.com/henrygd/beszel/internal/entities/system"
+	"github.com/pocketbase/pocketbase/core"
+	"github.com/pocketbase/pocketbase/tools/subscriptions"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNetworkMonitorProbePruning(t *testing.T) {
+	for _, tc := range []struct {
+		name     string
+		monitors map[string]monitor.Result
+		fail     bool
+		want     map[string]int64
+	}{
+		{"nil report", nil, false, map[string]int64{"monitor1": 1000, "monitor2": 1000}},
+		{"empty report", map[string]monitor.Result{}, false, map[string]int64{}},
+		{"removed monitor", map[string]monitor.Result{"monitor1": {LastProbeAt: 1000}}, false, map[string]int64{"monitor1": 1000}},
+		{"rolled back report", map[string]monitor.Result{}, true, map[string]int64{"monitor1": 1000, "monitor2": 1000}},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			sys, app := newTestSystemWithHub(t)
+			sys.lastSavedMonitorProbe = map[string]int64{"monitor1": 1000, "monitor2": 1000}
+			// Preserve the distinction between nil and empty across the agent transport.
+			encoded, err := cbor.Marshal(system.CombinedData{Monitors: tc.monitors})
+			require.NoError(t, err)
+			var data system.CombinedData
+			require.NoError(t, cbor.Unmarshal(encoded, &data))
+			if tc.fail {
+				_, err = app.DB().NewQuery(`CREATE TRIGGER fail_system_update BEFORE UPDATE ON systems BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute()
+				require.NoError(t, err)
+			}
+			_, err = sys.createRecords(&data)
+			if tc.fail {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+			assert.Equal(t, tc.want, sys.lastSavedMonitorProbe)
+		})
+	}
+}
+
+func TestNetworkMonitorStatsFreshness(t *testing.T) {
+	for _, realtime := range []bool{false, true} {
+		name := "sql"
+		if realtime {
+			name = "realtime"
+		}
+		t.Run(name, func(t *testing.T) {
+			sys, app := newTestSystemWithHub(t)
+			if realtime {
+				client := subscriptions.NewDefaultClient()
+				client.Subscribe("network_monitors/*")
+				app.SubscriptionsBroker().Register(client)
+				t.Cleanup(func() { app.SubscriptionsBroker().Unregister(client.Id()) })
+			}
+			col, err := app.FindCachedCollectionByNameOrId("network_monitors")
+			require.NoError(t, err)
+			for _, id := range []string{"monitor1", "monitor2"} {
+				record := core.NewRecord(col)
+				record.Id = id
+				record.Set("system", sys.Id)
+				require.NoError(t, app.SaveNoValidate(record))
+			}
+			data := &system.CombinedData{Monitors: map[string]monitor.Result{
+				"monitor1": {LastProbeAt: 1000, AvgResponse: 20, TotalCount: 6, SuccessCount: 6, ResponseSum: 123},
+				"monitor2": {LastProbeAt: 1000, PacketLoss: 100, TotalCount: 1},
+			}}
+			count := func(want int64) {
+				t.Helper()
+				got, err := app.CountRecords("network_monitor_stats")
+				require.NoError(t, err)
+				assert.Equal(t, want, got)
+			}
+			save := func() {
+				t.Helper()
+				_, err := sys.createRecords(data)
+				require.NoError(t, err)
+			}
+			save()
+			count(2)
+			stored, err := app.FindAllRecords("network_monitor_stats")
+			require.NoError(t, err)
+			for _, record := range stored {
+				result := data.Monitors[record.GetString("monitor")]
+				assert.EqualValues(t, result.TotalCount, record.GetInt("total_count"))
+				assert.EqualValues(t, result.SuccessCount, record.GetInt("success_count"))
+				assert.EqualValues(t, result.ResponseSum, record.GetInt("res_sum"))
+			}
+			// A resume can overlap the scheduled update with the same probe.
+			errs := make(chan error, 4)
+			for range 4 {
+				go func() {
+					_, err := sys.createRecords(data)
+					errs <- err
+				}()
+			}
+			for range 4 {
+				require.NoError(t, <-errs)
+			}
+			count(2)
+
+			// A rolling hourly value can change without a new probe.
+			result := data.Monitors["monitor1"]
+			result.AvgResponse1h = 42
+			data.Monitors["monitor1"] = result
+			save()
+			count(2)
+			record, err := app.FindRecordById("network_monitors", "monitor1")
+			require.NoError(t, err)
+			assert.Equal(t, 42, record.GetInt("resAvg1h"))
+
+			// Identical response values and failed probes still count as new measurements.
+			for id, result := range data.Monitors {
+				result.LastProbeAt = 301000
+				data.Monitors[id] = result
+			}
+			save()
+			count(4)
+
+			// A failed individual insert must remain retryable, even if others commit.
+			_, err = app.DB().NewQuery(`CREATE TRIGGER fail_monitor_insert BEFORE INSERT ON network_monitor_stats WHEN NEW.monitor = 'monitor1' BEGIN SELECT RAISE(ABORT, 'test insert failure'); END`).Execute()
+			require.NoError(t, err)
+			for id, result := range data.Monitors {
+				result.LastProbeAt = 601000
+				data.Monitors[id] = result
+			}
+			save()
+			count(5)
+			assert.Equal(t, int64(301000), sys.lastSavedMonitorProbe["monitor1"])
+			assert.Equal(t, int64(601000), sys.lastSavedMonitorProbe["monitor2"])
+			_, err = app.DB().NewQuery("DROP TRIGGER fail_monitor_insert").Execute()
+			require.NoError(t, err)
+			save()
+			count(6)
+
+			// Failure after inserting stats rolls back the whole transaction and its markers.
+			_, err = app.DB().NewQuery(`CREATE TRIGGER fail_system_update BEFORE UPDATE ON systems BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute()
+			require.NoError(t, err)
+			result = data.Monitors["monitor1"]
+			result.LastProbeAt = 901000
+			data.Monitors["monitor1"] = result
+			_, err = sys.createRecords(data)
+			require.Error(t, err)
+			count(6)
+			assert.Equal(t, int64(601000), sys.lastSavedMonitorProbe["monitor1"])
+			_, err = app.DB().NewQuery("DROP TRIGGER fail_system_update").Execute()
+			require.NoError(t, err)
+			save()
+			count(7)
+
+			// Clock rollback is a new probe identity, not a reason to stall writes.
+			result.LastProbeAt = 500
+			data.Monitors["monitor1"] = result
+			save()
+			count(8)
+
+			// Recreated systems intentionally accept the first result without restoring state.
+			sys = &System{Id: sys.Id, manager: sys.manager}
+			save()
+			count(10)
+		})
+	}
+}
+
+// Observes the committed DB through the hub, not the transaction's app.
+type monitorAlertHub struct {
+	stubHub
+	handle func(*core.Record, map[string]monitor.Result) error
+}
+
+func (h monitorAlertHub) HandleNetworkMonitorAlerts(record *core.Record, results map[string]monitor.Result) error {
+	return h.handle(record, results)
+}
+
+func TestNetworkMonitorAlertsAfterCommit(t *testing.T) {
+	for _, realtime := range []bool{false, true} {
+		t.Run(fmt.Sprint(realtime), func(t *testing.T) {
+			sys, app := newTestSystemWithHub(t)
+			if realtime {
+				client := subscriptions.NewDefaultClient()
+				client.Subscribe("network_monitors/*")
+				app.SubscriptionsBroker().Register(client)
+			}
+			collection, err := app.FindCachedCollectionByNameOrId("network_monitors")
+			require.NoError(t, err)
+			record := core.NewRecord(collection)
+			record.Set("system", sys.Id)
+			require.NoError(t, app.SaveNoValidate(record))
+			called := 0
+			result := monitor.Result{LastProbeAt: time.Now().UnixMilli(), SampleCount: 3, PacketLoss1h: 10}
+			sys.manager.hub = monitorAlertHub{stubHub: stubHub{app}, handle: func(systemRecord *core.Record, results map[string]monitor.Result) error {
+				called++
+				assert.Equal(t, sys.Id, systemRecord.Id)
+				assert.Equal(t, result, results[record.Id])
+				saved, err := app.FindRecordById("network_monitors", record.Id)
+				require.NoError(t, err)
+				assert.Equal(t, 10.0, saved.GetFloat("loss1h"))
+				return nil
+			}}
+			data := &system.CombinedData{Monitors: map[string]monitor.Result{record.Id: result}}
+			_, err = sys.createRecords(data)
+			require.NoError(t, err)
+			assert.Equal(t, 1, called)
+			// A transaction that fails after writing monitor stats must not notify.
+			_, err = app.DB().NewQuery(`CREATE TRIGGER fail_system BEFORE UPDATE ON systems BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute()
+			require.NoError(t, err)
+			_, err = sys.createRecords(data)
+			require.Error(t, err)
+			assert.Equal(t, 1, called)
+		})
+	}
+}

internal/hub/systems/network_monitor_sync_test.go

@@ -0,0 +1,144 @@
+//go:build testing
+
+package systems
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/blang/semver"
+	"github.com/fxamacker/cbor/v2"
+	"github.com/henrygd/beszel/internal/common"
+	"github.com/henrygd/beszel/internal/entities/monitor"
+	"github.com/henrygd/beszel/internal/hub/ws"
+	"github.com/lxzan/gws"
+	"github.com/pocketbase/pocketbase/core"
+	"github.com/stretchr/testify/require"
+)
+
+type monitorSyncClient struct {
+	gws.BuiltinEventHandler
+	requests chan common.HubRequest[monitor.SyncRequest]
+}
+
+func (c *monitorSyncClient) OnMessage(conn *gws.Conn, message *gws.Message) {
+	defer message.Close()
+	var req common.HubRequest[monitor.SyncRequest]
+	if err := cbor.Unmarshal(message.Bytes(), &req); err != nil {
+		return
+	}
+	c.requests <- req
+	data, _ := cbor.Marshal(monitor.SyncResponse{})
+	response, _ := cbor.Marshal(common.AgentResponse{Id: req.Id, Data: data})
+	_ = conn.WriteMessage(gws.OpcodeBinary, response)
+}
+
+// Avoid the production delayed disconnect notification; these tests explicitly
+// remove each connection from the manager before reconnecting.
+type monitorSyncServer struct{ ws.Handler }
+
+func (*monitorSyncServer) OnClose(*gws.Conn, error) {}
+
+func TestNetworkMonitorSyncSkipsOlderAgents(t *testing.T) {
+	for _, version := range []string{"0.0.0", "0.18.0", "0.19.0"} {
+		t.Run(version, func(t *testing.T) {
+			// No transport: attempting to send any request would fail.
+			sys := &System{agentVersion: semver.MustParse(version)}
+			require.NoError(t, sys.SyncNetworkMonitors(nil))
+			result, err := sys.UpsertNetworkMonitor(monitor.Config{ID: "test"}, true)
+			require.NoError(t, err)
+			require.Nil(t, result)
+			require.NoError(t, sys.DeleteNetworkMonitor("test"))
+		})
+	}
+}
+
+func TestNetworkMonitorReconnectSync(t *testing.T) {
+	for _, change := range []string{"delete", "disable"} {
+		t.Run(change, func(t *testing.T) {
+			sys, app := newTestSystemWithHub(t)
+			record, err := app.FindRecordById("systems", sys.Id)
+			require.NoError(t, err)
+			// Suppress unrelated system-stat requests while exercising reconnects.
+			record.Set("status", paused)
+			require.NoError(t, app.SaveNoValidate(record))
+			collection, err := app.FindCachedCollectionByNameOrId("network_monitors")
+			require.NoError(t, err)
+			probe := core.NewRecord(collection)
+			probe.Load(map[string]any{
+				"system": sys.Id, "target": "localhost", "protocol": "tcp",
+				"port": 80, "interval": 60, "enabled": true,
+			})
+			require.NoError(t, app.SaveNoValidate(probe))
+
+			sm := NewSystemManager(stubHub{app})
+			t.Cleanup(func() {
+				sm.cancel()
+				_ = sm.RemoveSystem(sys.Id)
+				sm.smartFetchMap.StopCleaner()
+				sm.zfsFetchMap.StopCleaner()
+			})
+			version := semver.MustParse("0.20.0")
+			connections := make(chan *ws.WsConn, 1)
+			upgrader := gws.NewUpgrader(&monitorSyncServer{}, nil)
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				conn, err := upgrader.Upgrade(w, r)
+				if err != nil {
+					t.Error(err)
+					return
+				}
+				wsConn := ws.NewWsConnection(conn, version)
+				conn.Session().Store("wsConn", wsConn)
+				connections <- wsConn
+				conn.ReadLoop()
+			}))
+			t.Cleanup(server.Close)
+			client := &monitorSyncClient{requests: make(chan common.HubRequest[monitor.SyncRequest], 2)}
+			connect := func() monitor.SyncRequest {
+				t.Helper()
+				conn, _, err := gws.NewClient(client, &gws.ClientOption{Addr: "ws" + strings.TrimPrefix(server.URL, "http")})
+				require.NoError(t, err)
+				t.Cleanup(func() { _ = conn.NetConn().Close() })
+				go conn.ReadLoop()
+				select {
+				case wsConn := <-connections:
+					require.NoError(t, sm.AddWebSocketSystem(sys.Id, version, wsConn))
+				case <-time.After(3 * time.Second):
+					t.Fatal("websocket connection was not established")
+				}
+				select {
+				case req := <-client.requests:
+					require.Equal(t, common.SyncNetworkMonitors, req.Action)
+					require.Equal(t, monitor.SyncActionReplace, req.Data.Action)
+					return req.Data
+				case <-time.After(3 * time.Second):
+					t.Fatal("reconnected agent did not receive a monitor replacement")
+					return monitor.SyncRequest{}
+				}
+			}
+
+			initial := connect()
+			require.Len(t, initial.Configs, 1)
+			require.Equal(t, probe.Id, initial.Configs[0].ID)
+			require.NoError(t, sm.RemoveSystem(sys.Id))
+			if change == "delete" {
+				require.NoError(t, app.Delete(probe))
+			} else {
+				probe.Set("enabled", false)
+				require.NoError(t, app.SaveNoValidate(probe))
+			}
+			require.Empty(t, connect().Configs, "reconnect must clear the agent's previous probe")
+		})
+	}
+}
+
+func TestGetMonitorConfigsForSystemQueryError(t *testing.T) {
+	sys, app := newTestSystemWithHub(t)
+	_, err := app.DB().NewQuery("DROP TABLE network_monitors").Execute()
+	require.NoError(t, err)
+	_, err = sys.manager.GetMonitorConfigsForSystem(sys.Id)
+	require.Error(t, err, "a failed query must not be treated as an empty monitor set")
+}

internal/hub/systems/network_monitors.go

@@ -0,0 +1,58 @@
+package systems
+
+import (
+	"context"
+	"time"
+
+	"github.com/henrygd/beszel"
+	"github.com/henrygd/beszel/internal/common"
+	"github.com/henrygd/beszel/internal/entities/monitor"
+)
+
+// SyncNetworkMonitors sends monitor configurations to the agent.
+func (sys *System) SyncNetworkMonitors(configs []monitor.Config) error {
+	_, err := sys.syncNetworkMonitors(monitor.SyncRequest{Action: monitor.SyncActionReplace, Configs: configs})
+	return err
+}
+
+// UpsertNetworkMonitor sends a single monitor configuration change to the agent.
+func (sys *System) UpsertNetworkMonitor(config monitor.Config, runNow bool) (*monitor.Result, error) {
+	resp, err := sys.syncNetworkMonitors(monitor.SyncRequest{
+		Action: monitor.SyncActionUpsert,
+		Config: config,
+		RunNow: runNow,
+	})
+	if err != nil {
+		return nil, err
+	}
+	if resp.Result == (monitor.Result{}) {
+		return nil, nil
+	}
+	result := resp.Result
+	return &result, nil
+}
+
+// DeleteNetworkMonitor removes a single monitor task from the agent.
+func (sys *System) DeleteNetworkMonitor(id string) error {
+	_, err := sys.syncNetworkMonitors(monitor.SyncRequest{
+		Action: monitor.SyncActionDelete,
+		Config: monitor.Config{ID: id},
+	})
+	return err
+}
+
+func (sys *System) syncNetworkMonitors(req monitor.SyncRequest) (monitor.SyncResponse, error) {
+	if sys.agentVersion.LT(beszel.MinVersionNetworkMonitors) {
+		return monitor.SyncResponse{}, nil
+	}
+	timeout := 5 * time.Second
+	if req.Action == monitor.SyncActionUpsert && req.RunNow {
+		// Allow the probe to finish, including a timeout result, while preserving
+		// the normal request budget for transport and response handling.
+		timeout += monitor.MaxProbeTimeout
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+	var result monitor.SyncResponse
+	return result, sys.request(ctx, common.SyncNetworkMonitors, req, &result)
+}

internal/hub/systems/system.go

@@ -9,6 +9,7 @@ import (
 	"math/rand"
 	"net"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -18,6 +19,7 @@ import (
 	"github.com/henrygd/beszel/internal/hub/ws"
 
 	"github.com/henrygd/beszel/internal/entities/container"
+	"github.com/henrygd/beszel/internal/entities/monitor"
 	"github.com/henrygd/beszel/internal/entities/smart"
 	"github.com/henrygd/beszel/internal/entities/system"
 	"github.com/henrygd/beszel/internal/entities/systemd"
@@ -30,6 +32,8 @@ import (
 	"github.com/lxzan/gws"
 	"github.com/pocketbase/dbx"
 	"github.com/pocketbase/pocketbase/core"
+	"github.com/pocketbase/pocketbase/tools/security"
+	"github.com/pocketbase/pocketbase/tools/types"
 	"golang.org/x/crypto/ssh"
 )
 
@@ -52,6 +56,10 @@ type System struct {
 	smartInterval  time.Duration              // Interval for periodic SMART data updates
 	zfsFetching    atomic.Bool                // True if ZFS pools are currently being fetched
 	zfsInterval    time.Duration              // Interval for periodic ZFS detail data updates
+	// Serialize persistence from scheduled updates and resumes through commit.
+	recordsMu sync.Mutex
+	// Protected by recordsMu; realtime reads don't consume probes.
+	lastSavedMonitorProbe map[string]int64
 }
 
 func (sm *SystemManager) NewSystem(systemId string) *System {
@@ -211,11 +219,15 @@ func (sys *System) handlePaused() {
 
 // createRecords updates the system record and adds system_stats and container_stats records
 func (sys *System) createRecords(data *system.CombinedData) (*core.Record, error) {
+	sys.recordsMu.Lock()
+	defer sys.recordsMu.Unlock()
+
 	systemRecord, err := sys.getRecord(sys.manager.hub)
 	if err != nil {
 		return nil, err
 	}
 	hub := sys.manager.hub
+	savedMonitorProbes := make(map[string]int64)
 	err = hub.RunInTransaction(func(txApp core.App) error {
 		// add system_stats record
 		systemStatsCollection, err := txApp.FindCachedCollectionByNameOrId("system_stats")
@@ -266,6 +278,12 @@ func (sys *System) createRecords(data *system.CombinedData) (*core.Record, error
 			}
 		}
 
+		if data.Monitors != nil {
+			if err := sys.updateNetworkMonitorsRecords(txApp, data.Monitors, savedMonitorProbes); err != nil {
+				return err
+			}
+		}
+
 		if err := sys.syncZfsPoolHealth(txApp, data.Stats.ZfsPools); err != nil {
 			return err
 		}
@@ -287,6 +305,29 @@ func (sys *System) createRecords(data *system.CombinedData) (*core.Record, error
 		return nil
 	})
 
+	// Publish only successful inserts after the entire transaction commits.
+	if err == nil && len(savedMonitorProbes) > 0 {
+		if sys.lastSavedMonitorProbe == nil {
+			sys.lastSavedMonitorProbe = savedMonitorProbes
+		} else {
+			for id, timestamp := range savedMonitorProbes {
+				sys.lastSavedMonitorProbe[id] = timestamp
+			}
+		}
+	}
+	// A non-nil report includes cached results for all remaining monitors.
+	if err == nil && data.Monitors != nil {
+		for id := range sys.lastSavedMonitorProbe {
+			if _, exists := data.Monitors[id]; !exists {
+				delete(sys.lastSavedMonitorProbe, id)
+			}
+		}
+	}
+	if err == nil {
+		if alertErr := hub.HandleNetworkMonitorAlerts(systemRecord, data.Monitors); alertErr != nil {
+			hub.Logger().Error("Error handling network monitor alerts", "err", alertErr)
+		}
+	}
 	return systemRecord, err
 }
 
@@ -337,7 +378,7 @@ func createSystemdStatsRecords(app core.App, data []*systemd.Service, systemId s
 		}
 		suffix := fmt.Sprintf("%d", i)
 		valueStrings = append(valueStrings, fmt.Sprintf("({:id%[1]s}, {:system}, {:name%[1]s}, {:state%[1]s}, {:sub%[1]s}, {:cpu%[1]s}, {:cpuPeak%[1]s}, {:memory%[1]s}, {:memPeak%[1]s}, {:updated})", suffix))
-		params["id"+suffix] = makeStableHashId(systemId, service.Name)
+		params["id"+suffix] = MakeStableHashId(systemId, service.Name)
 		params["name"+suffix] = service.Name
 		params["state"+suffix] = service.State
 		params["sub"+suffix] = service.Sub
@@ -363,6 +404,106 @@ func createSystemdStatsRecords(app core.App, data []*systemd.Service, systemId s
 	return err
 }
 
+func (sys *System) updateNetworkMonitorsRecords(app core.App, monitorResults map[string]monitor.Result, savedProbes map[string]int64) error {
+	if len(monitorResults) == 0 {
+		return nil
+	}
+	var err error
+	systemId := sys.Id
+	const monitorCollectionName = "network_monitors"
+
+	// If realtime updates are active, we save via PocketBase records to trigger realtime events.
+	// Otherwise we can do a more efficient direct update via SQL
+	realtimeActive := utils.RealtimeActiveForCollection(app, monitorCollectionName, func(filterQuery string) bool {
+		return !strings.Contains(filterQuery, "system") || strings.Contains(filterQuery, systemId)
+	})
+
+	now := time.Now().UTC()
+	nowMilli := now.UnixMilli()
+	nowString := now.Format(types.DefaultDateLayout)
+	var db dbx.Builder
+	var updateQuery *dbx.Query
+	if !realtimeActive {
+		db = app.DB()
+		monitorFields := []string{"res", "resMin1h", "resMax1h", "resAvg1h", "loss1h", "updated"}
+		setClauses := make([]string, len(monitorFields))
+		for i, f := range monitorFields {
+			setClauses[i] = fmt.Sprintf("%s={:%s}", f, f)
+		}
+		queryString := fmt.Sprintf("UPDATE %s SET %s WHERE id={:id}", monitorCollectionName, strings.Join(setClauses, ", "))
+		updateQuery = db.NewQuery(queryString)
+	}
+
+	// update network_monitors records
+	for id, result := range monitorResults {
+		monitorData := map[string]any{
+			"id":       id,
+			"res":      result.AvgResponse,
+			"resAvg1h": result.AvgResponse1h,
+			"resMin1h": result.MinResponse1h,
+			"resMax1h": result.MaxResponse1h,
+			"loss1h":   result.PacketLoss1h,
+			"updated":  nowString,
+		}
+		switch realtimeActive {
+		case true:
+			var record *core.Record
+			record, err = app.FindRecordById(monitorCollectionName, id)
+			if err == nil {
+				record.Load(monitorData)
+				err = app.SaveNoValidate(record)
+			}
+		default:
+			_, err = updateQuery.Bind(dbx.Params(monitorData)).Execute()
+		}
+		if err != nil {
+			app.Logger().Warn("Failed to update monitor", "system", systemId, "monitor", id, "err", err)
+		}
+	}
+
+	// handle stats collection — one record per monitor
+	const statsCollectionName = "network_monitor_stats"
+
+	var statsCollection *core.Collection
+	if realtimeActive {
+		statsCollection, _ = app.FindCachedCollectionByNameOrId(statsCollectionName)
+	}
+
+	for monitorId, result := range monitorResults {
+		// Compare identity, not ordering, so agent clock changes don't stall writes.
+		if result.LastProbeAt == sys.lastSavedMonitorProbe[monitorId] {
+			continue
+		}
+		statsRecordData := map[string]any{
+			"system":        systemId,
+			"monitor":       monitorId,
+			"type":          "1m",
+			"created":       nowMilli,
+			"res_min":       result.MinResponse,
+			"res_max":       result.MaxResponse,
+			"total_count":   result.TotalCount,
+			"success_count": result.SuccessCount,
+			"res_sum":       result.ResponseSum,
+		}
+		switch realtimeActive {
+		case true:
+			record := core.NewRecord(statsCollection)
+			record.Load(statsRecordData)
+			err = app.SaveNoValidate(record)
+		default:
+			statsRecordData["id"] = security.PseudorandomStringWithAlphabet(10, core.DefaultIdAlphabet)
+			_, err = db.Insert(statsCollectionName, dbx.Params(statsRecordData)).Execute()
+		}
+		if err != nil {
+			app.Logger().Error("Failed to update monitor stats", "system", systemId, "monitor", monitorId, "err", err)
+		} else {
+			savedProbes[monitorId] = result.LastProbeAt
+		}
+	}
+
+	return nil
+}
+
 // createContainerRecords creates container records
 func createContainerRecords(app core.App, data []*container.Stats, systemId string) error {
 	if len(data) == 0 {
@@ -622,7 +763,7 @@ func (sys *System) FetchZfsDataFromAgent(force bool) (*zfs.ZfsData, error) {
 	return &result, err
 }
 
-func makeStableHashId(strings ...string) string {
+func MakeStableHashId(strings ...string) string {
 	hash := fnv.New32a()
 	for _, str := range strings {
 		hash.Write([]byte(str))

internal/hub/systems/system_manager.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/henrygd/beszel/internal/hub/ws"
 
+	"github.com/henrygd/beszel/internal/entities/monitor"
 	"github.com/henrygd/beszel/internal/entities/system"
 	"github.com/henrygd/beszel/internal/hub/expirymap"
 
@@ -17,6 +18,7 @@ import (
 	"github.com/henrygd/beszel"
 
 	"github.com/blang/semver"
+	"github.com/pocketbase/dbx"
 	"github.com/pocketbase/pocketbase/core"
 	"github.com/pocketbase/pocketbase/tools/store"
 	"golang.org/x/crypto/ssh"
@@ -62,6 +64,7 @@ type hubLike interface {
 	core.App
 	GetSSHKey(dataDir string) (ssh.Signer, error)
 	HandleSystemAlerts(systemRecord *core.Record, data *system.CombinedData) error
+	HandleNetworkMonitorAlerts(systemRecord *core.Record, results map[string]monitor.Result) error
 	HandleStatusAlerts(status string, systemRecord *core.Record) error
 	HandleContainerAlerts(systemRecord *core.Record, data *system.CombinedData, fetchLogs func(containerID string) (string, error)) error
 	CancelPendingStatusAlerts(systemID string)
@@ -350,6 +353,20 @@ func (sm *SystemManager) AddWebSocketSystem(systemId string, agentVersion semver
 	if err := sm.AddRecord(systemRecord, system); err != nil {
 		return err
 	}
+
+	// Sync network monitors to the newly connected agent
+	go func() {
+		configs, err := sm.GetMonitorConfigsForSystem(systemId)
+		if err != nil {
+			sm.hub.Logger().Warn("failed to load monitors for agent", "system", systemId, "err", err)
+			return
+		}
+		// An empty set must also replace any probes retained across a disconnect.
+		if err := system.SyncNetworkMonitors(configs); err != nil {
+			sm.hub.Logger().Warn("failed to sync monitors to agent", "system", systemId, "err", err)
+		}
+	}()
+
 	return nil
 }
 
@@ -362,6 +379,16 @@ func (sm *SystemManager) resetFailedSmartFetchState(systemID string) {
 	}
 }
 
+// GetMonitorConfigsForSystem returns all enabled monitor configs for a system.
+func (sm *SystemManager) GetMonitorConfigsForSystem(systemID string) ([]monitor.Config, error) {
+	var configs []monitor.Config
+	err := sm.hub.DB().
+		NewQuery("SELECT id, target, protocol, port, interval FROM network_monitors WHERE system = {:system} AND enabled = true").
+		Bind(dbx.Params{"system": systemID}).
+		All(&configs)
+	return configs, err
+}
+
 // resetFailedZfsFetchState clears only failed ZFS cooldown entries so a fresh
 // agent reconnect retries ZFS discovery immediately after configuration changes.
 func (sm *SystemManager) resetFailedZfsFetchState(systemID string) {
@@ -397,11 +424,12 @@ func (sm *SystemManager) createSSHClientConfig() error {
 
 // deactivateAlerts finds all triggered alerts for a system and sets them to inactive.
 // This is called when a system is paused or goes offline to prevent continued alerts.
+// Monitor incidents remain open: a missing observation does not establish recovery.
 func deactivateAlerts(app core.App, systemID string) error {
 	// Note: Direct SQL updates don't trigger SSE, so we use the PocketBase API
 	// _, err := app.DB().NewQuery(fmt.Sprintf("UPDATE alerts SET triggered = false WHERE system = '%s'", systemID)).Execute()
 
-	alerts, err := app.FindRecordsByFilter("alerts", fmt.Sprintf("system = '%s' && triggered = 1", systemID), "", -1, 0)
+	alerts, err := app.FindRecordsByFilter("alerts", fmt.Sprintf("system = '%s' && triggered = 1 && name != 'NetworkMonitorLoss'", systemID), "", -1, 0)
 	if err != nil {
 		return err
 	}

internal/hub/systems/system_realtime.go

@@ -6,6 +6,8 @@ import (
 	"time"
 
 	"github.com/henrygd/beszel/internal/common"
+	"github.com/henrygd/beszel/internal/entities/monitor"
+	"github.com/henrygd/beszel/internal/entities/system"
 	"github.com/henrygd/beszel/internal/hub/utils"
 	"github.com/pocketbase/dbx"
 	"github.com/pocketbase/pocketbase/apis"
@@ -165,7 +167,7 @@ func (sm *SystemManager) fetchRealtimeDataAndNotify() {
 			if err != nil {
 				return
 			}
-			bytes, err := json.Marshal(data)
+			bytes, err := marshalRealtimeData(data)
 			if err == nil {
 				notify(sm.hub, system, fetch.subscription, bytes)
 			}
@@ -204,6 +206,22 @@ func (sm *SystemManager) finishRealtimeFetch(fetch realtimeFetch) {
 	}
 }
 
+// marshalRealtimeData marshals combined agent data for a realtime broadcast, converting
+// the per-monitor results into the derived metric fields the frontend charts expect.
+func marshalRealtimeData(data *system.CombinedData) ([]byte, error) {
+	if len(data.Monitors) == 0 {
+		return json.Marshal(data)
+	}
+	monitorStats := make(map[string]monitor.Stats, len(data.Monitors))
+	for id, result := range data.Monitors {
+		monitorStats[id] = monitor.Stats{}.FromResult(result)
+	}
+	return json.Marshal(struct {
+		*system.CombinedData
+		Monitors map[string]monitor.Stats `json:"Monitors"`
+	}{data, monitorStats})
+}
+
 // notify broadcasts realtime data to all clients subscribed to a specific subscription.
 // Custom topics bypass collection rules, so check current access for every
 // recipient, including clients whose authentication or membership was revoked.

internal/hub/systems/system_smart.go

@@ -77,7 +77,7 @@ func (sys *System) saveSmartDevices(smartData map[string]smart.SmartData, comple
 
 	currentIDs := make(map[string]struct{}, len(smartData))
 	for deviceKey := range smartData {
-		currentIDs[makeStableHashId(sys.Id, deviceKey)] = struct{}{}
+		currentIDs[MakeStableHashId(sys.Id, deviceKey)] = struct{}{}
 	}
 
 	err = hub.RunInTransaction(func(txApp core.App) error {
@@ -115,7 +115,7 @@ func (sys *System) saveSmartDevices(smartData map[string]smart.SmartData, comple
 }
 
 func (sys *System) upsertSmartDeviceRecord(app core.App, collection *core.Collection, deviceKey string, device smart.SmartData) error {
-	recordID := makeStableHashId(sys.Id, deviceKey)
+	recordID := MakeStableHashId(sys.Id, deviceKey)
 
 	record, err := app.FindRecordById(collection, recordID)
 	if err != nil {

internal/hub/systems/system_smart_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/henrygd/beszel/internal/entities/monitor"
 	"github.com/henrygd/beszel/internal/entities/smart"
 	esystem "github.com/henrygd/beszel/internal/entities/system"
 	"github.com/henrygd/beszel/internal/hub/expirymap"
@@ -28,7 +29,8 @@ func (stubHub) GetSSHKey(dataDir string) (ssh.Signer, error) { return nil, nil }
 func (stubHub) HandleSystemAlerts(systemRecord *core.Record, data *esystem.CombinedData) error {
 	return nil
 }
-func (stubHub) HandleStatusAlerts(status string, systemRecord *core.Record) error { return nil }
+func (stubHub) HandleNetworkMonitorAlerts(*core.Record, map[string]monitor.Result) error { return nil }
+func (stubHub) HandleStatusAlerts(status string, systemRecord *core.Record) error        { return nil }
 func (stubHub) HandleContainerAlerts(systemRecord *core.Record, data *esystem.CombinedData, fetchLogs func(containerID string) (string, error)) error {
 	return nil
 }
@@ -212,7 +214,7 @@ func TestSaveSmartDevices_IncompleteDataDoesNotRemoveDevices(t *testing.T) {
 	}, false))
 
 	assert.Len(t, countSmartDeviceRecords(t, testApp, sys.Id), 2)
-	recordA, err := testApp.FindRecordById("smart_devices", makeStableHashId(sys.Id, "AAA"))
+	recordA, err := testApp.FindRecordById("smart_devices", MakeStableHashId(sys.Id, "AAA"))
 	require.NoError(t, err)
 	assert.EqualValues(t, 42, recordA.GetInt("temp"))
 }

internal/hub/systems/system_systemd_test.go

@@ -14,9 +14,9 @@ func TestGetSystemdServiceId(t *testing.T) {
 		serviceName := "nginx.service"
 
 		// Call multiple times and ensure same result
-		id1 := makeStableHashId(systemId, serviceName)
-		id2 := makeStableHashId(systemId, serviceName)
-		id3 := makeStableHashId(systemId, serviceName)
+		id1 := MakeStableHashId(systemId, serviceName)
+		id2 := MakeStableHashId(systemId, serviceName)
+		id3 := MakeStableHashId(systemId, serviceName)
 
 		assert.Equal(t, id1, id2)
 		assert.Equal(t, id2, id3)
@@ -29,10 +29,10 @@ func TestGetSystemdServiceId(t *testing.T) {
 		serviceName1 := "nginx.service"
 		serviceName2 := "apache.service"
 
-		id1 := makeStableHashId(systemId1, serviceName1)
-		id2 := makeStableHashId(systemId2, serviceName1)
-		id3 := makeStableHashId(systemId1, serviceName2)
-		id4 := makeStableHashId(systemId2, serviceName2)
+		id1 := MakeStableHashId(systemId1, serviceName1)
+		id2 := MakeStableHashId(systemId2, serviceName1)
+		id3 := MakeStableHashId(systemId1, serviceName2)
+		id4 := MakeStableHashId(systemId2, serviceName2)
 
 		// All IDs should be different
 		assert.NotEqual(t, id1, id2)
@@ -56,14 +56,14 @@ func TestGetSystemdServiceId(t *testing.T) {
 		}
 
 		for _, tc := range testCases {
-			id := makeStableHashId(tc.systemId, tc.serviceName)
+			id := MakeStableHashId(tc.systemId, tc.serviceName)
 			// FNV-32 produces 8 hex characters
 			assert.Len(t, id, 8, "ID should be 8 characters for systemId='%s', serviceName='%s'", tc.systemId, tc.serviceName)
 		}
 	})
 
 	t.Run("hexadecimal output", func(t *testing.T) {
-		id := makeStableHashId("test-system", "test-service")
+		id := MakeStableHashId("test-system", "test-service")
 		assert.NotEmpty(t, id)
 
 		// Should only contain hexadecimal characters

internal/hub/systems/system_zfs.go

@@ -129,7 +129,7 @@ func (sys *System) saveZfsPools(zfsData *zfs.ZfsData) error {
 }
 
 func (sys *System) upsertZfsPoolRecord(app core.App, collection *core.Collection, pool *zfs.PoolDetail) error {
-	recordID := makeStableHashId(sys.Id, pool.Name)
+	recordID := MakeStableHashId(sys.Id, pool.Name)
 
 	record, err := app.FindRecordById(collection, recordID)
 	if err != nil {
@@ -171,7 +171,7 @@ func (sys *System) syncZfsPoolHealth(app core.App, pools map[string]*system.ZfsP
 		if pool == nil {
 			continue
 		}
-		recordID := makeStableHashId(sys.Id, name)
+		recordID := MakeStableHashId(sys.Id, name)
 		record, err := app.FindRecordById(collection, recordID)
 		if err != nil {
 			if !errors.Is(err, sql.ErrNoRows) {

internal/hub/systems/system_zfs_test.go

@@ -135,14 +135,14 @@ func TestSavePartialBackendInventory(t *testing.T) {
 				{Name: healthyKey, Alloc: 10}, {Name: failedKey, Alloc: 10},
 			}}
 			require.NoError(t, sys.saveZfsPools(initial))
-			failedID := makeStableHashId(sys.Id, failedKey)
+			failedID := MakeStableHashId(sys.Id, failedKey)
 			before, err := app.FindRecordById("zfs_pools", failedID)
 			require.NoError(t, err)
 			partial := &zfs.ZfsData{CompleteBackends: []string{healthy}, Pools: []*zfs.PoolDetail{
 				{Name: healthyKey, Alloc: 20}, {Name: failedKey, Alloc: 99},
 			}}
 			assert.ErrorIs(t, sys.saveZfsPools(partial), errIncompleteZfsData)
-			fresh, err := app.FindRecordById("zfs_pools", makeStableHashId(sys.Id, healthyKey))
+			fresh, err := app.FindRecordById("zfs_pools", MakeStableHashId(sys.Id, healthyKey))
 			require.NoError(t, err)
 			assert.EqualValues(t, 20, fresh.GetInt("alloc"))
 			cached, err := app.FindRecordById("zfs_pools", failedID)
@@ -169,7 +169,7 @@ func TestSyncZfsPoolHealthWritesOnlyTransitions(t *testing.T) {
 	require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{
 		"tank": {Total: 100, Used: 25, Health: "ONLINE"},
 	}))
-	record, err := app.FindRecordById(collection, makeStableHashId(sys.Id, "tank"))
+	record, err := app.FindRecordById(collection, MakeStableHashId(sys.Id, "tank"))
 	require.NoError(t, err)
 	firstUpdated := record.GetDateTime("updated")
 	assert.Equal(t, "ONLINE", record.GetString("health"))
@@ -193,7 +193,7 @@ func TestSyncZfsPoolHealthWritesOnlyTransitions(t *testing.T) {
 func TestZfsRawCapacityPersistence(t *testing.T) {
 	sys, app := newTestSystemWithHub(t)
 	require.NoError(t, sys.saveZfsPools(&zfs.ZfsData{Complete: true, Pools: []*zfs.PoolDetail{{Name: "btrfs", Size: 200, Alloc: 10, Raw: true}}}))
-	record, err := app.FindRecordById("zfs_pools", makeStableHashId(sys.Id, "btrfs"))
+	record, err := app.FindRecordById("zfs_pools", MakeStableHashId(sys.Id, "btrfs"))
 	require.NoError(t, err)
 	require.True(t, record.GetBool("raw"))
 	require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{"btrfs": {Total: 1, Used: 0.25}}))
@@ -210,7 +210,7 @@ func TestBtrfsDisplayNameKeepsRecordIdentity(t *testing.T) {
 		key:    {DisplayName: "tank", Health: "ONLINE"},
 		"tank": {Health: "ONLINE"},
 	}))
-	id := makeStableHashId(sys.Id, key)
+	id := MakeStableHashId(sys.Id, key)
 	record, err := app.FindRecordById("zfs_pools", id)
 	require.NoError(t, err)
 	assert.Equal(t, "tank", record.GetString("display_name"))
@@ -225,6 +225,6 @@ func TestBtrfsDisplayNameKeepsRecordIdentity(t *testing.T) {
 	record, err = app.FindRecordById("zfs_pools", id)
 	require.NoError(t, err)
 	assert.Equal(t, "detail name", record.GetString("display_name"))
-	_, err = app.FindRecordById("zfs_pools", makeStableHashId(sys.Id, "tank"))
+	_, err = app.FindRecordById("zfs_pools", MakeStableHashId(sys.Id, "tank"))
 	require.NoError(t, err)
 }