feat: network monitoring from agents (#2266, #1911)

Co-authored-by: Sven van Ginkel <svenvanginkel@icloud.com>
Co-authored-by: xiaomiku01 <xiaomiku01@outlook.com>

agent/agent.go

@@ -48,6 +48,7 @@ type Agent struct {
 	keys                      []gossh.PublicKey                                     // SSH public keys
 	smartManager              *SmartManager                                         // Manages SMART data
 	systemdManager            *systemdManager                                       // Manages systemd services
+	monitorManager            *MonitorManager                                       // Manages network monitors
 	storagePoolManager        *StoragePoolManager                                   // Manages storage pool and dataset data
 }
 
@@ -122,6 +123,9 @@ func NewAgent(dataDir ...string) (agent *Agent, err error) {
 	// initialize handler registry
 	agent.handlerRegistry = NewHandlerRegistry()
 
+	// initialize monitor manager
+	agent.monitorManager = newMonitorManager()
+
 	agent.storagePoolManager = newStoragePoolManager()
 
 	// Retain ZFS_INTERVAL for the shared storage pool detail refresh interval.
@@ -192,6 +196,11 @@ func (a *Agent) gatherStats(options common.DataRequestOptions) *system.CombinedD
 		}
 	}
 
+	if a.monitorManager != nil {
+		data.Monitors = a.monitorManager.GetResults(cacheTimeMs)
+		slog.Debug("Monitors", "data", data.Monitors)
+	}
+
 	// skip updating systemd services if cache time is not the default 60sec interval
 	if a.systemdManager != nil && cacheTimeMs == defaultDataCacheTimeMs {
 		totalCount := uint16(a.systemdManager.getServiceStatsCount())

agent/connection_manager.go

@@ -153,6 +153,7 @@ func (c *ConnectionManager) Start(serverOptions ServerOptions) error {
 //	}
 func (c *ConnectionManager) stop() error {
 	_ = c.agent.StopServer()
+	c.agent.monitorManager.Stop()
 	c.closeWebSocket()
 	return health.CleanUp()
 }

agent/gpu_test.go

@@ -1119,7 +1119,6 @@ func TestCalculateGPUAverage(t *testing.T) {
 }
 
 func TestGPUCapabilitiesAndLegacyPriority(t *testing.T) {
-	// Save original PATH
 	hasAmdSysfs := (&GPUManager{}).hasAmdSysfs()
 
 	tests := []struct {
@@ -1213,7 +1212,7 @@ echo "[]"`
 		{
 			name: "no gpu tools available",
 			setupCommands: func(_ string) error {
-				t.Setenv("PATH", "")
+				// The subtest already restricts PATH to its empty temporary directory.
 				return nil
 			},
 			wantErr: true,

agent/handlers.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/fxamacker/cbor/v2"
 	"github.com/henrygd/beszel/internal/common"
+	"github.com/henrygd/beszel/internal/entities/monitor"
 	"github.com/henrygd/beszel/internal/entities/smart"
 
 	"log/slog"
@@ -51,6 +52,7 @@ func NewHandlerRegistry() *HandlerRegistry {
 	registry.Register(common.GetContainerInfo, &GetContainerInfoHandler{})
 	registry.Register(common.GetSmartData, &GetSmartDataHandler{})
 	registry.Register(common.GetSystemdInfo, &GetSystemdInfoHandler{})
+	registry.Register(common.SyncNetworkMonitors, &SyncNetworkMonitorsHandler{})
 	registry.Register(common.GetZfsData, &GetZfsDataHandler{})
 
 	return registry
@@ -223,3 +225,21 @@ func (h *GetSystemdInfoHandler) Handle(hctx *HandlerContext) error {
 
 	return hctx.SendResponse(details, hctx.RequestID)
 }
+
+////////////////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////////////////
+
+// SyncNetworkMonitorsHandler handles monitor configuration sync from hub
+type SyncNetworkMonitorsHandler struct{}
+
+func (h *SyncNetworkMonitorsHandler) Handle(hctx *HandlerContext) error {
+	var req monitor.SyncRequest
+	if err := cbor.Unmarshal(hctx.Request.Data, &req); err != nil {
+		return err
+	}
+	resp, err := hctx.Agent.monitorManager.HandleSyncRequest(req)
+	if err != nil {
+		return err
+	}
+	return hctx.SendResponse(resp, hctx.RequestID)
+}

agent/network_monitor.go

@@ -0,0 +1,176 @@
+package agent
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/henrygd/beszel/internal/entities/monitor"
+)
+
+// MonitorManager manages network monitor configurations and task lifetimes.
+type MonitorManager struct {
+	mu          sync.RWMutex
+	monitors    map[string]*monitorTask // keyed by monitor ID
+	probe       monitorProbe
+	resumeGuard monitorResumeGuard
+}
+
+func newMonitorManager() *MonitorManager {
+	return newMonitorManagerWithProbe(networkMonitorProbe(&http.Client{Timeout: monitor.MaxProbeTimeout}))
+}
+
+func newMonitorManagerWithProbe(probe monitorProbe) *MonitorManager {
+	return &MonitorManager{monitors: make(map[string]*monitorTask), probe: probe}
+}
+
+// SyncMonitors replaces all monitor tasks with the given configs.
+func (pm *MonitorManager) SyncMonitors(configs []monitor.Config) {
+	pm.mu.Lock()
+	defer pm.mu.Unlock()
+
+	// Build set of new keys
+	newKeys := make(map[string]monitor.Config, len(configs))
+	for _, cfg := range configs {
+		if cfg.ID == "" {
+			continue
+		}
+		newKeys[cfg.ID] = cfg
+	}
+
+	// Stop removed monitors
+	for key, task := range pm.monitors {
+		if _, exists := newKeys[key]; !exists {
+			task.cancel()
+			delete(pm.monitors, key)
+		}
+	}
+
+	// Start new monitors and restart tasks whose config changed.
+	for key, cfg := range newKeys {
+		task, exists := pm.monitors[key]
+		if exists && task.config == cfg {
+			continue
+		}
+		if exists {
+			task.cancel()
+		}
+		task = newMonitorTaskFromExisting(cfg, task)
+		task.resumeGuard = &pm.resumeGuard
+		pm.resumeGuard.start()
+		pm.monitors[key] = task
+		pm.startMonitor(task)
+	}
+	if len(pm.monitors) == 0 {
+		pm.resumeGuard.shutdown()
+	}
+}
+
+// HandleSyncRequest applies a full or incremental monitor sync request.
+func (pm *MonitorManager) HandleSyncRequest(req monitor.SyncRequest) (monitor.SyncResponse, error) {
+	switch req.Action {
+	case monitor.SyncActionReplace:
+		pm.SyncMonitors(req.Configs)
+		return monitor.SyncResponse{}, nil
+	case monitor.SyncActionUpsert:
+		result, err := pm.UpsertMonitor(req.Config, req.RunNow)
+		if err != nil {
+			return monitor.SyncResponse{}, err
+		}
+		if result == nil {
+			return monitor.SyncResponse{}, nil
+		}
+		return monitor.SyncResponse{Result: *result}, nil
+	case monitor.SyncActionDelete:
+		if req.Config.ID == "" {
+			return monitor.SyncResponse{}, errors.New("missing monitor ID for delete")
+		}
+		pm.DeleteMonitor(req.Config.ID)
+		return monitor.SyncResponse{}, nil
+	default:
+		return monitor.SyncResponse{}, fmt.Errorf("unknown monitor sync action: %d", req.Action)
+	}
+}
+
+// UpsertMonitor creates or replaces a single monitor task.
+func (pm *MonitorManager) UpsertMonitor(config monitor.Config, runNow bool) (*monitor.Result, error) {
+	if config.ID == "" {
+		return nil, errors.New("missing monitor ID")
+	}
+
+	pm.mu.Lock()
+	task, exists := pm.monitors[config.ID]
+	if exists && task.config == config {
+		pm.mu.Unlock()
+		if !runNow {
+			return nil, nil
+		}
+		return task.runProbe(pm.probe), nil
+	}
+	if exists {
+		task.cancel()
+	}
+	task = newMonitorTaskFromExisting(config, task)
+	task.resumeGuard = &pm.resumeGuard
+	pm.resumeGuard.start()
+	pm.monitors[config.ID] = task
+	pm.mu.Unlock()
+
+	if runNow {
+		result := task.runProbe(pm.probe)
+		pm.startMonitor(task)
+		return result, nil
+	}
+	pm.startMonitor(task)
+	return nil, nil
+}
+
+// DeleteMonitor stops and removes a single monitor task.
+func (pm *MonitorManager) DeleteMonitor(id string) {
+	if id == "" {
+		return
+	}
+	pm.mu.Lock()
+	defer pm.mu.Unlock()
+	if task, exists := pm.monitors[id]; exists {
+		task.cancel()
+		delete(pm.monitors, id)
+	}
+	if len(pm.monitors) == 0 {
+		pm.resumeGuard.shutdown()
+	}
+}
+
+// GetResults returns aggregated results for all monitors over the last supplied duration in ms.
+func (pm *MonitorManager) GetResults(durationMs uint16) map[string]monitor.Result {
+	pm.mu.RLock()
+	defer pm.mu.RUnlock()
+
+	results := make(map[string]monitor.Result, len(pm.monitors))
+	now := time.Now()
+	duration := time.Duration(durationMs) * time.Millisecond
+
+	for _, task := range pm.monitors {
+		result, ok := task.history.result(duration, now)
+
+		if !ok {
+			continue
+		}
+		results[task.config.ID] = result
+	}
+
+	return results
+}
+
+// Stop stops all monitor tasks.
+func (pm *MonitorManager) Stop() {
+	pm.mu.Lock()
+	defer pm.mu.Unlock()
+	for key, task := range pm.monitors {
+		task.cancel()
+		delete(pm.monitors, key)
+	}
+	pm.resumeGuard.shutdown()
+}

agent/network_monitor_history.go

@@ -0,0 +1,274 @@
+package agent
+
+import (
+	"math"
+	"sync"
+	"time"
+
+	"github.com/henrygd/beszel/internal/entities/monitor"
+)
+
+// Monitors run at user-defined intervals (e.g., every 10s).
+// To keep memory usage low and constant, data is stored in two layers:
+// 1. Raw samples: The most recent individual results (kept for monitorRawRetention).
+// 2. Minute buckets: A ring buffer of 61 buckets, each representing one
+//    wall-clock minute. Samples collected within the same minute are aggregated
+//    (sum, min, max, count) into a single bucket.
+//
+// Short-term requests (<= 61s) use raw samples.
+// Long-term requests (up to 1h) use the minute buckets to avoid storing thousands
+// of individual data points.
+
+const (
+	// monitorRawRetention is the duration to keep individual samples
+	monitorRawRetention = 61 * time.Second
+	// monitorMinuteBucketLen is the number of 1-minute buckets to keep (1 hour + 1 for partials)
+	monitorMinuteBucketLen int32 = 61
+)
+
+// monitorHistory owns retention and aggregation, independently of probe execution.
+type monitorHistory struct {
+	mu          sync.Mutex
+	sampleCount int64
+	samples     []monitorSample
+	buckets     [monitorMinuteBucketLen]monitorBucket
+}
+
+func newMonitorHistory() *monitorHistory {
+	// Start small for typical intervals; append grows the buffer for faster probes.
+	return &monitorHistory{samples: make([]monitorSample, 0, 4)}
+}
+
+func (h *monitorHistory) clone() *monitorHistory {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	cloned := newMonitorHistory()
+	cloned.samples = append(cloned.samples, h.samples...)
+	cloned.buckets = h.buckets
+	cloned.sampleCount = h.sampleCount
+	return cloned
+}
+
+func (h *monitorHistory) result(duration time.Duration, now time.Time) (monitor.Result, bool) {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	return h.resultLocked(duration, now)
+}
+
+func (h *monitorHistory) record(sample monitorSample) monitor.Result {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	h.addSampleLocked(sample)
+	result, _ := h.resultLocked(time.Minute, sample.timestamp)
+	return result
+}
+
+// monitorSample stores one monitor attempt and its collection time.
+type monitorSample struct {
+	responseUs int64 // -1 means loss
+	timestamp  time.Time
+}
+
+// monitorBucket stores one minute of aggregated monitor data.
+type monitorBucket struct {
+	minute int32
+	filled bool
+	stats  monitorAggregate
+}
+
+// monitorAggregate accumulates successful response stats and total sample counts.
+type monitorAggregate struct {
+	sumUs        int64
+	minUs        int64
+	maxUs        int64
+	totalCount   int64
+	successCount int64
+}
+
+// newMonitorAggregate initializes an aggregate with an unset minimum value.
+func newMonitorAggregate() monitorAggregate {
+	return monitorAggregate{minUs: math.MaxInt64}
+}
+
+// addResponse folds a single monitor sample into the aggregate.
+func (agg *monitorAggregate) addResponse(responseUs int64) {
+	agg.totalCount++
+	if responseUs < 0 {
+		return
+	}
+	agg.successCount++
+	agg.sumUs += responseUs
+	if responseUs < agg.minUs {
+		agg.minUs = responseUs
+	}
+	if responseUs > agg.maxUs {
+		agg.maxUs = responseUs
+	}
+}
+
+// addAggregate merges another aggregate into this one.
+func (agg *monitorAggregate) addAggregate(other monitorAggregate) {
+	if other.totalCount == 0 {
+		return
+	}
+	agg.totalCount += other.totalCount
+	agg.successCount += other.successCount
+	agg.sumUs += other.sumUs
+	if other.successCount == 0 {
+		return
+	}
+	if agg.minUs == math.MaxInt64 || other.minUs < agg.minUs {
+		agg.minUs = other.minUs
+	}
+	if other.maxUs > agg.maxUs {
+		agg.maxUs = other.maxUs
+	}
+}
+
+// hasData reports whether the aggregate contains any samples.
+func (agg monitorAggregate) hasData() bool {
+	return agg.totalCount > 0
+}
+
+// result converts the aggregate into the monitor result format.
+func (agg monitorAggregate) result() monitor.Result {
+	avg := agg.avgResponse()
+	result := monitor.Result{
+		AvgResponse:  avg,
+		MinResponse:  agg.minUs,
+		MaxResponse:  agg.maxUs,
+		PacketLoss:   agg.lossPercentage(),
+		TotalCount:   agg.totalCount,
+		SuccessCount: agg.successCount,
+		ResponseSum:  agg.sumUs,
+	}
+	if agg.successCount == 0 {
+		result.MinResponse, result.MaxResponse = 0, 0
+	}
+	return result
+}
+
+// avgResponse returns the rounded average of successful samples.
+func (agg monitorAggregate) avgResponse() int64 {
+	if agg.successCount == 0 {
+		return 0
+	}
+	return agg.sumUs / agg.successCount
+
+}
+
+// lossPercentage returns the rounded failure rate for the aggregate.
+func (agg monitorAggregate) lossPercentage() float64 {
+	if agg.totalCount == 0 {
+		return 0
+	}
+	return math.Round(float64(agg.totalCount-agg.successCount)/float64(agg.totalCount)*10000) / 100
+}
+
+// resultLocked returns the aggregated monitor result for the requested duration along with a bool indicating whether any data was available.
+func (h *monitorHistory) resultLocked(duration time.Duration, now time.Time) (monitor.Result, bool) {
+	agg := h.aggregateLocked(duration, now)
+	if !agg.hasData() {
+		// short realtime windows (e.g. the 1s window used for 1m/realtime charts) often fall
+		// between monitor samples since monitors run at longer, user-defined intervals; fall back to
+		// the most recent sample so realtime requests still report current status.
+		agg = h.latestSampleAggregateLocked()
+	}
+	hourAgg := h.aggregateLocked(time.Hour, now)
+	if !agg.hasData() {
+		return monitor.Result{}, false
+	}
+
+	result := agg.result()
+	if len(h.samples) > 0 {
+		result.LastProbeAt = h.samples[len(h.samples)-1].timestamp.UnixMilli()
+	}
+
+	result.AvgResponse1h = hourAgg.avgResponse()
+	result.MinResponse1h = hourAgg.minUs
+	result.MaxResponse1h = hourAgg.maxUs
+	result.PacketLoss1h = hourAgg.lossPercentage()
+	result.SampleCount = h.sampleCount
+
+	if hourAgg.successCount == 0 {
+		result.MinResponse1h, result.MaxResponse1h = 0, 0
+	}
+	return result, true
+}
+
+// latestSampleAggregateLocked returns an aggregate containing only the most recent sample, if any.
+func (h *monitorHistory) latestSampleAggregateLocked() monitorAggregate {
+	agg := newMonitorAggregate()
+	if len(h.samples) == 0 {
+		return agg
+	}
+	agg.addResponse(h.samples[len(h.samples)-1].responseUs)
+	return agg
+}
+
+// aggregateLocked collects monitor data for the requested time window.
+func (h *monitorHistory) aggregateLocked(duration time.Duration, now time.Time) monitorAggregate {
+	cutoff := now.Add(-duration)
+	// Keep short windows exact; longer windows read from minute buckets to avoid raw-sample retention.
+	if duration <= monitorRawRetention {
+		return aggregateSamplesSince(h.samples, cutoff)
+	}
+	return aggregateBucketsSince(h.buckets[:], cutoff, now)
+}
+
+// aggregateSamplesSince aggregates raw samples newer than the cutoff.
+func aggregateSamplesSince(samples []monitorSample, cutoff time.Time) monitorAggregate {
+	agg := newMonitorAggregate()
+	for _, sample := range samples {
+		if sample.timestamp.Before(cutoff) {
+			continue
+		}
+		agg.addResponse(sample.responseUs)
+	}
+	return agg
+}
+
+// aggregateBucketsSince aggregates minute buckets overlapping the requested window.
+func aggregateBucketsSince(buckets []monitorBucket, cutoff, now time.Time) monitorAggregate {
+	agg := newMonitorAggregate()
+	startMinute := int32(cutoff.Unix() / 60)
+	endMinute := int32(now.Unix() / 60)
+	for _, bucket := range buckets {
+		if !bucket.filled || bucket.minute < startMinute || bucket.minute > endMinute {
+			continue
+		}
+		agg.addAggregate(bucket.stats)
+	}
+	return agg
+}
+
+// addSampleLocked stores a fresh sample in both raw and per-minute retention buffers.
+func (h *monitorHistory) addSampleLocked(sample monitorSample) {
+	h.sampleCount++
+	cutoff := sample.timestamp.Add(-monitorRawRetention)
+	start := 0
+	for i := range h.samples {
+		if !h.samples[i].timestamp.Before(cutoff) {
+			start = i
+			break
+		}
+		if i == len(h.samples)-1 {
+			start = len(h.samples)
+		}
+	}
+	if start > 0 {
+		size := copy(h.samples, h.samples[start:])
+		h.samples = h.samples[:size]
+	}
+	h.samples = append(h.samples, sample)
+
+	minute := int32(sample.timestamp.Unix() / 60)
+	// Each slot stores one wall-clock minute, so the ring stays fixed-size at ~1h per monitor.
+	bucket := &h.buckets[minute%monitorMinuteBucketLen]
+	if !bucket.filled || bucket.minute != minute {
+		bucket.minute = minute
+		bucket.filled = true
+		bucket.stats = newMonitorAggregate()
+	}
+	bucket.stats.addResponse(sample.responseUs)
+}

agent/network_monitor_history_test.go

@@ -0,0 +1,154 @@
+package agent
+
+import (
+	"testing"
+	"time"
+
+	"github.com/fxamacker/cbor/v2"
+	"github.com/henrygd/beszel/internal/entities/monitor"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMonitorHistoryWindowCounts(t *testing.T) {
+	history := newMonitorHistory()
+	now := time.Now()
+	// This older success counts toward lifetime warm-up, but not this window.
+	history.record(monitorSample{responseUs: 1000, timestamp: now.Add(-2 * time.Minute)})
+	history.record(monitorSample{responseUs: 10, timestamp: now.Add(-30 * time.Second)})
+	history.record(monitorSample{responseUs: 21, timestamp: now.Add(-20 * time.Second)})
+	history.record(monitorSample{responseUs: -1, timestamp: now.Add(-10 * time.Second)})
+	result, ok := history.result(time.Minute, now)
+	require.True(t, ok)
+	assert.EqualValues(t, 4, result.SampleCount)
+	assert.EqualValues(t, 3, result.TotalCount)
+	assert.EqualValues(t, 2, result.SuccessCount)
+	assert.EqualValues(t, 31, result.ResponseSum, "preserve the sum before average rounding")
+	assert.EqualValues(t, 15, result.AvgResponse)
+	assert.Equal(t, 33.33, result.PacketLoss)
+
+	encoded, err := cbor.Marshal(result)
+	require.NoError(t, err)
+	var decoded monitor.Result
+	require.NoError(t, cbor.Unmarshal(encoded, &decoded))
+	assert.Equal(t, result, decoded)
+	stats := monitor.Stats{}.FromResult(decoded)
+	assert.Equal(t, result.TotalCount, stats.TotalCount)
+	assert.Equal(t, result.SuccessCount, stats.SuccessCount)
+	assert.Equal(t, result.ResponseSum, stats.ResponseSum)
+
+	// Reads do not consume samples. A short window's latest-sample fallback
+	// carries the count for that single failure, not the minute or lifetime count.
+	repeated, _ := history.result(time.Minute, now)
+	assert.Equal(t, result, repeated)
+	fallback, ok := history.result(time.Second, now)
+	require.True(t, ok)
+	assert.EqualValues(t, 1, fallback.TotalCount)
+	assert.Zero(t, fallback.SuccessCount)
+	assert.Zero(t, fallback.ResponseSum)
+	assert.Equal(t, 100.0, fallback.PacketLoss)
+	assert.EqualValues(t, 4, fallback.SampleCount)
+}
+
+func TestMonitorHistoryAggregateLockedUsesRawSamplesForShortWindows(t *testing.T) {
+	now := time.Date(2026, time.April, 21, 12, 0, 0, 0, time.UTC)
+	history := newMonitorHistory()
+
+	history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-90 * time.Second)})
+	history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now.Add(-30 * time.Second)})
+	history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-10 * time.Second)})
+
+	agg := history.aggregateLocked(time.Minute, now)
+	require.True(t, agg.hasData())
+	assert.Equal(t, int64(2), agg.totalCount)
+	assert.Equal(t, int64(1), agg.successCount)
+	result := agg.result()
+	assert.Equal(t, int64(20), result.AvgResponse)
+	assert.Equal(t, int64(20), result.MinResponse)
+	assert.Equal(t, int64(20), result.MaxResponse)
+	assert.Equal(t, 50.0, result.PacketLoss)
+}
+
+func TestMonitorHistoryAggregateLockedUsesMinuteBucketsForLongWindows(t *testing.T) {
+	now := time.Date(2026, time.April, 21, 12, 0, 30, 0, time.UTC)
+	history := newMonitorHistory()
+
+	history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-11 * time.Minute)})
+	history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now.Add(-9 * time.Minute)})
+	history.addSampleLocked(monitorSample{responseUs: 40, timestamp: now.Add(-5 * time.Minute)})
+	history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-90 * time.Second)})
+	history.addSampleLocked(monitorSample{responseUs: 30, timestamp: now.Add(-30 * time.Second)})
+
+	agg := history.aggregateLocked(10*time.Minute, now)
+	require.True(t, agg.hasData())
+	assert.Equal(t, int64(4), agg.totalCount)
+	assert.Equal(t, int64(3), agg.successCount)
+	result := agg.result()
+	assert.Equal(t, int64(30), result.AvgResponse)
+	assert.Equal(t, int64(20), result.MinResponse)
+	assert.Equal(t, int64(40), result.MaxResponse)
+	assert.Equal(t, 25.0, result.PacketLoss)
+}
+
+func TestMonitorHistoryAddSampleLockedTrimsRawSamplesButKeepsBucketHistory(t *testing.T) {
+	now := time.Date(2026, time.April, 21, 12, 0, 0, 0, time.UTC)
+	history := newMonitorHistory()
+
+	history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-10 * time.Minute)})
+	history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now})
+
+	require.Len(t, history.samples, 1)
+	assert.Equal(t, int64(20), history.samples[0].responseUs)
+
+	agg := history.aggregateLocked(10*time.Minute, now)
+	require.True(t, agg.hasData())
+	assert.Equal(t, int64(2), agg.totalCount)
+	assert.Equal(t, int64(2), agg.successCount)
+	result := agg.result()
+	assert.Equal(t, int64(15), result.AvgResponse)
+	assert.Equal(t, int64(10), result.MinResponse)
+	assert.Equal(t, int64(20), result.MaxResponse)
+	assert.Equal(t, 0.0, result.PacketLoss)
+}
+
+func TestMonitorHistoryProbeTimestamp(t *testing.T) {
+	history := newMonitorHistory()
+	start := time.Date(2026, time.September, 14, 12, 0, 0, 0, time.UTC)
+	_, ok := history.result(time.Minute, start)
+	require.False(t, ok)
+	first := history.record(monitorSample{responseUs: 20, timestamp: start})
+	assert.Equal(t, start.UnixMilli(), first.LastProbeAt)
+	for minute := 0; minute < 5; minute++ {
+		now := start.Add(time.Duration(minute)*time.Minute + time.Second)
+		// Realtime reads must not consume freshness for the persistence request.
+		for _, window := range []time.Duration{time.Second, time.Minute} {
+			result, ok := history.result(window, now)
+			require.True(t, ok)
+			assert.Equal(t, first.LastProbeAt, result.LastProbeAt)
+			assert.Equal(t, int64(20), result.AvgResponse)
+		}
+	}
+	next := start.Add(5 * time.Minute)
+	failed := history.record(monitorSample{responseUs: -1, timestamp: next})
+	assert.Equal(t, next.UnixMilli(), failed.LastProbeAt)
+	assert.Equal(t, float64(100), failed.PacketLoss)
+	repeated, ok := history.result(time.Minute, next.Add(2*time.Minute))
+	require.True(t, ok)
+	assert.Equal(t, failed.LastProbeAt, repeated.LastProbeAt)
+	assert.Equal(t, float64(100), repeated.PacketLoss)
+}
+
+func TestMonitorHistorySampleCount(t *testing.T) {
+	history := newMonitorHistory()
+	now := time.Now()
+	// Both failed and successful probes count, including older samples so
+	// monitors with hourly intervals can finish warming up.
+	history.record(monitorSample{responseUs: -1, timestamp: now.Add(-2 * time.Hour)})
+	for i, response := range []int64{10, -1, 20} {
+		result := history.record(monitorSample{responseUs: response, timestamp: now.Add(time.Duration(i) * time.Second)})
+		assert.EqualValues(t, i+2, result.SampleCount)
+	}
+	result, ok := history.clone().result(time.Minute, now.Add(3*time.Second))
+	require.True(t, ok)
+	assert.EqualValues(t, 4, result.SampleCount)
+}

agent/network_monitor_ping.go

@@ -0,0 +1,312 @@
+package agent
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"errors"
+	"fmt"
+	"math"
+	"net"
+	"os"
+	"os/exec"
+	"regexp"
+	"runtime"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"golang.org/x/net/icmp"
+	"golang.org/x/net/ipv4"
+	"golang.org/x/net/ipv6"
+
+	"log/slog"
+)
+
+// Match the numeric RTT independently of the localized label used by Windows.
+var pingTimeRegex = regexp.MustCompile(`(?i)[=<]\s*([0-9]+(?:[.,][0-9]+)?)\s*ms\b`)
+
+var icmpSequence atomic.Uint32
+
+type icmpPacketConn interface {
+	Close() error
+}
+
+// icmpMethod tracks which ICMP approach to use. Once a method succeeds or
+// all native methods fail, the choice is cached so subsequent monitors skip
+// the trial-and-error overhead.
+type icmpMethod uint8
+
+const (
+	icmpUntried      icmpMethod = iota // haven't tried yet
+	icmpRaw                            // privileged raw socket
+	icmpDatagram                       // unprivileged datagram socket
+	icmpExecFallback                   // shell out to system ping command
+)
+
+// icmpFamily holds the network parameters and cached detection result for one address family.
+type icmpFamily struct {
+	rawNetwork   string    // e.g. "ip4:icmp" or "ip6:ipv6-icmp"
+	dgramNetwork string    // e.g. "udp4" or "udp6"
+	listenAddr   string    // "0.0.0.0" or "::"
+	echoType     icmp.Type // outgoing echo request type
+	replyType    icmp.Type // expected echo reply type
+	proto        int       // IANA protocol number for parsing replies
+	isIPv6       bool
+	mode         icmpMethod // cached detection result (guarded by icmpModeMu)
+}
+
+var (
+	icmpV4 = icmpFamily{
+		rawNetwork:   "ip4:icmp",
+		dgramNetwork: "udp4",
+		listenAddr:   "0.0.0.0",
+		echoType:     ipv4.ICMPTypeEcho,
+		replyType:    ipv4.ICMPTypeEchoReply,
+		proto:        1,
+	}
+	icmpV6 = icmpFamily{
+		rawNetwork:   "ip6:ipv6-icmp",
+		dgramNetwork: "udp6",
+		listenAddr:   "::",
+		echoType:     ipv6.ICMPTypeEchoRequest,
+		replyType:    ipv6.ICMPTypeEchoReply,
+		proto:        58,
+		isIPv6:       true,
+	}
+	icmpModeMu sync.Mutex
+	icmpListen = func(network, listenAddr string) (icmpPacketConn, error) {
+		return icmp.ListenPacket(network, listenAddr)
+	}
+)
+
+// monitorICMP sends an ICMP echo request and measures round-trip response.
+// Supports both IPv4 and IPv6 targets. The ICMP method (raw socket,
+// unprivileged datagram, or exec fallback) is detected once per address
+// family and cached for subsequent monitors.
+// Returns response in microseconds, or -1 and an error on failure.
+func monitorICMP(ctx context.Context, target string) (int64, error) {
+	ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
+	defer cancel()
+
+	family, ip, err := resolveICMPTarget(ctx, target)
+	if err != nil {
+		return -1, err
+	}
+
+	icmpModeMu.Lock()
+	if family.mode == icmpUntried {
+		family.mode = detectICMPMode(family, icmpListen)
+	}
+	mode := family.mode
+	icmpModeMu.Unlock()
+
+	switch mode {
+	case icmpRaw:
+		return monitorICMPNative(ctx, family.rawNetwork, family, &net.IPAddr{IP: ip})
+	case icmpDatagram:
+		return monitorICMPNative(ctx, family.dgramNetwork, family, &net.UDPAddr{IP: ip})
+	case icmpExecFallback:
+		return monitorICMPExec(ctx, ip.String(), family.isIPv6)
+	default:
+		return -1, errors.New("unsupported ICMP mode")
+	}
+}
+
+// resolveICMPTarget resolves a target hostname or IP to determine the address
+// family and concrete IP address. Prefers IPv4 for dual-stack hostnames.
+func resolveICMPTarget(ctx context.Context, target string) (*icmpFamily, net.IP, error) {
+	if ip := net.ParseIP(target); ip != nil {
+		if ip.To4() != nil {
+			return &icmpV4, ip.To4(), nil
+		}
+		return &icmpV6, ip, nil
+	}
+
+	ips, err := net.DefaultResolver.LookupIP(ctx, "ip", target)
+	if err != nil || len(ips) == 0 {
+		return nil, nil, err
+	}
+	for _, ip := range ips {
+		if v4 := ip.To4(); v4 != nil {
+			return &icmpV4, v4, nil
+		}
+	}
+	return &icmpV6, ips[0], nil
+}
+
+func detectICMPMode(family *icmpFamily, listen func(network, listenAddr string) (icmpPacketConn, error)) icmpMethod {
+	label := "IPv4"
+	if family.isIPv6 {
+		label = "IPv6"
+	}
+
+	conn, err := listen(family.rawNetwork, family.listenAddr)
+	slog.Debug("ICMP raw socket test", "family", label, "err", err)
+	if err == nil {
+		conn.Close()
+		return icmpRaw
+	}
+
+	conn, err = listen(family.dgramNetwork, family.listenAddr)
+	slog.Debug("ICMP datagram socket test", "family", label, "err", err)
+	if err == nil {
+		conn.Close()
+		return icmpDatagram
+	}
+
+	return icmpExecFallback
+}
+
+// monitorICMPNative sends an ICMP echo request using Go's x/net/icmp package.
+func monitorICMPNative(ctx context.Context, network string, family *icmpFamily, dst net.Addr) (int64, error) {
+	conn, err := icmp.ListenPacket(network, family.listenAddr)
+	if err != nil {
+		return -1, err
+	}
+	defer conn.Close()
+
+	return monitorICMPPacket(ctx, conn, family, dst)
+}
+
+func monitorICMPPacket(ctx context.Context, conn net.PacketConn, family *icmpFamily, dst net.Addr) (int64, error) {
+	if err := ctx.Err(); err != nil {
+		return -1, err
+	}
+	// Closing the socket interrupts both reads and writes on cancellation.
+	stop := context.AfterFunc(ctx, func() { _ = conn.Close() })
+	defer stop()
+
+	// Prepare correlation data before starting the round-trip timer. The token
+	// also distinguishes delayed replies after the 16-bit sequence wraps.
+	token := make([]byte, 16)
+	if _, err := rand.Read(token); err != nil {
+		return -1, err
+	}
+	echo := &icmp.Echo{
+		ID:   os.Getpid() & 0xffff,
+		Seq:  int(icmpSequence.Add(1) & 0xffff),
+		Data: token,
+	}
+	// Linux ping sockets replace the Echo ID with their bound port. Darwin
+	// datagram sockets and raw sockets preserve the supplied ID.
+	if local, ok := conn.LocalAddr().(*net.UDPAddr); ok && runtime.GOOS == "linux" {
+		echo.ID = local.Port
+	}
+	targetIP := icmpAddrIP(dst)
+	msg := &icmp.Message{
+		Type: family.echoType,
+		Code: 0,
+		Body: echo,
+	}
+	msgBytes, err := msg.Marshal(nil)
+	if err != nil {
+		return -1, err
+	}
+
+	// Set deadline before sending
+	if err := conn.SetDeadline(time.Now().Add(3 * time.Second)); err != nil {
+		return -1, err
+	}
+
+	buf := make([]byte, 1500)
+	start := time.Now()
+	if _, err := conn.WriteTo(msgBytes, dst); err != nil {
+		return -1, err
+	}
+
+	// Read reply
+	for {
+		n, peer, err := conn.ReadFrom(buf)
+		received := time.Now()
+		if err != nil {
+			return -1, err
+		}
+		if !targetIP.Equal(icmpAddrIP(peer)) {
+			continue
+		}
+
+		reply, err := icmp.ParseMessage(family.proto, buf[:n])
+		if err != nil || reply.Type != family.replyType || reply.Code != 0 {
+			continue
+		}
+
+		body, ok := reply.Body.(*icmp.Echo)
+		if ok && body.ID == echo.ID && body.Seq == echo.Seq && bytes.Equal(body.Data, echo.Data) {
+			return received.Sub(start).Microseconds(), nil
+		}
+		// Keep waiting for our reply without extending the original deadline.
+	}
+}
+
+func icmpAddrIP(addr net.Addr) net.IP {
+	switch addr := addr.(type) {
+	case *net.IPAddr:
+		return addr.IP
+	case *net.UDPAddr:
+		return addr.IP
+	default:
+		return nil
+	}
+}
+
+// pingCommand selects the executable and arguments for the supported agent platforms.
+// The context deadline enforces the timeout: -W has incompatible meanings across
+// Linux, BSD IPv4 ping, and macOS ping6.
+func pingCommand(goos, target string, isIPv6 bool) (string, []string, error) {
+	family := "-4"
+	if isIPv6 {
+		family = "-6"
+	}
+	switch goos {
+	case "windows":
+		return "ping", []string{family, "-n", "1", "-w", "3000", target}, nil
+	case "linux":
+		return "ping", []string{family, "-n", "-c", "1", target}, nil
+	case "darwin", "freebsd", "openbsd":
+		command := "ping"
+		if isIPv6 {
+			command = "ping6"
+		}
+		return command, []string{"-n", "-c", "1", target}, nil
+	default:
+		return "", nil, fmt.Errorf("ping fallback is unsupported on %s", goos)
+	}
+}
+
+// monitorICMPExec falls back to the system ping command. Returns -1 and an error on failure.
+func monitorICMPExec(ctx context.Context, target string, isIPv6 bool) (int64, error) {
+	ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
+	defer cancel()
+	name, args, err := pingCommand(runtime.GOOS, target, isIPv6)
+	if err != nil {
+		return -1, err
+	}
+	cmd := exec.CommandContext(ctx, name, args...)
+	// Keep Unix output and decimal formatting stable. Windows ignores LC_ALL.
+	cmd.Env = append(os.Environ(), "LC_ALL=C")
+	output, err := cmd.Output()
+	if ctx.Err() != nil {
+		return -1, ctx.Err()
+	}
+	if err != nil {
+		return -1, fmt.Errorf("%s failed: %w", name, err)
+	}
+	return parsePingResponse(output)
+}
+
+// parsePingResponse returns the reported RTT, never subprocess execution time.
+// For a bounded value such as Windows' time<1ms, retain the reported upper bound.
+func parsePingResponse(output []byte) (int64, error) {
+	matches := pingTimeRegex.FindSubmatch(output)
+	if len(matches) < 2 {
+		return -1, errors.New("ping output contains no round-trip time")
+	}
+	ms, err := strconv.ParseFloat(strings.ReplaceAll(string(matches[1]), ",", "."), 64)
+	if err != nil || math.IsInf(ms, 0) || ms >= float64(math.MaxInt64)/1000 {
+		return -1, errors.New("invalid round-trip time in ping output")
+	}
+	return int64(math.Round(ms * 1000)), nil
+}

agent/network_monitor_ping_test.go

@@ -0,0 +1,433 @@
+//go:build testing
+
+package agent
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/icmp"
+)
+
+type testICMPPacketConn struct{}
+
+func (testICMPPacketConn) Close() error { return nil }
+
+type blockingICMPConn struct {
+	net.PacketConn
+	reading chan struct{}
+}
+
+func (c *blockingICMPConn) WriteTo(p []byte, addr net.Addr) (int, error) {
+	return len(p), nil
+}
+
+func (c *blockingICMPConn) ReadFrom(p []byte) (int, net.Addr, error) {
+	close(c.reading)
+	return c.PacketConn.ReadFrom(p)
+}
+
+func TestMonitorICMPPacketCancellation(t *testing.T) {
+	conn, err := net.ListenPacket("udp4", "127.0.0.1:0")
+	require.NoError(t, err)
+	defer conn.Close()
+	blocking := &blockingICMPConn{PacketConn: conn, reading: make(chan struct{})}
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	done := make(chan error, 1)
+	go func() {
+		_, err := monitorICMPPacket(ctx, blocking, &icmpV4, conn.LocalAddr())
+		done <- err
+	}()
+	select {
+	case <-blocking.reading:
+	case <-time.After(time.Second):
+		t.Fatal("probe did not begin reading")
+	}
+	cancel()
+	select {
+	case err := <-done:
+		require.Error(t, err)
+	case <-time.After(time.Second):
+		t.Fatal("cancellation did not interrupt the socket read")
+	}
+}
+
+func TestMonitorICMPExecCancellation(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("test uses a POSIX shell stub for ping")
+	}
+	dir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "ping"), []byte("#!/bin/sh\nexec sleep 30\n"), 0o755))
+	t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+	done := make(chan error, 1)
+	go func() {
+		_, err := monitorICMPExec(ctx, "127.0.0.1", false)
+		done <- err
+	}()
+	select {
+	case err := <-done:
+		require.ErrorIs(t, err, context.DeadlineExceeded)
+	case <-time.After(time.Second):
+		t.Fatal("cancellation did not terminate ping")
+	}
+}
+
+func TestPingCommand(t *testing.T) {
+	for _, goos := range []string{"linux", "windows", "darwin", "freebsd", "openbsd"} {
+		for _, ipv6 := range []bool{false, true} {
+			t.Run(fmt.Sprintf("%s/ipv6=%t", goos, ipv6), func(t *testing.T) {
+				target, family := "192.0.2.1", "-4"
+				if ipv6 {
+					target, family = "2001:db8::1", "-6"
+				}
+				name, args, err := pingCommand(goos, target, ipv6)
+				require.NoError(t, err)
+				wantName := "ping"
+				wantArgs := []string{"-n", "-c", "1", target}
+				switch goos {
+				case "windows":
+					wantArgs = []string{family, "-n", "1", "-w", "3000", target}
+				case "linux":
+					wantArgs = append([]string{family}, wantArgs...)
+				default:
+					if ipv6 {
+						wantName = "ping6"
+					}
+				}
+				assert.Equal(t, wantName, name)
+				assert.Equal(t, wantArgs, args)
+			})
+		}
+	}
+	_, _, err := pingCommand("unsupported", "192.0.2.1", false)
+	require.Error(t, err)
+}
+
+func TestParsePingResponse(t *testing.T) {
+	for _, tc := range []struct {
+		name   string
+		output string
+		wantUs int64
+	}{
+		{"linux", "64 bytes from 192.0.2.1: icmp_seq=1 ttl=64 time=12.345 ms", 12345},
+		{"bsd", "64 bytes from 192.0.2.1: icmp_seq=0 ttl=64 time=0.023 ms", 23},
+		{"ipv6", "64 bytes from 2001:db8::1: icmp_seq=0 hlim=64 time=1.234 ms", 1234},
+		{"windows", "Reply from 192.0.2.1: bytes=32 time=12ms TTL=128", 12000},
+		{"windows submillisecond", "Reply from ::1: time<1ms", 1000},
+		{"localized windows", "Antwort von 192.0.2.1: Bytes=32 Zeit=12ms TTL=128", 12000},
+		{"decimal comma", "64 bytes from 192.0.2.1: time=1,234 ms", 1234},
+		{"rounding", "time=0.1236 ms", 124},
+		{"empty", "", -1},
+		{"timeout", "Request timed out.", -1},
+		{"unreachable", "Reply from 192.0.2.1: Destination host unreachable.", -1},
+		{"malformed", "time=oops ms", -1},
+		{"negative", "time=-1 ms", -1},
+		{"overflow", "time=999999999999999999999 ms", -1},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			responseUs, err := parsePingResponse([]byte(tc.output))
+			if tc.wantUs < 0 {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+			assert.Equal(t, tc.wantUs, responseUs)
+		})
+	}
+}
+
+func TestMonitorICMPExecOutput(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("test uses a POSIX shell stub for ping")
+	}
+	for _, tc := range []struct {
+		name   string
+		output string
+		exit   int
+		wantUs int64
+	}{
+		{"success", "time=1.234 ms", 0, 1234},
+		{"missing RTT", "unrecognized output", 0, -1},
+		{"failed command with RTT", "time=1.234 ms", 1, -1},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			dir := t.TempDir()
+			// Also verify an inherited locale cannot override the C locale.
+			script := fmt.Sprintf("#!/bin/sh\n[ \"$LC_ALL\" = C ] || exit 2\nprintf '%%s\\n' '%s'\nexit %d\n", tc.output, tc.exit)
+			require.NoError(t, os.WriteFile(filepath.Join(dir, "ping"), []byte(script), 0o755))
+			t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
+			t.Setenv("LC_ALL", "de_DE.UTF-8")
+			responseUs, err := monitorICMPExec(t.Context(), "127.0.0.1", false)
+			if tc.wantUs < 0 {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+			assert.Equal(t, tc.wantUs, responseUs)
+		})
+	}
+}
+
+type icmpTestReply struct {
+	data []byte
+	peer net.Addr
+}
+
+type scriptedICMPConn struct {
+	net.PacketConn
+	local        net.Addr
+	onWrite      func([]byte, net.Addr)
+	replies      []icmpTestReply
+	reads        int
+	deadlineSets int
+}
+
+func (c *scriptedICMPConn) LocalAddr() net.Addr { return c.local }
+
+func (c *scriptedICMPConn) SetDeadline(deadline time.Time) error {
+	c.deadlineSets++
+	return nil
+}
+
+func (c *scriptedICMPConn) WriteTo(data []byte, dst net.Addr) (int, error) {
+	c.onWrite(data, dst)
+	return len(data), nil
+}
+
+func (c *scriptedICMPConn) ReadFrom(buf []byte) (int, net.Addr, error) {
+	c.reads++
+	if len(c.replies) == 0 {
+		return 0, nil, os.ErrDeadlineExceeded
+	}
+	reply := c.replies[0]
+	c.replies = c.replies[1:]
+	return copy(buf, reply.data), reply.peer, nil
+}
+
+func TestMonitorICMPReplyCorrelation(t *testing.T) {
+	for _, family := range []*icmpFamily{&icmpV4, &icmpV6} {
+		for _, datagram := range []bool{false, true} {
+			network := family.rawNetwork
+			ip, other := net.ParseIP("192.0.2.1"), net.ParseIP("192.0.2.2")
+			if family.isIPv6 {
+				ip, other = net.ParseIP("2001:db8::1"), net.ParseIP("2001:db8::2")
+			}
+			var dst net.Addr = &net.IPAddr{IP: ip}
+			var wrongPeer net.Addr = &net.IPAddr{IP: other}
+			if datagram {
+				network = family.dgramNetwork
+				dst = &net.UDPAddr{IP: ip}
+				wrongPeer = &net.UDPAddr{IP: other}
+			}
+			for _, mismatch := range []string{"source", "id", "sequence", "payload", "type", "code", "malformed"} {
+				for _, eventuallyMatches := range []bool{false, true} {
+					ending := "timeout"
+					if eventuallyMatches {
+						ending = "success"
+					}
+					t.Run(network+"/"+mismatch+"/"+ending, func(t *testing.T) {
+						conn := &scriptedICMPConn{local: &net.IPAddr{IP: net.IPv4zero}}
+						if datagram {
+							conn.local = &net.UDPAddr{Port: 12345}
+							if runtime.GOOS == "linux" {
+								// Deliberately differ from the process ID.
+								conn.local = &net.UDPAddr{Port: (os.Getpid() % 65534) + 1}
+							}
+						}
+						conn.onWrite = func(data []byte, target net.Addr) {
+							require.Equal(t, dst, target)
+							request, err := icmp.ParseMessage(family.proto, data)
+							require.NoError(t, err)
+							echo := request.Body.(*icmp.Echo)
+							expectedID := os.Getpid() & 0xffff
+							if datagram && runtime.GOOS == "linux" {
+								expectedID = conn.local.(*net.UDPAddr).Port
+							}
+							require.Equal(t, expectedID, echo.ID)
+							reply := &icmp.Message{Type: family.replyType, Body: echo}
+							valid, err := reply.Marshal(nil)
+							require.NoError(t, err)
+							peer := dst
+							switch mismatch {
+							case "source":
+								peer = wrongPeer
+							case "id":
+								echo.ID ^= 1
+							case "sequence":
+								echo.Seq ^= 1
+							case "payload":
+								echo.Data[0] ^= 1
+							case "type":
+								reply.Type = family.echoType
+							case "code":
+								reply.Code = 1
+							}
+							invalid, err := reply.Marshal(nil)
+							require.NoError(t, err)
+							if mismatch == "malformed" {
+								invalid = invalid[:2]
+							}
+							conn.replies = []icmpTestReply{{invalid, peer}}
+							if eventuallyMatches {
+								conn.replies = append(conn.replies, icmpTestReply{valid, dst})
+							}
+						}
+						elapsed, err := monitorICMPPacket(context.Background(), conn, family, dst)
+						if eventuallyMatches {
+							require.NoError(t, err)
+							assert.GreaterOrEqual(t, elapsed, int64(0))
+						} else {
+							require.ErrorIs(t, err, os.ErrDeadlineExceeded)
+							assert.Equal(t, int64(-1), elapsed)
+						}
+						assert.Equal(t, 2, conn.reads)
+						assert.Equal(t, 1, conn.deadlineSets)
+					})
+				}
+			}
+		}
+	}
+}
+
+func TestMonitorICMPLoopback(t *testing.T) {
+	for _, family := range []*icmpFamily{&icmpV4, &icmpV6} {
+		for _, network := range []string{family.rawNetwork, family.dgramNetwork} {
+			t.Run(network, func(t *testing.T) {
+				conn, err := icmp.ListenPacket(network, family.listenAddr)
+				if err != nil {
+					t.Skipf("ICMP socket unavailable: %v", err)
+				}
+				defer conn.Close()
+				ip := net.ParseIP("127.0.0.1")
+				if family.isIPv6 {
+					ip = net.ParseIP("::1")
+				}
+				var dst net.Addr = &net.IPAddr{IP: ip}
+				if network == family.dgramNetwork {
+					dst = &net.UDPAddr{IP: ip}
+				}
+				elapsed, err := monitorICMPPacket(context.Background(), conn, family, dst)
+				require.NoError(t, err)
+				assert.GreaterOrEqual(t, elapsed, int64(0))
+			})
+		}
+	}
+}
+
+func TestDetectICMPMode(t *testing.T) {
+	tests := []struct {
+		name         string
+		family       *icmpFamily
+		rawErr       error
+		udpErr       error
+		want         icmpMethod
+		wantNetworks []string
+	}{
+		{
+			name:         "IPv4 prefers raw socket when available",
+			family:       &icmpV4,
+			want:         icmpRaw,
+			wantNetworks: []string{"ip4:icmp"},
+		},
+		{
+			name:         "IPv4 uses datagram when raw unavailable",
+			family:       &icmpV4,
+			rawErr:       errors.New("operation not permitted"),
+			want:         icmpDatagram,
+			wantNetworks: []string{"ip4:icmp", "udp4"},
+		},
+		{
+			name:         "IPv4 falls back to exec when both unavailable",
+			family:       &icmpV4,
+			rawErr:       errors.New("operation not permitted"),
+			udpErr:       errors.New("protocol not supported"),
+			want:         icmpExecFallback,
+			wantNetworks: []string{"ip4:icmp", "udp4"},
+		},
+		{
+			name:         "IPv6 prefers raw socket when available",
+			family:       &icmpV6,
+			want:         icmpRaw,
+			wantNetworks: []string{"ip6:ipv6-icmp"},
+		},
+		{
+			name:         "IPv6 uses datagram when raw unavailable",
+			family:       &icmpV6,
+			rawErr:       errors.New("operation not permitted"),
+			want:         icmpDatagram,
+			wantNetworks: []string{"ip6:ipv6-icmp", "udp6"},
+		},
+		{
+			name:         "IPv6 falls back to exec when both unavailable",
+			family:       &icmpV6,
+			rawErr:       errors.New("operation not permitted"),
+			udpErr:       errors.New("protocol not supported"),
+			want:         icmpExecFallback,
+			wantNetworks: []string{"ip6:ipv6-icmp", "udp6"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			calls := make([]string, 0, 2)
+			listen := func(network, listenAddr string) (icmpPacketConn, error) {
+				require.Equal(t, tt.family.listenAddr, listenAddr)
+				calls = append(calls, network)
+				switch network {
+				case tt.family.rawNetwork:
+					if tt.rawErr != nil {
+						return nil, tt.rawErr
+					}
+				case tt.family.dgramNetwork:
+					if tt.udpErr != nil {
+						return nil, tt.udpErr
+					}
+				default:
+					t.Fatalf("unexpected network %q", network)
+				}
+				return testICMPPacketConn{}, nil
+			}
+
+			assert.Equal(t, tt.want, detectICMPMode(tt.family, listen))
+			assert.Equal(t, tt.wantNetworks, calls)
+		})
+	}
+}
+
+func TestResolveICMPTarget(t *testing.T) {
+	t.Run("IPv4 literal", func(t *testing.T) {
+		family, ip, err := resolveICMPTarget(context.Background(), "127.0.0.1")
+		require.NoError(t, err)
+		require.NotNil(t, family)
+		assert.False(t, family.isIPv6)
+		assert.Equal(t, "127.0.0.1", ip.String())
+	})
+
+	t.Run("IPv6 literal", func(t *testing.T) {
+		family, ip, err := resolveICMPTarget(context.Background(), "::1")
+		require.NoError(t, err)
+		require.NotNil(t, family)
+		assert.True(t, family.isIPv6)
+		assert.Equal(t, "::1", ip.String())
+	})
+
+	t.Run("IPv4-mapped IPv6 resolves as IPv4", func(t *testing.T) {
+		family, ip, err := resolveICMPTarget(context.Background(), "::ffff:127.0.0.1")
+		require.NoError(t, err)
+		require.NotNil(t, family)
+		assert.False(t, family.isIPv6)
+		assert.Equal(t, "127.0.0.1", ip.String())
+	})
+}

agent/network_monitor_probe.go

@@ -0,0 +1,105 @@
+package agent
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/henrygd/beszel/internal/entities/monitor"
+)
+
+// monitorProbe performs one check. Errors are recorded as loss by the task runner.
+// Implementations must honor cancellation and bound their execution time.
+type monitorProbe func(context.Context, monitor.Config) (int64, error)
+
+func networkMonitorProbe(client *http.Client) monitorProbe {
+	return func(ctx context.Context, config monitor.Config) (int64, error) {
+		switch config.Protocol {
+		case "icmp":
+			return monitorICMP(ctx, config.Target)
+		case "tcp":
+			return monitorTCP(ctx, config.Target, config.Port)
+		case "http":
+			return monitorHTTP(ctx, client, config.Target)
+		case "dns":
+			return monitorDNS(ctx, config.Target)
+		default:
+			return -1, fmt.Errorf("unknown monitor protocol: %s", config.Protocol)
+		}
+	}
+}
+
+// monitorTCP measures connection establishment time, including address fallback
+// but excluding DNS resolution.
+// Returns -1 and an error on failure.
+func monitorTCP(ctx context.Context, target string, port uint16) (int64, error) {
+	ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
+	defer cancel()
+
+	// Resolve DNS first, outside the timing window but within the probe deadline.
+	ips, err := net.DefaultResolver.LookupHost(ctx, target)
+	if err != nil {
+		return -1, err
+	}
+	if len(ips) == 0 {
+		return -1, errors.New("no addresses resolved for TCP monitor")
+	}
+	portString := fmt.Sprintf("%d", port)
+	deadline, _ := ctx.Deadline()
+
+	// Share the remaining probe budget across addresses so an unresponsive
+	// first address cannot consume all the time available for alternatives.
+	start := time.Now()
+	for i, ip := range ips {
+		if err := ctx.Err(); err != nil {
+			return -1, err
+		}
+		dialer := net.Dialer{Timeout: time.Until(deadline) / time.Duration(len(ips)-i)}
+		var conn net.Conn
+		conn, err = dialer.DialContext(ctx, "tcp", net.JoinHostPort(ip, portString))
+		if err != nil {
+			continue
+		}
+		responseUs := time.Since(start).Microseconds()
+		conn.Close()
+		return responseUs, nil
+	}
+	return -1, err
+}
+
+// monitorDNS measures DNS resolution response time in microseconds. Returns -1 and an error on failure.
+func monitorDNS(ctx context.Context, target string) (int64, error) {
+	ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
+	defer cancel()
+
+	start := time.Now()
+	ips, err := net.DefaultResolver.LookupHost(ctx, target)
+	if err != nil || len(ips) == 0 {
+		return -1, err
+	}
+	return time.Since(start).Microseconds(), nil
+}
+
+// monitorHTTP measures HTTP GET request response in microseconds. Returns -1 and an error on failure.
+func monitorHTTP(ctx context.Context, client *http.Client, url string) (int64, error) {
+	if client == nil {
+		client = http.DefaultClient
+	}
+	start := time.Now()
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return -1, err
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return -1, err
+	}
+	resp.Body.Close()
+	if resp.StatusCode >= 400 {
+		return -1, fmt.Errorf("HTTP error: %s", resp.Status)
+	}
+	return time.Since(start).Microseconds(), nil
+}

agent/network_monitor_resume.go

@@ -0,0 +1,88 @@
+package agent
+
+import (
+	"sync"
+	"time"
+)
+
+const (
+	monitorResumeHeartbeat = 10 * time.Second
+	// Allow scheduling jitter without mistaking an ordinary tick for resume.
+	monitorResumeGap   = 2 * monitorResumeHeartbeat
+	monitorResumePause = 10 * time.Second
+)
+
+// monitorResumeGuard detects likely suspend/resume using wall time. A long
+// process stall or forward clock adjustment can also trigger the bounded pause.
+// One heartbeat is shared by all configured monitors.
+type monitorResumeGuard struct {
+	mu         sync.Mutex
+	stop       chan struct{}
+	lastTick   time.Time
+	pauseUntil time.Time
+	generation uint32
+}
+
+func (g *monitorResumeGuard) start() {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	if g.stop != nil {
+		return
+	}
+	stop := make(chan struct{})
+	g.stop = stop
+	g.lastTick = time.Now().Round(0)
+	g.pauseUntil = time.Time{}
+	go func() {
+		ticker := time.NewTicker(monitorResumeHeartbeat)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-stop:
+				return
+			case <-ticker.C:
+				g.mu.Lock()
+				if g.stop == stop {
+					g.observe(time.Now())
+				}
+				g.mu.Unlock()
+			}
+		}
+	}()
+}
+
+func (g *monitorResumeGuard) shutdown() {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	if g.stop != nil {
+		close(g.stop)
+		g.stop = nil
+		g.generation++
+	}
+}
+
+// observe requires mu. Strip the monotonic component because it can stop during
+// suspend. Read the current time rather than the ticker's queued timestamp.
+func (g *monitorResumeGuard) observe(now time.Time) {
+	now = now.Round(0)
+	if now.Sub(g.lastTick) > monitorResumeGap {
+		g.pauseUntil = now.Add(monitorResumePause)
+		g.generation++
+	}
+	g.lastTick = now
+}
+
+// snapshot also observes time so a probe waking before the heartbeat detects
+// resume itself. A changed generation invalidates probes spanning suspend.
+func (g *monitorResumeGuard) snapshot() (generation uint32, allowed bool) {
+	if g == nil {
+		return 0, true
+	}
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	if g.stop == nil {
+		return g.generation, true
+	}
+	g.observe(time.Now())
+	return g.generation, !g.lastTick.Before(g.pauseUntil)
+}

agent/network_monitor_resume_test.go

@@ -0,0 +1,121 @@
+//go:build testing
+
+package agent
+
+import (
+	"context"
+	"errors"
+	"sync/atomic"
+	"testing"
+	"testing/synctest"
+	"time"
+
+	"github.com/henrygd/beszel/internal/entities/monitor"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func simulateMonitorSleep(g *monitorResumeGuard) {
+	g.mu.Lock()
+	g.lastTick = time.Now().Add(-time.Hour).Round(0)
+	g.mu.Unlock()
+}
+
+func TestMonitorResumePause(t *testing.T) {
+	synctest.Test(t, func(t *testing.T) {
+		var g monitorResumeGuard
+		g.start()
+		defer g.shutdown()
+		generation, allowed := g.snapshot()
+		require.True(t, allowed)
+		// Heartbeats alone must keep the guard current between infrequent probes.
+		time.Sleep(time.Minute)
+		synctest.Wait()
+		steadyGeneration, allowed := g.snapshot()
+		require.True(t, allowed)
+		require.Equal(t, generation, steadyGeneration)
+		// The probe, rather than the heartbeat, must detect this gap.
+		simulateMonitorSleep(&g)
+		next, allowed := g.snapshot()
+		assert.False(t, allowed)
+		assert.NotEqual(t, generation, next)
+		time.Sleep(9 * time.Second)
+		_, allowed = g.snapshot()
+		assert.False(t, allowed)
+		time.Sleep(time.Second)
+		end, allowed := g.snapshot()
+		assert.True(t, allowed)
+		assert.Equal(t, next, end)
+	})
+}
+
+func TestMonitorResumeGuardLifecycle(t *testing.T) {
+	synctest.Test(t, func(t *testing.T) {
+		pm := newMonitorManagerWithProbe(func(context.Context, monitor.Config) (int64, error) { return 1, nil })
+		defer pm.Stop()
+		assert.Nil(t, pm.resumeGuard.stop)
+		pm.SyncMonitors([]monitor.Config{{ID: "a", Interval: 3600}, {ID: "b", Interval: 3600}})
+		stop := pm.resumeGuard.stop
+		require.NotNil(t, stop)
+		pm.DeleteMonitor("a")
+		assert.Equal(t, stop, pm.resumeGuard.stop)
+		pm.DeleteMonitor("b")
+		assert.Nil(t, pm.resumeGuard.stop)
+		select {
+		case <-stop:
+		default:
+			t.Fatal("heartbeat was not stopped")
+		}
+		time.Sleep(time.Hour)
+		_, err := pm.UpsertMonitor(monitor.Config{ID: "c", Interval: 3600}, false)
+		require.NoError(t, err)
+		_, allowed := pm.resumeGuard.snapshot()
+		assert.True(t, allowed, "idle time must not trigger a resume pause")
+		pm.SyncMonitors(nil)
+		assert.Nil(t, pm.resumeGuard.stop)
+	})
+}
+
+func TestMonitorResumeDiscardsInflightProbe(t *testing.T) {
+	synctest.Test(t, func(t *testing.T) {
+		var g monitorResumeGuard
+		g.start()
+		defer g.shutdown()
+		task := newMonitorTask(monitor.Config{ID: "test"})
+		defer task.cancel()
+		task.resumeGuard = &g
+		result := task.runProbe(func(context.Context, monitor.Config) (int64, error) {
+			simulateMonitorSleep(&g)
+			return 0, errors.New("network not ready")
+		})
+		assert.Nil(t, result)
+		assert.Empty(t, task.history.samples)
+		// Explicit requests may still run during the pause and record real failures.
+		result = task.runProbe(func(context.Context, monitor.Config) (int64, error) {
+			return 0, errors.New("unreachable")
+		})
+		require.NotNil(t, result)
+		assert.Equal(t, 100.0, result.PacketLoss)
+	})
+}
+
+func TestMonitorResumeSkipsScheduledProbes(t *testing.T) {
+	synctest.Test(t, func(t *testing.T) {
+		var calls atomic.Int32
+		pm := newMonitorManagerWithProbe(func(context.Context, monitor.Config) (int64, error) {
+			calls.Add(1)
+			return 1, nil
+		})
+		defer pm.Stop()
+		pm.SyncMonitors([]monitor.Config{{ID: "test", Interval: 1}})
+		simulateMonitorSleep(&pm.resumeGuard)
+		pm.resumeGuard.snapshot()
+		time.Sleep(9 * time.Second)
+		synctest.Wait()
+		assert.Zero(t, calls.Load())
+		assert.Empty(t, pm.GetResults(1000))
+		time.Sleep(2 * time.Second)
+		synctest.Wait()
+		assert.Positive(t, calls.Load())
+	})
+}

agent/network_monitor_schedule.go

@@ -0,0 +1,60 @@
+package agent
+
+import (
+	"context"
+	"log/slog"
+	"math/rand"
+	"time"
+)
+
+func (pm *MonitorManager) startMonitor(task *monitorTask) {
+	interval := time.Duration(task.config.Interval) * time.Second
+	if interval < time.Second {
+		interval = 30 * time.Second
+	}
+	delay := getStagger(interval.Milliseconds())
+	slog.Debug("starting monitor task", "target", task.config.Target, "delay", delay, "interval", interval)
+	go runMonitorSchedule(task.ctx, interval, delay, func() {
+		if _, allowed := task.resumeGuard.snapshot(); allowed {
+			task.runProbe(pm.probe)
+		}
+	})
+}
+
+// runMonitorSchedule owns only timing. Checks run serially, and slow checks
+// naturally drop missed ticks rather than building an execution backlog.
+func runMonitorSchedule(ctx context.Context, interval, delay time.Duration, run func()) {
+	timer := time.NewTimer(delay)
+	defer timer.Stop()
+	select {
+	case <-ctx.Done():
+		return
+	case <-timer.C:
+	}
+	if ctx.Err() != nil {
+		return
+	}
+	run()
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			if ctx.Err() != nil {
+				return
+			}
+			run()
+		}
+	}
+}
+
+// getStagger returns an initial delay between half an interval and one interval.
+func getStagger(intervalMilli int64) time.Duration {
+	delay := rand.Intn(int(intervalMilli))
+	if delay < int(intervalMilli)/2 {
+		delay += int(intervalMilli) / 2
+	}
+	return time.Duration(delay) * time.Millisecond
+}

agent/network_monitor_schedule_test.go

@@ -0,0 +1,167 @@
+//go:build testing
+
+package agent
+
+import (
+	"context"
+	"sync/atomic"
+	"testing"
+	"testing/synctest"
+	"time"
+
+	"github.com/henrygd/beszel/internal/entities/monitor"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMonitorScheduleTiming(t *testing.T) {
+	synctest.Test(t, func(t *testing.T) {
+		ctx, cancel := context.WithCancel(t.Context())
+		defer cancel()
+		var calls atomic.Int32
+		go runMonitorSchedule(ctx, 10*time.Second, 5*time.Second, func() { calls.Add(1) })
+		synctest.Wait()
+		time.Sleep(4 * time.Second)
+		synctest.Wait()
+		assert.Equal(t, 0, int(calls.Load()))
+		time.Sleep(time.Second)
+		synctest.Wait()
+		assert.Equal(t, 1, int(calls.Load()))
+		time.Sleep(10 * time.Second)
+		synctest.Wait()
+		assert.Equal(t, 2, int(calls.Load()))
+		cancel()
+		synctest.Wait()
+		time.Sleep(time.Minute)
+		synctest.Wait()
+		assert.Equal(t, 2, int(calls.Load()))
+	})
+}
+
+func TestMonitorScheduleSlowProbe(t *testing.T) {
+	synctest.Test(t, func(t *testing.T) {
+		ctx, cancel := context.WithCancel(t.Context())
+		defer cancel()
+		var calls atomic.Int32
+		release := make(chan struct{})
+		go runMonitorSchedule(ctx, time.Second, 0, func() {
+			calls.Add(1)
+			select {
+			case <-release:
+			case <-ctx.Done():
+			}
+		})
+		synctest.Wait()
+		assert.Equal(t, 1, int(calls.Load()))
+		time.Sleep(time.Minute)
+		synctest.Wait()
+		assert.Equal(t, 1, int(calls.Load()), "a slow probe must not spawn overlapping checks")
+		close(release)
+		synctest.Wait()
+		assert.Equal(t, 1, int(calls.Load()), "missed intervals must not accumulate a backlog")
+		time.Sleep(time.Second)
+		synctest.Wait()
+		assert.Equal(t, 2, int(calls.Load()))
+		cancel()
+		synctest.Wait()
+	})
+}
+
+func TestMonitorScheduledAndImmediateRequestsShareProbe(t *testing.T) {
+	synctest.Test(t, func(t *testing.T) {
+		var calls atomic.Int32
+		release := make(chan struct{})
+		cfg := monitor.Config{ID: "test", Interval: 10}
+		pm := newMonitorManagerWithProbe(func(ctx context.Context, config monitor.Config) (int64, error) {
+			assert.Equal(t, cfg, config)
+			calls.Add(1)
+			<-release
+			return 42, nil
+		})
+		defer pm.Stop()
+		task := newMonitorTask(cfg)
+		pm.monitors[cfg.ID] = task
+		go runMonitorSchedule(task.ctx, 10*time.Second, 0, func() { task.runProbe(pm.probe) })
+		synctest.Wait()
+		results := make(chan *monitor.Result, 2)
+		for range 2 {
+			go func() {
+				result, _ := pm.UpsertMonitor(cfg, true)
+				results <- result
+			}()
+		}
+		synctest.Wait()
+		assert.Equal(t, 1, int(calls.Load()))
+		assert.Empty(t, pm.GetResults(1000), "reading history must not wait for network I/O")
+		close(release)
+		synctest.Wait()
+		first, second := <-results, <-results
+		require.NotNil(t, first)
+		require.NotNil(t, second)
+		assert.Equal(t, int64(42), first.AvgResponse)
+		assert.Equal(t, first, second)
+		assert.NotSame(t, first, second, "callers must not share mutable result pointers")
+		assert.Len(t, task.history.samples, 1)
+		// A later explicit request must still perform a fresh probe.
+		_, err := pm.UpsertMonitor(cfg, true)
+		require.NoError(t, err)
+		assert.Equal(t, 2, int(calls.Load()))
+		assert.Len(t, task.history.samples, 2)
+	})
+}
+
+func TestMonitorReplacementCancelsSharedProbe(t *testing.T) {
+	synctest.Test(t, func(t *testing.T) {
+		cfg := monitor.Config{ID: "test", Interval: 10}
+		pm := newMonitorManagerWithProbe(func(ctx context.Context, config monitor.Config) (int64, error) {
+			if config.Interval == 10 {
+				<-ctx.Done()
+				return 0, ctx.Err()
+			}
+			return 30, nil
+		})
+		defer pm.Stop()
+		task := newMonitorTask(cfg)
+		task.history.record(monitorSample{responseUs: 10, timestamp: time.Now()})
+		pm.monitors[cfg.ID] = task
+		results := make(chan *monitor.Result, 2)
+		for range 2 {
+			go func() {
+				result, _ := pm.UpsertMonitor(cfg, true)
+				results <- result
+			}()
+		}
+		synctest.Wait()
+		updated := cfg
+		updated.Interval = 20
+		result, err := pm.UpsertMonitor(updated, true)
+		require.NoError(t, err)
+		require.NotNil(t, result)
+		assert.Equal(t, int64(20), result.AvgResponse)
+		assert.Zero(t, result.PacketLoss)
+		synctest.Wait()
+		assert.Nil(t, <-results)
+		assert.Nil(t, <-results)
+		assert.Len(t, task.history.samples, 1)
+		assert.Len(t, pm.monitors[cfg.ID].history.samples, 2)
+	})
+}
+
+func TestMonitorInjectedProbeTimeoutRecordsLoss(t *testing.T) {
+	synctest.Test(t, func(t *testing.T) {
+		pm := newMonitorManagerWithProbe(func(ctx context.Context, _ monitor.Config) (int64, error) {
+			ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
+			defer cancel()
+			<-ctx.Done()
+			return 0, ctx.Err()
+		})
+		defer pm.Stop()
+		start := time.Now()
+		result, err := pm.UpsertMonitor(monitor.Config{ID: "test", Interval: 3600}, true)
+		require.NoError(t, err)
+		require.NotNil(t, result)
+		assert.Equal(t, 3*time.Second, time.Since(start))
+		assert.Equal(t, 100.0, result.PacketLoss)
+		assert.NoError(t, pm.monitors["test"].ctx.Err())
+	})
+}

agent/network_monitor_task.go

@@ -0,0 +1,116 @@
+package agent
+
+import (
+	"context"
+	"log/slog"
+	"sync"
+	"time"
+
+	"github.com/henrygd/beszel/internal/entities/monitor"
+)
+
+const monitorFailureLogInterval = 5 * time.Minute
+
+// monitorTask coordinates a probe and its history for one immutable configuration.
+type monitorTask struct {
+	config         monitor.Config
+	ctx            context.Context
+	cancel         context.CancelFunc
+	history        *monitorHistory
+	resumeGuard    *monitorResumeGuard
+	runMu          sync.Mutex
+	inflight       *monitorRun
+	lastFailureLog int64 // Unix nanoseconds
+}
+
+type monitorRun struct {
+	done   chan struct{}
+	result *monitor.Result // published by closing done; never mutated afterwards
+}
+
+func newMonitorTask(config monitor.Config) *monitorTask {
+	ctx, cancel := context.WithCancel(context.Background())
+	task := &monitorTask{config: config, ctx: ctx, history: newMonitorHistory()}
+	// Serialize cancellation with publication, so canceled probes cannot enter
+	// history copied into a replacement task.
+	task.cancel = func() {
+		task.runMu.Lock()
+		cancel()
+		task.runMu.Unlock()
+	}
+	return task
+}
+
+func newMonitorTaskFromExisting(config monitor.Config, existing *monitorTask) *monitorTask {
+	task := newMonitorTask(config)
+	if existing != nil {
+		task.history = existing.history.clone()
+	}
+	return task
+}
+
+// runProbe shares an in-flight check between scheduled and immediate requests.
+// Every completed check contributes exactly one sample, regardless of how many
+// callers were waiting for it. No task or history lock is held during network I/O.
+func (task *monitorTask) runProbe(probe monitorProbe) *monitor.Result {
+	task.runMu.Lock()
+	if task.ctx.Err() != nil {
+		task.runMu.Unlock()
+		return nil
+	}
+	if run := task.inflight; run != nil {
+		task.runMu.Unlock()
+		select {
+		case <-task.ctx.Done():
+			return nil
+		case <-run.done:
+			if task.ctx.Err() != nil {
+				return nil
+			}
+			return copyMonitorResult(run.result)
+		}
+	}
+	run := &monitorRun{done: make(chan struct{})}
+	task.inflight = run
+	task.runMu.Unlock()
+
+	generation, _ := task.resumeGuard.snapshot()
+	responseUs, err := probe(task.ctx, task.config)
+	var logFailure bool
+	task.runMu.Lock()
+	currentGeneration, _ := task.resumeGuard.snapshot()
+	if task.ctx.Err() == nil && generation == currentGeneration {
+		now := time.Now()
+		if err != nil {
+			responseUs = -1
+			logAt := now.UnixNano()
+			if task.lastFailureLog == 0 || logAt < task.lastFailureLog || logAt-task.lastFailureLog >= int64(monitorFailureLogInterval) {
+				logFailure = true
+				task.lastFailureLog = logAt
+			}
+		} else {
+			task.lastFailureLog = 0
+		}
+		result := task.history.record(monitorSample{responseUs: responseUs, timestamp: now})
+		run.result = &result
+	}
+
+	task.inflight = nil
+	close(run.done)
+	task.runMu.Unlock()
+	if logFailure {
+		slog.Warn("monitor failed", "err", err, "target", task.config.Target, "protocol", task.config.Protocol)
+	}
+	if task.ctx.Err() != nil {
+		return nil
+	}
+	return copyMonitorResult(run.result)
+}
+
+func copyMonitorResult(result *monitor.Result) *monitor.Result {
+	if result == nil {
+		return nil
+	}
+	copy := *result
+	return &copy
+}

agent/network_monitor_task_test.go

@@ -0,0 +1,79 @@
+//go:build testing
+
+package agent
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"log/slog"
+	"testing"
+	"testing/synctest"
+	"time"
+
+	"github.com/henrygd/beszel/internal/entities/monitor"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMonitorFailureLogCooldown(t *testing.T) {
+	var logs bytes.Buffer
+	previous := slog.Default()
+	slog.SetDefault(slog.New(slog.NewTextHandler(&logs, nil)))
+	t.Cleanup(func() { slog.SetDefault(previous) })
+
+	synctest.Test(t, func(t *testing.T) {
+		task := newMonitorTask(monitor.Config{ID: "test", Target: "example.test", Protocol: "tcp"})
+		defer task.cancel()
+		failure := errors.New("connection refused")
+		probe := func(context.Context, monitor.Config) (int64, error) { return 42, failure }
+		var samples int64
+		check := func(wantLog bool) {
+			t.Helper()
+			logs.Reset()
+			result := task.runProbe(probe)
+			require.NotNil(t, result)
+			samples++
+			assert.Equal(t, samples, result.SampleCount, "suppressed warnings must still record samples")
+			if !wantLog {
+				assert.Empty(t, logs.String())
+			} else {
+				assert.Contains(t, logs.String(), `msg="monitor failed"`)
+				assert.Equal(t, 1, bytes.Count(logs.Bytes(), []byte("\n")))
+			}
+		}
+
+		check(true)
+		check(false)
+		time.Sleep(5*time.Minute - time.Nanosecond)
+		check(false)
+		time.Sleep(time.Nanosecond)
+		check(true)
+		check(false)
+		time.Sleep(5 * time.Minute)
+		check(true)
+		check(false)
+
+		// Recovery clears the cooldown.
+		failure = nil
+		check(false)
+		failure = errors.New("connection refused again")
+		check(true)
+
+		// Another monitor has its own cooldown.
+		other := newMonitorTask(task.config)
+		defer other.cancel()
+		logs.Reset()
+		require.NotNil(t, other.runProbe(probe))
+		assert.Contains(t, logs.String(), `msg="monitor failed"`)
+
+		// A canceled probe must not publish a failure or emit a warning.
+		logs.Reset()
+		result := other.runProbe(func(context.Context, monitor.Config) (int64, error) {
+			other.cancel()
+			return -1, context.Canceled
+		})
+		assert.Nil(t, result)
+		assert.Empty(t, logs.String())
+	})
+}

agent/network_monitor_test.go

@@ -0,0 +1,524 @@
+package agent
+
+import (
+	"context"
+	"encoding/binary"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/henrygd/beszel/internal/entities/monitor"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/dns/dnsmessage"
+)
+
+func TestMonitorManagerGetResultsIncludesHourResponseRange(t *testing.T) {
+	now := time.Now().UTC()
+	task := newMonitorTask(monitor.Config{ID: "monitor-1"})
+	task.history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-30 * time.Minute)})
+	task.history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now.Add(-9 * time.Minute)})
+	task.history.addSampleLocked(monitorSample{responseUs: 40, timestamp: now.Add(-5 * time.Minute)})
+	task.history.addSampleLocked(monitorSample{responseUs: 30, timestamp: now.Add(-50 * time.Second)})
+	task.history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-30 * time.Second)})
+
+	pm := newMonitorManager()
+	pm.monitors = map[string]*monitorTask{"icmp:example.com": task}
+
+	results := pm.GetResults(uint16(time.Minute / time.Millisecond))
+	result, ok := results["monitor-1"]
+	require.True(t, ok)
+	assert.Equal(t, int64(30), result.AvgResponse)
+	assert.Equal(t, int64(25), result.AvgResponse1h)
+	assert.Equal(t, int64(30), result.MinResponse)
+	assert.Equal(t, int64(10), result.MinResponse1h)
+	assert.Equal(t, int64(30), result.MaxResponse)
+	assert.Equal(t, int64(40), result.MaxResponse1h)
+	assert.Equal(t, 50.0, result.PacketLoss)
+	assert.Equal(t, 20.0, result.PacketLoss1h)
+}
+
+func TestMonitorManagerGetResultsIncludesLossOnlyHourData(t *testing.T) {
+	now := time.Now().UTC()
+	task := newMonitorTask(monitor.Config{ID: "monitor-1"})
+	task.history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-30 * time.Second)})
+	task.history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-10 * time.Second)})
+
+	pm := newMonitorManager()
+	pm.monitors = map[string]*monitorTask{"icmp:example.com": task}
+
+	results := pm.GetResults(uint16(time.Minute / time.Millisecond))
+	result, ok := results["monitor-1"]
+	require.True(t, ok)
+	assert.Equal(t, int64(0), result.AvgResponse)
+	assert.Equal(t, int64(0), result.AvgResponse1h)
+	assert.Equal(t, int64(0), result.MinResponse)
+	assert.Equal(t, int64(0), result.MinResponse1h)
+	assert.Equal(t, int64(0), result.MaxResponse)
+	assert.Equal(t, int64(0), result.MaxResponse1h)
+	assert.Equal(t, 100.0, result.PacketLoss)
+	assert.Equal(t, 100.0, result.PacketLoss1h)
+}
+
+func TestMonitorConfigResultKeyUsesSyncedID(t *testing.T) {
+	cfg := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 10}
+	assert.Equal(t, "monitor-1", cfg.ID)
+}
+
+func TestMonitorManagerSyncMonitorsSkipsConfigsWithoutStableID(t *testing.T) {
+	validCfg := monitor.Config{ID: "monitor-1", Target: "ignored", Protocol: "noop", Interval: 10}
+	invalidCfg := monitor.Config{Target: "ignored", Protocol: "noop", Interval: 10}
+
+	pm := newMonitorManager()
+	pm.SyncMonitors([]monitor.Config{validCfg, invalidCfg})
+	defer pm.Stop()
+
+	_, validExists := pm.monitors[validCfg.ID]
+	_, invalidExists := pm.monitors[invalidCfg.ID]
+	assert.True(t, validExists)
+	assert.False(t, invalidExists)
+}
+
+func TestMonitorManagerSyncMonitorsStopsRemovedTasksButKeepsExisting(t *testing.T) {
+	keepCfg := monitor.Config{ID: "monitor-1", Target: "ignored", Protocol: "noop", Interval: 10}
+	removeCfg := monitor.Config{ID: "monitor-2", Target: "ignored", Protocol: "noop", Interval: 10}
+
+	keptTask := newMonitorTask(keepCfg)
+	removedTask := newMonitorTask(removeCfg)
+	pm := newMonitorManager()
+	pm.monitors = map[string]*monitorTask{
+		keepCfg.ID:   keptTask,
+		removeCfg.ID: removedTask,
+	}
+
+	pm.SyncMonitors([]monitor.Config{keepCfg})
+
+	assert.Same(t, keptTask, pm.monitors[keepCfg.ID])
+	_, exists := pm.monitors[removeCfg.ID]
+	assert.False(t, exists)
+
+	select {
+	case <-removedTask.ctx.Done():
+	default:
+		t.Fatal("expected removed monitor task to be cancelled")
+	}
+
+	select {
+	case <-keptTask.ctx.Done():
+		t.Fatal("expected existing monitor task to remain active")
+	default:
+	}
+}
+
+func TestMonitorManagerSyncMonitorsRestartsChangedConfig(t *testing.T) {
+	originalCfg := monitor.Config{ID: "monitor-1", Target: "ignored-a", Protocol: "noop", Interval: 10}
+	updatedCfg := monitor.Config{ID: "monitor-1", Target: "ignored-b", Protocol: "noop", Interval: 10}
+	originalTask := newMonitorTask(originalCfg)
+	pm := newMonitorManager()
+	pm.monitors = map[string]*monitorTask{
+		originalCfg.ID: originalTask,
+	}
+
+	pm.SyncMonitors([]monitor.Config{updatedCfg})
+	defer pm.Stop()
+
+	restartedTask := pm.monitors[updatedCfg.ID]
+	assert.NotSame(t, originalTask, restartedTask)
+	assert.Equal(t, updatedCfg, restartedTask.config)
+
+	select {
+	case <-originalTask.ctx.Done():
+	default:
+		t.Fatal("expected changed monitor task to be cancelled")
+	}
+}
+
+func TestMonitorManagerApplySyncUpsertRunsImmediatelyAndReturnsResult(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusNoContent)
+	}))
+	defer server.Close()
+
+	pm := &MonitorManager{
+		monitors: make(map[string]*monitorTask),
+		probe:    networkMonitorProbe(server.Client()),
+	}
+
+	resp, err := pm.HandleSyncRequest(monitor.SyncRequest{
+		Action: monitor.SyncActionUpsert,
+		Config: monitor.Config{ID: "monitor-1", Target: server.URL, Protocol: "http", Interval: 10},
+		RunNow: true,
+	})
+	defer pm.Stop()
+
+	require.NoError(t, err)
+	assert.GreaterOrEqual(t, resp.Result.AvgResponse, int64(0))
+	assert.Equal(t, 0.0, resp.Result.PacketLoss)
+	assert.Equal(t, 0.0, resp.Result.PacketLoss1h)
+
+	task := pm.monitors["monitor-1"]
+	require.NotNil(t, task)
+	task.history.mu.Lock()
+	defer task.history.mu.Unlock()
+	require.Len(t, task.history.samples, 1)
+}
+
+func TestMonitorManagerUpsertMonitorKeepsHistoryWhenOnlyIntervalChanges(t *testing.T) {
+	originalCfg := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 10}
+	updatedCfg := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 30}
+	now := time.Now().UTC()
+
+	existingTask := newMonitorTask(originalCfg)
+	existingTask.history.addSampleLocked(monitorSample{responseUs: 12, timestamp: now.Add(-50 * time.Minute)})
+	existingTask.history.addSampleLocked(monitorSample{responseUs: 24, timestamp: now.Add(-30 * time.Second)})
+
+	pm := newMonitorManager()
+	pm.monitors = map[string]*monitorTask{originalCfg.ID: existingTask}
+
+	result, err := pm.UpsertMonitor(updatedCfg, false)
+	defer pm.Stop()
+
+	require.NoError(t, err)
+	assert.Nil(t, result)
+
+	updatedTask := pm.monitors[updatedCfg.ID]
+	require.NotNil(t, updatedTask)
+	assert.NotSame(t, existingTask, updatedTask)
+	assert.Equal(t, updatedCfg, updatedTask.config)
+
+	updatedTask.history.mu.Lock()
+	defer updatedTask.history.mu.Unlock()
+	require.Len(t, updatedTask.history.samples, 1)
+	assert.Equal(t, int64(24), updatedTask.history.samples[0].responseUs)
+
+	agg := updatedTask.history.aggregateLocked(time.Hour, now)
+	require.True(t, agg.hasData())
+	assert.Equal(t, int64(2), agg.totalCount)
+	assert.Equal(t, int64(2), agg.successCount)
+	assert.Equal(t, int64(18), agg.avgResponse())
+
+	select {
+	case <-existingTask.ctx.Done():
+	default:
+		t.Fatal("expected original monitor task to be cancelled")
+	}
+}
+
+func TestMonitorManagerApplySyncDeleteRemovesTask(t *testing.T) {
+	config := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 10}
+	task := newMonitorTask(config)
+	pm := newMonitorManager()
+	pm.monitors = map[string]*monitorTask{config.ID: task}
+
+	_, err := pm.HandleSyncRequest(monitor.SyncRequest{
+		Action: monitor.SyncActionDelete,
+		Config: monitor.Config{ID: config.ID},
+	})
+
+	require.NoError(t, err)
+	_, exists := pm.monitors[config.ID]
+	assert.False(t, exists)
+
+	select {
+	case <-task.ctx.Done():
+	default:
+		t.Fatal("expected deleted monitor task to be cancelled")
+	}
+}
+
+func TestMonitorManagerGetRandomDelay(t *testing.T) {
+	for i := 1000; i < 360_000; i += 1000 {
+		delay := getStagger(int64(i))
+		assert.GreaterOrEqual(t, delay, time.Duration(i/2)*time.Millisecond)
+		assert.LessOrEqual(t, delay, time.Duration(i)*time.Millisecond)
+	}
+}
+
+func TestMonitorHTTP(t *testing.T) {
+	t.Run("success", func(t *testing.T) {
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(http.StatusNoContent)
+		}))
+		defer server.Close()
+
+		responseUs, err := monitorHTTP(context.Background(), server.Client(), server.URL)
+		require.NoError(t, err)
+		assert.GreaterOrEqual(t, responseUs, int64(0))
+	})
+
+	t.Run("server error", func(t *testing.T) {
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			http.Error(w, "boom", http.StatusInternalServerError)
+		}))
+		defer server.Close()
+
+		responseUs, err := monitorHTTP(context.Background(), server.Client(), server.URL)
+		assert.Equal(t, int64(-1), responseUs)
+		require.Error(t, err)
+	})
+}
+
+func TestMonitorTCP(t *testing.T) {
+	t.Run("success", func(t *testing.T) {
+		listener, err := net.Listen("tcp", "127.0.0.1:0")
+		require.NoError(t, err)
+		defer listener.Close()
+
+		accepted := make(chan struct{})
+		go func() {
+			defer close(accepted)
+			conn, err := listener.Accept()
+			if err == nil {
+				_ = conn.Close()
+			}
+		}()
+
+		port := uint16(listener.Addr().(*net.TCPAddr).Port)
+		responseUs, err := monitorTCP(context.Background(), "127.0.0.1", port)
+		require.NoError(t, err)
+		assert.GreaterOrEqual(t, responseUs, int64(0))
+		<-accepted
+	})
+
+	t.Run("connection failure", func(t *testing.T) {
+		listener, err := net.Listen("tcp", "127.0.0.1:0")
+		require.NoError(t, err)
+
+		port := uint16(listener.Addr().(*net.TCPAddr).Port)
+		require.NoError(t, listener.Close())
+
+		responseUs, err := monitorTCP(context.Background(), "127.0.0.1", port)
+		assert.Equal(t, int64(-1), responseUs)
+		require.Error(t, err)
+	})
+}
+
+func TestMonitorTCPAddressFallback(t *testing.T) {
+	for _, tc := range []struct {
+		name string
+		ips  []string
+		loss bool
+	}{
+		{"first address fails", []string{"127.0.0.2", "127.0.0.1"}, false},
+		{"first address succeeds", []string{"127.0.0.1", "127.0.0.2"}, false},
+		{"all addresses fail", []string{"127.0.0.2", "127.0.0.3"}, true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			listener, err := net.Listen("tcp4", "127.0.0.1:0")
+			require.NoError(t, err)
+			defer listener.Close()
+			original := net.DefaultResolver
+			net.DefaultResolver = tcpMonitorTestResolver(tc.ips)
+			defer func() { net.DefaultResolver = original }()
+
+			// Verify the resolver preserves the intended order, so success cannot
+			// accidentally bypass the failed first address in the regression case.
+			ips, err := net.DefaultResolver.LookupHost(t.Context(), "tcp-monitor.invalid.")
+			require.NoError(t, err)
+			require.Equal(t, tc.ips, ips)
+			responseUs, err := monitorTCP(t.Context(), "tcp-monitor.invalid.", uint16(listener.Addr().(*net.TCPAddr).Port))
+			if tc.loss {
+				require.Error(t, err)
+				assert.Equal(t, int64(-1), responseUs)
+			} else {
+				require.NoError(t, err)
+				assert.GreaterOrEqual(t, responseUs, int64(0))
+			}
+		})
+	}
+}
+
+// tcpMonitorTestResolver supplies multiple A records without external DNS.
+func tcpMonitorTestResolver(ips []string) *net.Resolver {
+	return &net.Resolver{PreferGo: true, Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
+		client, server := net.Pipe()
+		go func() {
+			defer server.Close()
+			// net.Resolver uses TCP framing when its connection is not a PacketConn.
+			var size uint16
+			if err := binary.Read(server, binary.BigEndian, &size); err != nil {
+				return
+			}
+			packet := make([]byte, size)
+			if _, err := io.ReadFull(server, packet); err != nil {
+				return
+			}
+			var msg dnsmessage.Message
+			if err := msg.Unpack(packet); err != nil {
+				return
+			}
+			msg.Header.Response = true
+			msg.Header.RecursionAvailable = true
+			for _, question := range msg.Questions {
+				if question.Type != dnsmessage.TypeA {
+					continue
+				}
+				for _, ip := range ips {
+					msg.Answers = append(msg.Answers, dnsmessage.Resource{
+						Header: dnsmessage.ResourceHeader{Name: question.Name, Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET},
+						Body:   &dnsmessage.AResource{A: [4]byte(net.ParseIP(ip).To4())},
+					})
+				}
+			}
+			packet, err := msg.Pack()
+			if err != nil {
+				return
+			}
+			response := binary.BigEndian.AppendUint16(nil, uint16(len(packet)))
+			_, _ = server.Write(append(response, packet...))
+		}()
+		return client, nil
+	}}
+}
+
+func TestMonitorDNS(t *testing.T) {
+	t.Run("success", func(t *testing.T) {
+		responseUs, err := monitorDNS(context.Background(), "localhost")
+		require.NoError(t, err)
+		assert.GreaterOrEqual(t, responseUs, int64(0))
+	})
+
+	t.Run("lookup failure", func(t *testing.T) {
+		responseUs, err := monitorDNS(context.Background(), "")
+		assert.Equal(t, int64(-1), responseUs)
+		require.Error(t, err)
+	})
+}
+
+func TestMonitorManagerCancelsActiveProbe(t *testing.T) {
+	for _, action := range []string{"stop", "delete", "upsert", "sync replace", "sync remove"} {
+		t.Run(action, func(t *testing.T) {
+			started := make(chan struct{})
+			canceled := make(chan struct{})
+			release := make(chan struct{})
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				close(started)
+				select {
+				case <-r.Context().Done():
+					close(canceled)
+				case <-release:
+				}
+			}))
+			defer server.Close()
+			defer close(release)
+			pm := newMonitorManager()
+			defer pm.Stop()
+			cfg := monitor.Config{ID: "test", Protocol: "http", Target: server.URL, Interval: 3600}
+			task := newMonitorTask(cfg)
+			// Seed history to ensure a canceled RunNow does not return an old result.
+			task.history.addSampleLocked(monitorSample{responseUs: 123, timestamp: time.Now()})
+			pm.monitors[cfg.ID] = task
+			done := make(chan *monitor.Result, 1)
+			go func() {
+				result, _ := pm.UpsertMonitor(cfg, true)
+				done <- result
+			}()
+			select {
+			case <-started:
+			case <-time.After(time.Second):
+				t.Fatal("probe did not start")
+			}
+			updated := cfg
+			updated.Interval--
+			switch action {
+			case "stop":
+				pm.Stop()
+			case "delete":
+				pm.DeleteMonitor(cfg.ID)
+			case "upsert":
+				_, err := pm.UpsertMonitor(updated, false)
+				require.NoError(t, err)
+			case "sync replace":
+				pm.SyncMonitors([]monitor.Config{updated})
+			case "sync remove":
+				pm.SyncMonitors(nil)
+			}
+			select {
+			case <-canceled:
+			case <-time.After(time.Second):
+				t.Fatal("active HTTP request was not canceled")
+			}
+			select {
+			case result := <-done:
+				assert.Nil(t, result)
+			case <-time.After(time.Second):
+				t.Fatal("RunNow did not return after cancellation")
+			}
+			task.history.mu.Lock()
+			assert.Len(t, task.history.samples, 1, "cancellation must not record packet loss")
+			task.history.mu.Unlock()
+		})
+	}
+}
+
+func TestMonitorResolutionCancellation(t *testing.T) {
+	for _, protocol := range []string{"tcp", "dns", "icmp"} {
+		t.Run(protocol, func(t *testing.T) {
+			started := make(chan struct{}, 1)
+			original := net.DefaultResolver
+			net.DefaultResolver = &net.Resolver{PreferGo: true, Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
+				select {
+				case started <- struct{}{}:
+				default:
+				}
+				<-ctx.Done()
+				return nil, ctx.Err()
+			}}
+			defer func() { net.DefaultResolver = original }()
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+			done := make(chan error, 1)
+			go func() {
+				var err error
+				switch protocol {
+				case "tcp":
+					_, err = monitorTCP(ctx, "monitor-cancellation.invalid.", 80)
+				case "dns":
+					_, err = monitorDNS(ctx, "monitor-cancellation.invalid.")
+				case "icmp":
+					_, err = monitorICMP(ctx, "monitor-cancellation.invalid.")
+				}
+				done <- err
+			}()
+			select {
+			case <-started:
+			case <-time.After(time.Second):
+				t.Fatal("lookup did not start")
+			}
+			cancel()
+			select {
+			case err := <-done:
+				require.Error(t, err)
+			case <-time.After(time.Second):
+				t.Fatal("lookup did not cancel")
+			}
+		})
+	}
+}
+
+func TestMonitorProbeTimeoutRecordsLoss(t *testing.T) {
+	release := make(chan struct{})
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		select {
+		case <-r.Context().Done():
+		case <-release:
+		}
+	}))
+	defer server.Close()
+	defer close(release)
+	pm := newMonitorManager()
+	pm.probe = networkMonitorProbe(&http.Client{Timeout: 20 * time.Millisecond})
+	task := newMonitorTask(monitor.Config{ID: "timeout", Protocol: "http", Target: server.URL})
+	defer task.cancel()
+
+	result := task.runProbe(pm.probe)
+	require.NotNil(t, result)
+	assert.Equal(t, 100.0, result.PacketLoss)
+	assert.Equal(t, 100.0, result.PacketLoss1h)
+	require.Len(t, task.history.samples, 1)
+	assert.Equal(t, int64(-1), task.history.samples[0].responseUs)
+	assert.NoError(t, task.ctx.Err(), "a probe timeout must not cancel the task")
+}