feat(alerts): add alert for failed systemd services (#2173)

Adds a user-configurable "Failed Services" alert that notifies when any tracked systemd service enters the failed state, and again when all services recover. --------- Signed-off-by: Martin Stenröse <martin@stenrose.se> Co-authored-by: henrygd <hank@henrygd.me>
2026-09-21 08:57:48 +02:00 · 2026-09-02 02:41:48 +02:00
parent ed88e6efae
commit 097180e8d7
17 changed files with 891 additions and 52 deletions
--- a/internal/alerts/alerts.go
+++ b/internal/alerts/alerts.go
@@ -129,6 +129,9 @@ func (am *AlertManager) bindEvents() {
 		if err := resolveStatusAlerts(e.App); err != nil {
 			e.App.Logger().Error("Failed to resolve stale status alerts", "err", err)
 		}
+		if err := resolveSystemdAlerts(e.App); err != nil {
+			e.App.Logger().Error("Failed to resolve stale systemd alerts", "err", err)
+		}
 		if err := am.restorePendingStatusAlerts(); err != nil {
 			e.App.Logger().Error("Failed to restore pending status alerts", "err", err)
 		}
--- a/internal/alerts/alerts_system.go
+++ b/internal/alerts/alerts_system.go
@@ -37,7 +37,24 @@ func cpuStateAlertValue(name string, breakdown []float64) (float64, bool) {
 }

 func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *system.CombinedData) error {
-	alerts := am.alertsCache.GetAlertsExcludingNames(systemRecord.Id, "Status")
+	// Systemd alerts are binary state, not numeric thresholds, so they're handled
+	// separately. They read their own state from the database and don't use data.
+	// Read the confirmed empty state from the record being saved instead of data:
+	// dashboard polling can replace the system's in-memory payload concurrently.
+	var currentInfo system.Info
+	confirmedEmptySnapshot := false
+	if err := systemRecord.UnmarshalJSONField("info", &currentInfo); err == nil {
+		confirmedEmptySnapshot = len(currentInfo.Services) > 0 && currentInfo.Services[0] == 0
+	}
+	if err := am.HandleSystemdAlerts(systemRecord, confirmedEmptySnapshot); err != nil {
+		am.hub.Logger().Error("Error handling systemd alerts", "err", err)
+	}
+
+	if data == nil {
+		return nil
+	}
+
+	alerts := am.alertsCache.GetAlertsExcludingNames(systemRecord.Id, "Status", alertNameSystemdFailed)
 	if len(alerts) == 0 {
 		return nil
 	}
--- a/internal/alerts/alerts_systemd.go
+++ b/internal/alerts/alerts_systemd.go
@@ -0,0 +1,183 @@
+package alerts
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/henrygd/beszel/internal/entities/systemd"
+
+	"github.com/pocketbase/dbx"
+	"github.com/pocketbase/pocketbase/core"
+)
+
+// alertNameSystemdFailed is the alerts.name value for the failed systemd services alert.
+const alertNameSystemdFailed = "SystemdFailed"
+
+// maxListedServices caps how many service names are listed in a notification body.
+const maxListedServices = 10
+
+// HandleSystemdAlerts manages alerts for systemd services in the failed state.
+//
+// This is a binary state alert and fires on the first observation of a failed
+// service rather than using a delay. The agent only refreshes systemd state every
+// 10 minutes, so a shorter delay could never observe new data before expiring, and
+// that poll interval already hides services that fail and restart quickly.
+func (am *AlertManager) HandleSystemdAlerts(systemRecord *core.Record, confirmedEmptySnapshot bool) error {
+	alerts := am.alertsCache.GetAlertsByName(systemRecord.Id, alertNameSystemdFailed)
+	if len(alerts) == 0 {
+		return nil
+	}
+
+	// State is read from the systemd_services snapshot rather than the update payload.
+	// The payload is not a reliable source here: realtime dashboard subscriptions fetch
+	// from the agent with a shorter cache time, and the agent omits systemd services from
+	// those responses, overwriting the cached payload roughly once a second while a system
+	// is being viewed. The snapshot table is only written by the full update cycle.
+	total, failed, err := am.queryServiceStates(systemRecord.Id)
+	if err != nil {
+		return err
+	}
+	// No rows normally means no systemd data for this system (agent without systemd,
+	// or not yet reported), which must not be treated as a recovery. A fresh snapshot
+	// marker disambiguates that case from an agent explicitly reporting zero services.
+	if total == 0 && !confirmedEmptySnapshot {
+		return nil
+	}
+
+	systemName := systemRecord.GetString("name")
+
+	for _, alertData := range alerts {
+		triggered := len(failed) > 0
+		// Only notify on a change of state, so a service that stays failed across
+		// cycles doesn't re-notify every update.
+		if triggered == alertData.Triggered {
+			continue
+		}
+		if err := am.sendSystemdAlert(triggered, systemName, alertData, failed); err != nil {
+			am.hub.Logger().Error("Failed to send alert", "err", err)
+		}
+	}
+	return nil
+}
+
+// queryServiceStates returns the number of services reported in the most recent update
+// for a system, and the names of those in the failed state.
+//
+// Rows are restricted to the latest update because systemd_services is upserted, never
+// pruned on change: a service that no longer exists on the host stops being reported and
+// its row keeps its last known state until the retention sweep removes it. Every row
+// written in one cycle shares a single updated timestamp, so the newest timestamp
+// identifies exactly the services the agent last reported.
+func (am *AlertManager) queryServiceStates(systemID string) (total int, failed []string, err error) {
+	var rows []struct {
+		Name  string               `db:"name"`
+		State systemd.ServiceState `db:"state"`
+	}
+	err = am.hub.DB().
+		Select("name", "state").
+		From("systemd_services").
+		Where(dbx.NewExp(
+			"system={:system} AND updated=(SELECT MAX(updated) FROM systemd_services WHERE system={:system})",
+			dbx.Params{"system": systemID},
+		)).
+		OrderBy("name").
+		All(&rows)
+	if err != nil {
+		return 0, nil, err
+	}
+	for _, row := range rows {
+		if row.State == systemd.StatusFailed {
+			failed = append(failed, row.Name)
+		}
+	}
+	return len(rows), failed, nil
+}
+
+// sendSystemdAlert sends a failed or recovered systemd services alert to the alert's user.
+func (am *AlertManager) sendSystemdAlert(triggered bool, systemName string, alertData CachedAlertData, failed []string) error {
+	// Update trigger state for alert record before sending alert
+	if err := am.setAlertTriggered(alertData, triggered); err != nil {
+		return err
+	}
+
+	var title, message string
+	if triggered {
+		title = fmt.Sprintf("Failed services on %s %v", systemName, "\U0001F534") // Red alert emoji
+		message = fmt.Sprintf("%s on %s: %s", pluralizeServices(len(failed)), systemName, formatServiceList(failed))
+	} else {
+		title = fmt.Sprintf("Services recovered on %s %v", systemName, "✅") // Green checkmark emoji
+		message = fmt.Sprintf("No services are in the failed state on %s.", systemName)
+	}
+
+	systemID := alertData.SystemID
+
+	return am.SendAlert(AlertMessageData{
+		UserID:   alertData.UserID,
+		SystemID: systemID,
+		Title:    title,
+		Message:  message,
+		Link:     am.hub.MakeLink("system", systemID),
+		LinkText: "View " + systemName,
+	})
+}
+
+// pluralizeServices returns a count label like "1 failed service" or "3 failed services".
+func pluralizeServices(count int) string {
+	if count == 1 {
+		return "1 failed service"
+	}
+	return fmt.Sprintf("%d failed services", count)
+}
+
+// formatServiceList joins service names, truncating long lists.
+func formatServiceList(names []string) string {
+	if len(names) <= maxListedServices {
+		return strings.Join(names, ", ")
+	}
+	remaining := len(names) - maxListedServices
+	return fmt.Sprintf("%s and %d more", strings.Join(names[:maxListedServices], ", "), remaining)
+}
+
+// resolveSystemdAlerts resolves triggered systemd alerts for systems that no longer
+// have any failed services. This clears stale state left by a hub restart.
+func resolveSystemdAlerts(app core.App) error {
+	db := app.DB()
+	var alertIds []string
+	err := db.NewQuery(`
+		SELECT a.id
+		FROM alerts a
+		JOIN systems sys ON sys.id = a.system
+		WHERE a.name = {:name}
+		AND a.triggered = true
+		AND (
+			EXISTS (
+				SELECT 1 FROM systemd_services cur
+				WHERE cur.system = a.system
+				AND cur.updated = (SELECT MAX(updated) FROM systemd_services WHERE system = a.system)
+			)
+			OR json_extract(sys.info, '$.sv[0]') = 0
+		)
+		AND NOT EXISTS (
+			SELECT 1 FROM systemd_services s
+			WHERE s.system = a.system AND s.state = {:state}
+			AND s.updated = (SELECT MAX(updated) FROM systemd_services WHERE system = a.system)
+		)
+	`).Bind(dbx.Params{
+		"name":  alertNameSystemdFailed,
+		"state": systemd.StatusFailed,
+	}).Column(&alertIds)
+	if err != nil {
+		return err
+	}
+	for _, alertId := range alertIds {
+		alert, err := app.FindRecordById("alerts", alertId)
+		if err != nil {
+			return err
+		}
+		alert.Set("triggered", false)
+		if err := app.Save(alert); err != nil {
+			return err
+		}
+	}
+	return nil
+}
--- a/internal/alerts/alerts_systemd_test.go
+++ b/internal/alerts/alerts_systemd_test.go
@@ -0,0 +1,383 @@
+//go:build testing
+
+package alerts_test
+
+import (
+	"testing"
+	"time"
+
+	"github.com/henrygd/beszel/internal/alerts"
+	systemEntity "github.com/henrygd/beszel/internal/entities/system"
+	"github.com/henrygd/beszel/internal/entities/systemd"
+	beszelTests "github.com/henrygd/beszel/internal/tests"
+	"github.com/pocketbase/dbx"
+	"github.com/pocketbase/pocketbase/core"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// setSystemdServiceState upserts a systemd_services row mirroring the raw SQL write
+// path used by the hub (createSystemdStatsRecords), which bypasses record hooks.
+func setSystemdServiceState(t *testing.T, hub core.App, systemID, name string, state systemd.ServiceState, updated int64) {
+	t.Helper()
+
+	_, err := hub.DB().NewQuery(
+		"INSERT INTO systemd_services (id, system, name, state, sub, cpu, cpuPeak, memory, memPeak, updated) " +
+			"VALUES ({:id}, {:system}, {:name}, {:state}, 0, 0, 0, 0, 0, {:updated}) " +
+			"ON CONFLICT(id) DO UPDATE SET state = excluded.state, updated = excluded.updated",
+	).Bind(dbx.Params{
+		"id":      systemID + "-" + name,
+		"system":  systemID,
+		"name":    name,
+		"state":   state,
+		"updated": updated,
+	}).Execute()
+	require.NoError(t, err)
+}
+
+// seedServices writes a set of services into the systemd_services snapshot, which is the
+// source HandleSystemdAlerts reads from. All rows share one updated timestamp, matching
+// how the hub writes a batch in createSystemdStatsRecords.
+func seedServices(t *testing.T, hub core.App, systemID string, states ...systemd.ServiceState) {
+	t.Helper()
+	seedServicesAt(t, hub, systemID, time.Now().UTC().UnixMilli(), states...)
+}
+
+// seedServicesAt writes services with an explicit batch timestamp.
+func seedServicesAt(t *testing.T, hub core.App, systemID string, updated int64, states ...systemd.ServiceState) {
+	t.Helper()
+	for i, state := range states {
+		setSystemdServiceState(t, hub, systemID, serviceName(i), state, updated)
+	}
+}
+
+func serviceName(i int) string {
+	return string(rune('a'+i)) + ".service"
+}
+
+// systemdTestSetup creates a user with an email, a system, and a SystemdFailed alert.
+func systemdTestSetup(t *testing.T, triggered bool) (*beszelTests.TestHub, *core.Record, *core.Record) {
+	t.Helper()
+
+	hub, user := beszelTests.GetHubWithUser(t)
+
+	userSettings, err := hub.FindFirstRecordByFilter("user_settings", "user={:user}", map[string]any{"user": user.Id})
+	require.NoError(t, err)
+	userSettings.Set("settings", `{"emails":["test@example.com"],"webhooks":[]}`)
+	require.NoError(t, hub.Save(userSettings))
+
+	// "paused" avoids spawning a background updater goroutine that would outlive
+	// the test hub; these tests drive HandleSystemdAlerts directly.
+	systems, err := beszelTests.CreateSystems(hub, 1, user.Id, "paused")
+	require.NoError(t, err)
+	system := systems[0]
+
+	alert, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{
+		"name":      "SystemdFailed",
+		"system":    system.Id,
+		"user":      user.Id,
+		"triggered": triggered,
+	})
+	require.NoError(t, err)
+
+	return hub, system, alert
+}
+
+func TestSystemdAlertFiresImmediately(t *testing.T) {
+	hub, system, alert := systemdTestSetup(t, false)
+	defer hub.Cleanup()
+
+	initialEmailCount := hub.TestMailer.TotalSend()
+	am := alerts.NewTestAlertManagerWithoutWorker(hub)
+
+	seedServices(t, hub, system.Id, systemd.StatusFailed, systemd.StatusActive)
+	require.NoError(t, am.HandleSystemdAlerts(system, false))
+
+	assert.Equal(t, initialEmailCount+1, hub.TestMailer.TotalSend(), "failed service should notify on first observation")
+
+	messages := hub.TestMailer.Messages()
+	require.NotEmpty(t, messages)
+	last := messages[len(messages)-1]
+	assert.Contains(t, last.Subject, "Failed services")
+	assert.Contains(t, last.Text, "a.service", "notification should name the failed service")
+
+	alertRecord, err := hub.FindRecordById("alerts", alert.Id)
+	require.NoError(t, err)
+	assert.True(t, alertRecord.GetBool("triggered"), "alert should be marked triggered")
+
+	// history record should be created via the alerts update hook
+	historyCount, err := hub.CountRecords("alerts_history", dbx.HashExp{"resolved": ""})
+	require.NoError(t, err)
+	assert.EqualValues(t, 1, historyCount, "should have one unresolved alert history record")
+}
+
+func TestSystemdAlertFullCycle(t *testing.T) {
+	hub, system, alert := systemdTestSetup(t, false)
+	defer hub.Cleanup()
+
+	initialEmailCount := hub.TestMailer.TotalSend()
+	am := alerts.NewTestAlertManagerWithoutWorker(hub)
+
+	// Fail, then recover.
+	seedServices(t, hub, system.Id, systemd.StatusFailed)
+	require.NoError(t, am.HandleSystemdAlerts(system, false))
+	seedServices(t, hub, system.Id, systemd.StatusActive)
+	require.NoError(t, am.HandleSystemdAlerts(system, false))
+
+	assert.Equal(t, initialEmailCount+2, hub.TestMailer.TotalSend(), "should send a failure and a recovery notification")
+
+	messages := hub.TestMailer.Messages()
+	require.Len(t, messages, 2)
+	assert.Contains(t, messages[0].Subject, "Failed services")
+	assert.Contains(t, messages[1].Subject, "Services recovered")
+
+	alertRecord, err := hub.FindRecordById("alerts", alert.Id)
+	require.NoError(t, err)
+	assert.False(t, alertRecord.GetBool("triggered"), "alert should be cleared after recovery")
+
+	// history record should be resolved
+	historyCount, err := hub.CountRecords("alerts_history", dbx.HashExp{"resolved": ""})
+	require.NoError(t, err)
+	assert.Zero(t, historyCount, "alert history record should be resolved")
+}
+
+func TestSystemdAlertSendsRecoveryWhenTriggered(t *testing.T) {
+	hub, system, alert := systemdTestSetup(t, true)
+	defer hub.Cleanup()
+
+	initialEmailCount := hub.TestMailer.TotalSend()
+	am := alerts.NewTestAlertManagerWithoutWorker(hub)
+
+	seedServices(t, hub, system.Id, systemd.StatusActive, systemd.StatusInactive)
+	require.NoError(t, am.HandleSystemdAlerts(system, false))
+
+	assert.Equal(t, initialEmailCount+1, hub.TestMailer.TotalSend(), "recovery notification should be sent")
+	messages := hub.TestMailer.Messages()
+	require.NotEmpty(t, messages)
+	assert.Contains(t, messages[len(messages)-1].Subject, "Services recovered")
+
+	alertRecord, err := hub.FindRecordById("alerts", alert.Id)
+	require.NoError(t, err)
+	assert.False(t, alertRecord.GetBool("triggered"), "alert should be cleared after recovery")
+}
+
+func TestSystemdAlertDoesNotResendWhileTriggered(t *testing.T) {
+	hub, system, _ := systemdTestSetup(t, true)
+	defer hub.Cleanup()
+
+	initialEmailCount := hub.TestMailer.TotalSend()
+	am := alerts.NewTestAlertManagerWithoutWorker(hub)
+
+	// Still failing across several cycles — should not re-notify.
+	for range 3 {
+		seedServices(t, hub, system.Id, systemd.StatusFailed)
+		require.NoError(t, am.HandleSystemdAlerts(system, false))
+	}
+
+	assert.Equal(t, initialEmailCount, hub.TestMailer.TotalSend(), "should not re-notify while still triggered")
+}
+
+func TestSystemdAlertRepeatedFailureNotifiesOnce(t *testing.T) {
+	hub, system, _ := systemdTestSetup(t, false)
+	defer hub.Cleanup()
+
+	initialEmailCount := hub.TestMailer.TotalSend()
+	am := alerts.NewTestAlertManagerWithoutWorker(hub)
+
+	for range 3 {
+		seedServices(t, hub, system.Id, systemd.StatusFailed)
+		require.NoError(t, am.HandleSystemdAlerts(system, false))
+	}
+
+	assert.Equal(t, initialEmailCount+1, hub.TestMailer.TotalSend(), "repeated failures should only notify once")
+}
+
+// A service that no longer exists on the host stops being reported, but its row stays
+// in systemd_services with its last known state until the retention sweep. That stale
+// row must not keep the alert triggered.
+func TestSystemdAlertIgnoresServicesNoLongerReported(t *testing.T) {
+	hub, system, alert := systemdTestSetup(t, true)
+	defer hub.Cleanup()
+
+	initialEmailCount := hub.TestMailer.TotalSend()
+	am := alerts.NewTestAlertManagerWithoutWorker(hub)
+
+	now := time.Now().UTC().UnixMilli()
+	// Older batch still holding a failed service that has since been removed.
+	setSystemdServiceState(t, hub, system.Id, "gone.service", systemd.StatusFailed, now-60_000)
+	// Current batch reports only healthy services.
+	seedServicesAt(t, hub, system.Id, now, systemd.StatusActive, systemd.StatusActive)
+
+	require.NoError(t, am.HandleSystemdAlerts(system, false))
+
+	assert.Equal(t, initialEmailCount+1, hub.TestMailer.TotalSend(), "stale failed row should not block recovery")
+	alertRecord, err := hub.FindRecordById("alerts", alert.Id)
+	require.NoError(t, err)
+	assert.False(t, alertRecord.GetBool("triggered"), "alert should resolve once the service stops being reported")
+}
+
+func TestResolveSystemdAlertsIgnoresStaleFailedRows(t *testing.T) {
+	hub, system, alert := systemdTestSetup(t, true)
+	defer hub.Cleanup()
+
+	now := time.Now().UTC().UnixMilli()
+	setSystemdServiceState(t, hub, system.Id, "gone.service", systemd.StatusFailed, now-60_000)
+	seedServicesAt(t, hub, system.Id, now, systemd.StatusActive)
+
+	require.NoError(t, alerts.ResolveSystemdAlerts(hub))
+
+	alertRecord, err := hub.FindRecordById("alerts", alert.Id)
+	require.NoError(t, err)
+	assert.False(t, alertRecord.GetBool("triggered"), "stale failed row should not keep the alert triggered")
+}
+
+func TestSystemdAlertNoSystemdDataIsIgnored(t *testing.T) {
+	hub, system, alert := systemdTestSetup(t, true)
+	defer hub.Cleanup()
+
+	initialEmailCount := hub.TestMailer.TotalSend()
+	am := alerts.NewTestAlertManagerWithoutWorker(hub)
+
+	// A system with no systemd_services rows (agent without systemd, or nothing
+	// reported yet) must not be treated as a recovery.
+	require.NoError(t, am.HandleSystemdAlerts(system, false))
+	require.NoError(t, am.HandleSystemdAlerts(system, false))
+
+	assert.Equal(t, initialEmailCount, hub.TestMailer.TotalSend(), "missing systemd data should not send a recovery")
+	alertRecord, err := hub.FindRecordById("alerts", alert.Id)
+	require.NoError(t, err)
+	assert.True(t, alertRecord.GetBool("triggered"), "triggered state should be preserved when data is absent")
+}
+
+func TestSystemdAlertFreshEmptySnapshotResolves(t *testing.T) {
+	hub, system, alert := systemdTestSetup(t, true)
+	defer hub.Cleanup()
+
+	initialEmailCount := hub.TestMailer.TotalSend()
+	am := alerts.NewTestAlertManagerWithoutWorker(hub)
+
+	// An explicit zero service count on the saved system record distinguishes a
+	// confirmed empty snapshot from an agent response that omitted systemd data.
+	system.Set("info", systemEntity.Info{Services: []uint16{0, 0}})
+	require.NoError(t, am.HandleSystemAlerts(system, nil))
+
+	assert.Equal(t, initialEmailCount+1, hub.TestMailer.TotalSend(), "fresh empty snapshot should send a recovery")
+	alertRecord, err := hub.FindRecordById("alerts", alert.Id)
+	require.NoError(t, err)
+	assert.False(t, alertRecord.GetBool("triggered"), "fresh empty snapshot should resolve the alert")
+}
+
+func TestSystemdAlertNoAlertRecord(t *testing.T) {
+	hub, user := beszelTests.GetHubWithUser(t)
+	defer hub.Cleanup()
+
+	systems, err := beszelTests.CreateSystems(hub, 1, user.Id, "paused")
+	require.NoError(t, err)
+	system := systems[0]
+
+	initialEmailCount := hub.TestMailer.TotalSend()
+	am := alerts.NewTestAlertManagerWithoutWorker(hub)
+
+	seedServices(t, hub, system.Id, systemd.StatusFailed)
+	require.NoError(t, am.HandleSystemdAlerts(system, false))
+	assert.Equal(t, initialEmailCount, hub.TestMailer.TotalSend(), "no email when no alert record exists")
+}
+
+func TestResolveSystemdAlertsClearsStaleTriggered(t *testing.T) {
+	hub, system, alert := systemdTestSetup(t, true)
+	defer hub.Cleanup()
+
+	// No failed services in the snapshot, but the alert is still marked triggered
+	// (e.g. the hub restarted while the alert was active).
+	setSystemdServiceState(t, hub, system.Id, "a.service", systemd.StatusActive, time.Now().UTC().UnixMilli())
+
+	require.NoError(t, alerts.ResolveSystemdAlerts(hub))
+
+	alertRecord, err := hub.FindRecordById("alerts", alert.Id)
+	require.NoError(t, err)
+	assert.False(t, alertRecord.GetBool("triggered"), "stale triggered flag should be cleared")
+}
+
+func TestResolveSystemdAlertsKeepsTriggeredWithoutSystemdData(t *testing.T) {
+	hub, _, alert := systemdTestSetup(t, true)
+	defer hub.Cleanup()
+
+	// Missing rows do not prove recovery. This can happen when a system is offline
+	// and its last service snapshot has been removed by retention.
+	require.NoError(t, alerts.ResolveSystemdAlerts(hub))
+
+	alertRecord, err := hub.FindRecordById("alerts", alert.Id)
+	require.NoError(t, err)
+	assert.True(t, alertRecord.GetBool("triggered"), "missing systemd data should preserve triggered state")
+}
+
+func TestResolveSystemdAlertsClearsConfirmedEmptySnapshot(t *testing.T) {
+	hub, system, alert := systemdTestSetup(t, true)
+	defer hub.Cleanup()
+
+	// Update the persisted snapshot directly so record hooks don't alter alert state
+	// before the startup resolver is exercised.
+	_, err := hub.DB().NewQuery(
+		"UPDATE systems SET info = {:info} WHERE id = {:id}",
+	).Bind(dbx.Params{"info": `{"sv":[0,0]}`, "id": system.Id}).Execute()
+	require.NoError(t, err)
+	require.NoError(t, alerts.ResolveSystemdAlerts(hub))
+
+	alertRecord, err := hub.FindRecordById("alerts", alert.Id)
+	require.NoError(t, err)
+	assert.False(t, alertRecord.GetBool("triggered"), "confirmed empty snapshot should clear triggered state")
+}
+
+func TestResolveSystemdAlertsKeepsStillFailing(t *testing.T) {
+	hub, system, alert := systemdTestSetup(t, true)
+	defer hub.Cleanup()
+
+	setSystemdServiceState(t, hub, system.Id, "a.service", systemd.StatusFailed, time.Now().UTC().UnixMilli())
+
+	require.NoError(t, alerts.ResolveSystemdAlerts(hub))
+
+	alertRecord, err := hub.FindRecordById("alerts", alert.Id)
+	require.NoError(t, err)
+	assert.True(t, alertRecord.GetBool("triggered"), "alert should stay triggered while a service is still failed")
+}
+
+func TestSystemdAlertMultipleUsersRespectOwnAlerts(t *testing.T) {
+	hub, user1 := beszelTests.GetHubWithUser(t)
+	defer hub.Cleanup()
+
+	setStatusAlertEmail(t, hub, user1.Id, "user1@example.com")
+
+	user2, err := beszelTests.CreateUser(hub, "user2@example.com", "password")
+	require.NoError(t, err)
+	_, err = beszelTests.CreateRecord(hub, "user_settings", map[string]any{
+		"user": user2.Id,
+		"settings": map[string]any{
+			"emails":   []string{"user2@example.com"},
+			"webhooks": []string{},
+		},
+	})
+	require.NoError(t, err)
+
+	system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
+		"name":  "shared-system",
+		"users": []string{user1.Id, user2.Id},
+		"host":  "127.0.0.1",
+	})
+	require.NoError(t, err)
+
+	for _, user := range []*core.Record{user1, user2} {
+		_, err = beszelTests.CreateRecord(hub, "alerts", map[string]any{
+			"name":   "SystemdFailed",
+			"system": system.Id,
+			"user":   user.Id,
+		})
+		require.NoError(t, err)
+	}
+
+	am := alerts.NewTestAlertManagerWithoutWorker(hub)
+	seedServices(t, hub, system.Id, systemd.StatusFailed)
+	require.NoError(t, am.HandleSystemdAlerts(system, false))
+
+	messages := hub.TestMailer.Messages()
+	require.Len(t, messages, 2, "each user should receive their own alert")
+}
--- a/internal/alerts/alerts_test_helpers.go
+++ b/internal/alerts/alerts_test_helpers.go
@@ -88,6 +88,10 @@ func ResolveStatusAlerts(app core.App) error {
 	return resolveStatusAlerts(app)
 }

+func ResolveSystemdAlerts(app core.App) error {
+	return resolveSystemdAlerts(app)
+}
+
 func (am *AlertManager) RestorePendingStatusAlerts() error {
 	return am.restorePendingStatusAlerts()
 }