feat(alerts): add alert for failed systemd services (#2173)

Adds a user-configurable "Failed Services" alert that notifies when any
tracked systemd service enters the failed state, and again when all services
recover.

---------

Signed-off-by: Martin Stenröse <martin@stenrose.se>
Co-authored-by: henrygd <hank@henrygd.me>

internal/alerts/alerts_systemd.go

@@ -0,0 +1,183 @@
+package alerts
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/henrygd/beszel/internal/entities/systemd"
+
+	"github.com/pocketbase/dbx"
+	"github.com/pocketbase/pocketbase/core"
+)
+
+// alertNameSystemdFailed is the alerts.name value for the failed systemd services alert.
+const alertNameSystemdFailed = "SystemdFailed"
+
+// maxListedServices caps how many service names are listed in a notification body.
+const maxListedServices = 10
+
+// HandleSystemdAlerts manages alerts for systemd services in the failed state.
+//
+// This is a binary state alert and fires on the first observation of a failed
+// service rather than using a delay. The agent only refreshes systemd state every
+// 10 minutes, so a shorter delay could never observe new data before expiring, and
+// that poll interval already hides services that fail and restart quickly.
+func (am *AlertManager) HandleSystemdAlerts(systemRecord *core.Record, confirmedEmptySnapshot bool) error {
+	alerts := am.alertsCache.GetAlertsByName(systemRecord.Id, alertNameSystemdFailed)
+	if len(alerts) == 0 {
+		return nil
+	}
+
+	// State is read from the systemd_services snapshot rather than the update payload.
+	// The payload is not a reliable source here: realtime dashboard subscriptions fetch
+	// from the agent with a shorter cache time, and the agent omits systemd services from
+	// those responses, overwriting the cached payload roughly once a second while a system
+	// is being viewed. The snapshot table is only written by the full update cycle.
+	total, failed, err := am.queryServiceStates(systemRecord.Id)
+	if err != nil {
+		return err
+	}
+	// No rows normally means no systemd data for this system (agent without systemd,
+	// or not yet reported), which must not be treated as a recovery. A fresh snapshot
+	// marker disambiguates that case from an agent explicitly reporting zero services.
+	if total == 0 && !confirmedEmptySnapshot {
+		return nil
+	}
+
+	systemName := systemRecord.GetString("name")
+
+	for _, alertData := range alerts {
+		triggered := len(failed) > 0
+		// Only notify on a change of state, so a service that stays failed across
+		// cycles doesn't re-notify every update.
+		if triggered == alertData.Triggered {
+			continue
+		}
+		if err := am.sendSystemdAlert(triggered, systemName, alertData, failed); err != nil {
+			am.hub.Logger().Error("Failed to send alert", "err", err)
+		}
+	}
+	return nil
+}
+
+// queryServiceStates returns the number of services reported in the most recent update
+// for a system, and the names of those in the failed state.
+//
+// Rows are restricted to the latest update because systemd_services is upserted, never
+// pruned on change: a service that no longer exists on the host stops being reported and
+// its row keeps its last known state until the retention sweep removes it. Every row
+// written in one cycle shares a single updated timestamp, so the newest timestamp
+// identifies exactly the services the agent last reported.
+func (am *AlertManager) queryServiceStates(systemID string) (total int, failed []string, err error) {
+	var rows []struct {
+		Name  string               `db:"name"`
+		State systemd.ServiceState `db:"state"`
+	}
+	err = am.hub.DB().
+		Select("name", "state").
+		From("systemd_services").
+		Where(dbx.NewExp(
+			"system={:system} AND updated=(SELECT MAX(updated) FROM systemd_services WHERE system={:system})",
+			dbx.Params{"system": systemID},
+		)).
+		OrderBy("name").
+		All(&rows)
+	if err != nil {
+		return 0, nil, err
+	}
+	for _, row := range rows {
+		if row.State == systemd.StatusFailed {
+			failed = append(failed, row.Name)
+		}
+	}
+	return len(rows), failed, nil
+}
+
+// sendSystemdAlert sends a failed or recovered systemd services alert to the alert's user.
+func (am *AlertManager) sendSystemdAlert(triggered bool, systemName string, alertData CachedAlertData, failed []string) error {
+	// Update trigger state for alert record before sending alert
+	if err := am.setAlertTriggered(alertData, triggered); err != nil {
+		return err
+	}
+
+	var title, message string
+	if triggered {
+		title = fmt.Sprintf("Failed services on %s %v", systemName, "\U0001F534") // Red alert emoji
+		message = fmt.Sprintf("%s on %s: %s", pluralizeServices(len(failed)), systemName, formatServiceList(failed))
+	} else {
+		title = fmt.Sprintf("Services recovered on %s %v", systemName, "✅") // Green checkmark emoji
+		message = fmt.Sprintf("No services are in the failed state on %s.", systemName)
+	}
+
+	systemID := alertData.SystemID
+
+	return am.SendAlert(AlertMessageData{
+		UserID:   alertData.UserID,
+		SystemID: systemID,
+		Title:    title,
+		Message:  message,
+		Link:     am.hub.MakeLink("system", systemID),
+		LinkText: "View " + systemName,
+	})
+}
+
+// pluralizeServices returns a count label like "1 failed service" or "3 failed services".
+func pluralizeServices(count int) string {
+	if count == 1 {
+		return "1 failed service"
+	}
+	return fmt.Sprintf("%d failed services", count)
+}
+
+// formatServiceList joins service names, truncating long lists.
+func formatServiceList(names []string) string {
+	if len(names) <= maxListedServices {
+		return strings.Join(names, ", ")
+	}
+	remaining := len(names) - maxListedServices
+	return fmt.Sprintf("%s and %d more", strings.Join(names[:maxListedServices], ", "), remaining)
+}
+
+// resolveSystemdAlerts resolves triggered systemd alerts for systems that no longer
+// have any failed services. This clears stale state left by a hub restart.
+func resolveSystemdAlerts(app core.App) error {
+	db := app.DB()
+	var alertIds []string
+	err := db.NewQuery(`
+		SELECT a.id
+		FROM alerts a
+		JOIN systems sys ON sys.id = a.system
+		WHERE a.name = {:name}
+		AND a.triggered = true
+		AND (
+			EXISTS (
+				SELECT 1 FROM systemd_services cur
+				WHERE cur.system = a.system
+				AND cur.updated = (SELECT MAX(updated) FROM systemd_services WHERE system = a.system)
+			)
+			OR json_extract(sys.info, '$.sv[0]') = 0
+		)
+		AND NOT EXISTS (
+			SELECT 1 FROM systemd_services s
+			WHERE s.system = a.system AND s.state = {:state}
+			AND s.updated = (SELECT MAX(updated) FROM systemd_services WHERE system = a.system)
+		)
+	`).Bind(dbx.Params{
+		"name":  alertNameSystemdFailed,
+		"state": systemd.StatusFailed,
+	}).Column(&alertIds)
+	if err != nil {
+		return err
+	}
+	for _, alertId := range alertIds {
+		alert, err := app.FindRecordById("alerts", alertId)
+		if err != nil {
+			return err
+		}
+		alert.Set("triggered", false)
+		if err := app.Save(alert); err != nil {
+			return err
+		}
+	}
+	return nil
+}