feat(alerts): add container health alerts with log excerpt on notifications (#2225)

Add a new "ContainerHealth" alert type that fires when a Docker container's
health check reports unhealthy, and resolves when it recovers. This mirrors
the existing Status (up/down) alert pattern: an alert can be armed per system
and honors the "min minutes" delay before firing.

When the alert fires, the notification (email and any configured webhook,
including Discord via shoutrrr) includes a log excerpt fetched live from the
agent for up to 2 of the unhealthy containers, prioritizing lines containing
"error" or "fatal" (falling back to the log tail if none match), capped to
keep the message well under Discord's size limit.

---------

Co-authored-by: hank <hank@henrygd.me>

internal/alerts/alerts.go

@@ -20,10 +20,10 @@ type hubLike interface {
 }
 
 type AlertManager struct {
-	hub           hubLike
-	stopOnce      sync.Once
-	pendingAlerts sync.Map
-	alertsCache   *AlertsCache
+	hub                    hubLike
+	stopOnce               sync.Once
+	pendingAlerts          sync.Map
+	alertsCache            *AlertsCache
 }
 
 type AlertMessageData struct {

internal/alerts/alerts_cache.go

@@ -1,6 +1,8 @@
 package alerts
 
 import (
+	"time"
+
 	"github.com/pocketbase/dbx"
 	"github.com/pocketbase/pocketbase/core"
 	"github.com/pocketbase/pocketbase/tools/store"
@@ -8,13 +10,14 @@ import (
 
 // CachedAlertData represents the relevant fields of an alert record for status checking and updates.
 type CachedAlertData struct {
-	Id        string
-	SystemID  string
-	UserID    string
-	Name      string
-	Value     float64
-	Triggered bool
-	Min       uint8
+	Id           string
+	SystemID     string
+	UserID       string
+	Name         string
+	Value        float64
+	Triggered    bool
+	Min          uint8
+	PendingSince time.Time
 	// Created   types.DateTime
 }
 
@@ -26,6 +29,7 @@ func (a *CachedAlertData) PopulateFromRecord(record *core.Record) {
 	a.Value = record.GetFloat("value")
 	a.Triggered = record.GetBool("triggered")
 	a.Min = uint8(record.GetInt("min"))
+	a.PendingSince = record.GetDateTime("pending_since").Time()
 	// a.Created = record.GetDateTime("created")
 }
 

internal/alerts/alerts_container.go

@@ -0,0 +1,318 @@
+package alerts
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/henrygd/beszel/internal/entities/container"
+	"github.com/henrygd/beszel/internal/entities/system"
+
+	"github.com/pocketbase/pocketbase/core"
+)
+
+const (
+	// containerAlertName is the value stored in the alerts.name field for this alert type.
+	containerAlertName = "ContainerHealth"
+
+	// containerLogMaxLines caps how many matched (error/fatal) log lines are kept.
+	containerLogMaxLines = 12
+	// containerLogFallbackLines is how many trailing raw log lines are used when no
+	// line matches "error" or "fatal", so the notification still carries some context.
+	containerLogFallbackLines = 6
+	// containerLogExcerptMaxChars bounds a single container's log excerpt so a
+	// handful of containers can't blow past Discord's message size limit.
+	containerLogExcerptMaxChars = 500
+	// containerAlertMaxLogged is the max number of unhealthy containers we fetch
+	// and embed logs for in a single alert message.
+	containerAlertMaxLogged = 2
+	// containerAlertMessageMaxChars is a final safety cap on the whole message body.
+	containerAlertMessageMaxChars = 1800
+)
+
+// FetchContainerLogsFunc retrieves recent logs for a container ID from its
+// connected agent. Implementations should apply their own timeout. This is a
+// type alias (not a defined type) so it satisfies the hubLike interface in
+// internal/hub/systems, which declares the same func signature without
+// importing this package.
+type FetchContainerLogsFunc = func(containerID string) (string, error)
+
+// containerAlertTarget is an immutable snapshot of the fields needed after the
+// alert fires. Keeping agent-owned container records out of notification work
+// avoids retaining and concurrently reading data that is refreshed in place.
+type containerAlertTarget struct {
+	id   string
+	name string
+}
+
+// HandleContainerAlerts checks configured "ContainerHealth" alerts for a system
+// against the Docker container health data included in the latest agent update.
+// It persists when containers first become unhealthy, fires from a fresh poll
+// once the configured delay has elapsed, and resolves once containers recover.
+// fetchLogs is used when an alert actually fires so the notification can include
+// a log excerpt (prioritizing lines containing "error"/"fatal") for context.
+func (am *AlertManager) HandleContainerAlerts(systemRecord *core.Record, data *system.CombinedData, fetchLogs FetchContainerLogsFunc) error {
+	alerts := am.alertsCache.GetAlertsByName(systemRecord.Id, containerAlertName)
+	if len(alerts) == 0 {
+		return nil
+	}
+	if data.Containers == nil {
+		// An unknown Docker state must not resolve a triggered alert or count
+		// toward the minimum unhealthy duration.
+		var result error
+		for _, alertData := range alerts {
+			if err := am.clearPendingContainerAlert(alertData); err != nil {
+				result = errors.Join(result, err)
+			}
+		}
+		return result
+	}
+
+	var unhealthy []*container.Stats
+	for _, c := range data.Containers {
+		if c.Health == container.DockerHealthUnhealthy {
+			unhealthy = append(unhealthy, c)
+		}
+	}
+
+	systemName := systemRecord.GetString("name")
+	now := time.Now().UTC()
+	var result error
+	for _, alertData := range alerts {
+		if len(unhealthy) > 0 {
+			if alertData.Triggered {
+				continue
+			}
+			min := max(1, int(alertData.Min))
+			if alertData.PendingSince.IsZero() {
+				pendingSince, err := am.setPendingContainerAlert(alertData, now)
+				if err != nil {
+					result = errors.Join(result, err)
+					continue
+				}
+				if pendingSince.IsZero() {
+					continue
+				}
+				alertData.PendingSince = pendingSince
+				if min > 1 {
+					continue
+				}
+			}
+			if min > 1 && now.Before(alertData.PendingSince.Add(time.Duration(min)*time.Minute)) {
+				continue
+			}
+			if err := am.sendContainerHealthAlert(true, systemName, alertData, snapshotContainerAlertTargets(unhealthy), fetchLogs); err != nil {
+				result = errors.Join(result, err)
+			}
+			continue
+		}
+
+		// no unhealthy containers right now
+		if err := am.clearPendingContainerAlert(alertData); err != nil {
+			result = errors.Join(result, err)
+		}
+		if !alertData.Triggered {
+			continue
+		}
+		if err := am.sendContainerHealthAlert(false, systemName, alertData, nil, fetchLogs); err != nil {
+			result = errors.Join(result, err)
+		}
+	}
+	return result
+}
+
+func snapshotContainerAlertTargets(containers []*container.Stats) []containerAlertTarget {
+	targets := make([]containerAlertTarget, len(containers))
+	for i, c := range containers {
+		targets[i] = containerAlertTarget{id: c.Id, name: c.Name}
+	}
+	return targets
+}
+
+// setPendingContainerAlert durably records the first unhealthy observation and
+// returns the persisted generation used to claim delivery.
+func (am *AlertManager) setPendingContainerAlert(alertData CachedAlertData, since time.Time) (time.Time, error) {
+	record, err := am.hub.FindRecordById("alerts", alertData.Id)
+	if err != nil {
+		return time.Time{}, err
+	}
+	if record.GetBool("triggered") {
+		return time.Time{}, nil
+	}
+	if pendingSince := record.GetDateTime("pending_since").Time(); !pendingSince.IsZero() {
+		return pendingSince, nil
+	}
+	// PocketBase date fields are persisted with millisecond precision. Normalize
+	// before saving so the update-hook cache and a subsequent database read agree.
+	since = since.Truncate(time.Millisecond)
+	record.Set("pending_since", since)
+	return since, am.hub.Save(record)
+}
+
+func (am *AlertManager) clearPendingContainerAlert(alertData CachedAlertData) error {
+	if alertData.PendingSince.IsZero() {
+		return nil
+	}
+	record, err := am.hub.FindRecordById("alerts", alertData.Id)
+	if err != nil {
+		return err
+	}
+	if record.GetDateTime("pending_since").Time().IsZero() {
+		return nil
+	}
+	record.Set("pending_since", nil)
+	return am.hub.Save(record)
+}
+
+// claimPendingContainerAlert marks an alert triggered only if the pending
+// generation is still current. A healthy/unknown update can clear the timestamp
+// while logs are being fetched, causing this claim to become a no-op.
+func (am *AlertManager) claimPendingContainerAlert(alertData CachedAlertData) (bool, error) {
+	record, err := am.hub.FindRecordById("alerts", alertData.Id)
+	if err != nil {
+		return false, err
+	}
+	pendingSince := record.GetDateTime("pending_since").Time()
+	if record.GetBool("triggered") || pendingSince.IsZero() || pendingSince.UnixMilli() != alertData.PendingSince.UnixMilli() {
+		return false, nil
+	}
+	record.Set("pending_since", nil)
+	record.Set("triggered", true)
+	return true, am.hub.Save(record)
+}
+
+// CancelPendingContainerAlerts clears pending container-health durations for a
+// system. Called when monitoring pauses or the system goes down.
+func (am *AlertManager) CancelPendingContainerAlerts(systemID string) {
+	for _, alertData := range am.alertsCache.GetAlertsByName(systemID, containerAlertName) {
+		if err := am.clearPendingContainerAlert(alertData); err != nil {
+			am.hub.Logger().Error("Failed to clear pending container alert", "err", err)
+		}
+	}
+}
+
+// sendContainerHealthAlert updates the alert's triggered state and sends the
+// notification. When unhealthy is true, it embeds a log excerpt (prioritizing
+// error/fatal lines) for up to containerAlertMaxLogged of the affected containers.
+func (am *AlertManager) sendContainerHealthAlert(unhealthy bool, systemName string, alertData CachedAlertData, containers []containerAlertTarget, fetchLogs FetchContainerLogsFunc) error {
+	link := am.hub.MakeLink("system", alertData.SystemID)
+	linkText := "View " + systemName
+
+	if !unhealthy {
+		if err := am.setAlertTriggered(alertData, false); err != nil {
+			return err
+		}
+		title := fmt.Sprintf("%s containers are healthy ✅", systemName)
+		return am.SendAlert(AlertMessageData{
+			UserID:   alertData.UserID,
+			SystemID: alertData.SystemID,
+			Title:    title,
+			Message:  strings.TrimSuffix(title, " ✅"),
+			Link:     link,
+			LinkText: linkText,
+		})
+	}
+
+	names := make([]string, len(containers))
+	for i, c := range containers {
+		names[i] = c.name
+	}
+
+	var title string
+	if len(names) == 1 {
+		title = fmt.Sprintf("Unhealthy container %s on %s \U0001F534", names[0], systemName)
+	} else {
+		title = fmt.Sprintf("%d unhealthy containers on %s \U0001F534", len(names), systemName)
+	}
+
+	var body strings.Builder
+	fmt.Fprintf(&body, "Unhealthy: %s", strings.Join(names, ", "))
+	body.WriteString(am.buildContainerLogsSection(containers, fetchLogs))
+
+	message := body.String()
+	if len(message) > containerAlertMessageMaxChars {
+		message = message[:containerAlertMessageMaxChars] + "\n…(truncated)"
+	}
+
+	claimed, err := am.claimPendingContainerAlert(alertData)
+	if err != nil || !claimed {
+		return err
+	}
+
+	return am.SendAlert(AlertMessageData{
+		UserID:   alertData.UserID,
+		SystemID: alertData.SystemID,
+		Title:    title,
+		Message:  message,
+		Link:     link,
+		LinkText: linkText,
+	})
+}
+
+// buildContainerLogsSection attempts to fetch and format log excerpts for up to
+// containerAlertMaxLogged unhealthy containers, to append to an alert message.
+func (am *AlertManager) buildContainerLogsSection(containers []containerAlertTarget, fetchLogs FetchContainerLogsFunc) string {
+	if fetchLogs == nil {
+		return ""
+	}
+
+	var section strings.Builder
+	attempts := min(len(containers), containerAlertMaxLogged)
+	for _, c := range containers[:attempts] {
+		rawLogs, err := fetchLogs(c.id)
+		if err != nil {
+			am.hub.Logger().Warn("Failed to fetch container logs for alert", "container", c.name, "err", err)
+			continue
+		}
+		excerpt := buildContainerLogExcerpt(rawLogs)
+		if excerpt == "" {
+			continue
+		}
+		fmt.Fprintf(&section, "\n\n%s logs:\n```\n%s\n```", c.name, excerpt)
+	}
+
+	if len(containers) > containerAlertMaxLogged {
+		fmt.Fprintf(&section, "\n\n(+%d more unhealthy container(s), logs omitted)", len(containers)-containerAlertMaxLogged)
+	}
+
+	return section.String()
+}
+
+// buildContainerLogExcerpt filters raw container log output down to the lines
+// most likely to explain why the container is unhealthy: lines containing
+// "error" or "fatal" (case-insensitive) are preferred. If none match, the tail
+// of the raw output is used instead so the notification still carries context.
+func buildContainerLogExcerpt(raw string) string {
+	raw = strings.TrimSpace(raw)
+	if raw == "" {
+		return ""
+	}
+	lines := strings.Split(raw, "\n")
+
+	var matched []string
+	for _, line := range lines {
+		line = strings.TrimRight(line, "\r")
+		if line == "" {
+			continue
+		}
+		lower := strings.ToLower(line)
+		if strings.Contains(lower, "error") || strings.Contains(lower, "fatal") {
+			matched = append(matched, line)
+		}
+	}
+
+	selected := matched
+	if len(selected) == 0 {
+		start := max(0, len(lines)-containerLogFallbackLines)
+		selected = lines[start:]
+	} else if len(selected) > containerLogMaxLines {
+		selected = selected[len(selected)-containerLogMaxLines:]
+	}
+
+	excerpt := strings.TrimSpace(strings.Join(selected, "\n"))
+	if len(excerpt) > containerLogExcerptMaxChars {
+		excerpt = "…" + excerpt[len(excerpt)-containerLogExcerptMaxChars:]
+	}
+	return excerpt
+}

internal/alerts/alerts_container_test.go

@@ -0,0 +1,349 @@
+//go:build testing
+
+package alerts_test
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+	"testing/synctest"
+	"time"
+
+	"github.com/henrygd/beszel/internal/alerts"
+	"github.com/henrygd/beszel/internal/entities/container"
+	"github.com/henrygd/beszel/internal/entities/system"
+	beszelTests "github.com/henrygd/beszel/internal/tests"
+	"github.com/pocketbase/pocketbase/core"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type containerAlertTestFixture struct {
+	hub          *beszelTests.TestHub
+	am           *alerts.AlertManager
+	alertID      string
+	systemRecord *core.Record
+}
+
+func newContainerAlertTestFixture(t *testing.T, min int) *containerAlertTestFixture {
+	t.Helper()
+
+	hub, user := beszelTests.GetHubWithUser(t)
+
+	systems, err := beszelTests.CreateSystems(hub, 1, user.Id, "up")
+	require.NoError(t, err)
+	systemRecord := systems[0]
+
+	userSettings, err := hub.FindFirstRecordByFilter("user_settings", "user={:user}", map[string]any{"user": user.Id})
+	require.NoError(t, err)
+	userSettings.Set("settings", `{"emails":["test@example.com"],"webhooks":[]}`)
+	require.NoError(t, hub.Save(userSettings))
+
+	alertRecord, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{
+		"name":   "ContainerHealth",
+		"system": systemRecord.Id,
+		"user":   user.Id,
+		"min":    min,
+	})
+	require.NoError(t, err)
+	assert.False(t, alertRecord.GetBool("triggered"), "Alert should not be triggered initially")
+
+	return &containerAlertTestFixture{
+		hub:          hub,
+		am:           alerts.NewTestAlertManagerWithoutWorker(hub),
+		alertID:      alertRecord.Id,
+		systemRecord: systemRecord,
+	}
+}
+
+func (f *containerAlertTestFixture) cleanup() {
+	f.hub.Cleanup()
+}
+
+func (f *containerAlertTestFixture) submit(t *testing.T, containers []*container.Stats, fetchLogs alerts.FetchContainerLogsFunc) {
+	t.Helper()
+	data := &system.CombinedData{Containers: containers}
+	require.NoError(t, f.am.HandleContainerAlerts(f.systemRecord, data, fetchLogs))
+}
+
+func (f *containerAlertTestFixture) submitInvalid(t *testing.T) {
+	t.Helper()
+	require.NoError(t, f.am.HandleContainerAlerts(f.systemRecord, &system.CombinedData{}, nil))
+}
+
+func (f *containerAlertTestFixture) assertTriggered(t *testing.T, triggered bool, message string) {
+	t.Helper()
+	alertRecord, err := f.hub.FindRecordById("alerts", f.alertID)
+	require.NoError(t, err)
+	assert.Equal(t, triggered, alertRecord.GetBool("triggered"), message)
+}
+
+func (f *containerAlertTestFixture) assertPending(t *testing.T, pending bool) {
+	t.Helper()
+	alertRecord, err := f.hub.FindRecordById("alerts", f.alertID)
+	require.NoError(t, err)
+	assert.Equal(t, pending, !alertRecord.GetDateTime("pending_since").Time().IsZero())
+}
+
+func waitForContainerAlert(d time.Duration) {
+	time.Sleep(d)
+	synctest.Wait()
+}
+
+func healthyContainer(name string) *container.Stats {
+	return &container.Stats{Name: name, Id: "abc123def456", Health: container.DockerHealthHealthy}
+}
+
+func unhealthyContainer(name string) *container.Stats {
+	return &container.Stats{Name: name, Id: "abc123def456", Health: container.DockerHealthUnhealthy}
+}
+
+func TestContainerHealthAlertTriggersAndResolves(t *testing.T) {
+	fixture := newContainerAlertTestFixture(t, 1)
+	defer fixture.cleanup()
+
+	synctest.Test(t, func(t *testing.T) {
+		fixture.submit(t, []*container.Stats{unhealthyContainer("web")}, nil)
+
+		fixture.assertTriggered(t, true, "A one-minute alert should trigger on the first unhealthy update")
+		require.Equal(t, 1, fixture.hub.TestMailer.TotalSend(), "An email should have been sent")
+
+		msg := fixture.hub.TestMailer.LastMessage()
+		assert.Contains(t, msg.Subject, "web", "Subject should name the unhealthy container")
+		assert.Contains(t, strings.ToLower(msg.Subject), "unhealthy")
+
+		fixture.submit(t, []*container.Stats{unhealthyContainer("web")}, nil)
+		fixture.assertPending(t, false)
+
+		fixture.submitInvalid(t)
+		fixture.assertTriggered(t, true, "An invalid container snapshot should not resolve the alert")
+		assert.Equal(t, 1, fixture.hub.TestMailer.TotalSend(), "An invalid snapshot should not send a recovery")
+
+		fixture.submit(t, []*container.Stats{}, nil)
+		waitForContainerAlert(time.Second)
+
+		fixture.assertTriggered(t, false, "Alert should resolve once the container is healthy again")
+		assert.Equal(t, 2, fixture.hub.TestMailer.TotalSend(), "A second email should have been sent for the recovery")
+		assert.Contains(t, fixture.hub.TestMailer.LastMessage().Subject, " healthy")
+	})
+}
+
+func TestContainerHealthAlertInvalidSnapshotCancelsPending(t *testing.T) {
+	fixture := newContainerAlertTestFixture(t, 5)
+	defer fixture.cleanup()
+
+	synctest.Test(t, func(t *testing.T) {
+		fixture.submit(t, []*container.Stats{unhealthyContainer("db")}, nil)
+		fixture.assertPending(t, true)
+		waitForContainerAlert(time.Minute)
+		fixture.submitInvalid(t)
+		fixture.assertPending(t, false)
+		waitForContainerAlert(10 * time.Minute)
+		fixture.submit(t, []*container.Stats{unhealthyContainer("db")}, nil)
+
+		fixture.assertTriggered(t, false, "Stale unhealthy data should not trigger an alert")
+		fixture.assertPending(t, true)
+		assert.Equal(t, 0, fixture.hub.TestMailer.TotalSend())
+	})
+}
+
+func TestContainerHealthAlertSystemDownCancelsPending(t *testing.T) {
+	fixture := newContainerAlertTestFixture(t, 5)
+	defer fixture.cleanup()
+
+	// Use the hub's alert manager because the system-manager status hook invokes
+	// cancellation on that instance.
+	am := fixture.hub.GetAlertManager()
+	require.NoError(t, am.HandleContainerAlerts(
+		fixture.systemRecord,
+		&system.CombinedData{Containers: []*container.Stats{unhealthyContainer("db")}},
+		nil,
+	))
+	fixture.assertPending(t, true)
+
+	fixture.systemRecord.Set("status", "down")
+	require.NoError(t, fixture.hub.Save(fixture.systemRecord))
+
+	fixture.assertPending(t, false)
+}
+
+func TestContainerHealthAlertResolvesBeforeMinDelayCancelsPending(t *testing.T) {
+	fixture := newContainerAlertTestFixture(t, 5)
+	defer fixture.cleanup()
+
+	synctest.Test(t, func(t *testing.T) {
+		fixture.submit(t, []*container.Stats{unhealthyContainer("db")}, nil)
+		waitForContainerAlert(time.Minute)
+
+		fixture.assertTriggered(t, false, "Alert should not fire until the min delay elapses")
+		fixture.assertPending(t, true)
+		assert.Equal(t, 0, fixture.hub.TestMailer.TotalSend())
+
+		// container recovers before the 5 minute delay elapses
+		fixture.submit(t, []*container.Stats{healthyContainer("db")}, nil)
+		waitForContainerAlert(10 * time.Minute)
+		fixture.submit(t, []*container.Stats{healthyContainer("db")}, nil)
+
+		fixture.assertTriggered(t, false, "Alert should remain untriggered")
+		fixture.assertPending(t, false)
+		assert.Equal(t, 0, fixture.hub.TestMailer.TotalSend(), "No email should be sent for a container that recovered before the delay")
+	})
+}
+
+func TestContainerHealthAlertPreservesPendingDurationAcrossManagerRestart(t *testing.T) {
+	fixture := newContainerAlertTestFixture(t, 2)
+	defer fixture.cleanup()
+
+	synctest.Test(t, func(t *testing.T) {
+		fixture.submit(t, []*container.Stats{unhealthyContainer("db")}, nil)
+		waitForContainerAlert(30 * time.Second)
+
+		restarted := alerts.NewTestAlertManagerWithoutWorker(fixture.hub)
+		waitForContainerAlert(91 * time.Second)
+		require.NoError(t, restarted.HandleContainerAlerts(
+			fixture.systemRecord,
+			&system.CombinedData{Containers: []*container.Stats{unhealthyContainer("db")}},
+			nil,
+		))
+
+		fixture.assertTriggered(t, true, "Restart should preserve the original unhealthy start time")
+		fixture.assertPending(t, false)
+		assert.Equal(t, 1, fixture.hub.TestMailer.TotalSend())
+	})
+}
+
+func TestContainerHealthAlertClaimsPendingTimestampAtDatabasePrecision(t *testing.T) {
+	fixture := newContainerAlertTestFixture(t, 1)
+	defer fixture.cleanup()
+
+	alertRecord, err := fixture.hub.FindRecordById("alerts", fixture.alertID)
+	require.NoError(t, err)
+	// PocketBase persists dates to milliseconds, while record update hooks can
+	// retain the original sub-millisecond value in the in-memory alert cache.
+	alertRecord.Set("pending_since", time.Now().UTC().Add(-2*time.Minute).Truncate(time.Millisecond).Add(123*time.Nanosecond))
+	require.NoError(t, fixture.hub.Save(alertRecord))
+
+	fixture.submit(t, []*container.Stats{unhealthyContainer("db")}, nil)
+
+	fixture.assertTriggered(t, true, "Equivalent persisted and cached timestamps should claim the alert")
+	fixture.assertPending(t, false)
+	assert.Equal(t, 1, fixture.hub.TestMailer.TotalSend())
+}
+
+func TestContainerHealthAlertRecoveryWhileFetchingLogsCancelsDelivery(t *testing.T) {
+	fixture := newContainerAlertTestFixture(t, 1)
+	defer fixture.cleanup()
+
+	synctest.Test(t, func(t *testing.T) {
+		fetchLogs := func(containerID string) (string, error) {
+			fixture.submit(t, []*container.Stats{healthyContainer("api")}, nil)
+			return "FATAL stale failure", nil
+		}
+		fixture.submit(t, []*container.Stats{unhealthyContainer("api")}, fetchLogs)
+
+		fixture.assertTriggered(t, false, "Recovery should cancel delivery while logs are fetched")
+		fixture.assertPending(t, false)
+		assert.Equal(t, 0, fixture.hub.TestMailer.TotalSend())
+	})
+}
+
+func TestContainerHealthAlertIncludesLogExcerpt(t *testing.T) {
+	fixture := newContainerAlertTestFixture(t, 1)
+	defer fixture.cleanup()
+
+	rawLogs := strings.Join([]string{
+		"2026-08-16T10:00:00Z booting",
+		"2026-08-16T10:00:01Z ERROR could not reach upstream",
+		"2026-08-16T10:00:02Z FATAL giving up after 3 retries",
+	}, "\n")
+	fetchLogs := func(containerID string) (string, error) {
+		assert.Equal(t, "abc123def456", containerID)
+		return rawLogs, nil
+	}
+
+	synctest.Test(t, func(t *testing.T) {
+		fixture.submit(t, []*container.Stats{unhealthyContainer("api")}, fetchLogs)
+
+		fixture.assertTriggered(t, true, "Alert should be triggered")
+		require.Equal(t, 1, fixture.hub.TestMailer.TotalSend())
+
+		body := fixture.hub.TestMailer.LastMessage().Text
+		assert.Contains(t, body, "could not reach upstream")
+		assert.Contains(t, body, "giving up after 3 retries")
+		assert.NotContains(t, body, "booting", "non error/fatal lines should be dropped when matches exist")
+	})
+}
+
+func TestContainerHealthAlertSkipsLogsOnFetchError(t *testing.T) {
+	fixture := newContainerAlertTestFixture(t, 1)
+	defer fixture.cleanup()
+
+	fetchLogs := func(containerID string) (string, error) {
+		return "", fmt.Errorf("agent unreachable")
+	}
+
+	synctest.Test(t, func(t *testing.T) {
+		fixture.submit(t, []*container.Stats{unhealthyContainer("api")}, fetchLogs)
+
+		fixture.assertTriggered(t, true, "Alert should still be triggered even if logs can't be fetched")
+		require.Equal(t, 1, fixture.hub.TestMailer.TotalSend())
+	})
+}
+
+func TestContainerHealthAlertCapsLogFetchAttempts(t *testing.T) {
+	fixture := newContainerAlertTestFixture(t, 1)
+	defer fixture.cleanup()
+
+	containers := make([]*container.Stats, 100)
+	for i := range containers {
+		containers[i] = &container.Stats{
+			Name:   fmt.Sprintf("container-%d", i),
+			Id:     fmt.Sprintf("id-%d", i),
+			Health: container.DockerHealthUnhealthy,
+		}
+	}
+	attempts := 0
+	fetchLogs := func(containerID string) (string, error) {
+		attempts++
+		return "", fmt.Errorf("agent unreachable")
+	}
+
+	synctest.Test(t, func(t *testing.T) {
+		fixture.submit(t, containers, fetchLogs)
+
+		fixture.assertTriggered(t, true, "Alert should still fire when log retrieval fails")
+		assert.Equal(t, 2, attempts, "Log retrieval should attempt at most two containers")
+		require.Equal(t, 1, fixture.hub.TestMailer.TotalSend())
+	})
+}
+
+func TestBuildContainerLogExcerptPrefersErrorAndFatalLines(t *testing.T) {
+	raw := strings.Join([]string{
+		"2026-08-16T10:00:00Z starting up",
+		"2026-08-16T10:00:01Z listening on :8080",
+		"2026-08-16T10:00:02Z ERROR failed to connect to db",
+		"2026-08-16T10:00:03Z retrying connection",
+		"2026-08-16T10:00:04Z FATAL could not recover, exiting",
+	}, "\n")
+
+	excerpt := alerts.BuildContainerLogExcerpt(raw)
+	assert.Contains(t, excerpt, "failed to connect to db")
+	assert.Contains(t, excerpt, "could not recover, exiting")
+	assert.NotContains(t, excerpt, "starting up", "non-matching lines should be dropped when error/fatal lines exist")
+}
+
+func TestBuildContainerLogExcerptFallsBackToTailWhenNoMatches(t *testing.T) {
+	var lines []string
+	for i := range 20 {
+		lines = append(lines, fmt.Sprintf("line %d: all good here", i))
+	}
+	raw := strings.Join(lines, "\n")
+
+	excerpt := alerts.BuildContainerLogExcerpt(raw)
+	assert.Contains(t, excerpt, "line 19", "should keep the tail of the output")
+	assert.NotContains(t, excerpt, "line 0:", "should not keep the very start when falling back to a short tail")
+}
+
+func TestBuildContainerLogExcerptEmpty(t *testing.T) {
+	assert.Equal(t, "", alerts.BuildContainerLogExcerpt("   \n  \n"))
+}

internal/alerts/alerts_system.go

@@ -47,7 +47,7 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
 		return nil
 	}
 
-	alerts := am.alertsCache.GetAlertsExcludingNames(systemRecord.Id, "Status", alertNameSystemdFailed)
+	alerts := am.alertsCache.GetAlertsExcludingNames(systemRecord.Id, "Status", alertNameSystemdFailed, containerAlertName)
 	if len(alerts) == 0 {
 		return nil
 	}

internal/alerts/alerts_test_helpers.go

@@ -103,3 +103,8 @@ func (am *AlertManager) SetAlertTriggered(alert CachedAlertData, triggered bool)
 func IsInternalURL(rawURL string) (bool, error) {
 	return isInternalURL(rawURL)
 }
+
+// BuildContainerLogExcerpt exposes buildContainerLogExcerpt for testing.
+func BuildContainerLogExcerpt(raw string) string {
+	return buildContainerLogExcerpt(raw)
+}