mirror of
https://github.com/henrygd/beszel.git
synced 2026-09-21 08:57:48 +02:00
5969d36856
Add a new "ContainerHealth" alert type that fires when a Docker container's health check reports unhealthy, and resolves when it recovers. This mirrors the existing Status (up/down) alert pattern: an alert can be armed per system and honors the "min minutes" delay before firing. When the alert fires, the notification (email and any configured webhook, including Discord via shoutrrr) includes a log excerpt fetched live from the agent for up to 2 of the unhealthy containers, prioritizing lines containing "error" or "fatal" (falling back to the log tail if none match), capped to keep the message well under Discord's size limit. --------- Co-authored-by: hank <hank@henrygd.me>
350 lines
12 KiB
Go
350 lines
12 KiB
Go
//go:build testing
|
|
|
|
package alerts_test
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
"testing/synctest"
|
|
"time"
|
|
|
|
"github.com/henrygd/beszel/internal/alerts"
|
|
"github.com/henrygd/beszel/internal/entities/container"
|
|
"github.com/henrygd/beszel/internal/entities/system"
|
|
beszelTests "github.com/henrygd/beszel/internal/tests"
|
|
"github.com/pocketbase/pocketbase/core"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
type containerAlertTestFixture struct {
|
|
hub *beszelTests.TestHub
|
|
am *alerts.AlertManager
|
|
alertID string
|
|
systemRecord *core.Record
|
|
}
|
|
|
|
func newContainerAlertTestFixture(t *testing.T, min int) *containerAlertTestFixture {
|
|
t.Helper()
|
|
|
|
hub, user := beszelTests.GetHubWithUser(t)
|
|
|
|
systems, err := beszelTests.CreateSystems(hub, 1, user.Id, "up")
|
|
require.NoError(t, err)
|
|
systemRecord := systems[0]
|
|
|
|
userSettings, err := hub.FindFirstRecordByFilter("user_settings", "user={:user}", map[string]any{"user": user.Id})
|
|
require.NoError(t, err)
|
|
userSettings.Set("settings", `{"emails":["test@example.com"],"webhooks":[]}`)
|
|
require.NoError(t, hub.Save(userSettings))
|
|
|
|
alertRecord, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{
|
|
"name": "ContainerHealth",
|
|
"system": systemRecord.Id,
|
|
"user": user.Id,
|
|
"min": min,
|
|
})
|
|
require.NoError(t, err)
|
|
assert.False(t, alertRecord.GetBool("triggered"), "Alert should not be triggered initially")
|
|
|
|
return &containerAlertTestFixture{
|
|
hub: hub,
|
|
am: alerts.NewTestAlertManagerWithoutWorker(hub),
|
|
alertID: alertRecord.Id,
|
|
systemRecord: systemRecord,
|
|
}
|
|
}
|
|
|
|
func (f *containerAlertTestFixture) cleanup() {
|
|
f.hub.Cleanup()
|
|
}
|
|
|
|
func (f *containerAlertTestFixture) submit(t *testing.T, containers []*container.Stats, fetchLogs alerts.FetchContainerLogsFunc) {
|
|
t.Helper()
|
|
data := &system.CombinedData{Containers: containers}
|
|
require.NoError(t, f.am.HandleContainerAlerts(f.systemRecord, data, fetchLogs))
|
|
}
|
|
|
|
func (f *containerAlertTestFixture) submitInvalid(t *testing.T) {
|
|
t.Helper()
|
|
require.NoError(t, f.am.HandleContainerAlerts(f.systemRecord, &system.CombinedData{}, nil))
|
|
}
|
|
|
|
func (f *containerAlertTestFixture) assertTriggered(t *testing.T, triggered bool, message string) {
|
|
t.Helper()
|
|
alertRecord, err := f.hub.FindRecordById("alerts", f.alertID)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, triggered, alertRecord.GetBool("triggered"), message)
|
|
}
|
|
|
|
func (f *containerAlertTestFixture) assertPending(t *testing.T, pending bool) {
|
|
t.Helper()
|
|
alertRecord, err := f.hub.FindRecordById("alerts", f.alertID)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, pending, !alertRecord.GetDateTime("pending_since").Time().IsZero())
|
|
}
|
|
|
|
func waitForContainerAlert(d time.Duration) {
|
|
time.Sleep(d)
|
|
synctest.Wait()
|
|
}
|
|
|
|
func healthyContainer(name string) *container.Stats {
|
|
return &container.Stats{Name: name, Id: "abc123def456", Health: container.DockerHealthHealthy}
|
|
}
|
|
|
|
func unhealthyContainer(name string) *container.Stats {
|
|
return &container.Stats{Name: name, Id: "abc123def456", Health: container.DockerHealthUnhealthy}
|
|
}
|
|
|
|
func TestContainerHealthAlertTriggersAndResolves(t *testing.T) {
|
|
fixture := newContainerAlertTestFixture(t, 1)
|
|
defer fixture.cleanup()
|
|
|
|
synctest.Test(t, func(t *testing.T) {
|
|
fixture.submit(t, []*container.Stats{unhealthyContainer("web")}, nil)
|
|
|
|
fixture.assertTriggered(t, true, "A one-minute alert should trigger on the first unhealthy update")
|
|
require.Equal(t, 1, fixture.hub.TestMailer.TotalSend(), "An email should have been sent")
|
|
|
|
msg := fixture.hub.TestMailer.LastMessage()
|
|
assert.Contains(t, msg.Subject, "web", "Subject should name the unhealthy container")
|
|
assert.Contains(t, strings.ToLower(msg.Subject), "unhealthy")
|
|
|
|
fixture.submit(t, []*container.Stats{unhealthyContainer("web")}, nil)
|
|
fixture.assertPending(t, false)
|
|
|
|
fixture.submitInvalid(t)
|
|
fixture.assertTriggered(t, true, "An invalid container snapshot should not resolve the alert")
|
|
assert.Equal(t, 1, fixture.hub.TestMailer.TotalSend(), "An invalid snapshot should not send a recovery")
|
|
|
|
fixture.submit(t, []*container.Stats{}, nil)
|
|
waitForContainerAlert(time.Second)
|
|
|
|
fixture.assertTriggered(t, false, "Alert should resolve once the container is healthy again")
|
|
assert.Equal(t, 2, fixture.hub.TestMailer.TotalSend(), "A second email should have been sent for the recovery")
|
|
assert.Contains(t, fixture.hub.TestMailer.LastMessage().Subject, " healthy")
|
|
})
|
|
}
|
|
|
|
func TestContainerHealthAlertInvalidSnapshotCancelsPending(t *testing.T) {
|
|
fixture := newContainerAlertTestFixture(t, 5)
|
|
defer fixture.cleanup()
|
|
|
|
synctest.Test(t, func(t *testing.T) {
|
|
fixture.submit(t, []*container.Stats{unhealthyContainer("db")}, nil)
|
|
fixture.assertPending(t, true)
|
|
waitForContainerAlert(time.Minute)
|
|
fixture.submitInvalid(t)
|
|
fixture.assertPending(t, false)
|
|
waitForContainerAlert(10 * time.Minute)
|
|
fixture.submit(t, []*container.Stats{unhealthyContainer("db")}, nil)
|
|
|
|
fixture.assertTriggered(t, false, "Stale unhealthy data should not trigger an alert")
|
|
fixture.assertPending(t, true)
|
|
assert.Equal(t, 0, fixture.hub.TestMailer.TotalSend())
|
|
})
|
|
}
|
|
|
|
func TestContainerHealthAlertSystemDownCancelsPending(t *testing.T) {
|
|
fixture := newContainerAlertTestFixture(t, 5)
|
|
defer fixture.cleanup()
|
|
|
|
// Use the hub's alert manager because the system-manager status hook invokes
|
|
// cancellation on that instance.
|
|
am := fixture.hub.GetAlertManager()
|
|
require.NoError(t, am.HandleContainerAlerts(
|
|
fixture.systemRecord,
|
|
&system.CombinedData{Containers: []*container.Stats{unhealthyContainer("db")}},
|
|
nil,
|
|
))
|
|
fixture.assertPending(t, true)
|
|
|
|
fixture.systemRecord.Set("status", "down")
|
|
require.NoError(t, fixture.hub.Save(fixture.systemRecord))
|
|
|
|
fixture.assertPending(t, false)
|
|
}
|
|
|
|
func TestContainerHealthAlertResolvesBeforeMinDelayCancelsPending(t *testing.T) {
|
|
fixture := newContainerAlertTestFixture(t, 5)
|
|
defer fixture.cleanup()
|
|
|
|
synctest.Test(t, func(t *testing.T) {
|
|
fixture.submit(t, []*container.Stats{unhealthyContainer("db")}, nil)
|
|
waitForContainerAlert(time.Minute)
|
|
|
|
fixture.assertTriggered(t, false, "Alert should not fire until the min delay elapses")
|
|
fixture.assertPending(t, true)
|
|
assert.Equal(t, 0, fixture.hub.TestMailer.TotalSend())
|
|
|
|
// container recovers before the 5 minute delay elapses
|
|
fixture.submit(t, []*container.Stats{healthyContainer("db")}, nil)
|
|
waitForContainerAlert(10 * time.Minute)
|
|
fixture.submit(t, []*container.Stats{healthyContainer("db")}, nil)
|
|
|
|
fixture.assertTriggered(t, false, "Alert should remain untriggered")
|
|
fixture.assertPending(t, false)
|
|
assert.Equal(t, 0, fixture.hub.TestMailer.TotalSend(), "No email should be sent for a container that recovered before the delay")
|
|
})
|
|
}
|
|
|
|
func TestContainerHealthAlertPreservesPendingDurationAcrossManagerRestart(t *testing.T) {
|
|
fixture := newContainerAlertTestFixture(t, 2)
|
|
defer fixture.cleanup()
|
|
|
|
synctest.Test(t, func(t *testing.T) {
|
|
fixture.submit(t, []*container.Stats{unhealthyContainer("db")}, nil)
|
|
waitForContainerAlert(30 * time.Second)
|
|
|
|
restarted := alerts.NewTestAlertManagerWithoutWorker(fixture.hub)
|
|
waitForContainerAlert(91 * time.Second)
|
|
require.NoError(t, restarted.HandleContainerAlerts(
|
|
fixture.systemRecord,
|
|
&system.CombinedData{Containers: []*container.Stats{unhealthyContainer("db")}},
|
|
nil,
|
|
))
|
|
|
|
fixture.assertTriggered(t, true, "Restart should preserve the original unhealthy start time")
|
|
fixture.assertPending(t, false)
|
|
assert.Equal(t, 1, fixture.hub.TestMailer.TotalSend())
|
|
})
|
|
}
|
|
|
|
func TestContainerHealthAlertClaimsPendingTimestampAtDatabasePrecision(t *testing.T) {
|
|
fixture := newContainerAlertTestFixture(t, 1)
|
|
defer fixture.cleanup()
|
|
|
|
alertRecord, err := fixture.hub.FindRecordById("alerts", fixture.alertID)
|
|
require.NoError(t, err)
|
|
// PocketBase persists dates to milliseconds, while record update hooks can
|
|
// retain the original sub-millisecond value in the in-memory alert cache.
|
|
alertRecord.Set("pending_since", time.Now().UTC().Add(-2*time.Minute).Truncate(time.Millisecond).Add(123*time.Nanosecond))
|
|
require.NoError(t, fixture.hub.Save(alertRecord))
|
|
|
|
fixture.submit(t, []*container.Stats{unhealthyContainer("db")}, nil)
|
|
|
|
fixture.assertTriggered(t, true, "Equivalent persisted and cached timestamps should claim the alert")
|
|
fixture.assertPending(t, false)
|
|
assert.Equal(t, 1, fixture.hub.TestMailer.TotalSend())
|
|
}
|
|
|
|
func TestContainerHealthAlertRecoveryWhileFetchingLogsCancelsDelivery(t *testing.T) {
|
|
fixture := newContainerAlertTestFixture(t, 1)
|
|
defer fixture.cleanup()
|
|
|
|
synctest.Test(t, func(t *testing.T) {
|
|
fetchLogs := func(containerID string) (string, error) {
|
|
fixture.submit(t, []*container.Stats{healthyContainer("api")}, nil)
|
|
return "FATAL stale failure", nil
|
|
}
|
|
fixture.submit(t, []*container.Stats{unhealthyContainer("api")}, fetchLogs)
|
|
|
|
fixture.assertTriggered(t, false, "Recovery should cancel delivery while logs are fetched")
|
|
fixture.assertPending(t, false)
|
|
assert.Equal(t, 0, fixture.hub.TestMailer.TotalSend())
|
|
})
|
|
}
|
|
|
|
func TestContainerHealthAlertIncludesLogExcerpt(t *testing.T) {
|
|
fixture := newContainerAlertTestFixture(t, 1)
|
|
defer fixture.cleanup()
|
|
|
|
rawLogs := strings.Join([]string{
|
|
"2026-08-16T10:00:00Z booting",
|
|
"2026-08-16T10:00:01Z ERROR could not reach upstream",
|
|
"2026-08-16T10:00:02Z FATAL giving up after 3 retries",
|
|
}, "\n")
|
|
fetchLogs := func(containerID string) (string, error) {
|
|
assert.Equal(t, "abc123def456", containerID)
|
|
return rawLogs, nil
|
|
}
|
|
|
|
synctest.Test(t, func(t *testing.T) {
|
|
fixture.submit(t, []*container.Stats{unhealthyContainer("api")}, fetchLogs)
|
|
|
|
fixture.assertTriggered(t, true, "Alert should be triggered")
|
|
require.Equal(t, 1, fixture.hub.TestMailer.TotalSend())
|
|
|
|
body := fixture.hub.TestMailer.LastMessage().Text
|
|
assert.Contains(t, body, "could not reach upstream")
|
|
assert.Contains(t, body, "giving up after 3 retries")
|
|
assert.NotContains(t, body, "booting", "non error/fatal lines should be dropped when matches exist")
|
|
})
|
|
}
|
|
|
|
func TestContainerHealthAlertSkipsLogsOnFetchError(t *testing.T) {
|
|
fixture := newContainerAlertTestFixture(t, 1)
|
|
defer fixture.cleanup()
|
|
|
|
fetchLogs := func(containerID string) (string, error) {
|
|
return "", fmt.Errorf("agent unreachable")
|
|
}
|
|
|
|
synctest.Test(t, func(t *testing.T) {
|
|
fixture.submit(t, []*container.Stats{unhealthyContainer("api")}, fetchLogs)
|
|
|
|
fixture.assertTriggered(t, true, "Alert should still be triggered even if logs can't be fetched")
|
|
require.Equal(t, 1, fixture.hub.TestMailer.TotalSend())
|
|
})
|
|
}
|
|
|
|
func TestContainerHealthAlertCapsLogFetchAttempts(t *testing.T) {
|
|
fixture := newContainerAlertTestFixture(t, 1)
|
|
defer fixture.cleanup()
|
|
|
|
containers := make([]*container.Stats, 100)
|
|
for i := range containers {
|
|
containers[i] = &container.Stats{
|
|
Name: fmt.Sprintf("container-%d", i),
|
|
Id: fmt.Sprintf("id-%d", i),
|
|
Health: container.DockerHealthUnhealthy,
|
|
}
|
|
}
|
|
attempts := 0
|
|
fetchLogs := func(containerID string) (string, error) {
|
|
attempts++
|
|
return "", fmt.Errorf("agent unreachable")
|
|
}
|
|
|
|
synctest.Test(t, func(t *testing.T) {
|
|
fixture.submit(t, containers, fetchLogs)
|
|
|
|
fixture.assertTriggered(t, true, "Alert should still fire when log retrieval fails")
|
|
assert.Equal(t, 2, attempts, "Log retrieval should attempt at most two containers")
|
|
require.Equal(t, 1, fixture.hub.TestMailer.TotalSend())
|
|
})
|
|
}
|
|
|
|
func TestBuildContainerLogExcerptPrefersErrorAndFatalLines(t *testing.T) {
|
|
raw := strings.Join([]string{
|
|
"2026-08-16T10:00:00Z starting up",
|
|
"2026-08-16T10:00:01Z listening on :8080",
|
|
"2026-08-16T10:00:02Z ERROR failed to connect to db",
|
|
"2026-08-16T10:00:03Z retrying connection",
|
|
"2026-08-16T10:00:04Z FATAL could not recover, exiting",
|
|
}, "\n")
|
|
|
|
excerpt := alerts.BuildContainerLogExcerpt(raw)
|
|
assert.Contains(t, excerpt, "failed to connect to db")
|
|
assert.Contains(t, excerpt, "could not recover, exiting")
|
|
assert.NotContains(t, excerpt, "starting up", "non-matching lines should be dropped when error/fatal lines exist")
|
|
}
|
|
|
|
func TestBuildContainerLogExcerptFallsBackToTailWhenNoMatches(t *testing.T) {
|
|
var lines []string
|
|
for i := range 20 {
|
|
lines = append(lines, fmt.Sprintf("line %d: all good here", i))
|
|
}
|
|
raw := strings.Join(lines, "\n")
|
|
|
|
excerpt := alerts.BuildContainerLogExcerpt(raw)
|
|
assert.Contains(t, excerpt, "line 19", "should keep the tail of the output")
|
|
assert.NotContains(t, excerpt, "line 0:", "should not keep the very start when falling back to a short tail")
|
|
}
|
|
|
|
func TestBuildContainerLogExcerptEmpty(t *testing.T) {
|
|
assert.Equal(t, "", alerts.BuildContainerLogExcerpt(" \n \n"))
|
|
}
|