feat(alerts): add container health alerts with log excerpt on notifications (#2225)

Add a new "ContainerHealth" alert type that fires when a Docker container's
health check reports unhealthy, and resolves when it recovers. This mirrors
the existing Status (up/down) alert pattern: an alert can be armed per system
and honors the "min minutes" delay before firing.

When the alert fires, the notification (email and any configured webhook,
including Discord via shoutrrr) includes a log excerpt fetched live from the
agent for up to 2 of the unhealthy containers, prioritizing lines containing
"error" or "fatal" (falling back to the log tail if none match), capped to
keep the message well under Discord's size limit.

---------

Co-authored-by: hank <hank@henrygd.me>
This commit is contained in:
Michał Mleczko
2026-09-02 18:46:23 +02:00
committed by GitHub
parent b1895247ba
commit 5969d36856
44 changed files with 1245 additions and 16 deletions

View File

@@ -729,6 +729,7 @@ func TestGetDockerStatsChecksDockerVersionAfterContainerList(t *testing.T) {
stats, err := dm.getDockerStats(defaultCacheTimeMs)
require.NoError(t, err)
require.NotNil(t, stats, "A successful empty snapshot must remain distinguishable from a collection failure")
assert.Empty(t, stats)
assert.True(t, dm.dockerVersionChecked)
assert.Equal(t, tt.expectedGood, dm.goodDockerVersion)
@@ -742,6 +743,7 @@ func TestGetDockerStatsChecksDockerVersionAfterContainerList(t *testing.T) {
stats, err = dm.getDockerStats(defaultCacheTimeMs)
require.NoError(t, err)
require.NotNil(t, stats, "A successful empty snapshot must remain distinguishable from a collection failure")
assert.Empty(t, stats)
assert.Equal(t, tt.expectedGood, dm.goodDockerVersion)
assert.Equal(t, tt.expectedPodman, dm.usingPodman)