jnd/beszel-ipv6
1
0
Fork 0
mirror of https://github.com/henrygd/beszel.git synced 2026-09-21 17:07:47 +02:00
Code Issues Packages Projects Releases 2 Wiki Activity

feat(alerts): add container health alerts with log excerpt on notifications (#2225)

Browse Source 
Add a new "ContainerHealth" alert type that fires when a Docker container's
health check reports unhealthy, and resolves when it recovers. This mirrors
the existing Status (up/down) alert pattern: an alert can be armed per system
and honors the "min minutes" delay before firing.

When the alert fires, the notification (email and any configured webhook,
including Discord via shoutrrr) includes a log excerpt fetched live from the
agent for up to 2 of the unhealthy containers, prioritizing lines containing
"error" or "fatal" (falling back to the log tail if none match), capped to
keep the message well under Discord's size limit.

---------

Co-authored-by: hank <hank@henrygd.me>
This commit is contained in:
Michał Mleczko
2026-09-02 18:46:23 +02:00
committed by GitHub
parent b1895247ba
commit 5969d36856
44 changed files with 1245 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
internal/hub/systems/system_manager.go
View File

@@ -58,7 +58,9 @@ type hubLike interface {
GetSSHKey(dataDir string) (ssh.Signer, error)
HandleSystemAlerts(systemRecord *core.Record, data *system.CombinedData) error
HandleStatusAlerts(status string, systemRecord *core.Record) error
HandleContainerAlerts(systemRecord *core.Record, data *system.CombinedData, fetchLogs func(containerID string) (string, error)) error
CancelPendingStatusAlerts(systemID string)
CancelPendingContainerAlerts(systemID string)
}
// NewSystemManager creates a new SystemManager instance with the provided hub.
@@ -189,7 +191,7 @@ func (sm *SystemManager) onRecordUpdate(e *core.RecordEvent) error {
// - paused: Closes SSH connection and deactivates alerts
// - pending: Starts monitoring (reuses WebSocket if available)
// - up: Triggers system alerts
// - down: Triggers status change alerts
// - down: Cancels pending container alerts and triggers status change alerts
func (sm *SystemManager) onRecordAfterUpdateSuccess(e *core.RecordEvent) error {
newStatus := e.Record.GetString("status")
prevStatus := pending
@@ -207,6 +209,7 @@ func (sm *SystemManager) onRecordAfterUpdateSuccess(e *core.RecordEvent) error {
}
_ = deactivateAlerts(e.App, e.Record.Id)
sm.hub.CancelPendingStatusAlerts(e.Record.Id)
sm.hub.CancelPendingContainerAlerts(e.Record.Id)
return e.Next()
case pending:
// Resume monitoring, preferring existing WebSocket connection
@@ -220,6 +223,10 @@ func (sm *SystemManager) onRecordAfterUpdateSuccess(e *core.RecordEvent) error {
}
_ = deactivateAlerts(e.App, e.Record.Id)
return e.Next()
case down:
// Docker state is unknown while the system is unreachable. Do not let a
// delayed container-health alert fire from the last received snapshot.
sm.hub.CancelPendingContainerAlerts(e.Record.Id)
}
// Handle systems not in manager
@@ -232,6 +239,9 @@ func (sm *SystemManager) onRecordAfterUpdateSuccess(e *core.RecordEvent) error {
if err := sm.hub.HandleSystemAlerts(e.Record, system.data); err != nil {
e.App.Logger().Error("Error handling system alerts", "err", err)
}
if err := sm.hub.HandleContainerAlerts(e.Record, system.data, system.FetchContainerLogsFromAgent); err != nil {
e.App.Logger().Error("Error handling container alerts", "err", err)
}
}
// Trigger status change alerts for up/down transitions

6
internal/hub/systems/system_smart_test.go
View File

@@ -29,7 +29,11 @@ func (stubHub) HandleSystemAlerts(systemRecord *core.Record, data *esystem.Combi
return nil
}
func (stubHub) HandleStatusAlerts(status string, systemRecord *core.Record) error { return nil }
func (stubHub) CancelPendingStatusAlerts(systemID string) {}
func (stubHub) HandleContainerAlerts(systemRecord *core.Record, data *esystem.CombinedData, fetchLogs func(containerID string) (string, error)) error {
return nil
}
func (stubHub) CancelPendingStatusAlerts(systemID string) {}
func (stubHub) CancelPendingContainerAlerts(systemID string) {}
// newTestSystemWithHub creates a System backed by a real (temp) database, along
// with a matching "systems" record, for tests that need to exercise DB reads/writes.