Alert history updates

This commit is contained in:
henrygd
2025-07-21 20:07:52 -04:00
parent 9d7fb8ab80
commit 18d9258907
29 changed files with 1301 additions and 643 deletions

View File

@@ -93,10 +93,18 @@ func NewAlertManager(app hubLike) *AlertManager {
alertQueue: make(chan alertTask),
stopChan: make(chan struct{}),
}
am.bindEvents()
go am.startWorker()
return am
}
// Bind events to the alerts collection lifecycle
func (am *AlertManager) bindEvents() {
am.hub.OnRecordAfterUpdateSuccess("alerts").BindFunc(updateHistoryOnAlertUpdate)
am.hub.OnRecordAfterDeleteSuccess("alerts").BindFunc(resolveHistoryOnAlertDelete)
}
// SendAlert sends an alert to the user
func (am *AlertManager) SendAlert(data AlertMessageData) error {
// get user settings
record, err := am.hub.FindFirstRecordByFilter(

View File

@@ -7,90 +7,79 @@ import (
"github.com/pocketbase/pocketbase/core"
)
func (am *AlertManager) RecordAlertHistory(alert SystemAlertData) {
// Get alert, user, system, name, value
alertId := alert.alertRecord.Id
userId := ""
if errs := am.hub.ExpandRecord(alert.alertRecord, []string{"user"}, nil); len(errs) == 0 {
if user := alert.alertRecord.ExpandedOne("user"); user != nil {
userId = user.Id
}
}
systemId := alert.systemRecord.Id
name := alert.name
value := alert.val
now := time.Now().UTC()
if alert.triggered {
// Create new alerts_history record
collection, err := am.hub.FindCollectionByNameOrId("alerts_history")
if err == nil {
history := core.NewRecord(collection)
history.Set("alert", alertId)
history.Set("user", userId)
history.Set("system", systemId)
history.Set("name", name)
history.Set("value", value)
history.Set("state", "active")
history.Set("created_date", now)
history.Set("solved_date", nil)
_ = am.hub.Save(history)
}
} else {
// Find latest active alerts_history record for this alert and set to solved
record, err := am.hub.FindFirstRecordByFilter(
"alerts_history",
"alert={:alert} && state='active'",
dbx.Params{"alert": alertId},
)
if err == nil && record != nil {
record.Set("state", "solved")
record.Set("solved_date", now)
_ = am.hub.Save(record)
}
// On triggered alert record delete, set matching alert history record to resolved
func resolveHistoryOnAlertDelete(e *core.RecordEvent) error {
if !e.Record.GetBool("triggered") {
return e.Next()
}
_ = resolveAlertHistoryRecord(e.App, e.Record)
return e.Next()
}
// DeleteOldAlertHistory deletes alerts_history records older than the given retention duration
func (am *AlertManager) DeleteOldAlertHistory(retention time.Duration) {
now := time.Now().UTC()
cutoff := now.Add(-retention)
_, err := am.hub.DB().NewQuery(
"DELETE FROM alerts_history WHERE solved_date IS NOT NULL AND solved_date < {:cutoff}",
).Bind(dbx.Params{"cutoff": cutoff}).Execute()
// On alert record update, update alert history record
func updateHistoryOnAlertUpdate(e *core.RecordEvent) error {
original := e.Record.Original()
new := e.Record
originalTriggered := original.GetBool("triggered")
newTriggered := new.GetBool("triggered")
// no need to update alert history if triggered state has not changed
if originalTriggered == newTriggered {
return e.Next()
}
// if new state is triggered, create new alert history record
if newTriggered {
_, _ = createAlertHistoryRecord(e.App, new)
return e.Next()
}
// if new state is not triggered, check for matching alert history record and set it to resolved
_ = resolveAlertHistoryRecord(e.App, new)
return e.Next()
}
// resolveAlertHistoryRecord sets the resolved field to the current time
func resolveAlertHistoryRecord(app core.App, alertRecord *core.Record) error {
alertHistoryRecords, err := app.FindRecordsByFilter(
"alerts_history",
"alert_id={:alert_id} && resolved=null",
"-created",
1,
0,
dbx.Params{"alert_id": alertRecord.Id},
)
if err != nil {
am.hub.Logger().Error("failed to delete old alerts_history records", "error", err)
return err
}
}
// Helper to get retention duration from user settings
func getAlertHistoryRetention(settings map[string]interface{}) time.Duration {
retStr, _ := settings["alertHistoryRetention"].(string)
switch retStr {
case "1m":
return 30 * 24 * time.Hour
case "3m":
return 90 * 24 * time.Hour
case "6m":
return 180 * 24 * time.Hour
case "1y":
return 365 * 24 * time.Hour
default:
return 90 * 24 * time.Hour // default 3 months
if len(alertHistoryRecords) == 0 {
return nil
}
}
// CleanUpAllAlertHistory deletes old alerts_history records for each user based on their retention setting
func (am *AlertManager) CleanUpAllAlertHistory() {
records, err := am.hub.FindAllRecords("user_settings")
alertHistoryRecord := alertHistoryRecords[0] // there should be only one record
alertHistoryRecord.Set("resolved", time.Now().UTC())
err = app.Save(alertHistoryRecord)
if err != nil {
return
}
for _, record := range records {
var settings map[string]interface{}
if err := record.UnmarshalJSONField("settings", &settings); err != nil {
continue
}
am.DeleteOldAlertHistory(getAlertHistoryRetention(settings))
app.Logger().Error("Failed to resolve alert history", "err", err)
}
return err
}
// createAlertHistoryRecord creates a new alert history record
func createAlertHistoryRecord(app core.App, alertRecord *core.Record) (alertHistoryRecord *core.Record, err error) {
alertHistoryCollection, err := app.FindCachedCollectionByNameOrId("alerts_history")
if err != nil {
return nil, err
}
alertHistoryRecord = core.NewRecord(alertHistoryCollection)
alertHistoryRecord.Set("alert_id", alertRecord.Id)
alertHistoryRecord.Set("user", alertRecord.GetString("user"))
alertHistoryRecord.Set("system", alertRecord.GetString("system"))
alertHistoryRecord.Set("name", alertRecord.GetString("name"))
alertHistoryRecord.Set("value", alertRecord.GetFloat("value"))
err = app.Save(alertHistoryRecord)
if err != nil {
app.Logger().Error("Failed to save alert history", "err", err)
}
return alertHistoryRecord, err
}

View File

@@ -136,6 +136,14 @@ func (am *AlertManager) handleSystemUp(systemName string, alertRecords []*core.R
// sendStatusAlert sends a status alert ("up" or "down") to the users associated with the alert records.
func (am *AlertManager) sendStatusAlert(alertStatus string, systemName string, alertRecord *core.Record) error {
switch alertStatus {
case "up":
alertRecord.Set("triggered", false)
case "down":
alertRecord.Set("triggered", true)
}
am.hub.Save(alertRecord)
var emoji string
if alertStatus == "up" {
emoji = "\u2705" // Green checkmark emoji
@@ -146,16 +154,16 @@ func (am *AlertManager) sendStatusAlert(alertStatus string, systemName string, a
title := fmt.Sprintf("Connection to %s is %s %v", systemName, alertStatus, emoji)
message := strings.TrimSuffix(title, emoji)
if errs := am.hub.ExpandRecord(alertRecord, []string{"user"}, nil); len(errs) > 0 {
return errs["user"]
}
user := alertRecord.ExpandedOne("user")
if user == nil {
return nil
}
// if errs := am.hub.ExpandRecord(alertRecord, []string{"user"}, nil); len(errs) > 0 {
// return errs["user"]
// }
// user := alertRecord.ExpandedOne("user")
// if user == nil {
// return nil
// }
return am.SendAlert(AlertMessageData{
UserID: user.Id,
UserID: alertRecord.GetString("user"),
Title: title,
Message: message,
Link: am.hub.MakeLink("system", systemName),

View File

@@ -15,7 +15,7 @@ import (
func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *system.CombinedData) error {
alertRecords, err := am.hub.FindAllRecords("alerts",
dbx.NewExp("system={:system}", dbx.Params{"system": systemRecord.Id}),
dbx.NewExp("system={:system} AND name!='Status'", dbx.Params{"system": systemRecord.Id}),
)
if err != nil || len(alertRecords) == 0 {
// log.Println("no alerts found for system")
@@ -293,10 +293,6 @@ func (am *AlertManager) sendSystemAlert(alert SystemAlertData) {
// app.Logger().Error("failed to save alert record", "err", err)
return
}
// Create Alert History
am.RecordAlertHistory(alert)
// expand the user relation and send the alert
if errs := am.hub.ExpandRecord(alert.alertRecord, []string{"user"}, nil); len(errs) > 0 {
// app.Logger().Error("failed to expand user relation", "errs", errs)