Compare commits

..

62 Commits

Author SHA1 Message Date
henrygd
d80a2f49f9 fix(ui): hide stray 0 in info bar for paused systems 2026-09-24 19:56:23 -04:00
spatiumstas
21b648a005 fix(site): prevent scrollbar layout shifts (#2406) 2026-09-24 19:25:56 -04:00
henrygd
151423ac63 fix(ui): size monitors system column by systems in the table
Base the network monitors system column width on the longest name among
systems that have monitors in the table, rather than all systems.
2026-09-24 19:13:48 -04:00
Sven van Ginkel
f7528a0208 feat: Add option to define which DNS server a DNS monitor queries (#2389) 2026-09-24 19:01:10 -04:00
henrygd
a20a7d2edc fix(ui): correct navbar users link 2026-09-24 18:21:17 -04:00
henrygd
f2adb9cf94 fix(ui): correct navbar users link 2026-09-24 18:18:00 -04:00
Sven van Ginkel
4d10ea2e03 feat: add SYNC_SYSTEM_NAMES env var to sync system names with hostname (#1917) 2026-09-24 17:45:59 -04:00
Ludwig J. Marx
b5ef015451 fix(agent): handle 32-bit wrap of disk I/O time counters (#2407) 2026-09-24 17:39:53 -04:00
Miłosz Kolber
6141b15f03 fix(agent): SKIP_GPU excludes GPU hwmon from temperatures and fans (#2313) 2026-09-24 13:59:52 -04:00
henrygd
c21412f45d i18n: update strings 2026-09-24 12:29:12 -04:00
spatiumstas
367d2f39da fix(i18n): translate system actions as complete phrases (#2403) 2026-09-24 12:23:42 -04:00
hank
eabd9a950a New Crowdin updates (#2384) 2026-09-24 12:21:49 -04:00
Sven van Ginkel
0be9882b34 feat(ui): simplify Services column in systems table (#2399)
Co-authored-by: hank <hank@henrygd.me>
2026-09-24 12:05:26 -04:00
Mario Mohar
e4b84b72ab fix(site): measure the longest string width instead of estimating it (#2411)
Co-authored-by: henrygd <hank@henrygd.me>
2026-09-24 11:32:21 -04:00
Mario Mohar
fc33e62736 fix(hub): restart beszel-hub.service after update (#2410)
restartService only looked for beszel.service, but install-hub.sh installs
the unit as beszel-hub.service, so the restart was skipped on a standard
install and the hub kept running the old binary.

Try the installer's name first and keep beszel.service as a fallback for
hand written units. Same for the OpenRC branch.
2026-09-24 10:00:18 -04:00
Sven van Ginkel
a99fe5e997 feat: add TLS certificate expiry check to HTTPS network monitors (#2401)
Co-authored-by: henrygd <hank@henrygd.me>
2026-09-23 17:07:29 -04:00
spatiumstas
0870716052 fix(i18n): extract bulk monitor format hint for translation (#2362) 2026-09-23 15:19:30 -04:00
Sven van Ginkel
b1270e341c feat(ui): Add view settings to network monitors table (#2396) 2026-09-23 13:47:42 -04:00
Adrien Boullé
9042a8c5c8 fix(site): handle missing default chart time (#2397)
Co-authored-by: henrygd <hank@henrygd.me>
2026-09-23 12:29:31 -04:00
henrygd
8bf6917fe0 fix: prevent WebSocket reconnect loops on slow agent collections (#2294)
The hub waited only 5s (the request manager default) for stats over
WebSocket and closed the connection on any error. On hosts where
`zpool list` stalls (seen on Proxmox, up to its 10s timeout), collection
exceeded that limit, so the hub sent a close (code 1000) and the agent
reconnected. The refresh ran every other cycle, which caused a
disconnect roughly every 2 minutes.

Hub:
- Wait up to 30s for WebSocket stats responses.
- Keep the connection open when a stats request times out; only close
  it (and fall back to SSH) for other errors.

Agent:
- After the first collection, refresh `zpool list` pool stats and
  `zfs list` dataset usage in the background and serve cached values
  meanwhile, so a hung utility cannot delay the stats response.
2026-09-23 11:21:51 -04:00
spatiumstas
2d5ea3fa08 fix: preserve status alerts through pending recovery (#2375) 2026-09-22 19:14:54 -04:00
Sven van Ginkel
627d364071 fix(ui): disable Add Monitor button when no eligible systems exist (#2392) 2026-09-22 19:04:48 -04:00
Sven van Ginkel
8047f005d4 fix(ui): preserve paused systems in network monitors table (#2391) 2026-09-22 18:40:46 -04:00
Sven van Ginkel
4a4610bbc3 fix(ui): reflect target response status in network monitor status dot (#2390)
Co-authored-by: henrygd <hank@henrygd.me>
2026-09-22 18:12:19 -04:00
henrygd
1aaabfc255 fix(agent): check all image repository digests (#2393) 2026-09-22 18:00:13 -04:00
henrygd
97ea3c16cb add custom user agent to HTTP monitors 2026-09-21 19:53:53 -04:00
Sven van Ginkel
c9de35fad2 feat: change network packet loss to a red line (#2377) 2026-09-21 14:49:11 -04:00
henrygd
a5f216f425 exit 0 without key on Windows to avoid Winget Validation-Executable-Error (#2376, #2247) 2026-09-21 11:27:37 -04:00
henrygd
cbe4824ac3 add env var to disable container image update checks (#2371) 2026-09-21 10:49:53 -04:00
henrygd
2c69197d2d fix(install): improve handling of openwrt user account (#2370)
- Allocate unused UID/GIDs instead of using 999
- Detect existing ID collisions and repair missing shadow entries
- Support userdel and deluser during uninstall
- Fix Docker group membership handling
2026-09-20 18:04:07 -04:00
henrygd
97e6f64bdc fix(ui): swap usage chart shown twice in tabs view (#2356) 2026-09-19 13:27:00 -04:00
henrygd
4a5915b141 add SMART to readme features 2026-09-19 12:11:10 -04:00
henrygd
e68372dce4 update readme features 2026-09-19 12:09:26 -04:00
henrygd
c52f3acb94 fix(docker): enable Debian contrib for ZFS utilities in slim NVIDIA image 2026-09-19 12:03:05 -04:00
hank
c09eb8c6df chore(helm): update app version to 0.20.0 (#2355)
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
2026-09-19 11:39:13 -04:00
henrygd
a0dc19eacf fix(ui): sort containers alphabetically within systems 2026-09-19 11:18:17 -04:00
henrygd
912bc50874 docker: update alpine version and remove unnecessary GOGC 2026-09-19 00:19:02 -04:00
henrygd
c54dbfba7c test: wait for hub setup in universal token integration test 2026-09-19 00:14:19 -04:00
henrygd
dd3f7d58b5 test: use virtual time to eliminate MQTT notification timeout waits 2026-09-19 00:10:02 -04:00
henrygd
0509053a69 test: wait for hub system status in agent WebSocket integration test 2026-09-19 00:02:29 -04:00
henrygd
187dc886a9 fix: resync network monitors after SSH reconnect and retry failed syncs 2026-09-18 23:49:54 -04:00
henrygd
2784460621 fix(agent): avoid RAID health warnings during healthy scrubs (#2109) 2026-09-18 21:05:49 -04:00
henrygd
b0bc727411 update migration files for 0.20.0 2026-09-18 18:51:42 -04:00
henrygd
6937453282 chore: update changelog 2026-09-18 18:46:45 -04:00
henrygd
97de1471d7 fix: prevent variant Docker images from overwriting latest tags 2026-09-18 18:38:18 -04:00
hank
bd7e359dcd New Crowdin updates (#2352) 2026-09-18 18:28:28 -04:00
henrygd
d2352e8882 upgrade go packages 2026-09-18 17:10:56 -04:00
henrygd
f7fd3ef403 force oauth created users to start with user role 2026-09-18 17:10:37 -04:00
henrygd
db3afeabd9 i18n: update translations 2026-09-18 14:34:16 -04:00
henrygd
b347599928 update translation strings 2026-09-18 14:00:20 -04:00
hank
6e5440c21f New Crowdin updates (#2323) 2026-09-18 13:40:00 -04:00
hank
bb1b39928e feat: network monitoring from agents (#2266, #1911)
Co-authored-by: Sven van Ginkel <svenvanginkel@icloud.com>
Co-authored-by: xiaomiku01 <xiaomiku01@outlook.com>
2026-09-18 13:22:50 -04:00
José M. Requena Plens
4bf70700f2 feat(hub): add TRUSTED_PROXY_IPS allowlist for TRUSTED_AUTH_HEADER (#2327)
With TRUSTED_AUTH_HEADER set, the hub authenticates a request from the
header alone, whichever address it comes from. That is right when every
request passes through the reverse proxy, and not when the hub can also
be reached directly: anyone who can reach it sets the header themselves.

TRUSTED_PROXY_IPS takes a comma-separated list of IPs or CIDR ranges.
When set, the header is only honored on requests whose peer address is
in the list; other requests fall through to the normal authentication.
When unset, nothing changes.

The check uses the connection's RemoteAddr, not a forwarded header, so
the list names the proxy itself. IPv4-mapped IPv6 entries are treated as
IPv4. Entries that do not parse are skipped with a warning on the
console; a list with no valid entry trusts nobody, so a typo narrows the
allowlist instead of widening it.
2026-09-17 13:48:46 -04:00
henrygd
18f7a4bbc0 agent: revert #2275 warning on certain SMART attributes (#2296, #2308, #2347) 2026-09-17 11:48:01 -04:00
Sven van Ginkel
f0f1f7985c feat: persist view preferences and language to user settings (#1831)
Co-authored-by: ChangkeunJ <reiot92@gmail.com>
2026-09-16 20:29:36 -04:00
Luís Palma
50f6fc075d Merge commit from fork
* fix: make first-user bootstrap atomic

* add tests

---------

Co-authored-by: henrygd <hank@henrygd.me>
2026-09-16 20:13:32 -04:00
henrygd
a0bf338796 hub: raise max batch requests and lower max batch body size 2026-09-16 19:39:07 -04:00
Santhi Prakash
982101743e fix(zfs): skip zpool list when /dev/zfs unavailable in Linux (#2325)
Co-authored-by: henrygd <hank@henrygd.me>
2026-09-16 13:28:47 -04:00
Petr Rajtslegr
6a7b2772d9 fix(site): switch theme live when system preference changes (#2328)
Listen for prefers-color-scheme changes in ThemeProvider and expose resolvedTheme so the login border color follows the active theme without a hard refresh.
2026-09-13 12:40:21 -04:00
David Jang
086091a0fe fix(site): discard pending history when switching to live charts (#2333) 2026-09-13 12:32:04 -04:00
Bruno Bousquet
f204dc17e6 feat(agent): Add docker image update available flag (#2211)
Co-authored-by: henrygd <hank@henrygd.me>
2026-09-10 19:39:40 -04:00
Alec Rubin
5fe1583655 fix(agent): don't warn about unset HUB_URL in SSH-only mode (#2316) 2026-09-10 16:58:12 -04:00
199 changed files with 22824 additions and 1664 deletions

View File

@@ -1,6 +1,6 @@
# Node.js dependencies # Node.js dependencies
node_modules node_modules/
internalsite/node_modules **/node_modules/
# Go build artifacts and binaries # Go build artifacts and binaries
build build

View File

@@ -29,6 +29,7 @@ jobs:
# henrygd/beszel-agent:alpine # henrygd/beszel-agent:alpine
- image: henrygd/beszel-agent - image: henrygd/beszel-agent
dockerfile: ./internal/dockerfile_agent_alpine dockerfile: ./internal/dockerfile_agent_alpine
flavor: latest=false
registry: docker.io registry: docker.io
username_secret: DOCKERHUB_USERNAME username_secret: DOCKERHUB_USERNAME
password_secret: DOCKERHUB_TOKEN password_secret: DOCKERHUB_TOKEN
@@ -55,6 +56,7 @@ jobs:
# henrygd/beszel-agent-nvidia:slim # henrygd/beszel-agent-nvidia:slim
- image: henrygd/beszel-agent-nvidia - image: henrygd/beszel-agent-nvidia
dockerfile: ./internal/dockerfile_agent_nvidia_slim dockerfile: ./internal/dockerfile_agent_nvidia_slim
flavor: latest=false
platforms: linux/amd64,linux/arm64 platforms: linux/amd64,linux/arm64
registry: docker.io registry: docker.io
username_secret: DOCKERHUB_USERNAME username_secret: DOCKERHUB_USERNAME
@@ -123,6 +125,7 @@ jobs:
# ghcr.io/henrygd/beszel-agent-nvidia:slim # ghcr.io/henrygd/beszel-agent-nvidia:slim
- image: ghcr.io/${{ github.repository }}/beszel-agent-nvidia - image: ghcr.io/${{ github.repository }}/beszel-agent-nvidia
dockerfile: ./internal/dockerfile_agent_nvidia_slim dockerfile: ./internal/dockerfile_agent_nvidia_slim
flavor: latest=false
platforms: linux/amd64,linux/arm64 platforms: linux/amd64,linux/arm64
registry: ghcr.io registry: ghcr.io
username: ${{ github.actor }} username: ${{ github.actor }}
@@ -150,6 +153,7 @@ jobs:
# ghcr.io/henrygd/beszel-agent:alpine # ghcr.io/henrygd/beszel-agent:alpine
- image: ghcr.io/${{ github.repository }}/beszel-agent - image: ghcr.io/${{ github.repository }}/beszel-agent
dockerfile: ./internal/dockerfile_agent_alpine dockerfile: ./internal/dockerfile_agent_alpine
flavor: latest=false
registry: ghcr.io registry: ghcr.io
username: ${{ github.actor }} username: ${{ github.actor }}
password_secret: GITHUB_TOKEN password_secret: GITHUB_TOKEN
@@ -159,7 +163,7 @@ jobs:
type=semver,pattern={{major}}.{{minor}}-alpine type=semver,pattern={{major}}.{{minor}}-alpine
type=semver,pattern={{major}}-alpine type=semver,pattern={{major}}-alpine
# henrygd/beszel-agent (keep at bottom so it gets built after :alpine and gets the latest tag) # henrygd/beszel-agent
- image: henrygd/beszel-agent - image: henrygd/beszel-agent
dockerfile: ./internal/dockerfile_agent dockerfile: ./internal/dockerfile_agent
registry: docker.io registry: docker.io
@@ -200,6 +204,8 @@ jobs:
uses: docker/metadata-action@v6 uses: docker/metadata-action@v6
with: with:
images: ${{ matrix.image }} images: ${{ matrix.image }}
# Variant images must not overwrite the standard image's latest tag.
flavor: ${{ matrix.flavor || 'latest=auto' }}
tags: ${{ matrix.tags }} tags: ${{ matrix.tags }}
# https://github.com/docker/login-action # https://github.com/docker/login-action

View File

@@ -48,6 +48,7 @@ type Agent struct {
keys []gossh.PublicKey // SSH public keys keys []gossh.PublicKey // SSH public keys
smartManager *SmartManager // Manages SMART data smartManager *SmartManager // Manages SMART data
systemdManager *systemdManager // Manages systemd services systemdManager *systemdManager // Manages systemd services
monitorManager *MonitorManager // Manages network monitors
storagePoolManager *StoragePoolManager // Manages storage pool and dataset data storagePoolManager *StoragePoolManager // Manages storage pool and dataset data
} }
@@ -122,6 +123,9 @@ func NewAgent(dataDir ...string) (agent *Agent, err error) {
// initialize handler registry // initialize handler registry
agent.handlerRegistry = NewHandlerRegistry() agent.handlerRegistry = NewHandlerRegistry()
// initialize monitor manager
agent.monitorManager = newMonitorManager()
agent.storagePoolManager = newStoragePoolManager() agent.storagePoolManager = newStoragePoolManager()
// Retain ZFS_INTERVAL for the shared storage pool detail refresh interval. // Retain ZFS_INTERVAL for the shared storage pool detail refresh interval.
@@ -192,6 +196,11 @@ func (a *Agent) gatherStats(options common.DataRequestOptions) *system.CombinedD
} }
} }
if a.monitorManager != nil {
data.Monitors = a.monitorManager.GetResults(cacheTimeMs)
slog.Debug("Monitors", "data", data.Monitors)
}
// skip updating systemd services if cache time is not the default 60sec interval // skip updating systemd services if cache time is not the default 60sec interval
if a.systemdManager != nil && cacheTimeMs == defaultDataCacheTimeMs { if a.systemdManager != nil && cacheTimeMs == defaultDataCacheTimeMs {
totalCount := uint16(a.systemdManager.getServiceStatsCount()) totalCount := uint16(a.systemdManager.getServiceStatsCount())
@@ -243,7 +252,11 @@ func (a *Agent) gatherStats(options common.DataRequestOptions) *system.CombinedD
// Start initializes and starts the agent with optional WebSocket connection // Start initializes and starts the agent with optional WebSocket connection
func (a *Agent) Start(serverOptions ServerOptions) error { func (a *Agent) Start(serverOptions ServerOptions) error {
a.keys = serverOptions.Keys a.keys = serverOptions.Keys
return a.connectionManager.Start(serverOptions) err := a.connectionManager.Start(serverOptions)
if err != nil {
a.cleanupSensorShadow()
}
return err
} }
func (a *Agent) getFingerprint() string { func (a *Agent) getFingerprint() string {

View File

@@ -30,6 +30,11 @@ const (
wsDeadline = 120 * time.Second wsDeadline = 120 * time.Second
) )
// errNoHubURL is returned when HUB_URL is unset. This is not a failure
// condition: an agent configured with only a public key runs in SSH-only mode,
// where the hub dials the agent and no outbound WebSocket client is expected.
var errNoHubURL = errors.New("HUB_URL environment variable not set")
type caCertFileError struct { type caCertFileError struct {
err error err error
} }
@@ -63,7 +68,7 @@ type WebSocketClient struct {
func newWebSocketClient(agent *Agent) (client *WebSocketClient, err error) { func newWebSocketClient(agent *Agent) (client *WebSocketClient, err error) {
hubURLStr, exists := utils.GetEnv("HUB_URL") hubURLStr, exists := utils.GetEnv("HUB_URL")
if !exists { if !exists {
return nil, errors.New("HUB_URL environment variable not set") return nil, errNoHubURL
} }
client = &WebSocketClient{} client = &WebSocketClient{}

View File

@@ -32,6 +32,28 @@ import (
"golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh"
) )
// TestNewWebSocketClientNoHubURL verifies that an unset HUB_URL returns the
// errNoHubURL sentinel rather than an opaque error. Callers rely on this to
// distinguish SSH-only mode -- a supported configuration in which the hub dials
// the agent -- from an actual misconfiguration.
func TestNewWebSocketClientNoHubURL(t *testing.T) {
agent := createTestAgent(t)
// t.Setenv registers restoration of the original value; unset afterwards so
// GetEnv's LookupEnv reports the variable as absent rather than empty.
t.Setenv("BESZEL_AGENT_HUB_URL", "")
os.Unsetenv("BESZEL_AGENT_HUB_URL")
t.Setenv("HUB_URL", "")
os.Unsetenv("HUB_URL")
t.Setenv("BESZEL_AGENT_TOKEN", "test-token")
client, err := newWebSocketClient(agent)
require.Error(t, err)
assert.Nil(t, client)
assert.ErrorIs(t, err, errNoHubURL)
}
// TestNewWebSocketClient tests WebSocket client creation // TestNewWebSocketClient tests WebSocket client creation
func TestNewWebSocketClient(t *testing.T) { func TestNewWebSocketClient(t *testing.T) {
agent := createTestAgent(t) agent := createTestAgent(t)

View File

@@ -91,7 +91,15 @@ func (c *ConnectionManager) Start(serverOptions ServerOptions) error {
if errors.As(err, &caCertErr) { if errors.As(err, &caCertErr) {
return err return err
} }
slog.Warn("Error creating WebSocket client", "err", err) disableSSH, _ := utils.GetEnv("DISABLE_SSH")
if errors.Is(err, errNoHubURL) && disableSSH != "true" {
// SSH-only mode: the hub dials the agent, so there is nothing to warn
// about. With SSH also disabled there is no connection method at all,
// so that case still warns.
slog.Debug("WebSocket client not configured", "err", err)
} else {
slog.Warn("Error creating WebSocket client", "err", err)
}
} }
c.wsClient = wsClient c.wsClient = wsClient
@@ -145,7 +153,9 @@ func (c *ConnectionManager) Start(serverOptions ServerOptions) error {
// } // }
func (c *ConnectionManager) stop() error { func (c *ConnectionManager) stop() error {
_ = c.agent.StopServer() _ = c.agent.StopServer()
c.agent.monitorManager.Stop()
c.closeWebSocket() c.closeWebSocket()
c.agent.cleanupSensorShadow()
return health.CleanUp() return health.CleanUp()
} }

View File

@@ -3,6 +3,7 @@ package agent
import ( import (
"context" "context"
"log/slog" "log/slog"
"math"
"os" "os"
"path/filepath" "path/filepath"
"runtime" "runtime"
@@ -686,25 +687,27 @@ func (a *Agent) updateDiskIo(cacheTimeMs uint16, systemStats *system.Stats) {
// This is the total number of milliseconds spent by all reads (as // This is the total number of milliseconds spent by all reads (as
// measured from __make_request() to end_that_request_last()). // measured from __make_request() to end_that_request_last()).
// https://www.kernel.org/doc/Documentation/iostats.txt (fields 4, 8) // https://www.kernel.org/doc/Documentation/iostats.txt (fields 4, 8)
diskReadTime := utils.TwoDecimals(float64(d.ReadTime-prev.readTime) / float64(msElapsed) * 100) deltaReadTime := ioTimeDelta(d.ReadTime, prev.readTime)
diskWriteTime := utils.TwoDecimals(float64(d.WriteTime-prev.writeTime) / float64(msElapsed) * 100) deltaWriteTime := ioTimeDelta(d.WriteTime, prev.writeTime)
diskReadTime := utils.TwoDecimals(float64(deltaReadTime) / float64(msElapsed) * 100)
diskWriteTime := utils.TwoDecimals(float64(deltaWriteTime) / float64(msElapsed) * 100)
// I/O utilization %: fraction of wall time the device had any I/O in progress (0-100). // I/O utilization %: fraction of wall time the device had any I/O in progress (0-100).
diskIoUtilPct := utils.TwoDecimals(float64(d.IoTime-prev.ioTime) / float64(msElapsed) * 100) diskIoUtilPct := utils.TwoDecimals(float64(ioTimeDelta(d.IoTime, prev.ioTime)) / float64(msElapsed) * 100)
// Weighted I/O: queue-depth weighted I/O time, normalized to interval (can exceed 100%). // Weighted I/O: queue-depth weighted I/O time, normalized to interval (can exceed 100%).
// Linux kernel field 11: incremented by iops_in_progress × ms_since_last_update. // Linux kernel field 11: incremented by iops_in_progress × ms_since_last_update.
// Used to display queue depth. Multipled by 100 to increase accuracy of digit truncation (divided by 100 in UI). // Used to display queue depth. Multipled by 100 to increase accuracy of digit truncation (divided by 100 in UI).
diskWeightedIO := utils.TwoDecimals(float64(d.WeightedIO-prev.weightedIO) / float64(msElapsed) * 100) diskWeightedIO := utils.TwoDecimals(float64(ioTimeDelta(d.WeightedIO, prev.weightedIO)) / float64(msElapsed) * 100)
// r_await / w_await: average time per read/write operation in milliseconds. // r_await / w_await: average time per read/write operation in milliseconds.
// Equivalent to r_await and w_await in iostat. // Equivalent to r_await and w_await in iostat.
var rAwait, wAwait float64 var rAwait, wAwait float64
if deltaReadCount := d.ReadCount - prev.readCount; deltaReadCount > 0 { if deltaReadCount := d.ReadCount - prev.readCount; deltaReadCount > 0 {
rAwait = utils.TwoDecimals(float64(d.ReadTime-prev.readTime) / float64(deltaReadCount)) rAwait = utils.TwoDecimals(float64(deltaReadTime) / float64(deltaReadCount))
} }
if deltaWriteCount := d.WriteCount - prev.writeCount; deltaWriteCount > 0 { if deltaWriteCount := d.WriteCount - prev.writeCount; deltaWriteCount > 0 {
wAwait = utils.TwoDecimals(float64(d.WriteTime-prev.writeTime) / float64(deltaWriteCount)) wAwait = utils.TwoDecimals(float64(deltaWriteTime) / float64(deltaWriteCount))
} }
// Update global fsStats baseline for cross-interval correctness // Update global fsStats baseline for cross-interval correctness
@@ -740,6 +743,21 @@ func (a *Agent) updateDiskIo(cacheTimeMs uint16, systemStats *system.Stats) {
} }
} }
// ioTimeDelta returns the increase of a cumulative millisecond counter from
// the disk I/O stats. Linux prints these fields of /proc/diskstats as 32-bit
// unsigned ints, so they wrap to zero at 2^32. A busy disk reaches that in
// days for the weighted I/O time. Other platforms report 64-bit counters,
// so a lower value there is a reset.
func ioTimeDelta(current, previous uint64) uint64 {
if current >= previous {
return current - previous
}
if runtime.GOOS == "linux" && previous <= math.MaxUint32 {
return current + (math.MaxUint32 + 1 - previous)
}
return 0
}
// getRootMountPoint returns the appropriate root mount point for the system. // getRootMountPoint returns the appropriate root mount point for the system.
// On Windows it returns the system drive (e.g. "C:"). // On Windows it returns the system drive (e.g. "C:").
// For immutable systems like Fedora Silverblue, it returns /sysroot instead of / // For immutable systems like Fedora Silverblue, it returns /sysroot instead of /

View File

@@ -0,0 +1,75 @@
//go:build linux
package agent
import (
"fmt"
"os"
"path/filepath"
"testing"
"time"
"github.com/henrygd/beszel/internal/entities/system"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// Linux prints four millisecond fields of /proc/diskstats as 32-bit unsigned ints:
// read time, write time, io time and weighted io time. They wrap to zero at 2^32.
func TestUpdateDiskIoTimeCounterWrap(t *testing.T) {
const wrap = uint64(1) << 32
tests := []struct {
name string
base uint64 // added to every previous time counter
}{
{"no wrap", 0},
{"32-bit wrap", wrap - 1000},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Deltas over 60s: read 300ms / 10 ops, write 400ms / 20 ops,
// io time 1200ms, weighted io 3000ms.
prev := prevDisk{
readBytes: 20000 * 512,
writeBytes: 10000 * 512,
readTime: tt.base + 900,
writeTime: tt.base + 700,
ioTime: tt.base + 400,
weightedIO: tt.base,
readCount: 1000,
writeCount: 500,
at: time.Now().Add(-60 * time.Second),
}
cur := func(v uint64) uint64 { return v % wrap }
line := fmt.Sprintf(" 8 0 sda %d 0 %d %d %d 0 %d %d 0 %d %d\n",
1010, 21200, cur(prev.readTime+300),
520, 10400, cur(prev.writeTime+400),
cur(prev.ioTime+1200), cur(prev.weightedIO+3000))
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "diskstats"), []byte(line), 0o644))
t.Setenv("HOST_PROC", dir)
t.Setenv("HOST_SYS", dir)
t.Setenv("HOST_DEV", dir)
t.Setenv("HOST_RUN", dir)
fs := &system.FsStats{Root: true}
a := &Agent{
fsNames: []string{"sda"},
fsStats: map[string]*system.FsStats{"sda": fs},
diskPrev: map[uint16]map[string]prevDisk{60000: {"sda": prev}},
}
var stats system.Stats
a.updateDiskIo(60000, &stats)
// Same order as DiskIoStats in system.FsStats.
want := [6]float64{0.5, 0.67, 2, 30, 20, 5}
for i := range want {
assert.InDelta(t, want[i], fs.DiskIoStats[i], 0.01, "DiskIoStats[%d]", i)
assert.InDelta(t, want[i], stats.DiskIoStats[i], 0.01, "system DiskIoStats[%d]", i)
}
})
}
}

View File

@@ -3,7 +3,9 @@
package agent package agent
import ( import (
"math"
"os" "os"
"runtime"
"strings" "strings"
"testing" "testing"
"time" "time"
@@ -1041,3 +1043,17 @@ func TestInitializeDiskIoStatsResetsTrackedDevices(t *testing.T) {
assert.Equal(t, uint64(50), agent.fsStats["sdb"].TotalRead) assert.Equal(t, uint64(50), agent.fsStats["sdb"].TotalRead)
assert.Equal(t, uint64(60), agent.fsStats["sdb"].TotalWrite) assert.Equal(t, uint64(60), agent.fsStats["sdb"].TotalWrite)
} }
func TestIoTimeDelta(t *testing.T) {
assert.Equal(t, uint64(300), ioTimeDelta(1200, 900))
// A lower value is a 32-bit wrap only on Linux. Other platforms
// report 64-bit counters, so there it is a reset.
var want uint64
if runtime.GOOS == "linux" {
want = 1200
}
assert.Equal(t, want, ioTimeDelta(200, math.MaxUint32+1-1000))
assert.Equal(t, uint64(0), ioTimeDelta(200, math.MaxUint32+1000))
}

View File

@@ -65,10 +65,15 @@ type dockerManager struct {
dockerVersionChecked bool // Whether a version probe has completed successfully dockerVersionChecked bool // Whether a version probe has completed successfully
isWindows bool // Whether the Docker Engine API is running on Windows isWindows bool // Whether the Docker Engine API is running on Windows
buf *bytes.Buffer // Buffer to store and read response bodies buf *bytes.Buffer // Buffer to store and read response bodies
apiStats *container.ApiStats // Reusable API stats object
excludeContainers []string // Patterns to exclude containers by name excludeContainers []string // Patterns to exclude containers by name
usingPodman bool // Whether the Docker Engine API is running on Podman usingPodman bool // Whether the Docker Engine API is running on Podman
registryClient *http.Client // Client for registry requests; nil uses a client with a 10-second timeout
imageUpdatesDisabled bool // Whether image update checks are disabled by configuration
imageUpdatesMutex sync.RWMutex // Protects imageUpdates, its entries, and imageUpdatesRunning
imageUpdates map[string]*imageUpdateStatus // Shared update status keyed by normalized image reference
imageUpdatesRunning bool // Whether a background image-update batch is in progress
// Cache-time-aware tracking for CPU stats (similar to cpu.go) // Cache-time-aware tracking for CPU stats (similar to cpu.go)
// Maps cache time intervals to container-specific CPU usage tracking // Maps cache time intervals to container-specific CPU usage tracking
lastCpuContainer map[uint16]map[string]uint64 // cacheTimeMs -> containerId -> last cpu container usage lastCpuContainer map[uint16]map[string]uint64 // cacheTimeMs -> containerId -> last cpu container usage
@@ -161,6 +166,9 @@ func (dm *dockerManager) getDockerStats(cacheTimeMs uint16) ([]*container.Stats,
clear(dm.validIds) clear(dm.validIds)
} }
// Only schedule auxiliary work here; metrics never wait for image discovery.
dm.refreshImageUpdates(dm.apiContainerList, time.Now())
var failedContainers []*container.ApiInfo var failedContainers []*container.ApiInfo
for _, ctr := range dm.apiContainerList { for _, ctr := range dm.apiContainerList {
@@ -506,6 +514,17 @@ func (dm *dockerManager) updateContainerStats(ctr *container.ApiInfo, cacheTimeM
} }
} }
// Read and decode the response before locking shared stats to avoid blocking
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("container stats request failed: %s", resp.Status)
}
res := &container.ApiStats{}
if err := json.NewDecoder(resp.Body).Decode(res); err != nil {
return err
}
updateAvailable := dm.cachedImageUpdate(ctr.Image)
dm.containerStatsMutex.Lock() dm.containerStatsMutex.Lock()
defer dm.containerStatsMutex.Unlock() defer dm.containerStatsMutex.Unlock()
@@ -520,6 +539,9 @@ func (dm *dockerManager) updateContainerStats(ctr *container.ApiInfo, cacheTimeM
stats.Status = statusText stats.Status = statusText
stats.Health = health stats.Health = health
stats.Image = ctr.Image
stats.UpdateAvailable = updateAvailable
if len(ctr.Ports) > 0 { if len(ctr.Ports) > 0 {
stats.Ports = convertContainerPortsToString(ctr) stats.Ports = convertContainerPortsToString(ctr)
} }
@@ -532,12 +554,6 @@ func (dm *dockerManager) updateContainerStats(ctr *container.ApiInfo, cacheTimeM
stats.NetworkSent = 0 stats.NetworkSent = 0
stats.NetworkRecv = 0 stats.NetworkRecv = 0
res := dm.apiStats
res.Networks = nil
if err := dm.decode(resp, res); err != nil {
return err
}
// Initialize CPU tracking for this cache time interval // Initialize CPU tracking for this cache time interval
dm.initializeCpuTracking(cacheTimeMs) dm.initializeCpuTracking(cacheTimeMs)
@@ -673,6 +689,8 @@ func newDockerManager(agent *Agent) *dockerManager {
userAgent: "Docker-Client/", userAgent: "Docker-Client/",
} }
dockerImageCheck, _ := utils.GetEnv("DOCKER_IMAGE_CHECK")
// Read container exclusion patterns from environment variable // Read container exclusion patterns from environment variable
var excludeContainers []string var excludeContainers []string
if excludeStr, set := utils.GetEnv("EXCLUDE_CONTAINERS"); set && excludeStr != "" { if excludeStr, set := utils.GetEnv("EXCLUDE_CONTAINERS"); set && excludeStr != "" {
@@ -692,11 +710,11 @@ func newDockerManager(agent *Agent) *dockerManager {
Timeout: timeout, Timeout: timeout,
Transport: userAgentTransport, Transport: userAgentTransport,
}, },
containerStatsMap: make(map[string]*container.Stats), containerStatsMap: make(map[string]*container.Stats),
sem: make(chan struct{}, 5), sem: make(chan struct{}, 5),
apiContainerList: []*container.ApiInfo{}, apiContainerList: []*container.ApiInfo{},
apiStats: &container.ApiStats{}, excludeContainers: excludeContainers,
excludeContainers: excludeContainers, imageUpdatesDisabled: dockerImageCheck == "false",
// Initialize cache-time-aware tracking structures // Initialize cache-time-aware tracking structures
lastCpuContainer: make(map[uint16]map[string]uint64), lastCpuContainer: make(map[uint16]map[string]uint64),

View File

@@ -0,0 +1,108 @@
package agent
import (
"log/slog"
"sync"
"time"
"github.com/distribution/reference"
"github.com/henrygd/beszel/internal/entities/container"
)
const imageUpdateInterval = time.Hour
type imageUpdateStatus struct {
available bool
checkedAt time.Time
}
func normalizedImageReference(image string) string {
named, err := reference.ParseNormalizedNamed(image)
if err != nil {
return ""
}
// Digest-pinned references cannot move to a new version.
if _, pinned := named.(reference.Digested); pinned {
return ""
}
return reference.TagNameOnly(named).String()
}
// refreshImageUpdates starts at most one background batch. Neither its network
// work nor its completion is part of the container metrics wait group.
func (dm *dockerManager) refreshImageUpdates(containers []*container.ApiInfo, now time.Time) {
if dm.imageUpdatesDisabled {
return
}
dm.imageUpdatesMutex.Lock()
defer dm.imageUpdatesMutex.Unlock()
if dm.imageUpdatesRunning {
return
}
if dm.imageUpdates == nil {
dm.imageUpdates = make(map[string]*imageUpdateStatus)
}
active := make(map[string]struct{}, len(containers))
pending := make(map[string]*imageUpdateStatus)
for _, ctr := range containers {
if len(ctr.Names) > 0 && dm.shouldExcludeContainer(ctr.Names[0][1:]) {
continue
}
key := normalizedImageReference(ctr.Image)
if key == "" {
continue
}
active[key] = struct{}{}
entry := dm.imageUpdates[key]
if entry == nil {
entry = &imageUpdateStatus{}
dm.imageUpdates[key] = entry
}
if entry.checkedAt.IsZero() || now.Sub(entry.checkedAt) >= imageUpdateInterval {
pending[key] = entry
}
}
for key := range dm.imageUpdates {
if _, ok := active[key]; !ok {
delete(dm.imageUpdates, key)
}
}
if len(pending) == 0 {
return
}
dm.imageUpdatesRunning = true
go func() {
// Limit auxiliary requests even on hosts running many different images.
sem := make(chan struct{}, 2)
var wg sync.WaitGroup
for key, entry := range pending {
sem <- struct{}{}
wg.Add(1)
go func() {
defer wg.Done()
defer func() { <-sem }()
available, err := dm.checkImageUpdate(key)
if err != nil {
available = false
slog.Debug("Image update check failed", "image", key, "err", err)
}
dm.imageUpdatesMutex.Lock()
entry.available = available
entry.checkedAt = time.Now()
dm.imageUpdatesMutex.Unlock()
}()
}
wg.Wait()
dm.imageUpdatesMutex.Lock()
dm.imageUpdatesRunning = false
dm.imageUpdatesMutex.Unlock()
}()
}
func (dm *dockerManager) cachedImageUpdate(image string) bool {
key := normalizedImageReference(image)
dm.imageUpdatesMutex.RLock()
defer dm.imageUpdatesMutex.RUnlock()
entry := dm.imageUpdates[key]
return entry != nil && entry.available
}

View File

@@ -0,0 +1,248 @@
//go:build testing
package agent
import (
"encoding/json"
"fmt"
"github.com/fxamacker/cbor/v2"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/henrygd/beszel/internal/entities/container"
"github.com/stretchr/testify/require"
)
func waitForImageUpdates(t *testing.T, dm *dockerManager) {
t.Helper()
require.Eventually(t, func() bool {
dm.imageUpdatesMutex.RLock()
defer dm.imageUpdatesMutex.RUnlock()
return !dm.imageUpdatesRunning
}, time.Second*3, time.Millisecond)
}
func TestDisableDockerImageUpdateCheck(t *testing.T) {
t.Setenv("BESZEL_AGENT_DOCKER_IMAGE_CHECK", "false")
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/version" {
fmt.Fprint(w, `{"Version":"25.0.0"}`)
return
}
http.NotFound(w, r)
}))
defer server.Close()
t.Setenv("BESZEL_AGENT_DOCKER_HOST", server.URL)
dm := newDockerManager(nil)
require.True(t, dm.imageUpdatesDisabled)
dm.registryClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
t.Fatal("disabled image update check made a registry request")
return nil, nil
})}
dm.refreshImageUpdates([]*container.ApiInfo{{Image: "nginx", Names: []string{"/nginx"}}}, time.Now())
require.False(t, dm.imageUpdatesRunning)
require.Nil(t, dm.imageUpdates)
}
func TestImageUpdateCacheAndStats(t *testing.T) {
local := "sha256:" + strings.Repeat("a", 64)
remote := "sha256:" + strings.Repeat("b", 64)
var inspections, lookups atomic.Int32
var fail atomic.Bool
var upToDate atomic.Bool
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case strings.HasPrefix(r.URL.Path, "/images/"):
inspections.Add(1)
fmt.Fprintf(w, `{"RepoDigests":["docker.io/library/nginx@%s"]}`, local)
case r.URL.Path == "/containers/json":
fmt.Fprint(w, `[{"Id":"aaaaaaaaaaaa","Names":["/one"],"Image":"nginx","Status":"Up 2 hours"},{"Id":"bbbbbbbbbbbb","Names":["/two"],"Image":"docker.io/library/nginx:latest","Status":"Up 2 hours"}]`)
case strings.Contains(r.URL.Path, "/stats"):
fmt.Fprint(w, `{"memory_stats":{"usage":1048576},"cpu_stats":{},"networks":{}}`)
default:
http.NotFound(w, r)
}
}))
defer server.Close()
dm := newDockerManagerForVersionTest(server)
dm.dockerVersionChecked = true
dm.registryClient = &http.Client{Timeout: time.Second, Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
if fail.Load() {
return nil, fmt.Errorf("registry unavailable")
}
response := &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"token":"test"}`))}
if r.Method == http.MethodHead {
lookups.Add(1)
digest := remote
if upToDate.Load() {
digest = local
}
response.Header.Set("Docker-Content-Digest", digest)
}
return response, nil
})}
stats, err := dm.getDockerStats(defaultCacheTimeMs)
require.NoError(t, err)
require.Len(t, stats, 2)
waitForImageUpdates(t, dm)
require.EqualValues(t, 1, lookups.Load())
require.EqualValues(t, 1, inspections.Load())
stats, err = dm.getDockerStats(defaultCacheTimeMs)
require.NoError(t, err)
for _, stat := range stats {
require.True(t, stat.UpdateAvailable)
if stat.Id == "aaaaaaaaaaaa" {
require.Equal(t, "nginx", stat.Image)
} else {
require.Equal(t, "docker.io/library/nginx:latest", stat.Image)
}
}
require.EqualValues(t, 1, lookups.Load())
expire := func() {
dm.imageUpdatesMutex.Lock()
dm.imageUpdates["docker.io/library/nginx:latest"].checkedAt = time.Now().Add(-imageUpdateInterval)
dm.imageUpdatesMutex.Unlock()
}
upToDate.Store(true)
expire()
_, err = dm.getDockerStats(defaultCacheTimeMs)
require.NoError(t, err)
waitForImageUpdates(t, dm)
require.EqualValues(t, 2, lookups.Load())
require.False(t, dm.cachedImageUpdate("nginx:latest"))
// An expired positive result is cleared on failure, and the failure itself
// is cached so realtime stats do not retry a broken registry every second.
dm.imageUpdatesMutex.Lock()
dm.imageUpdates["docker.io/library/nginx:latest"].available = true
dm.imageUpdatesMutex.Unlock()
fail.Store(true)
expire()
_, err = dm.getDockerStats(defaultCacheTimeMs)
require.NoError(t, err)
waitForImageUpdates(t, dm)
failedInspections := inspections.Load()
stats, err = dm.getDockerStats(defaultCacheTimeMs)
require.NoError(t, err)
require.Len(t, stats, 2)
require.Equal(t, failedInspections, inspections.Load())
for _, stat := range stats {
require.False(t, stat.UpdateAvailable)
require.Equal(t, 1.0, stat.Mem)
}
}
func TestImageDiscoveryDoesNotBlockStats(t *testing.T) {
started := make(chan struct{}, 1)
release := make(chan struct{})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasPrefix(r.URL.Path, "/images/") {
fmt.Fprintf(w, `{"RepoDigests":["example.com/app@sha256:%s"]}`, strings.Repeat("a", 64))
} else {
fmt.Fprint(w, `{"memory_stats":{"usage":1048576}}`)
}
}))
defer server.Close()
dm := newDockerManagerForVersionTest(server)
defer func() { close(release); waitForImageUpdates(t, dm) }()
dm.registryClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
started <- struct{}{}
<-release
return nil, fmt.Errorf("timeout")
})}
ctr := &container.ApiInfo{IdShort: "aaaaaaaaaaaa", Image: "example.com/app", Names: []string{"/one"}}
dm.refreshImageUpdates([]*container.ApiInfo{ctr}, time.Now())
select {
case <-started:
case <-time.After(3 * time.Second):
t.Fatal("check did not start")
}
done := make(chan error, 1)
go func() { done <- dm.updateContainerStats(ctr, defaultCacheTimeMs) }()
select {
case err := <-done:
require.NoError(t, err)
case <-time.After(time.Second):
t.Fatal("registry blocked stats")
}
dm.imageUpdatesMutex.RLock()
require.True(t, dm.imageUpdatesRunning)
dm.imageUpdatesMutex.RUnlock()
}
func TestNormalizeImageUpdateReferences(t *testing.T) {
require.Equal(t, normalizedImageReference("nginx"), normalizedImageReference("docker.io/library/nginx:latest"))
require.Empty(t, normalizedImageReference("bad reference"))
require.Empty(t, normalizedImageReference("nginx@sha256:"+strings.Repeat("a", 64)))
}
// A stats request can return headers promptly and then stall while reading its
// body. The stats-map mutex must remain available during that read.
func TestStatsResponseBodyDoesNotHoldStatsLock(t *testing.T) {
started := make(chan struct{})
release := make(chan struct{})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
w.(http.Flusher).Flush()
close(started)
<-release
fmt.Fprint(w, `{"memory_stats":{"usage":1048576}}`)
}))
defer server.Close()
dm := newDockerManagerForVersionTest(server)
done := make(chan error, 1)
go func() {
done <- dm.updateContainerStats(&container.ApiInfo{IdShort: "aaaaaaaaaaaa", Names: []string{"/one"}, Image: "nginx"}, defaultCacheTimeMs)
}()
<-started
locked := make(chan struct{})
go func() { dm.containerStatsMutex.Lock(); dm.containerStatsMutex.Unlock(); close(locked) }()
select {
case <-locked:
case <-time.After(time.Second):
close(release)
<-done
t.Fatal("Docker response body held the stats mutex")
}
close(release)
require.NoError(t, <-done)
}
func TestImageUpdateStatsEncoding(t *testing.T) {
original := container.Stats{Image: "nginx:latest", UpdateAvailable: true}
encoded, err := cbor.Marshal(original)
require.NoError(t, err)
var fields map[int]any
require.NoError(t, cbor.Unmarshal(encoded, &fields))
require.Equal(t, true, fields[11])
require.Equal(t, "nginx:latest", fields[8])
var decoded container.Stats
require.NoError(t, cbor.Unmarshal(encoded, &decoded))
require.True(t, decoded.UpdateAvailable)
require.Equal(t, original.Image, decoded.Image)
encoded, err = json.Marshal(original)
require.NoError(t, err)
require.Contains(t, string(encoded), `"u":true`)
}
func TestImageUpdateCacheExpiryBoundaryAndPruning(t *testing.T) {
now := time.Now()
key := normalizedImageReference("nginx")
dm := &dockerManager{imageUpdates: map[string]*imageUpdateStatus{
key: {available: true, checkedAt: now},
"unused.example/image:latest": {checkedAt: now},
}}
dm.refreshImageUpdates([]*container.ApiInfo{{Image: "nginx"}}, now.Add(imageUpdateInterval-time.Nanosecond))
require.False(t, dm.imageUpdatesRunning)
require.Len(t, dm.imageUpdates, 1)
require.True(t, dm.cachedImageUpdate("nginx:latest"))
dm.refreshImageUpdates(nil, now)
require.Empty(t, dm.imageUpdates)
}

224
agent/docker_registry.go Normal file
View File

@@ -0,0 +1,224 @@
package agent
import (
_ "crypto/sha256"
"encoding/json"
"fmt"
"net/http"
"net/url"
"slices"
"strings"
"time"
"github.com/distribution/reference"
"github.com/opencontainers/go-digest"
)
const imageRegistryTimeout = 10 * time.Second
const imageManifestAccept = "application/vnd.docker.distribution.manifest.list.v2+json, " +
"application/vnd.docker.distribution.manifest.v2+json, " +
"application/vnd.oci.image.manifest.v1+json, " +
"application/vnd.oci.image.index.v1+json"
// checkImageUpdate compares the digest recorded by Docker for image with the
// digest currently advertised by its registry. A digest-pinned reference is
// immutable and therefore never has an update available.
func (dm *dockerManager) checkImageUpdate(image string) (bool, error) {
named, err := reference.ParseNormalizedNamed(image)
if err != nil {
return false, fmt.Errorf("parse image reference %q: %w", image, err)
}
if _, pinned := named.(reference.Digested); pinned {
return false, nil
}
named = reference.TagNameOnly(named)
registry := reference.Domain(named)
repository := reference.Path(named)
tag := named.(reference.Tagged).Tag()
localDigests, err := dm.inspectImageDigests(image, registry, repository)
if err != nil {
return false, err
}
remoteDigest, err := dm.registryImageDigest(registry, repository, tag)
if err != nil {
return false, err
}
return !slices.Contains(localDigests, remoteDigest), nil
}
// inspectImageDigests reads Docker's image metadata without using dm.decode.
// The checker runs in the image-discovery goroutine, so it must not hold any
// of the container statistics locks while waiting on the Docker API.
func (dm *dockerManager) inspectImageDigests(image, registry, repository string) ([]string, error) {
if dm.client == nil {
return nil, fmt.Errorf("inspect image %q: Docker client is unavailable", image)
}
endpoint := "http://localhost/images/" + url.PathEscape(image) + "/json"
resp, err := dm.client.Get(endpoint)
if err != nil {
return nil, fmt.Errorf("inspect image %q: %w", image, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("inspect image %q failed: %s", image, responseStatus(resp))
}
var inspect struct {
RepoDigests []string `json:"RepoDigests"`
}
if err := json.NewDecoder(resp.Body).Decode(&inspect); err != nil {
return nil, fmt.Errorf("decode image inspect %q: %w", image, err)
}
if len(inspect.RepoDigests) == 0 {
return nil, fmt.Errorf("inspect image %q returned no repository digests", image)
}
localDigests := matchingRepositoryDigests(inspect.RepoDigests, registry, repository)
if len(localDigests) == 0 {
return nil, fmt.Errorf("inspect image %q returned no valid digest for %s/%s", image, registry, repository)
}
return localDigests, nil
}
// matchingRepositoryDigests returns all valid digests belonging to the requested
// repository. Container engines can return both index and platform manifest digests for one
// local image, in either order.
func matchingRepositoryDigests(repoDigests []string, registry, repository string) []string {
var digests []string
for _, repoDigest := range repoDigests {
repoDigest = strings.TrimSpace(repoDigest)
at := strings.LastIndexByte(repoDigest, '@')
if at <= 0 || at == len(repoDigest)-1 || strings.Contains(repoDigest[:at], "@") {
continue
}
repoRef, err := reference.ParseNormalizedNamed(repoDigest[:at])
if err != nil || reference.Path(repoRef) != repository || !sameRegistry(reference.Domain(repoRef), registry) {
continue
}
if _, hasTag := repoRef.(reference.Tagged); hasTag {
continue
}
d, err := digest.Parse(repoDigest[at+1:])
if err != nil {
continue
}
digests = append(digests, d.String())
}
return digests
}
func sameRegistry(left, right string) bool {
left = canonicalRegistry(left)
right = canonicalRegistry(right)
return left == right ||
(left == "ghcr.io" && right == "lscr.io") ||
(left == "lscr.io" && right == "ghcr.io")
}
func canonicalRegistry(registry string) string {
if registry == "index.docker.io" {
return "docker.io"
}
return registry
}
func (dm *dockerManager) registryImageDigest(registry, repository, tag string) (string, error) {
client := dm.registryClient
if client == nil {
client = &http.Client{Timeout: imageRegistryTimeout}
}
token, err := dm.registryToken(client, registry, repository)
if err != nil {
return "", err
}
host := registry
if registry == "docker.io" {
host = "registry-1.docker.io"
}
manifestURL := "https://" + host + "/v2/" + repository + "/manifests/" + url.PathEscape(tag)
req, err := http.NewRequest(http.MethodHead, manifestURL, nil)
if err != nil {
return "", fmt.Errorf("create manifest request: %w", err)
}
req.Header.Set("Accept", imageManifestAccept)
if token != "" {
req.Header.Set("Authorization", "Bearer "+token)
}
resp, err := client.Do(req)
if err != nil {
return "", fmt.Errorf("fetch manifest %s:%s: %w", registry, repository, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("manifest request for %s:%s failed: %s", repository, tag, responseStatus(resp))
}
remote := strings.TrimSpace(resp.Header.Get("Docker-Content-Digest"))
d, err := digest.Parse(remote)
if err != nil {
return "", fmt.Errorf("manifest request for %s:%s returned invalid digest: %w", repository, tag, err)
}
return d.String(), nil
}
func (dm *dockerManager) registryToken(client *http.Client, registry, repository string) (string, error) {
var authURL string
switch registry {
case "docker.io":
authURL = "https://auth.docker.io/token?service=registry.docker.io&scope=" + url.QueryEscape("repository:"+repository+":pull")
case "ghcr.io", "lscr.io":
// lscr.io is the LinuxServer alias for its GHCR-backed images.
authURL = "https://ghcr.io/token?service=ghcr.io&scope=" + url.QueryEscape("repository:"+repository+":pull")
default:
// Anonymous registries remain supported, as they were before the
// authenticated Docker Hub and GHCR paths were added.
return "", nil
}
req, err := http.NewRequest(http.MethodGet, authURL, nil)
if err != nil {
return "", fmt.Errorf("create registry auth request: %w", err)
}
resp, err := client.Do(req)
if err != nil {
return "", fmt.Errorf("fetch registry auth token for %s: %w", repository, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("registry auth request for %s failed: %s", repository, responseStatus(resp))
}
var tokenResponse struct {
Token string `json:"token"`
AccessToken string `json:"access_token"`
}
if err := json.NewDecoder(resp.Body).Decode(&tokenResponse); err != nil {
return "", fmt.Errorf("decode registry auth response for %s: %w", repository, err)
}
token := strings.TrimSpace(tokenResponse.Token)
if token == "" {
token = strings.TrimSpace(tokenResponse.AccessToken)
}
if token == "" {
return "", fmt.Errorf("registry auth response for %s contained no token", repository)
}
return token, nil
}
func responseStatus(resp *http.Response) string {
if resp.Status != "" {
return resp.Status
}
return http.StatusText(resp.StatusCode)
}

View File

@@ -0,0 +1,243 @@
//go:build testing
package agent
import (
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"github.com/stretchr/testify/require"
)
type registryTransportFunc func(*http.Request) (*http.Response, error)
func (fn registryTransportFunc) RoundTrip(req *http.Request) (*http.Response, error) {
return fn(req)
}
func registryResponse(status int, body string) *http.Response {
return &http.Response{
StatusCode: status,
Status: fmt.Sprintf("%d %s", status, http.StatusText(status)),
Header: make(http.Header),
Body: io.NopCloser(strings.NewReader(body)),
}
}
func registryDigest(fill byte) string {
return "sha256:" + strings.Repeat(string(fill), 64)
}
func newRegistryChecker(t *testing.T, inspectBody string, transport http.RoundTripper) *dockerManager {
t.Helper()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasPrefix(r.URL.Path, "/images/") {
w.Header().Set("Content-Type", "application/json")
_, _ = io.WriteString(w, inspectBody)
return
}
http.NotFound(w, r)
}))
t.Cleanup(server.Close)
return &dockerManager{
client: newDockerManagerForVersionTest(server).client,
registryClient: &http.Client{Transport: transport},
}
}
func TestCheckImageUpdateUsesInspectAndManifestDigests(t *testing.T) {
local := registryDigest('a')
remote := registryDigest('b')
var authCalls, manifestCalls atomic.Int32
dm := newRegistryChecker(t, fmt.Sprintf(`{"RepoDigests":["docker.io/library/alpine@%s"]}`, local), registryTransportFunc(func(req *http.Request) (*http.Response, error) {
switch {
case req.Method == http.MethodGet && req.URL.Host == "auth.docker.io":
authCalls.Add(1)
require.Equal(t, "/token", req.URL.Path)
return registryResponse(http.StatusOK, `{"token":"test-token"}`), nil
case req.Method == http.MethodHead && req.URL.Host == "registry-1.docker.io":
manifestCalls.Add(1)
require.Equal(t, "/v2/library/alpine/manifests/latest", req.URL.Path)
require.Equal(t, "Bearer test-token", req.Header.Get("Authorization"))
resp := registryResponse(http.StatusOK, "")
resp.Header.Set("Docker-Content-Digest", remote)
return resp, nil
default:
return registryResponse(http.StatusNotFound, ""), nil
}
}))
available, err := dm.checkImageUpdate("alpine")
require.NoError(t, err)
require.True(t, available)
require.EqualValues(t, 1, authCalls.Load())
require.EqualValues(t, 1, manifestCalls.Load())
}
func TestCheckImageUpdateMatchesAnyRepositoryDigest(t *testing.T) {
platform := registryDigest('a')
index := registryDigest('b')
other := registryDigest('c')
for _, test := range []struct {
name string
digests []string
remote string
available bool
}{
{name: "platform then index, remote index", digests: []string{platform, index}, remote: index},
{name: "index then platform, remote index", digests: []string{index, platform}, remote: index},
{name: "platform then index, remote platform", digests: []string{platform, index}, remote: platform},
{name: "index then platform, remote platform", digests: []string{index, platform}, remote: platform},
{name: "neither matches", digests: []string{platform, index}, remote: other, available: true},
} {
t.Run(test.name, func(t *testing.T) {
inspect := fmt.Sprintf(`{"RepoDigests":["docker.io/library/busybox@%s","docker.io/library/alpine@%s","docker.io/library/alpine@sha256:invalid","docker.io/library/alpine@%s"]}`, test.remote, test.digests[0], test.digests[1])
var manifestCalls atomic.Int32
dm := newRegistryChecker(t, inspect, registryTransportFunc(func(req *http.Request) (*http.Response, error) {
if req.Method == http.MethodGet {
return registryResponse(http.StatusOK, `{"token":"test"}`), nil
}
manifestCalls.Add(1)
require.Equal(t, http.MethodHead, req.Method)
resp := registryResponse(http.StatusOK, "")
resp.Header.Set("Docker-Content-Digest", test.remote)
return resp, nil
}))
available, err := dm.checkImageUpdate("alpine")
require.NoError(t, err)
require.Equal(t, test.available, available)
require.EqualValues(t, 1, manifestCalls.Load())
})
}
}
func TestCheckImageUpdateReportsUnknownInspectState(t *testing.T) {
for _, test := range []struct {
name string
body string
}{
{name: "missing field", body: `{}`},
{name: "empty field", body: `{"RepoDigests":[]}`},
{name: "malformed reference", body: `{"RepoDigests":["not-a-repo-digest"]}`},
{name: "wrong repository", body: `{"RepoDigests":["docker.io/library/busybox@` + registryDigest('a') + `"]}`},
{name: "malformed digest", body: `{"RepoDigests":["docker.io/library/alpine@sha256:not-a-digest"]}`},
} {
t.Run(test.name, func(t *testing.T) {
var registryCalls atomic.Int32
dm := newRegistryChecker(t, test.body, registryTransportFunc(func(req *http.Request) (*http.Response, error) {
registryCalls.Add(1)
return registryResponse(http.StatusOK, `{"token":"unexpected"}`), nil
}))
available, err := dm.checkImageUpdate("alpine")
require.Error(t, err)
require.False(t, available)
require.EqualValues(t, 0, registryCalls.Load(), "invalid local state must not query a registry")
})
}
}
func TestCheckImageUpdateChecksInspectAuthAndManifestStatuses(t *testing.T) {
local := registryDigest('a')
validInspect := fmt.Sprintf(`{"RepoDigests":["docker.io/library/alpine@%s"]}`, local)
tests := []struct {
name string
inspectCode int
authCode int
manifestCode int
remote string
want string
}{
{name: "inspect status", inspectCode: http.StatusNotFound, want: "inspect image"},
{name: "auth status", inspectCode: http.StatusOK, authCode: http.StatusUnauthorized, want: "registry auth"},
{name: "manifest status", inspectCode: http.StatusOK, authCode: http.StatusOK, manifestCode: http.StatusNotFound, remote: local, want: "manifest request"},
{name: "missing digest", inspectCode: http.StatusOK, authCode: http.StatusOK, manifestCode: http.StatusOK, want: "invalid digest"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if test.inspectCode != http.StatusOK && strings.HasPrefix(r.URL.Path, "/images/") {
w.WriteHeader(test.inspectCode)
return
}
_, _ = io.WriteString(w, validInspect)
}))
t.Cleanup(server.Close)
calls := 0
dm := &dockerManager{client: newDockerManagerForVersionTest(server).client, registryClient: &http.Client{Transport: registryTransportFunc(func(req *http.Request) (*http.Response, error) {
calls++
if req.Method == http.MethodGet {
return registryResponse(test.authCode, `{"token":"test"}`), nil
}
response := registryResponse(test.manifestCode, "")
response.Header.Set("Docker-Content-Digest", test.remote)
return response, nil
})}}
_, err := dm.checkImageUpdate("alpine")
require.Error(t, err)
require.Contains(t, err.Error(), test.want)
if test.inspectCode != http.StatusOK {
require.Zero(t, calls)
}
})
}
}
func TestCheckImageUpdateSupportsAnonymousAndLSCRRegistries(t *testing.T) {
t.Run("anonymous registry", func(t *testing.T) {
local := registryDigest('a')
var calls atomic.Int32
dm := newRegistryChecker(t, fmt.Sprintf(`{"RepoDigests":["example.com/app@%s"]}`, local), registryTransportFunc(func(req *http.Request) (*http.Response, error) {
calls.Add(1)
require.Equal(t, http.MethodHead, req.Method)
require.Equal(t, "example.com", req.URL.Host)
resp := registryResponse(http.StatusOK, "")
resp.Header.Set("Docker-Content-Digest", local)
return resp, nil
}))
available, err := dm.checkImageUpdate("example.com/app")
require.NoError(t, err)
require.False(t, available)
require.EqualValues(t, 1, calls.Load())
})
t.Run("lscr ghcr alias", func(t *testing.T) {
local := registryDigest('a')
var authCalls, manifestCalls atomic.Int32
dm := newRegistryChecker(t, fmt.Sprintf(`{"RepoDigests":["ghcr.io/linuxserver/app@%s"]}`, local), registryTransportFunc(func(req *http.Request) (*http.Response, error) {
if req.Method == http.MethodGet {
authCalls.Add(1)
return registryResponse(http.StatusOK, `{"token":"test"}`), nil
}
manifestCalls.Add(1)
require.Equal(t, "lscr.io", req.URL.Host)
resp := registryResponse(http.StatusOK, "")
resp.Header.Set("Docker-Content-Digest", local)
return resp, nil
}))
available, err := dm.checkImageUpdate("lscr.io/linuxserver/app")
require.NoError(t, err)
require.False(t, available)
require.EqualValues(t, 1, authCalls.Load())
require.EqualValues(t, 1, manifestCalls.Load())
})
}
func TestCheckImageUpdateSkipsPinnedDigest(t *testing.T) {
image := "docker.io/library/alpine@" + registryDigest('a')
dm := &dockerManager{}
available, err := dm.checkImageUpdate(image)
require.NoError(t, err)
require.False(t, available)
}

View File

@@ -1184,7 +1184,6 @@ func TestUpdateContainerStatsPodmanCpuCalculation(t *testing.T) {
} }
})}, })},
containerStatsMap: make(map[string]*container.Stats), containerStatsMap: make(map[string]*container.Stats),
apiStats: &container.ApiStats{},
usingPodman: true, usingPodman: true,
lastCpuContainer: map[uint16]map[string]uint64{ lastCpuContainer: map[uint16]map[string]uint64{
defaultCacheTimeMs: {"0123456789ab": prevCpuUsage}, defaultCacheTimeMs: {"0123456789ab": prevCpuUsage},
@@ -1676,7 +1675,6 @@ func TestUpdateContainerStatsUsesPodmanInspectHealthFallback(t *testing.T) {
} }
})}, })},
containerStatsMap: make(map[string]*container.Stats), containerStatsMap: make(map[string]*container.Stats),
apiStats: &container.ApiStats{},
usingPodman: true, usingPodman: true,
lastCpuContainer: make(map[uint16]map[string]uint64), lastCpuContainer: make(map[uint16]map[string]uint64),
lastCpuSystem: make(map[uint16]map[string]uint64), lastCpuSystem: make(map[uint16]map[string]uint64),

View File

@@ -12,7 +12,7 @@ import (
) )
type fanSensor struct { type fanSensor struct {
key, path string key, path, chip string
} }
var getFanSensors = newFanSensorCache(hwmonRoot) var getFanSensors = newFanSensorCache(hwmonRoot)
@@ -34,6 +34,10 @@ func (a *Agent) updateFans(systemStats *system.Stats) {
slog.Debug("Error reading fans", "err", err) slog.Debug("Error reading fans", "err", err)
return return
} }
// Filter before reading fan*_input: each read can wake an idle GPU.
if a.sensorConfig != nil && a.sensorConfig.skipGPU {
sensors = filterGpuFans(sensors)
}
fans := readFanSensors(sensors) fans := readFanSensors(sensors)
if len(fans) == 0 { if len(fans) == 0 {
return return
@@ -100,7 +104,7 @@ func discoverHwmonFans(root string) ([]fanSensor, error) {
if label != "" { if label != "" {
key = chipName + "_" + label key = chipName + "_" + label
} }
sensors = append(sensors, fanSensor{key, inputPath}) sensors = append(sensors, fanSensor{key, inputPath, chipName})
} }
} }
return sensors, nil return sensors, nil
@@ -115,3 +119,15 @@ func readFanSensors(sensors []fanSensor) map[string]uint16 {
} }
return fans return fans
} }
// filterGpuFans drops GPU chips without touching the shared cache backing array.
func filterGpuFans(sensors []fanSensor) []fanSensor {
kept := make([]fanSensor, 0, len(sensors))
for _, sensor := range sensors {
if isGpuChipName(sensor.chip) {
continue
}
kept = append(kept, sensor)
}
return kept
}

View File

@@ -103,3 +103,20 @@ func TestFanDiscoveryCache(t *testing.T) {
fans = readFanSensors(sensors) fans = readFanSensors(sensors)
assert.Equal(t, map[string]uint16{"chip_fan1": 1200}, fans) assert.Equal(t, map[string]uint16{"chip_fan1": 1200}, fans)
} }
func TestFilterGpuFans(t *testing.T) {
root := t.TempDir()
writeFile(t, filepath.Join(root, "hwmon0", "name"), "xe\n")
writeFile(t, filepath.Join(root, "hwmon0", "fan1_input"), "1200\n")
writeFile(t, filepath.Join(root, "hwmon1", "name"), "nct6798\n")
writeFile(t, filepath.Join(root, "hwmon1", "fan1_input"), "800\n")
discovered, err := discoverHwmonFans(root)
require.NoError(t, err)
require.Len(t, discovered, 2)
filtered := filterGpuFans(discovered)
require.Len(t, filtered, 1)
assert.Equal(t, "nct6798_fan1", filtered[0].key)
assert.Len(t, discovered, 2)
}

View File

@@ -454,8 +454,8 @@ func (gm *GPUManager) storeSnapshot(id string, gpu *system.GPUData, cacheKey uin
// It only reports capability presence and does not apply policy decisions. // It only reports capability presence and does not apply policy decisions.
func (gm *GPUManager) discoverGpuCapabilities() gpuCapabilities { func (gm *GPUManager) discoverGpuCapabilities() gpuCapabilities {
caps := gpuCapabilities{ caps := gpuCapabilities{
hasAmdSysfs: gm.hasAmdSysfs(), hasAmdSysfs: gm.hasAmdSysfs(),
hasXe: gm.hasXe(), hasXe: gm.hasXe(),
hasIntelSysfs: gm.hasIntelSysfs(), hasIntelSysfs: gm.hasIntelSysfs(),
} }
if _, err := exec.LookPath(nvidiaSmiCmd); err == nil { if _, err := exec.LookPath(nvidiaSmiCmd); err == nil {
@@ -750,9 +750,36 @@ func (gm *GPUManager) resolveLegacyCollectorPriority(caps gpuCapabilities) []col
return priorities return priorities
} }
// gpuHwmonChips are hwmon chip names belonging to GPUs. Sensor reads on some
// of these drivers (notably Intel Xe, where each read is a runtime PM resume)
// wake the card, so SKIP_GPU must avoid touching them, not just hide them.
var gpuHwmonChips = []string{"xe", "i915", "amdgpu", "radeon", "nvidia", "nouveau"}
func isGpuChipName(name string) bool {
name = strings.ToLower(strings.TrimSpace(name))
for _, chip := range gpuHwmonChips {
if name == chip {
return true
}
}
return false
}
// SensorKeys are "<chip>" or "<chip>_<label>".
func isGpuSensorKey(key string) bool {
key = strings.ToLower(strings.TrimSpace(key))
for _, chip := range gpuHwmonChips {
if key == chip || strings.HasPrefix(key, chip+"_") {
return true
}
}
return false
}
// NewGPUManager creates and initializes a new GPUManager // NewGPUManager creates and initializes a new GPUManager
func NewGPUManager() (*GPUManager, error) { func NewGPUManager() (*GPUManager, error) {
if skipGPU, _ := utils.GetEnv("SKIP_GPU"); skipGPU == "true" { if skipGPU, _ := utils.GetEnv("SKIP_GPU"); skipGPU == "true" {
slog.Info("SKIP_GPU enabled, skipping GPU monitoring (collectors, temperatures, and fans)")
return nil, nil return nil, nil
} }
var gm GPUManager var gm GPUManager

View File

@@ -1119,7 +1119,6 @@ func TestCalculateGPUAverage(t *testing.T) {
} }
func TestGPUCapabilitiesAndLegacyPriority(t *testing.T) { func TestGPUCapabilitiesAndLegacyPriority(t *testing.T) {
// Save original PATH
hasAmdSysfs := (&GPUManager{}).hasAmdSysfs() hasAmdSysfs := (&GPUManager{}).hasAmdSysfs()
tests := []struct { tests := []struct {
@@ -1213,7 +1212,7 @@ echo "[]"`
{ {
name: "no gpu tools available", name: "no gpu tools available",
setupCommands: func(_ string) error { setupCommands: func(_ string) error {
t.Setenv("PATH", "") // The subtest already restricts PATH to its empty temporary directory.
return nil return nil
}, },
wantErr: true, wantErr: true,

View File

@@ -7,6 +7,7 @@ import (
"github.com/fxamacker/cbor/v2" "github.com/fxamacker/cbor/v2"
"github.com/henrygd/beszel/internal/common" "github.com/henrygd/beszel/internal/common"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/entities/smart" "github.com/henrygd/beszel/internal/entities/smart"
"log/slog" "log/slog"
@@ -51,6 +52,7 @@ func NewHandlerRegistry() *HandlerRegistry {
registry.Register(common.GetContainerInfo, &GetContainerInfoHandler{}) registry.Register(common.GetContainerInfo, &GetContainerInfoHandler{})
registry.Register(common.GetSmartData, &GetSmartDataHandler{}) registry.Register(common.GetSmartData, &GetSmartDataHandler{})
registry.Register(common.GetSystemdInfo, &GetSystemdInfoHandler{}) registry.Register(common.GetSystemdInfo, &GetSystemdInfoHandler{})
registry.Register(common.SyncNetworkMonitors, &SyncNetworkMonitorsHandler{})
registry.Register(common.GetZfsData, &GetZfsDataHandler{}) registry.Register(common.GetZfsData, &GetZfsDataHandler{})
return registry return registry
@@ -223,3 +225,21 @@ func (h *GetSystemdInfoHandler) Handle(hctx *HandlerContext) error {
return hctx.SendResponse(details, hctx.RequestID) return hctx.SendResponse(details, hctx.RequestID)
} }
////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////
// SyncNetworkMonitorsHandler handles monitor configuration sync from hub
type SyncNetworkMonitorsHandler struct{}
func (h *SyncNetworkMonitorsHandler) Handle(hctx *HandlerContext) error {
var req monitor.SyncRequest
if err := cbor.Unmarshal(hctx.Request.Data, &req); err != nil {
return err
}
resp, err := hctx.Agent.monitorManager.HandleSyncRequest(req)
if err != nil {
return err
}
return hctx.SendResponse(resp, hctx.RequestID)
}

View File

@@ -201,12 +201,9 @@ func mdraidSmartStatus(health mdraidHealth) string {
if health.mismatchCnt > 0 { if health.mismatchCnt > 0 {
return "WARNING" return "WARNING"
} }
// "check" scans for consistency problems without repairing mismatches. // "check" and "repair" are requested consistency scans, not evidence of
// With no mismatches, keep it green while reporting progress attributes. // array failure. With no health issues above, keep scrubbing green while
switch syncAction { // reporting the sync action and progress attributes.
case "repair":
return "WARNING"
}
switch state { switch state {
case "clean", "active", "active-idle", "write-pending", "read-auto", "readonly": case "clean", "active", "active-idle", "write-pending", "read-auto", "readonly":
return "PASSED" return "PASSED"

View File

@@ -174,8 +174,25 @@ func TestMdraidSmartStatus(t *testing.T) {
if got := mdraidSmartStatus(mdraidHealth{arrayState: "clean", mismatchCnt: 1}); got != "WARNING" { if got := mdraidSmartStatus(mdraidHealth{arrayState: "clean", mismatchCnt: 1}); got != "WARNING" {
t.Fatalf("mdraidSmartStatus(clean+mismatch) = %q, want WARNING", got) t.Fatalf("mdraidSmartStatus(clean+mismatch) = %q, want WARNING", got)
} }
if got := mdraidSmartStatus(mdraidHealth{arrayState: "clean", syncAction: "repair"}); got != "WARNING" { for _, tc := range []struct {
t.Fatalf("mdraidSmartStatus(repair) = %q, want WARNING", got) name string
health mdraidHealth
want string
}{
{"clean", mdraidHealth{arrayState: "clean"}, "PASSED"},
{"active", mdraidHealth{arrayState: "active"}, "PASSED"},
{"mismatch", mdraidHealth{arrayState: "active", mismatchCnt: 1}, "WARNING"},
{"degraded", mdraidHealth{arrayState: "active", degraded: 1}, "FAILED"},
{"faulty member", mdraidHealth{arrayState: "active", faultyDisks: 1}, "FAILED"},
{"inactive", mdraidHealth{arrayState: "inactive"}, "FAILED"},
{"unknown", mdraidHealth{arrayState: "unknown"}, "UNKNOWN"},
} {
t.Run("repair/"+tc.name, func(t *testing.T) {
tc.health.syncAction = "repair"
if got := mdraidSmartStatus(tc.health); got != tc.want {
t.Fatalf("mdraidSmartStatus(%+v) = %q, want %s", tc.health, got, tc.want)
}
})
} }
if got := mdraidSmartStatus(mdraidHealth{arrayState: "clean"}); got != "PASSED" { if got := mdraidSmartStatus(mdraidHealth{arrayState: "clean"}); got != "PASSED" {
t.Fatalf("mdraidSmartStatus(clean) = %q, want PASSED", got) t.Fatalf("mdraidSmartStatus(clean) = %q, want PASSED", got)

195
agent/network_monitor.go Normal file
View File

@@ -0,0 +1,195 @@
package agent
import (
"errors"
"fmt"
"net/http"
"sync"
"time"
"github.com/henrygd/beszel/internal/entities/monitor"
)
// MonitorManager manages network monitor configurations and task lifetimes.
type MonitorManager struct {
mu sync.RWMutex
monitors map[string]*monitorTask // keyed by monitor ID
probe monitorProbe
certCheck certChecker
resumeGuard monitorResumeGuard
}
func newMonitorManager() *MonitorManager {
return newMonitorManagerWithProbe(networkMonitorProbe(&http.Client{Timeout: monitor.MaxProbeTimeout}))
}
func newMonitorManagerWithProbe(probe monitorProbe) *MonitorManager {
return &MonitorManager{monitors: make(map[string]*monitorTask), probe: probe, certCheck: checkCert}
}
// SyncMonitors replaces all monitor tasks with the given configs.
func (pm *MonitorManager) SyncMonitors(configs []monitor.Config) {
pm.mu.Lock()
defer pm.mu.Unlock()
// Build set of new keys
newKeys := make(map[string]monitor.Config, len(configs))
for _, cfg := range configs {
if cfg.ID == "" {
continue
}
newKeys[cfg.ID] = cfg
}
// Stop removed monitors
for key, task := range pm.monitors {
if _, exists := newKeys[key]; !exists {
task.cancel()
delete(pm.monitors, key)
}
}
// Start new monitors and restart tasks whose config changed.
for key, cfg := range newKeys {
task, exists := pm.monitors[key]
if exists && task.config == cfg {
continue
}
if exists {
task.cancel()
}
task = newMonitorTaskFromExisting(cfg, task)
task.resumeGuard = &pm.resumeGuard
pm.resumeGuard.start()
pm.monitors[key] = task
pm.startMonitor(task)
}
if len(pm.monitors) == 0 {
pm.resumeGuard.shutdown()
}
}
// HandleSyncRequest applies a full or incremental monitor sync request.
func (pm *MonitorManager) HandleSyncRequest(req monitor.SyncRequest) (monitor.SyncResponse, error) {
switch req.Action {
case monitor.SyncActionReplace:
pm.SyncMonitors(req.Configs)
return monitor.SyncResponse{}, nil
case monitor.SyncActionUpsert:
result, err := pm.UpsertMonitor(req.Config, req.RunNow)
if err != nil {
return monitor.SyncResponse{}, err
}
if result == nil {
return monitor.SyncResponse{}, nil
}
return monitor.SyncResponse{Result: *result}, nil
case monitor.SyncActionDelete:
if req.Config.ID == "" {
return monitor.SyncResponse{}, errors.New("missing monitor ID for delete")
}
pm.DeleteMonitor(req.Config.ID)
return monitor.SyncResponse{}, nil
default:
return monitor.SyncResponse{}, fmt.Errorf("unknown monitor sync action: %d", req.Action)
}
}
// UpsertMonitor creates or replaces a single monitor task.
func (pm *MonitorManager) UpsertMonitor(config monitor.Config, runNow bool) (*monitor.Result, error) {
if config.ID == "" {
return nil, errors.New("missing monitor ID")
}
pm.mu.Lock()
task, exists := pm.monitors[config.ID]
if exists && task.config == config {
pm.mu.Unlock()
if !runNow {
return nil, nil
}
return pm.runNow(task), nil
}
if exists {
task.cancel()
}
task = newMonitorTaskFromExisting(config, task)
task.resumeGuard = &pm.resumeGuard
pm.resumeGuard.start()
pm.monitors[config.ID] = task
pm.mu.Unlock()
if runNow {
result := pm.runNow(task)
pm.startMonitor(task)
return result, nil
}
pm.startMonitor(task)
return nil, nil
}
// runNow runs a probe and any due certificate check concurrently, so the
// response fits within the hub's single probe timeout budget.
func (pm *MonitorManager) runNow(task *monitorTask) *monitor.Result {
var wg sync.WaitGroup
wg.Go(func() { task.refreshCert(pm.certCheck) })
result := task.runProbe(pm.probe)
wg.Wait()
if result != nil {
result.Cert = task.certInfo()
}
return result
}
// DeleteMonitor stops and removes a single monitor task.
func (pm *MonitorManager) DeleteMonitor(id string) {
if id == "" {
return
}
pm.mu.Lock()
defer pm.mu.Unlock()
if task, exists := pm.monitors[id]; exists {
task.cancel()
delete(pm.monitors, id)
}
if len(pm.monitors) == 0 {
pm.resumeGuard.shutdown()
}
}
// GetResults returns aggregated results for all monitors over the last supplied duration in ms.
func (pm *MonitorManager) GetResults(durationMs uint16) map[string]monitor.Result {
pm.mu.RLock()
defer pm.mu.RUnlock()
results := make(map[string]monitor.Result, len(pm.monitors))
now := time.Now()
duration := time.Duration(durationMs) * time.Millisecond
for _, task := range pm.monitors {
result, ok := task.history.result(duration, now)
if !ok {
continue
}
// Only the default interval updates monitor records on the hub, so
// realtime requests must not consume the unsent certificate.
if durationMs == defaultDataCacheTimeMs {
result.Cert = task.takeUnsentCert()
}
results[task.config.ID] = result
}
return results
}
// Stop stops all monitor tasks.
func (pm *MonitorManager) Stop() {
pm.mu.Lock()
defer pm.mu.Unlock()
for key, task := range pm.monitors {
task.cancel()
delete(pm.monitors, key)
}
pm.resumeGuard.shutdown()
}

View File

@@ -0,0 +1,74 @@
package agent
import (
"context"
"crypto/tls"
"errors"
"fmt"
"net"
"net/url"
"strings"
"time"
"github.com/henrygd/beszel/internal/entities/monitor"
)
const (
certCheckInterval = 24 * time.Hour
certCheckRetryInterval = time.Hour
)
// certChecker fetches the leaf certificate for an HTTPS target.
type certChecker func(context.Context, string) (monitor.CertInfo, error)
// certCheckEnabled reports whether a monitor's certificate is checked, which is
// the case for every HTTP monitor with an https target.
func certCheckEnabled(config monitor.Config) bool {
return config.Protocol == "http" && len(config.Target) > 8 && strings.EqualFold(config.Target[:8], "https://")
}
// checkCert reads the leaf certificate presented by an HTTPS target. The chain is
// not verified, so expired or self-signed certificates are still reported.
func checkCert(ctx context.Context, target string) (monitor.CertInfo, error) {
address, host, err := certAddress(target)
if err != nil {
return monitor.CertInfo{}, err
}
ctx, cancel := context.WithTimeout(ctx, monitor.MaxProbeTimeout)
defer cancel()
dialer := tls.Dialer{Config: &tls.Config{ServerName: host, InsecureSkipVerify: true}}
conn, err := dialer.DialContext(ctx, "tcp", address)
if err != nil {
return monitor.CertInfo{}, err
}
defer conn.Close()
certs := conn.(*tls.Conn).ConnectionState().PeerCertificates
if len(certs) == 0 {
return monitor.CertInfo{}, errors.New("no peer certificates")
}
leaf := certs[0]
return monitor.CertInfo{
Expires: leaf.NotAfter.UnixMilli(),
Issuer: leaf.Issuer.CommonName,
}, nil
}
// certAddress returns the dial address and server name for an HTTPS URL.
func certAddress(target string) (address, host string, err error) {
u, err := url.Parse(target)
if err != nil {
return "", "", err
}
if !strings.EqualFold(u.Scheme, "https") {
return "", "", fmt.Errorf("certificate check requires an https target: %s", target)
}
host = u.Hostname()
if host == "" {
return "", "", fmt.Errorf("missing host in target: %s", target)
}
port := u.Port()
if port == "" {
port = "443"
}
return net.JoinHostPort(host, port), host, nil
}

View File

@@ -0,0 +1,184 @@
//go:build testing
package agent
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"testing"
"testing/synctest"
"time"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestCheckCertReadsUnverifiedLeaf(t *testing.T) {
server := httptest.NewTLSServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
defer server.Close()
// httptest uses a self-signed certificate, which must still be reported.
info, err := checkCert(context.Background(), server.URL)
require.NoError(t, err)
leaf := server.Certificate()
assert.Equal(t, leaf.NotAfter.UnixMilli(), info.Expires)
assert.Equal(t, leaf.Issuer.CommonName, info.Issuer)
}
func TestCertAddress(t *testing.T) {
tests := []struct {
target, address, host string
wantErr bool
}{
{target: "https://example.com", address: "example.com:443", host: "example.com"},
{target: "https://example.com:8443/path?q=1", address: "example.com:8443", host: "example.com"},
{target: "HTTPS://[::1]:9443", address: "[::1]:9443", host: "::1"},
{target: "http://example.com", wantErr: true},
{target: "https://", wantErr: true},
}
for _, tt := range tests {
address, host, err := certAddress(tt.target)
if tt.wantErr {
assert.Error(t, err, tt.target)
continue
}
require.NoError(t, err, tt.target)
assert.Equal(t, tt.address, address)
assert.Equal(t, tt.host, host)
}
}
func TestRefreshCertCadence(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
task := newMonitorTask(monitor.Config{ID: "test", Target: "https://example.test", Protocol: "http"})
defer task.cancel()
var calls int
var fail error
// Far enough out that the regular interval applies for the whole test.
expires := time.Now().Add(365 * 24 * time.Hour).UnixMilli()
check := func(context.Context, string) (monitor.CertInfo, error) {
calls++
if fail != nil {
return monitor.CertInfo{}, fail
}
return monitor.CertInfo{Expires: expires + int64(calls)}, nil
}
task.refreshCert(check)
require.NotNil(t, task.certInfo())
assert.Equal(t, expires+1, task.certInfo().Expires)
// Not due again until the check interval passes.
time.Sleep(certCheckInterval - time.Second)
task.refreshCert(check)
assert.Equal(t, 1, calls)
time.Sleep(time.Second)
task.refreshCert(check)
assert.Equal(t, 2, calls)
// Failures keep the last known certificate and retry sooner.
fail = errors.New("connection refused")
time.Sleep(certCheckInterval)
task.refreshCert(check)
assert.Equal(t, 3, calls)
assert.Equal(t, expires+2, task.certInfo().Expires)
time.Sleep(certCheckRetryInterval)
fail = nil
task.refreshCert(check)
assert.Equal(t, 4, calls)
assert.Equal(t, expires+4, task.certInfo().Expires)
})
}
func TestRefreshCertRetriesSoonerNearExpiry(t *testing.T) {
for _, tc := range []struct {
name string
expires time.Duration // relative to the check
interval time.Duration
}{
{"expired", -time.Hour, certCheckRetryInterval},
{"expires before next regular check", certCheckInterval - time.Minute, certCheckRetryInterval},
{"expires after next regular check", certCheckInterval + time.Minute, certCheckInterval},
} {
t.Run(tc.name, func(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
task := newMonitorTask(monitor.Config{ID: "test", Target: "https://example.test", Protocol: "http"})
defer task.cancel()
var calls int
check := func(context.Context, string) (monitor.CertInfo, error) {
calls++
return monitor.CertInfo{Expires: time.Now().Add(tc.expires).UnixMilli()}, nil
}
task.refreshCert(check)
time.Sleep(tc.interval - time.Second)
task.refreshCert(check)
assert.Equal(t, 1, calls)
time.Sleep(time.Second)
task.refreshCert(check)
assert.Equal(t, 2, calls)
})
})
}
}
func TestCertCheckEnabled(t *testing.T) {
tests := []struct {
protocol, target string
want bool
}{
{"http", "https://example.com", true},
{"http", "HTTPS://example.com:8443/path", true},
{"http", "http://example.com", false},
{"http", "https://", false},
{"tcp", "https://example.com", false},
{"icmp", "example.com", false},
}
for _, tt := range tests {
assert.Equal(t, tt.want, certCheckEnabled(monitor.Config{Protocol: tt.protocol, Target: tt.target}), tt.protocol+" "+tt.target)
}
}
func TestRefreshCertSkipsNonHTTPS(t *testing.T) {
task := newMonitorTask(monitor.Config{ID: "test", Target: "http://example.test", Protocol: "http"})
defer task.cancel()
task.refreshCert(func(context.Context, string) (monitor.CertInfo, error) {
t.Fatal("certificate check must not run for non-https targets")
return monitor.CertInfo{}, nil
})
assert.Nil(t, task.certInfo())
}
func TestUpsertMonitorRunNowIncludesCert(t *testing.T) {
server := httptest.NewTLSServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
defer server.Close()
pm := newMonitorManagerWithProbe(func(context.Context, monitor.Config) (int64, error) { return 100, nil })
defer pm.Stop()
config := monitor.Config{ID: "cert", Target: server.URL, Protocol: "http", Interval: 60}
result, err := pm.UpsertMonitor(config, true)
require.NoError(t, err)
require.NotNil(t, result)
require.NotNil(t, result.Cert)
assert.Equal(t, server.Certificate().NotAfter.UnixMilli(), result.Cert.Expires)
// Realtime results never carry the certificate, and the default interval
// sends it only once per check.
assert.Nil(t, pm.GetResults(1000)["cert"].Cert)
results := pm.GetResults(defaultDataCacheTimeMs)
require.NotNil(t, results["cert"].Cert)
assert.Equal(t, result.Cert.Expires, results["cert"].Cert.Expires)
assert.Nil(t, pm.GetResults(defaultDataCacheTimeMs)["cert"].Cert)
// Changing the interval keeps the known certificate without resending it.
config.Interval = 30
_, err = pm.UpsertMonitor(config, false)
require.NoError(t, err)
pm.mu.RLock()
task := pm.monitors["cert"]
pm.mu.RUnlock()
assert.NotNil(t, task.certInfo())
assert.Nil(t, pm.GetResults(defaultDataCacheTimeMs)["cert"].Cert)
}

View File

@@ -0,0 +1,274 @@
package agent
import (
"math"
"sync"
"time"
"github.com/henrygd/beszel/internal/entities/monitor"
)
// Monitors run at user-defined intervals (e.g., every 10s).
// To keep memory usage low and constant, data is stored in two layers:
// 1. Raw samples: The most recent individual results (kept for monitorRawRetention).
// 2. Minute buckets: A ring buffer of 61 buckets, each representing one
// wall-clock minute. Samples collected within the same minute are aggregated
// (sum, min, max, count) into a single bucket.
//
// Short-term requests (<= 61s) use raw samples.
// Long-term requests (up to 1h) use the minute buckets to avoid storing thousands
// of individual data points.
const (
// monitorRawRetention is the duration to keep individual samples
monitorRawRetention = 61 * time.Second
// monitorMinuteBucketLen is the number of 1-minute buckets to keep (1 hour + 1 for partials)
monitorMinuteBucketLen int32 = 61
)
// monitorHistory owns retention and aggregation, independently of probe execution.
type monitorHistory struct {
mu sync.Mutex
sampleCount int64
samples []monitorSample
buckets [monitorMinuteBucketLen]monitorBucket
}
func newMonitorHistory() *monitorHistory {
// Start small for typical intervals; append grows the buffer for faster probes.
return &monitorHistory{samples: make([]monitorSample, 0, 4)}
}
func (h *monitorHistory) clone() *monitorHistory {
h.mu.Lock()
defer h.mu.Unlock()
cloned := newMonitorHistory()
cloned.samples = append(cloned.samples, h.samples...)
cloned.buckets = h.buckets
cloned.sampleCount = h.sampleCount
return cloned
}
func (h *monitorHistory) result(duration time.Duration, now time.Time) (monitor.Result, bool) {
h.mu.Lock()
defer h.mu.Unlock()
return h.resultLocked(duration, now)
}
func (h *monitorHistory) record(sample monitorSample) monitor.Result {
h.mu.Lock()
defer h.mu.Unlock()
h.addSampleLocked(sample)
result, _ := h.resultLocked(time.Minute, sample.timestamp)
return result
}
// monitorSample stores one monitor attempt and its collection time.
type monitorSample struct {
responseUs int64 // -1 means loss
timestamp time.Time
}
// monitorBucket stores one minute of aggregated monitor data.
type monitorBucket struct {
minute int32
filled bool
stats monitorAggregate
}
// monitorAggregate accumulates successful response stats and total sample counts.
type monitorAggregate struct {
sumUs int64
minUs int64
maxUs int64
totalCount int64
successCount int64
}
// newMonitorAggregate initializes an aggregate with an unset minimum value.
func newMonitorAggregate() monitorAggregate {
return monitorAggregate{minUs: math.MaxInt64}
}
// addResponse folds a single monitor sample into the aggregate.
func (agg *monitorAggregate) addResponse(responseUs int64) {
agg.totalCount++
if responseUs < 0 {
return
}
agg.successCount++
agg.sumUs += responseUs
if responseUs < agg.minUs {
agg.minUs = responseUs
}
if responseUs > agg.maxUs {
agg.maxUs = responseUs
}
}
// addAggregate merges another aggregate into this one.
func (agg *monitorAggregate) addAggregate(other monitorAggregate) {
if other.totalCount == 0 {
return
}
agg.totalCount += other.totalCount
agg.successCount += other.successCount
agg.sumUs += other.sumUs
if other.successCount == 0 {
return
}
if agg.minUs == math.MaxInt64 || other.minUs < agg.minUs {
agg.minUs = other.minUs
}
if other.maxUs > agg.maxUs {
agg.maxUs = other.maxUs
}
}
// hasData reports whether the aggregate contains any samples.
func (agg monitorAggregate) hasData() bool {
return agg.totalCount > 0
}
// result converts the aggregate into the monitor result format.
func (agg monitorAggregate) result() monitor.Result {
avg := agg.avgResponse()
result := monitor.Result{
AvgResponse: avg,
MinResponse: agg.minUs,
MaxResponse: agg.maxUs,
PacketLoss: agg.lossPercentage(),
TotalCount: agg.totalCount,
SuccessCount: agg.successCount,
ResponseSum: agg.sumUs,
}
if agg.successCount == 0 {
result.MinResponse, result.MaxResponse = 0, 0
}
return result
}
// avgResponse returns the rounded average of successful samples.
func (agg monitorAggregate) avgResponse() int64 {
if agg.successCount == 0 {
return 0
}
return agg.sumUs / agg.successCount
}
// lossPercentage returns the rounded failure rate for the aggregate.
func (agg monitorAggregate) lossPercentage() float64 {
if agg.totalCount == 0 {
return 0
}
return math.Round(float64(agg.totalCount-agg.successCount)/float64(agg.totalCount)*10000) / 100
}
// resultLocked returns the aggregated monitor result for the requested duration along with a bool indicating whether any data was available.
func (h *monitorHistory) resultLocked(duration time.Duration, now time.Time) (monitor.Result, bool) {
agg := h.aggregateLocked(duration, now)
if !agg.hasData() {
// short realtime windows (e.g. the 1s window used for 1m/realtime charts) often fall
// between monitor samples since monitors run at longer, user-defined intervals; fall back to
// the most recent sample so realtime requests still report current status.
agg = h.latestSampleAggregateLocked()
}
hourAgg := h.aggregateLocked(time.Hour, now)
if !agg.hasData() {
return monitor.Result{}, false
}
result := agg.result()
if len(h.samples) > 0 {
result.LastProbeAt = h.samples[len(h.samples)-1].timestamp.UnixMilli()
}
result.AvgResponse1h = hourAgg.avgResponse()
result.MinResponse1h = hourAgg.minUs
result.MaxResponse1h = hourAgg.maxUs
result.PacketLoss1h = hourAgg.lossPercentage()
result.SampleCount = h.sampleCount
if hourAgg.successCount == 0 {
result.MinResponse1h, result.MaxResponse1h = 0, 0
}
return result, true
}
// latestSampleAggregateLocked returns an aggregate containing only the most recent sample, if any.
func (h *monitorHistory) latestSampleAggregateLocked() monitorAggregate {
agg := newMonitorAggregate()
if len(h.samples) == 0 {
return agg
}
agg.addResponse(h.samples[len(h.samples)-1].responseUs)
return agg
}
// aggregateLocked collects monitor data for the requested time window.
func (h *monitorHistory) aggregateLocked(duration time.Duration, now time.Time) monitorAggregate {
cutoff := now.Add(-duration)
// Keep short windows exact; longer windows read from minute buckets to avoid raw-sample retention.
if duration <= monitorRawRetention {
return aggregateSamplesSince(h.samples, cutoff)
}
return aggregateBucketsSince(h.buckets[:], cutoff, now)
}
// aggregateSamplesSince aggregates raw samples newer than the cutoff.
func aggregateSamplesSince(samples []monitorSample, cutoff time.Time) monitorAggregate {
agg := newMonitorAggregate()
for _, sample := range samples {
if sample.timestamp.Before(cutoff) {
continue
}
agg.addResponse(sample.responseUs)
}
return agg
}
// aggregateBucketsSince aggregates minute buckets overlapping the requested window.
func aggregateBucketsSince(buckets []monitorBucket, cutoff, now time.Time) monitorAggregate {
agg := newMonitorAggregate()
startMinute := int32(cutoff.Unix() / 60)
endMinute := int32(now.Unix() / 60)
for _, bucket := range buckets {
if !bucket.filled || bucket.minute < startMinute || bucket.minute > endMinute {
continue
}
agg.addAggregate(bucket.stats)
}
return agg
}
// addSampleLocked stores a fresh sample in both raw and per-minute retention buffers.
func (h *monitorHistory) addSampleLocked(sample monitorSample) {
h.sampleCount++
cutoff := sample.timestamp.Add(-monitorRawRetention)
start := 0
for i := range h.samples {
if !h.samples[i].timestamp.Before(cutoff) {
start = i
break
}
if i == len(h.samples)-1 {
start = len(h.samples)
}
}
if start > 0 {
size := copy(h.samples, h.samples[start:])
h.samples = h.samples[:size]
}
h.samples = append(h.samples, sample)
minute := int32(sample.timestamp.Unix() / 60)
// Each slot stores one wall-clock minute, so the ring stays fixed-size at ~1h per monitor.
bucket := &h.buckets[minute%monitorMinuteBucketLen]
if !bucket.filled || bucket.minute != minute {
bucket.minute = minute
bucket.filled = true
bucket.stats = newMonitorAggregate()
}
bucket.stats.addResponse(sample.responseUs)
}

View File

@@ -0,0 +1,154 @@
package agent
import (
"testing"
"time"
"github.com/fxamacker/cbor/v2"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestMonitorHistoryWindowCounts(t *testing.T) {
history := newMonitorHistory()
now := time.Now()
// This older success counts toward lifetime warm-up, but not this window.
history.record(monitorSample{responseUs: 1000, timestamp: now.Add(-2 * time.Minute)})
history.record(monitorSample{responseUs: 10, timestamp: now.Add(-30 * time.Second)})
history.record(monitorSample{responseUs: 21, timestamp: now.Add(-20 * time.Second)})
history.record(monitorSample{responseUs: -1, timestamp: now.Add(-10 * time.Second)})
result, ok := history.result(time.Minute, now)
require.True(t, ok)
assert.EqualValues(t, 4, result.SampleCount)
assert.EqualValues(t, 3, result.TotalCount)
assert.EqualValues(t, 2, result.SuccessCount)
assert.EqualValues(t, 31, result.ResponseSum, "preserve the sum before average rounding")
assert.EqualValues(t, 15, result.AvgResponse)
assert.Equal(t, 33.33, result.PacketLoss)
encoded, err := cbor.Marshal(result)
require.NoError(t, err)
var decoded monitor.Result
require.NoError(t, cbor.Unmarshal(encoded, &decoded))
assert.Equal(t, result, decoded)
stats := monitor.Stats{}.FromResult(decoded)
assert.Equal(t, result.TotalCount, stats.TotalCount)
assert.Equal(t, result.SuccessCount, stats.SuccessCount)
assert.Equal(t, result.ResponseSum, stats.ResponseSum)
// Reads do not consume samples. A short window's latest-sample fallback
// carries the count for that single failure, not the minute or lifetime count.
repeated, _ := history.result(time.Minute, now)
assert.Equal(t, result, repeated)
fallback, ok := history.result(time.Second, now)
require.True(t, ok)
assert.EqualValues(t, 1, fallback.TotalCount)
assert.Zero(t, fallback.SuccessCount)
assert.Zero(t, fallback.ResponseSum)
assert.Equal(t, 100.0, fallback.PacketLoss)
assert.EqualValues(t, 4, fallback.SampleCount)
}
func TestMonitorHistoryAggregateLockedUsesRawSamplesForShortWindows(t *testing.T) {
now := time.Date(2026, time.April, 21, 12, 0, 0, 0, time.UTC)
history := newMonitorHistory()
history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-90 * time.Second)})
history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now.Add(-30 * time.Second)})
history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-10 * time.Second)})
agg := history.aggregateLocked(time.Minute, now)
require.True(t, agg.hasData())
assert.Equal(t, int64(2), agg.totalCount)
assert.Equal(t, int64(1), agg.successCount)
result := agg.result()
assert.Equal(t, int64(20), result.AvgResponse)
assert.Equal(t, int64(20), result.MinResponse)
assert.Equal(t, int64(20), result.MaxResponse)
assert.Equal(t, 50.0, result.PacketLoss)
}
func TestMonitorHistoryAggregateLockedUsesMinuteBucketsForLongWindows(t *testing.T) {
now := time.Date(2026, time.April, 21, 12, 0, 30, 0, time.UTC)
history := newMonitorHistory()
history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-11 * time.Minute)})
history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now.Add(-9 * time.Minute)})
history.addSampleLocked(monitorSample{responseUs: 40, timestamp: now.Add(-5 * time.Minute)})
history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-90 * time.Second)})
history.addSampleLocked(monitorSample{responseUs: 30, timestamp: now.Add(-30 * time.Second)})
agg := history.aggregateLocked(10*time.Minute, now)
require.True(t, agg.hasData())
assert.Equal(t, int64(4), agg.totalCount)
assert.Equal(t, int64(3), agg.successCount)
result := agg.result()
assert.Equal(t, int64(30), result.AvgResponse)
assert.Equal(t, int64(20), result.MinResponse)
assert.Equal(t, int64(40), result.MaxResponse)
assert.Equal(t, 25.0, result.PacketLoss)
}
func TestMonitorHistoryAddSampleLockedTrimsRawSamplesButKeepsBucketHistory(t *testing.T) {
now := time.Date(2026, time.April, 21, 12, 0, 0, 0, time.UTC)
history := newMonitorHistory()
history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-10 * time.Minute)})
history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now})
require.Len(t, history.samples, 1)
assert.Equal(t, int64(20), history.samples[0].responseUs)
agg := history.aggregateLocked(10*time.Minute, now)
require.True(t, agg.hasData())
assert.Equal(t, int64(2), agg.totalCount)
assert.Equal(t, int64(2), agg.successCount)
result := agg.result()
assert.Equal(t, int64(15), result.AvgResponse)
assert.Equal(t, int64(10), result.MinResponse)
assert.Equal(t, int64(20), result.MaxResponse)
assert.Equal(t, 0.0, result.PacketLoss)
}
func TestMonitorHistoryProbeTimestamp(t *testing.T) {
history := newMonitorHistory()
start := time.Date(2026, time.September, 14, 12, 0, 0, 0, time.UTC)
_, ok := history.result(time.Minute, start)
require.False(t, ok)
first := history.record(monitorSample{responseUs: 20, timestamp: start})
assert.Equal(t, start.UnixMilli(), first.LastProbeAt)
for minute := 0; minute < 5; minute++ {
now := start.Add(time.Duration(minute)*time.Minute + time.Second)
// Realtime reads must not consume freshness for the persistence request.
for _, window := range []time.Duration{time.Second, time.Minute} {
result, ok := history.result(window, now)
require.True(t, ok)
assert.Equal(t, first.LastProbeAt, result.LastProbeAt)
assert.Equal(t, int64(20), result.AvgResponse)
}
}
next := start.Add(5 * time.Minute)
failed := history.record(monitorSample{responseUs: -1, timestamp: next})
assert.Equal(t, next.UnixMilli(), failed.LastProbeAt)
assert.Equal(t, float64(100), failed.PacketLoss)
repeated, ok := history.result(time.Minute, next.Add(2*time.Minute))
require.True(t, ok)
assert.Equal(t, failed.LastProbeAt, repeated.LastProbeAt)
assert.Equal(t, float64(100), repeated.PacketLoss)
}
func TestMonitorHistorySampleCount(t *testing.T) {
history := newMonitorHistory()
now := time.Now()
// Both failed and successful probes count, including older samples so
// monitors with hourly intervals can finish warming up.
history.record(monitorSample{responseUs: -1, timestamp: now.Add(-2 * time.Hour)})
for i, response := range []int64{10, -1, 20} {
result := history.record(monitorSample{responseUs: response, timestamp: now.Add(time.Duration(i) * time.Second)})
assert.EqualValues(t, i+2, result.SampleCount)
}
result, ok := history.clone().result(time.Minute, now.Add(3*time.Second))
require.True(t, ok)
assert.EqualValues(t, 4, result.SampleCount)
}

View File

@@ -0,0 +1,312 @@
package agent
import (
"bytes"
"context"
"crypto/rand"
"errors"
"fmt"
"math"
"net"
"os"
"os/exec"
"regexp"
"runtime"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"golang.org/x/net/icmp"
"golang.org/x/net/ipv4"
"golang.org/x/net/ipv6"
"log/slog"
)
// Match the numeric RTT independently of the localized label used by Windows.
var pingTimeRegex = regexp.MustCompile(`(?i)[=<]\s*([0-9]+(?:[.,][0-9]+)?)\s*ms\b`)
var icmpSequence atomic.Uint32
type icmpPacketConn interface {
Close() error
}
// icmpMethod tracks which ICMP approach to use. Once a method succeeds or
// all native methods fail, the choice is cached so subsequent monitors skip
// the trial-and-error overhead.
type icmpMethod uint8
const (
icmpUntried icmpMethod = iota // haven't tried yet
icmpRaw // privileged raw socket
icmpDatagram // unprivileged datagram socket
icmpExecFallback // shell out to system ping command
)
// icmpFamily holds the network parameters and cached detection result for one address family.
type icmpFamily struct {
rawNetwork string // e.g. "ip4:icmp" or "ip6:ipv6-icmp"
dgramNetwork string // e.g. "udp4" or "udp6"
listenAddr string // "0.0.0.0" or "::"
echoType icmp.Type // outgoing echo request type
replyType icmp.Type // expected echo reply type
proto int // IANA protocol number for parsing replies
isIPv6 bool
mode icmpMethod // cached detection result (guarded by icmpModeMu)
}
var (
icmpV4 = icmpFamily{
rawNetwork: "ip4:icmp",
dgramNetwork: "udp4",
listenAddr: "0.0.0.0",
echoType: ipv4.ICMPTypeEcho,
replyType: ipv4.ICMPTypeEchoReply,
proto: 1,
}
icmpV6 = icmpFamily{
rawNetwork: "ip6:ipv6-icmp",
dgramNetwork: "udp6",
listenAddr: "::",
echoType: ipv6.ICMPTypeEchoRequest,
replyType: ipv6.ICMPTypeEchoReply,
proto: 58,
isIPv6: true,
}
icmpModeMu sync.Mutex
icmpListen = func(network, listenAddr string) (icmpPacketConn, error) {
return icmp.ListenPacket(network, listenAddr)
}
)
// monitorICMP sends an ICMP echo request and measures round-trip response.
// Supports both IPv4 and IPv6 targets. The ICMP method (raw socket,
// unprivileged datagram, or exec fallback) is detected once per address
// family and cached for subsequent monitors.
// Returns response in microseconds, or -1 and an error on failure.
func monitorICMP(ctx context.Context, target string) (int64, error) {
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
family, ip, err := resolveICMPTarget(ctx, target)
if err != nil {
return -1, err
}
icmpModeMu.Lock()
if family.mode == icmpUntried {
family.mode = detectICMPMode(family, icmpListen)
}
mode := family.mode
icmpModeMu.Unlock()
switch mode {
case icmpRaw:
return monitorICMPNative(ctx, family.rawNetwork, family, &net.IPAddr{IP: ip})
case icmpDatagram:
return monitorICMPNative(ctx, family.dgramNetwork, family, &net.UDPAddr{IP: ip})
case icmpExecFallback:
return monitorICMPExec(ctx, ip.String(), family.isIPv6)
default:
return -1, errors.New("unsupported ICMP mode")
}
}
// resolveICMPTarget resolves a target hostname or IP to determine the address
// family and concrete IP address. Prefers IPv4 for dual-stack hostnames.
func resolveICMPTarget(ctx context.Context, target string) (*icmpFamily, net.IP, error) {
if ip := net.ParseIP(target); ip != nil {
if ip.To4() != nil {
return &icmpV4, ip.To4(), nil
}
return &icmpV6, ip, nil
}
ips, err := net.DefaultResolver.LookupIP(ctx, "ip", target)
if err != nil || len(ips) == 0 {
return nil, nil, err
}
for _, ip := range ips {
if v4 := ip.To4(); v4 != nil {
return &icmpV4, v4, nil
}
}
return &icmpV6, ips[0], nil
}
func detectICMPMode(family *icmpFamily, listen func(network, listenAddr string) (icmpPacketConn, error)) icmpMethod {
label := "IPv4"
if family.isIPv6 {
label = "IPv6"
}
conn, err := listen(family.rawNetwork, family.listenAddr)
slog.Debug("ICMP raw socket test", "family", label, "err", err)
if err == nil {
conn.Close()
return icmpRaw
}
conn, err = listen(family.dgramNetwork, family.listenAddr)
slog.Debug("ICMP datagram socket test", "family", label, "err", err)
if err == nil {
conn.Close()
return icmpDatagram
}
return icmpExecFallback
}
// monitorICMPNative sends an ICMP echo request using Go's x/net/icmp package.
func monitorICMPNative(ctx context.Context, network string, family *icmpFamily, dst net.Addr) (int64, error) {
conn, err := icmp.ListenPacket(network, family.listenAddr)
if err != nil {
return -1, err
}
defer conn.Close()
return monitorICMPPacket(ctx, conn, family, dst)
}
func monitorICMPPacket(ctx context.Context, conn net.PacketConn, family *icmpFamily, dst net.Addr) (int64, error) {
if err := ctx.Err(); err != nil {
return -1, err
}
// Closing the socket interrupts both reads and writes on cancellation.
stop := context.AfterFunc(ctx, func() { _ = conn.Close() })
defer stop()
// Prepare correlation data before starting the round-trip timer. The token
// also distinguishes delayed replies after the 16-bit sequence wraps.
token := make([]byte, 16)
if _, err := rand.Read(token); err != nil {
return -1, err
}
echo := &icmp.Echo{
ID: os.Getpid() & 0xffff,
Seq: int(icmpSequence.Add(1) & 0xffff),
Data: token,
}
// Linux ping sockets replace the Echo ID with their bound port. Darwin
// datagram sockets and raw sockets preserve the supplied ID.
if local, ok := conn.LocalAddr().(*net.UDPAddr); ok && runtime.GOOS == "linux" {
echo.ID = local.Port
}
targetIP := icmpAddrIP(dst)
msg := &icmp.Message{
Type: family.echoType,
Code: 0,
Body: echo,
}
msgBytes, err := msg.Marshal(nil)
if err != nil {
return -1, err
}
// Set deadline before sending
if err := conn.SetDeadline(time.Now().Add(3 * time.Second)); err != nil {
return -1, err
}
buf := make([]byte, 1500)
start := time.Now()
if _, err := conn.WriteTo(msgBytes, dst); err != nil {
return -1, err
}
// Read reply
for {
n, peer, err := conn.ReadFrom(buf)
received := time.Now()
if err != nil {
return -1, err
}
if !targetIP.Equal(icmpAddrIP(peer)) {
continue
}
reply, err := icmp.ParseMessage(family.proto, buf[:n])
if err != nil || reply.Type != family.replyType || reply.Code != 0 {
continue
}
body, ok := reply.Body.(*icmp.Echo)
if ok && body.ID == echo.ID && body.Seq == echo.Seq && bytes.Equal(body.Data, echo.Data) {
return received.Sub(start).Microseconds(), nil
}
// Keep waiting for our reply without extending the original deadline.
}
}
func icmpAddrIP(addr net.Addr) net.IP {
switch addr := addr.(type) {
case *net.IPAddr:
return addr.IP
case *net.UDPAddr:
return addr.IP
default:
return nil
}
}
// pingCommand selects the executable and arguments for the supported agent platforms.
// The context deadline enforces the timeout: -W has incompatible meanings across
// Linux, BSD IPv4 ping, and macOS ping6.
func pingCommand(goos, target string, isIPv6 bool) (string, []string, error) {
family := "-4"
if isIPv6 {
family = "-6"
}
switch goos {
case "windows":
return "ping", []string{family, "-n", "1", "-w", "3000", target}, nil
case "linux":
return "ping", []string{family, "-n", "-c", "1", target}, nil
case "darwin", "freebsd", "openbsd":
command := "ping"
if isIPv6 {
command = "ping6"
}
return command, []string{"-n", "-c", "1", target}, nil
default:
return "", nil, fmt.Errorf("ping fallback is unsupported on %s", goos)
}
}
// monitorICMPExec falls back to the system ping command. Returns -1 and an error on failure.
func monitorICMPExec(ctx context.Context, target string, isIPv6 bool) (int64, error) {
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
name, args, err := pingCommand(runtime.GOOS, target, isIPv6)
if err != nil {
return -1, err
}
cmd := exec.CommandContext(ctx, name, args...)
// Keep Unix output and decimal formatting stable. Windows ignores LC_ALL.
cmd.Env = append(os.Environ(), "LC_ALL=C")
output, err := cmd.Output()
if ctx.Err() != nil {
return -1, ctx.Err()
}
if err != nil {
return -1, fmt.Errorf("%s failed: %w", name, err)
}
return parsePingResponse(output)
}
// parsePingResponse returns the reported RTT, never subprocess execution time.
// For a bounded value such as Windows' time<1ms, retain the reported upper bound.
func parsePingResponse(output []byte) (int64, error) {
matches := pingTimeRegex.FindSubmatch(output)
if len(matches) < 2 {
return -1, errors.New("ping output contains no round-trip time")
}
ms, err := strconv.ParseFloat(strings.ReplaceAll(string(matches[1]), ",", "."), 64)
if err != nil || math.IsInf(ms, 0) || ms >= float64(math.MaxInt64)/1000 {
return -1, errors.New("invalid round-trip time in ping output")
}
return int64(math.Round(ms * 1000)), nil
}

View File

@@ -0,0 +1,433 @@
//go:build testing
package agent
import (
"context"
"errors"
"fmt"
"net"
"os"
"path/filepath"
"runtime"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/net/icmp"
)
type testICMPPacketConn struct{}
func (testICMPPacketConn) Close() error { return nil }
type blockingICMPConn struct {
net.PacketConn
reading chan struct{}
}
func (c *blockingICMPConn) WriteTo(p []byte, addr net.Addr) (int, error) {
return len(p), nil
}
func (c *blockingICMPConn) ReadFrom(p []byte) (int, net.Addr, error) {
close(c.reading)
return c.PacketConn.ReadFrom(p)
}
func TestMonitorICMPPacketCancellation(t *testing.T) {
conn, err := net.ListenPacket("udp4", "127.0.0.1:0")
require.NoError(t, err)
defer conn.Close()
blocking := &blockingICMPConn{PacketConn: conn, reading: make(chan struct{})}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
done := make(chan error, 1)
go func() {
_, err := monitorICMPPacket(ctx, blocking, &icmpV4, conn.LocalAddr())
done <- err
}()
select {
case <-blocking.reading:
case <-time.After(time.Second):
t.Fatal("probe did not begin reading")
}
cancel()
select {
case err := <-done:
require.Error(t, err)
case <-time.After(time.Second):
t.Fatal("cancellation did not interrupt the socket read")
}
}
func TestMonitorICMPExecCancellation(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("test uses a POSIX shell stub for ping")
}
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "ping"), []byte("#!/bin/sh\nexec sleep 30\n"), 0o755))
t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
defer cancel()
done := make(chan error, 1)
go func() {
_, err := monitorICMPExec(ctx, "127.0.0.1", false)
done <- err
}()
select {
case err := <-done:
require.ErrorIs(t, err, context.DeadlineExceeded)
case <-time.After(time.Second):
t.Fatal("cancellation did not terminate ping")
}
}
func TestPingCommand(t *testing.T) {
for _, goos := range []string{"linux", "windows", "darwin", "freebsd", "openbsd"} {
for _, ipv6 := range []bool{false, true} {
t.Run(fmt.Sprintf("%s/ipv6=%t", goos, ipv6), func(t *testing.T) {
target, family := "192.0.2.1", "-4"
if ipv6 {
target, family = "2001:db8::1", "-6"
}
name, args, err := pingCommand(goos, target, ipv6)
require.NoError(t, err)
wantName := "ping"
wantArgs := []string{"-n", "-c", "1", target}
switch goos {
case "windows":
wantArgs = []string{family, "-n", "1", "-w", "3000", target}
case "linux":
wantArgs = append([]string{family}, wantArgs...)
default:
if ipv6 {
wantName = "ping6"
}
}
assert.Equal(t, wantName, name)
assert.Equal(t, wantArgs, args)
})
}
}
_, _, err := pingCommand("unsupported", "192.0.2.1", false)
require.Error(t, err)
}
func TestParsePingResponse(t *testing.T) {
for _, tc := range []struct {
name string
output string
wantUs int64
}{
{"linux", "64 bytes from 192.0.2.1: icmp_seq=1 ttl=64 time=12.345 ms", 12345},
{"bsd", "64 bytes from 192.0.2.1: icmp_seq=0 ttl=64 time=0.023 ms", 23},
{"ipv6", "64 bytes from 2001:db8::1: icmp_seq=0 hlim=64 time=1.234 ms", 1234},
{"windows", "Reply from 192.0.2.1: bytes=32 time=12ms TTL=128", 12000},
{"windows submillisecond", "Reply from ::1: time<1ms", 1000},
{"localized windows", "Antwort von 192.0.2.1: Bytes=32 Zeit=12ms TTL=128", 12000},
{"decimal comma", "64 bytes from 192.0.2.1: time=1,234 ms", 1234},
{"rounding", "time=0.1236 ms", 124},
{"empty", "", -1},
{"timeout", "Request timed out.", -1},
{"unreachable", "Reply from 192.0.2.1: Destination host unreachable.", -1},
{"malformed", "time=oops ms", -1},
{"negative", "time=-1 ms", -1},
{"overflow", "time=999999999999999999999 ms", -1},
} {
t.Run(tc.name, func(t *testing.T) {
responseUs, err := parsePingResponse([]byte(tc.output))
if tc.wantUs < 0 {
require.Error(t, err)
} else {
require.NoError(t, err)
}
assert.Equal(t, tc.wantUs, responseUs)
})
}
}
func TestMonitorICMPExecOutput(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("test uses a POSIX shell stub for ping")
}
for _, tc := range []struct {
name string
output string
exit int
wantUs int64
}{
{"success", "time=1.234 ms", 0, 1234},
{"missing RTT", "unrecognized output", 0, -1},
{"failed command with RTT", "time=1.234 ms", 1, -1},
} {
t.Run(tc.name, func(t *testing.T) {
dir := t.TempDir()
// Also verify an inherited locale cannot override the C locale.
script := fmt.Sprintf("#!/bin/sh\n[ \"$LC_ALL\" = C ] || exit 2\nprintf '%%s\\n' '%s'\nexit %d\n", tc.output, tc.exit)
require.NoError(t, os.WriteFile(filepath.Join(dir, "ping"), []byte(script), 0o755))
t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
t.Setenv("LC_ALL", "de_DE.UTF-8")
responseUs, err := monitorICMPExec(t.Context(), "127.0.0.1", false)
if tc.wantUs < 0 {
require.Error(t, err)
} else {
require.NoError(t, err)
}
assert.Equal(t, tc.wantUs, responseUs)
})
}
}
type icmpTestReply struct {
data []byte
peer net.Addr
}
type scriptedICMPConn struct {
net.PacketConn
local net.Addr
onWrite func([]byte, net.Addr)
replies []icmpTestReply
reads int
deadlineSets int
}
func (c *scriptedICMPConn) LocalAddr() net.Addr { return c.local }
func (c *scriptedICMPConn) SetDeadline(deadline time.Time) error {
c.deadlineSets++
return nil
}
func (c *scriptedICMPConn) WriteTo(data []byte, dst net.Addr) (int, error) {
c.onWrite(data, dst)
return len(data), nil
}
func (c *scriptedICMPConn) ReadFrom(buf []byte) (int, net.Addr, error) {
c.reads++
if len(c.replies) == 0 {
return 0, nil, os.ErrDeadlineExceeded
}
reply := c.replies[0]
c.replies = c.replies[1:]
return copy(buf, reply.data), reply.peer, nil
}
func TestMonitorICMPReplyCorrelation(t *testing.T) {
for _, family := range []*icmpFamily{&icmpV4, &icmpV6} {
for _, datagram := range []bool{false, true} {
network := family.rawNetwork
ip, other := net.ParseIP("192.0.2.1"), net.ParseIP("192.0.2.2")
if family.isIPv6 {
ip, other = net.ParseIP("2001:db8::1"), net.ParseIP("2001:db8::2")
}
var dst net.Addr = &net.IPAddr{IP: ip}
var wrongPeer net.Addr = &net.IPAddr{IP: other}
if datagram {
network = family.dgramNetwork
dst = &net.UDPAddr{IP: ip}
wrongPeer = &net.UDPAddr{IP: other}
}
for _, mismatch := range []string{"source", "id", "sequence", "payload", "type", "code", "malformed"} {
for _, eventuallyMatches := range []bool{false, true} {
ending := "timeout"
if eventuallyMatches {
ending = "success"
}
t.Run(network+"/"+mismatch+"/"+ending, func(t *testing.T) {
conn := &scriptedICMPConn{local: &net.IPAddr{IP: net.IPv4zero}}
if datagram {
conn.local = &net.UDPAddr{Port: 12345}
if runtime.GOOS == "linux" {
// Deliberately differ from the process ID.
conn.local = &net.UDPAddr{Port: (os.Getpid() % 65534) + 1}
}
}
conn.onWrite = func(data []byte, target net.Addr) {
require.Equal(t, dst, target)
request, err := icmp.ParseMessage(family.proto, data)
require.NoError(t, err)
echo := request.Body.(*icmp.Echo)
expectedID := os.Getpid() & 0xffff
if datagram && runtime.GOOS == "linux" {
expectedID = conn.local.(*net.UDPAddr).Port
}
require.Equal(t, expectedID, echo.ID)
reply := &icmp.Message{Type: family.replyType, Body: echo}
valid, err := reply.Marshal(nil)
require.NoError(t, err)
peer := dst
switch mismatch {
case "source":
peer = wrongPeer
case "id":
echo.ID ^= 1
case "sequence":
echo.Seq ^= 1
case "payload":
echo.Data[0] ^= 1
case "type":
reply.Type = family.echoType
case "code":
reply.Code = 1
}
invalid, err := reply.Marshal(nil)
require.NoError(t, err)
if mismatch == "malformed" {
invalid = invalid[:2]
}
conn.replies = []icmpTestReply{{invalid, peer}}
if eventuallyMatches {
conn.replies = append(conn.replies, icmpTestReply{valid, dst})
}
}
elapsed, err := monitorICMPPacket(context.Background(), conn, family, dst)
if eventuallyMatches {
require.NoError(t, err)
assert.GreaterOrEqual(t, elapsed, int64(0))
} else {
require.ErrorIs(t, err, os.ErrDeadlineExceeded)
assert.Equal(t, int64(-1), elapsed)
}
assert.Equal(t, 2, conn.reads)
assert.Equal(t, 1, conn.deadlineSets)
})
}
}
}
}
}
func TestMonitorICMPLoopback(t *testing.T) {
for _, family := range []*icmpFamily{&icmpV4, &icmpV6} {
for _, network := range []string{family.rawNetwork, family.dgramNetwork} {
t.Run(network, func(t *testing.T) {
conn, err := icmp.ListenPacket(network, family.listenAddr)
if err != nil {
t.Skipf("ICMP socket unavailable: %v", err)
}
defer conn.Close()
ip := net.ParseIP("127.0.0.1")
if family.isIPv6 {
ip = net.ParseIP("::1")
}
var dst net.Addr = &net.IPAddr{IP: ip}
if network == family.dgramNetwork {
dst = &net.UDPAddr{IP: ip}
}
elapsed, err := monitorICMPPacket(context.Background(), conn, family, dst)
require.NoError(t, err)
assert.GreaterOrEqual(t, elapsed, int64(0))
})
}
}
}
func TestDetectICMPMode(t *testing.T) {
tests := []struct {
name string
family *icmpFamily
rawErr error
udpErr error
want icmpMethod
wantNetworks []string
}{
{
name: "IPv4 prefers raw socket when available",
family: &icmpV4,
want: icmpRaw,
wantNetworks: []string{"ip4:icmp"},
},
{
name: "IPv4 uses datagram when raw unavailable",
family: &icmpV4,
rawErr: errors.New("operation not permitted"),
want: icmpDatagram,
wantNetworks: []string{"ip4:icmp", "udp4"},
},
{
name: "IPv4 falls back to exec when both unavailable",
family: &icmpV4,
rawErr: errors.New("operation not permitted"),
udpErr: errors.New("protocol not supported"),
want: icmpExecFallback,
wantNetworks: []string{"ip4:icmp", "udp4"},
},
{
name: "IPv6 prefers raw socket when available",
family: &icmpV6,
want: icmpRaw,
wantNetworks: []string{"ip6:ipv6-icmp"},
},
{
name: "IPv6 uses datagram when raw unavailable",
family: &icmpV6,
rawErr: errors.New("operation not permitted"),
want: icmpDatagram,
wantNetworks: []string{"ip6:ipv6-icmp", "udp6"},
},
{
name: "IPv6 falls back to exec when both unavailable",
family: &icmpV6,
rawErr: errors.New("operation not permitted"),
udpErr: errors.New("protocol not supported"),
want: icmpExecFallback,
wantNetworks: []string{"ip6:ipv6-icmp", "udp6"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
calls := make([]string, 0, 2)
listen := func(network, listenAddr string) (icmpPacketConn, error) {
require.Equal(t, tt.family.listenAddr, listenAddr)
calls = append(calls, network)
switch network {
case tt.family.rawNetwork:
if tt.rawErr != nil {
return nil, tt.rawErr
}
case tt.family.dgramNetwork:
if tt.udpErr != nil {
return nil, tt.udpErr
}
default:
t.Fatalf("unexpected network %q", network)
}
return testICMPPacketConn{}, nil
}
assert.Equal(t, tt.want, detectICMPMode(tt.family, listen))
assert.Equal(t, tt.wantNetworks, calls)
})
}
}
func TestResolveICMPTarget(t *testing.T) {
t.Run("IPv4 literal", func(t *testing.T) {
family, ip, err := resolveICMPTarget(context.Background(), "127.0.0.1")
require.NoError(t, err)
require.NotNil(t, family)
assert.False(t, family.isIPv6)
assert.Equal(t, "127.0.0.1", ip.String())
})
t.Run("IPv6 literal", func(t *testing.T) {
family, ip, err := resolveICMPTarget(context.Background(), "::1")
require.NoError(t, err)
require.NotNil(t, family)
assert.True(t, family.isIPv6)
assert.Equal(t, "::1", ip.String())
})
t.Run("IPv4-mapped IPv6 resolves as IPv4", func(t *testing.T) {
family, ip, err := resolveICMPTarget(context.Background(), "::ffff:127.0.0.1")
require.NoError(t, err)
require.NotNil(t, family)
assert.False(t, family.isIPv6)
assert.Equal(t, "127.0.0.1", ip.String())
})
}

View File

@@ -0,0 +1,133 @@
package agent
import (
"context"
"errors"
"fmt"
"net"
"net/http"
"time"
"github.com/henrygd/beszel"
"github.com/henrygd/beszel/internal/entities/monitor"
)
const networkMonitorUserAgent = "Beszel-Agent/" + beszel.Version + " (+https://beszel.dev)"
// monitorProbe performs one check. Errors are recorded as loss by the task runner.
// Implementations must honor cancellation and bound their execution time.
type monitorProbe func(context.Context, monitor.Config) (int64, error)
func networkMonitorProbe(client *http.Client) monitorProbe {
return func(ctx context.Context, config monitor.Config) (int64, error) {
switch config.Protocol {
case "icmp":
return monitorICMP(ctx, config.Target)
case "tcp":
return monitorTCP(ctx, config.Target, config.Port)
case "http":
return monitorHTTP(ctx, client, config.Target)
case "dns":
return monitorDNS(ctx, config.Target, config.Server)
default:
return -1, fmt.Errorf("unknown monitor protocol: %s", config.Protocol)
}
}
}
// monitorTCP measures connection establishment time, including address fallback
// but excluding DNS resolution.
// Returns -1 and an error on failure.
func monitorTCP(ctx context.Context, target string, port uint16) (int64, error) {
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
// Resolve DNS first, outside the timing window but within the probe deadline.
ips, err := net.DefaultResolver.LookupHost(ctx, target)
if err != nil {
return -1, err
}
if len(ips) == 0 {
return -1, errors.New("no addresses resolved for TCP monitor")
}
portString := fmt.Sprintf("%d", port)
deadline, _ := ctx.Deadline()
// Share the remaining probe budget across addresses so an unresponsive
// first address cannot consume all the time available for alternatives.
start := time.Now()
for i, ip := range ips {
if err := ctx.Err(); err != nil {
return -1, err
}
dialer := net.Dialer{Timeout: time.Until(deadline) / time.Duration(len(ips)-i)}
var conn net.Conn
conn, err = dialer.DialContext(ctx, "tcp", net.JoinHostPort(ip, portString))
if err != nil {
continue
}
responseUs := time.Since(start).Microseconds()
conn.Close()
return responseUs, nil
}
return -1, err
}
// monitorDNS measures DNS resolution response time in microseconds. If server is
// non-empty, the lookup is sent to that DNS server (host or host:port, default
// port 53) instead of the system resolver. Returns -1 and an error on failure.
func monitorDNS(ctx context.Context, target, server string) (int64, error) {
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
resolver := net.DefaultResolver
if server != "" {
resolver = dnsResolverForServer(server)
}
start := time.Now()
ips, err := resolver.LookupHost(ctx, target)
if err != nil || len(ips) == 0 {
return -1, err
}
return time.Since(start).Microseconds(), nil
}
// dnsResolverForServer builds a resolver that sends lookups to the given DNS
// server address instead of the system resolver. server may be a bare host or
// host:port; when no port is given, the standard DNS port 53 is used.
func dnsResolverForServer(server string) *net.Resolver {
address := server
if _, _, err := net.SplitHostPort(server); err != nil {
address = net.JoinHostPort(server, "53")
}
return &net.Resolver{
PreferGo: true,
Dial: func(ctx context.Context, network, _ string) (net.Conn, error) {
var dialer net.Dialer
return dialer.DialContext(ctx, network, address)
},
}
}
// monitorHTTP measures HTTP GET request response in microseconds. Returns -1 and an error on failure.
func monitorHTTP(ctx context.Context, client *http.Client, url string) (int64, error) {
if client == nil {
client = http.DefaultClient
}
start := time.Now()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return -1, err
}
req.Header.Set("User-Agent", networkMonitorUserAgent)
resp, err := client.Do(req)
if err != nil {
return -1, err
}
resp.Body.Close()
if resp.StatusCode >= 400 {
return -1, fmt.Errorf("HTTP error: %s", resp.Status)
}
return time.Since(start).Microseconds(), nil
}

View File

@@ -0,0 +1,88 @@
package agent
import (
"sync"
"time"
)
const (
monitorResumeHeartbeat = 10 * time.Second
// Allow scheduling jitter without mistaking an ordinary tick for resume.
monitorResumeGap = 2 * monitorResumeHeartbeat
monitorResumePause = 10 * time.Second
)
// monitorResumeGuard detects likely suspend/resume using wall time. A long
// process stall or forward clock adjustment can also trigger the bounded pause.
// One heartbeat is shared by all configured monitors.
type monitorResumeGuard struct {
mu sync.Mutex
stop chan struct{}
lastTick time.Time
pauseUntil time.Time
generation uint32
}
func (g *monitorResumeGuard) start() {
g.mu.Lock()
defer g.mu.Unlock()
if g.stop != nil {
return
}
stop := make(chan struct{})
g.stop = stop
g.lastTick = time.Now().Round(0)
g.pauseUntil = time.Time{}
go func() {
ticker := time.NewTicker(monitorResumeHeartbeat)
defer ticker.Stop()
for {
select {
case <-stop:
return
case <-ticker.C:
g.mu.Lock()
if g.stop == stop {
g.observe(time.Now())
}
g.mu.Unlock()
}
}
}()
}
func (g *monitorResumeGuard) shutdown() {
g.mu.Lock()
defer g.mu.Unlock()
if g.stop != nil {
close(g.stop)
g.stop = nil
g.generation++
}
}
// observe requires mu. Strip the monotonic component because it can stop during
// suspend. Read the current time rather than the ticker's queued timestamp.
func (g *monitorResumeGuard) observe(now time.Time) {
now = now.Round(0)
if now.Sub(g.lastTick) > monitorResumeGap {
g.pauseUntil = now.Add(monitorResumePause)
g.generation++
}
g.lastTick = now
}
// snapshot also observes time so a probe waking before the heartbeat detects
// resume itself. A changed generation invalidates probes spanning suspend.
func (g *monitorResumeGuard) snapshot() (generation uint32, allowed bool) {
if g == nil {
return 0, true
}
g.mu.Lock()
defer g.mu.Unlock()
if g.stop == nil {
return g.generation, true
}
g.observe(time.Now())
return g.generation, !g.lastTick.Before(g.pauseUntil)
}

View File

@@ -0,0 +1,121 @@
//go:build testing
package agent
import (
"context"
"errors"
"sync/atomic"
"testing"
"testing/synctest"
"time"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func simulateMonitorSleep(g *monitorResumeGuard) {
g.mu.Lock()
g.lastTick = time.Now().Add(-time.Hour).Round(0)
g.mu.Unlock()
}
func TestMonitorResumePause(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
var g monitorResumeGuard
g.start()
defer g.shutdown()
generation, allowed := g.snapshot()
require.True(t, allowed)
// Heartbeats alone must keep the guard current between infrequent probes.
time.Sleep(time.Minute)
synctest.Wait()
steadyGeneration, allowed := g.snapshot()
require.True(t, allowed)
require.Equal(t, generation, steadyGeneration)
// The probe, rather than the heartbeat, must detect this gap.
simulateMonitorSleep(&g)
next, allowed := g.snapshot()
assert.False(t, allowed)
assert.NotEqual(t, generation, next)
time.Sleep(9 * time.Second)
_, allowed = g.snapshot()
assert.False(t, allowed)
time.Sleep(time.Second)
end, allowed := g.snapshot()
assert.True(t, allowed)
assert.Equal(t, next, end)
})
}
func TestMonitorResumeGuardLifecycle(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
pm := newMonitorManagerWithProbe(func(context.Context, monitor.Config) (int64, error) { return 1, nil })
defer pm.Stop()
assert.Nil(t, pm.resumeGuard.stop)
pm.SyncMonitors([]monitor.Config{{ID: "a", Interval: 3600}, {ID: "b", Interval: 3600}})
stop := pm.resumeGuard.stop
require.NotNil(t, stop)
pm.DeleteMonitor("a")
assert.Equal(t, stop, pm.resumeGuard.stop)
pm.DeleteMonitor("b")
assert.Nil(t, pm.resumeGuard.stop)
select {
case <-stop:
default:
t.Fatal("heartbeat was not stopped")
}
time.Sleep(time.Hour)
_, err := pm.UpsertMonitor(monitor.Config{ID: "c", Interval: 3600}, false)
require.NoError(t, err)
_, allowed := pm.resumeGuard.snapshot()
assert.True(t, allowed, "idle time must not trigger a resume pause")
pm.SyncMonitors(nil)
assert.Nil(t, pm.resumeGuard.stop)
})
}
func TestMonitorResumeDiscardsInflightProbe(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
var g monitorResumeGuard
g.start()
defer g.shutdown()
task := newMonitorTask(monitor.Config{ID: "test"})
defer task.cancel()
task.resumeGuard = &g
result := task.runProbe(func(context.Context, monitor.Config) (int64, error) {
simulateMonitorSleep(&g)
return 0, errors.New("network not ready")
})
assert.Nil(t, result)
assert.Empty(t, task.history.samples)
// Explicit requests may still run during the pause and record real failures.
result = task.runProbe(func(context.Context, monitor.Config) (int64, error) {
return 0, errors.New("unreachable")
})
require.NotNil(t, result)
assert.Equal(t, 100.0, result.PacketLoss)
})
}
func TestMonitorResumeSkipsScheduledProbes(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
var calls atomic.Int32
pm := newMonitorManagerWithProbe(func(context.Context, monitor.Config) (int64, error) {
calls.Add(1)
return 1, nil
})
defer pm.Stop()
pm.SyncMonitors([]monitor.Config{{ID: "test", Interval: 1}})
simulateMonitorSleep(&pm.resumeGuard)
pm.resumeGuard.snapshot()
time.Sleep(9 * time.Second)
synctest.Wait()
assert.Zero(t, calls.Load())
assert.Empty(t, pm.GetResults(1000))
time.Sleep(2 * time.Second)
synctest.Wait()
assert.Positive(t, calls.Load())
})
}

View File

@@ -0,0 +1,63 @@
package agent
import (
"context"
"log/slog"
"math/rand"
"time"
)
func (pm *MonitorManager) startMonitor(task *monitorTask) {
interval := time.Duration(task.config.Interval) * time.Second
if interval < time.Second {
interval = 30 * time.Second
}
delay := getStagger(interval.Milliseconds())
slog.Debug("starting monitor task", "target", task.config.Target, "delay", delay, "interval", interval)
// Certificate checks piggyback on probe ticks, so they run at most once per
// probe interval after they become due.
go runMonitorSchedule(task.ctx, interval, delay, func() {
if _, allowed := task.resumeGuard.snapshot(); allowed {
task.runProbe(pm.probe)
task.refreshCert(pm.certCheck)
}
})
}
// runMonitorSchedule owns only timing. Checks run serially, and slow checks
// naturally drop missed ticks rather than building an execution backlog.
func runMonitorSchedule(ctx context.Context, interval, delay time.Duration, run func()) {
timer := time.NewTimer(delay)
defer timer.Stop()
select {
case <-ctx.Done():
return
case <-timer.C:
}
if ctx.Err() != nil {
return
}
run()
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if ctx.Err() != nil {
return
}
run()
}
}
}
// getStagger returns an initial delay between half an interval and one interval.
func getStagger(intervalMilli int64) time.Duration {
delay := rand.Intn(int(intervalMilli))
if delay < int(intervalMilli)/2 {
delay += int(intervalMilli) / 2
}
return time.Duration(delay) * time.Millisecond
}

View File

@@ -0,0 +1,167 @@
//go:build testing
package agent
import (
"context"
"sync/atomic"
"testing"
"testing/synctest"
"time"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestMonitorScheduleTiming(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
var calls atomic.Int32
go runMonitorSchedule(ctx, 10*time.Second, 5*time.Second, func() { calls.Add(1) })
synctest.Wait()
time.Sleep(4 * time.Second)
synctest.Wait()
assert.Equal(t, 0, int(calls.Load()))
time.Sleep(time.Second)
synctest.Wait()
assert.Equal(t, 1, int(calls.Load()))
time.Sleep(10 * time.Second)
synctest.Wait()
assert.Equal(t, 2, int(calls.Load()))
cancel()
synctest.Wait()
time.Sleep(time.Minute)
synctest.Wait()
assert.Equal(t, 2, int(calls.Load()))
})
}
func TestMonitorScheduleSlowProbe(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
var calls atomic.Int32
release := make(chan struct{})
go runMonitorSchedule(ctx, time.Second, 0, func() {
calls.Add(1)
select {
case <-release:
case <-ctx.Done():
}
})
synctest.Wait()
assert.Equal(t, 1, int(calls.Load()))
time.Sleep(time.Minute)
synctest.Wait()
assert.Equal(t, 1, int(calls.Load()), "a slow probe must not spawn overlapping checks")
close(release)
synctest.Wait()
assert.Equal(t, 1, int(calls.Load()), "missed intervals must not accumulate a backlog")
time.Sleep(time.Second)
synctest.Wait()
assert.Equal(t, 2, int(calls.Load()))
cancel()
synctest.Wait()
})
}
func TestMonitorScheduledAndImmediateRequestsShareProbe(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
var calls atomic.Int32
release := make(chan struct{})
cfg := monitor.Config{ID: "test", Interval: 10}
pm := newMonitorManagerWithProbe(func(ctx context.Context, config monitor.Config) (int64, error) {
assert.Equal(t, cfg, config)
calls.Add(1)
<-release
return 42, nil
})
defer pm.Stop()
task := newMonitorTask(cfg)
pm.monitors[cfg.ID] = task
go runMonitorSchedule(task.ctx, 10*time.Second, 0, func() { task.runProbe(pm.probe) })
synctest.Wait()
results := make(chan *monitor.Result, 2)
for range 2 {
go func() {
result, _ := pm.UpsertMonitor(cfg, true)
results <- result
}()
}
synctest.Wait()
assert.Equal(t, 1, int(calls.Load()))
assert.Empty(t, pm.GetResults(1000), "reading history must not wait for network I/O")
close(release)
synctest.Wait()
first, second := <-results, <-results
require.NotNil(t, first)
require.NotNil(t, second)
assert.Equal(t, int64(42), first.AvgResponse)
assert.Equal(t, first, second)
assert.NotSame(t, first, second, "callers must not share mutable result pointers")
assert.Len(t, task.history.samples, 1)
// A later explicit request must still perform a fresh probe.
_, err := pm.UpsertMonitor(cfg, true)
require.NoError(t, err)
assert.Equal(t, 2, int(calls.Load()))
assert.Len(t, task.history.samples, 2)
})
}
func TestMonitorReplacementCancelsSharedProbe(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
cfg := monitor.Config{ID: "test", Interval: 10}
pm := newMonitorManagerWithProbe(func(ctx context.Context, config monitor.Config) (int64, error) {
if config.Interval == 10 {
<-ctx.Done()
return 0, ctx.Err()
}
return 30, nil
})
defer pm.Stop()
task := newMonitorTask(cfg)
task.history.record(monitorSample{responseUs: 10, timestamp: time.Now()})
pm.monitors[cfg.ID] = task
results := make(chan *monitor.Result, 2)
for range 2 {
go func() {
result, _ := pm.UpsertMonitor(cfg, true)
results <- result
}()
}
synctest.Wait()
updated := cfg
updated.Interval = 20
result, err := pm.UpsertMonitor(updated, true)
require.NoError(t, err)
require.NotNil(t, result)
assert.Equal(t, int64(20), result.AvgResponse)
assert.Zero(t, result.PacketLoss)
synctest.Wait()
assert.Nil(t, <-results)
assert.Nil(t, <-results)
assert.Len(t, task.history.samples, 1)
assert.Len(t, pm.monitors[cfg.ID].history.samples, 2)
})
}
func TestMonitorInjectedProbeTimeoutRecordsLoss(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
pm := newMonitorManagerWithProbe(func(ctx context.Context, _ monitor.Config) (int64, error) {
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
<-ctx.Done()
return 0, ctx.Err()
})
defer pm.Stop()
start := time.Now()
result, err := pm.UpsertMonitor(monitor.Config{ID: "test", Interval: 3600}, true)
require.NoError(t, err)
require.NotNil(t, result)
assert.Equal(t, 3*time.Second, time.Since(start))
assert.Equal(t, 100.0, result.PacketLoss)
assert.NoError(t, pm.monitors["test"].ctx.Err())
})
}

View File

@@ -0,0 +1,191 @@
package agent
import (
"context"
"log/slog"
"sync"
"time"
"github.com/henrygd/beszel/internal/entities/monitor"
)
const monitorFailureLogInterval = 5 * time.Minute
// monitorTask coordinates a probe and its history for one immutable configuration.
type monitorTask struct {
config monitor.Config
ctx context.Context
cancel context.CancelFunc
history *monitorHistory
resumeGuard *monitorResumeGuard
runMu sync.Mutex
inflight *monitorRun
lastFailureLog int64 // Unix nanoseconds
certMu sync.Mutex
cert *monitor.CertInfo
certUnsent bool // cert has not been included in a stats result yet
certChecking bool
nextCertCheck time.Time
}
type monitorRun struct {
done chan struct{}
result *monitor.Result // published by closing done; never mutated afterwards
}
func newMonitorTask(config monitor.Config) *monitorTask {
ctx, cancel := context.WithCancel(context.Background())
task := &monitorTask{config: config, ctx: ctx, history: newMonitorHistory()}
// Serialize cancellation with publication, so canceled probes cannot enter
// history copied into a replacement task.
task.cancel = func() {
task.runMu.Lock()
cancel()
task.runMu.Unlock()
}
return task
}
func newMonitorTaskFromExisting(config monitor.Config, existing *monitorTask) *monitorTask {
task := newMonitorTask(config)
if existing != nil {
task.history = existing.history.clone()
// Keep the last known certificate, but check again soon for the new config.
// The hub already stores it, so it is not marked unsent.
if config.Target == existing.config.Target {
task.cert = existing.certInfo()
}
}
return task
}
// runProbe shares an in-flight check between scheduled and immediate requests.
// Every completed check contributes exactly one sample, regardless of how many
// callers were waiting for it. No task or history lock is held during network I/O.
func (task *monitorTask) runProbe(probe monitorProbe) *monitor.Result {
task.runMu.Lock()
if task.ctx.Err() != nil {
task.runMu.Unlock()
return nil
}
if run := task.inflight; run != nil {
task.runMu.Unlock()
select {
case <-task.ctx.Done():
return nil
case <-run.done:
if task.ctx.Err() != nil {
return nil
}
return copyMonitorResult(run.result)
}
}
run := &monitorRun{done: make(chan struct{})}
task.inflight = run
task.runMu.Unlock()
generation, _ := task.resumeGuard.snapshot()
responseUs, err := probe(task.ctx, task.config)
var logFailure bool
task.runMu.Lock()
currentGeneration, _ := task.resumeGuard.snapshot()
if task.ctx.Err() == nil && generation == currentGeneration {
now := time.Now()
if err != nil {
responseUs = -1
logAt := now.UnixNano()
if task.lastFailureLog == 0 || logAt < task.lastFailureLog || logAt-task.lastFailureLog >= int64(monitorFailureLogInterval) {
logFailure = true
task.lastFailureLog = logAt
}
} else {
task.lastFailureLog = 0
}
result := task.history.record(monitorSample{responseUs: responseUs, timestamp: now})
run.result = &result
}
task.inflight = nil
close(run.done)
task.runMu.Unlock()
if logFailure {
slog.Warn("monitor failed", "err", err, "target", task.config.Target, "protocol", task.config.Protocol)
}
if task.ctx.Err() != nil {
return nil
}
return copyMonitorResult(run.result)
}
// refreshCert checks the certificate of an HTTPS target when due. A failed
// check keeps the last known certificate and retries sooner, as does a
// certificate that expires before the next regular check, so renewals show up
// quickly. Concurrent callers skip rather than wait, and no lock is held during
// network I/O.
func (task *monitorTask) refreshCert(check certChecker) {
if check == nil || !certCheckEnabled(task.config) {
return
}
task.certMu.Lock()
if task.certChecking || time.Now().Before(task.nextCertCheck) {
task.certMu.Unlock()
return
}
task.certChecking = true
task.certMu.Unlock()
info, err := check(task.ctx, task.config.Target)
task.certMu.Lock()
defer task.certMu.Unlock()
task.certChecking = false
if task.ctx.Err() != nil {
return
}
if err != nil {
task.nextCertCheck = time.Now().Add(certCheckRetryInterval)
slog.Warn("certificate check failed", "err", err, "target", task.config.Target)
return
}
task.cert = &info
task.certUnsent = true
now := time.Now()
interval := certCheckInterval
if time.UnixMilli(info.Expires).Before(now.Add(certCheckInterval)) {
interval = certCheckRetryInterval
}
task.nextCertCheck = now.Add(interval)
}
// certInfo returns a copy of the latest certificate info, or nil if unknown.
func (task *monitorTask) certInfo() *monitor.CertInfo {
task.certMu.Lock()
defer task.certMu.Unlock()
if task.cert == nil {
return nil
}
cert := *task.cert
return &cert
}
// takeUnsentCert returns the latest certificate info once after each successful
// check, so unchanged info is not resent with every stats result.
func (task *monitorTask) takeUnsentCert() *monitor.CertInfo {
task.certMu.Lock()
defer task.certMu.Unlock()
if !task.certUnsent {
return nil
}
task.certUnsent = false
cert := *task.cert
return &cert
}
func copyMonitorResult(result *monitor.Result) *monitor.Result {
if result == nil {
return nil
}
copy := *result
return &copy
}

View File

@@ -0,0 +1,79 @@
//go:build testing
package agent
import (
"bytes"
"context"
"errors"
"log/slog"
"testing"
"testing/synctest"
"time"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestMonitorFailureLogCooldown(t *testing.T) {
var logs bytes.Buffer
previous := slog.Default()
slog.SetDefault(slog.New(slog.NewTextHandler(&logs, nil)))
t.Cleanup(func() { slog.SetDefault(previous) })
synctest.Test(t, func(t *testing.T) {
task := newMonitorTask(monitor.Config{ID: "test", Target: "example.test", Protocol: "tcp"})
defer task.cancel()
failure := errors.New("connection refused")
probe := func(context.Context, monitor.Config) (int64, error) { return 42, failure }
var samples int64
check := func(wantLog bool) {
t.Helper()
logs.Reset()
result := task.runProbe(probe)
require.NotNil(t, result)
samples++
assert.Equal(t, samples, result.SampleCount, "suppressed warnings must still record samples")
if !wantLog {
assert.Empty(t, logs.String())
} else {
assert.Contains(t, logs.String(), `msg="monitor failed"`)
assert.Equal(t, 1, bytes.Count(logs.Bytes(), []byte("\n")))
}
}
check(true)
check(false)
time.Sleep(5*time.Minute - time.Nanosecond)
check(false)
time.Sleep(time.Nanosecond)
check(true)
check(false)
time.Sleep(5 * time.Minute)
check(true)
check(false)
// Recovery clears the cooldown.
failure = nil
check(false)
failure = errors.New("connection refused again")
check(true)
// Another monitor has its own cooldown.
other := newMonitorTask(task.config)
defer other.cancel()
logs.Reset()
require.NotNil(t, other.runProbe(probe))
assert.Contains(t, logs.String(), `msg="monitor failed"`)
// A canceled probe must not publish a failure or emit a warning.
logs.Reset()
result := other.runProbe(func(context.Context, monitor.Config) (int64, error) {
other.cancel()
return -1, context.Canceled
})
assert.Nil(t, result)
assert.Empty(t, logs.String())
})
}

View File

@@ -0,0 +1,590 @@
package agent
import (
"context"
"encoding/binary"
"io"
"net"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/henrygd/beszel"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/net/dns/dnsmessage"
)
func TestMonitorManagerGetResultsIncludesHourResponseRange(t *testing.T) {
now := time.Now().UTC()
task := newMonitorTask(monitor.Config{ID: "monitor-1"})
task.history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-30 * time.Minute)})
task.history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now.Add(-9 * time.Minute)})
task.history.addSampleLocked(monitorSample{responseUs: 40, timestamp: now.Add(-5 * time.Minute)})
task.history.addSampleLocked(monitorSample{responseUs: 30, timestamp: now.Add(-50 * time.Second)})
task.history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-30 * time.Second)})
pm := newMonitorManager()
pm.monitors = map[string]*monitorTask{"icmp:example.com": task}
results := pm.GetResults(uint16(time.Minute / time.Millisecond))
result, ok := results["monitor-1"]
require.True(t, ok)
assert.Equal(t, int64(30), result.AvgResponse)
assert.Equal(t, int64(25), result.AvgResponse1h)
assert.Equal(t, int64(30), result.MinResponse)
assert.Equal(t, int64(10), result.MinResponse1h)
assert.Equal(t, int64(30), result.MaxResponse)
assert.Equal(t, int64(40), result.MaxResponse1h)
assert.Equal(t, 50.0, result.PacketLoss)
assert.Equal(t, 20.0, result.PacketLoss1h)
}
func TestMonitorManagerGetResultsIncludesLossOnlyHourData(t *testing.T) {
now := time.Now().UTC()
task := newMonitorTask(monitor.Config{ID: "monitor-1"})
task.history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-30 * time.Second)})
task.history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-10 * time.Second)})
pm := newMonitorManager()
pm.monitors = map[string]*monitorTask{"icmp:example.com": task}
results := pm.GetResults(uint16(time.Minute / time.Millisecond))
result, ok := results["monitor-1"]
require.True(t, ok)
assert.Equal(t, int64(0), result.AvgResponse)
assert.Equal(t, int64(0), result.AvgResponse1h)
assert.Equal(t, int64(0), result.MinResponse)
assert.Equal(t, int64(0), result.MinResponse1h)
assert.Equal(t, int64(0), result.MaxResponse)
assert.Equal(t, int64(0), result.MaxResponse1h)
assert.Equal(t, 100.0, result.PacketLoss)
assert.Equal(t, 100.0, result.PacketLoss1h)
}
func TestMonitorConfigResultKeyUsesSyncedID(t *testing.T) {
cfg := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 10}
assert.Equal(t, "monitor-1", cfg.ID)
}
func TestMonitorManagerSyncMonitorsSkipsConfigsWithoutStableID(t *testing.T) {
validCfg := monitor.Config{ID: "monitor-1", Target: "ignored", Protocol: "noop", Interval: 10}
invalidCfg := monitor.Config{Target: "ignored", Protocol: "noop", Interval: 10}
pm := newMonitorManager()
pm.SyncMonitors([]monitor.Config{validCfg, invalidCfg})
defer pm.Stop()
_, validExists := pm.monitors[validCfg.ID]
_, invalidExists := pm.monitors[invalidCfg.ID]
assert.True(t, validExists)
assert.False(t, invalidExists)
}
func TestMonitorManagerSyncMonitorsStopsRemovedTasksButKeepsExisting(t *testing.T) {
keepCfg := monitor.Config{ID: "monitor-1", Target: "ignored", Protocol: "noop", Interval: 10}
removeCfg := monitor.Config{ID: "monitor-2", Target: "ignored", Protocol: "noop", Interval: 10}
keptTask := newMonitorTask(keepCfg)
removedTask := newMonitorTask(removeCfg)
pm := newMonitorManager()
pm.monitors = map[string]*monitorTask{
keepCfg.ID: keptTask,
removeCfg.ID: removedTask,
}
pm.SyncMonitors([]monitor.Config{keepCfg})
assert.Same(t, keptTask, pm.monitors[keepCfg.ID])
_, exists := pm.monitors[removeCfg.ID]
assert.False(t, exists)
select {
case <-removedTask.ctx.Done():
default:
t.Fatal("expected removed monitor task to be cancelled")
}
select {
case <-keptTask.ctx.Done():
t.Fatal("expected existing monitor task to remain active")
default:
}
}
func TestMonitorManagerSyncMonitorsRestartsChangedConfig(t *testing.T) {
originalCfg := monitor.Config{ID: "monitor-1", Target: "ignored-a", Protocol: "noop", Interval: 10}
updatedCfg := monitor.Config{ID: "monitor-1", Target: "ignored-b", Protocol: "noop", Interval: 10}
originalTask := newMonitorTask(originalCfg)
pm := newMonitorManager()
pm.monitors = map[string]*monitorTask{
originalCfg.ID: originalTask,
}
pm.SyncMonitors([]monitor.Config{updatedCfg})
defer pm.Stop()
restartedTask := pm.monitors[updatedCfg.ID]
assert.NotSame(t, originalTask, restartedTask)
assert.Equal(t, updatedCfg, restartedTask.config)
select {
case <-originalTask.ctx.Done():
default:
t.Fatal("expected changed monitor task to be cancelled")
}
}
func TestMonitorManagerApplySyncUpsertRunsImmediatelyAndReturnsResult(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNoContent)
}))
defer server.Close()
pm := &MonitorManager{
monitors: make(map[string]*monitorTask),
probe: networkMonitorProbe(server.Client()),
}
resp, err := pm.HandleSyncRequest(monitor.SyncRequest{
Action: monitor.SyncActionUpsert,
Config: monitor.Config{ID: "monitor-1", Target: server.URL, Protocol: "http", Interval: 10},
RunNow: true,
})
defer pm.Stop()
require.NoError(t, err)
assert.GreaterOrEqual(t, resp.Result.AvgResponse, int64(0))
assert.Equal(t, 0.0, resp.Result.PacketLoss)
assert.Equal(t, 0.0, resp.Result.PacketLoss1h)
task := pm.monitors["monitor-1"]
require.NotNil(t, task)
task.history.mu.Lock()
defer task.history.mu.Unlock()
require.Len(t, task.history.samples, 1)
}
func TestMonitorManagerUpsertMonitorKeepsHistoryWhenOnlyIntervalChanges(t *testing.T) {
originalCfg := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 10}
updatedCfg := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 30}
now := time.Now().UTC()
existingTask := newMonitorTask(originalCfg)
existingTask.history.addSampleLocked(monitorSample{responseUs: 12, timestamp: now.Add(-50 * time.Minute)})
existingTask.history.addSampleLocked(monitorSample{responseUs: 24, timestamp: now.Add(-30 * time.Second)})
pm := newMonitorManager()
pm.monitors = map[string]*monitorTask{originalCfg.ID: existingTask}
result, err := pm.UpsertMonitor(updatedCfg, false)
defer pm.Stop()
require.NoError(t, err)
assert.Nil(t, result)
updatedTask := pm.monitors[updatedCfg.ID]
require.NotNil(t, updatedTask)
assert.NotSame(t, existingTask, updatedTask)
assert.Equal(t, updatedCfg, updatedTask.config)
updatedTask.history.mu.Lock()
defer updatedTask.history.mu.Unlock()
require.Len(t, updatedTask.history.samples, 1)
assert.Equal(t, int64(24), updatedTask.history.samples[0].responseUs)
agg := updatedTask.history.aggregateLocked(time.Hour, now)
require.True(t, agg.hasData())
assert.Equal(t, int64(2), agg.totalCount)
assert.Equal(t, int64(2), agg.successCount)
assert.Equal(t, int64(18), agg.avgResponse())
select {
case <-existingTask.ctx.Done():
default:
t.Fatal("expected original monitor task to be cancelled")
}
}
func TestMonitorManagerApplySyncDeleteRemovesTask(t *testing.T) {
config := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 10}
task := newMonitorTask(config)
pm := newMonitorManager()
pm.monitors = map[string]*monitorTask{config.ID: task}
_, err := pm.HandleSyncRequest(monitor.SyncRequest{
Action: monitor.SyncActionDelete,
Config: monitor.Config{ID: config.ID},
})
require.NoError(t, err)
_, exists := pm.monitors[config.ID]
assert.False(t, exists)
select {
case <-task.ctx.Done():
default:
t.Fatal("expected deleted monitor task to be cancelled")
}
}
func TestMonitorManagerGetRandomDelay(t *testing.T) {
for i := 1000; i < 360_000; i += 1000 {
delay := getStagger(int64(i))
assert.GreaterOrEqual(t, delay, time.Duration(i/2)*time.Millisecond)
assert.LessOrEqual(t, delay, time.Duration(i)*time.Millisecond)
}
}
func TestMonitorHTTP(t *testing.T) {
t.Run("success", func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, "Beszel-Agent/"+beszel.Version+" (+https://beszel.dev)", r.Header.Get("User-Agent"))
w.WriteHeader(http.StatusNoContent)
}))
defer server.Close()
responseUs, err := monitorHTTP(context.Background(), server.Client(), server.URL)
require.NoError(t, err)
assert.GreaterOrEqual(t, responseUs, int64(0))
})
t.Run("server error", func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "boom", http.StatusInternalServerError)
}))
defer server.Close()
responseUs, err := monitorHTTP(context.Background(), server.Client(), server.URL)
assert.Equal(t, int64(-1), responseUs)
require.Error(t, err)
})
}
func TestMonitorTCP(t *testing.T) {
t.Run("success", func(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
defer listener.Close()
accepted := make(chan struct{})
go func() {
defer close(accepted)
conn, err := listener.Accept()
if err == nil {
_ = conn.Close()
}
}()
port := uint16(listener.Addr().(*net.TCPAddr).Port)
responseUs, err := monitorTCP(context.Background(), "127.0.0.1", port)
require.NoError(t, err)
assert.GreaterOrEqual(t, responseUs, int64(0))
<-accepted
})
t.Run("connection failure", func(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
port := uint16(listener.Addr().(*net.TCPAddr).Port)
require.NoError(t, listener.Close())
responseUs, err := monitorTCP(context.Background(), "127.0.0.1", port)
assert.Equal(t, int64(-1), responseUs)
require.Error(t, err)
})
}
func TestMonitorTCPAddressFallback(t *testing.T) {
for _, tc := range []struct {
name string
ips []string
loss bool
}{
{"first address fails", []string{"127.0.0.2", "127.0.0.1"}, false},
{"first address succeeds", []string{"127.0.0.1", "127.0.0.2"}, false},
{"all addresses fail", []string{"127.0.0.2", "127.0.0.3"}, true},
} {
t.Run(tc.name, func(t *testing.T) {
listener, err := net.Listen("tcp4", "127.0.0.1:0")
require.NoError(t, err)
defer listener.Close()
original := net.DefaultResolver
net.DefaultResolver = tcpMonitorTestResolver(tc.ips)
defer func() { net.DefaultResolver = original }()
// Verify the resolver preserves the intended order, so success cannot
// accidentally bypass the failed first address in the regression case.
ips, err := net.DefaultResolver.LookupHost(t.Context(), "tcp-monitor.invalid.")
require.NoError(t, err)
require.Equal(t, tc.ips, ips)
responseUs, err := monitorTCP(t.Context(), "tcp-monitor.invalid.", uint16(listener.Addr().(*net.TCPAddr).Port))
if tc.loss {
require.Error(t, err)
assert.Equal(t, int64(-1), responseUs)
} else {
require.NoError(t, err)
assert.GreaterOrEqual(t, responseUs, int64(0))
}
})
}
}
// tcpMonitorTestResolver supplies multiple A records without external DNS.
func tcpMonitorTestResolver(ips []string) *net.Resolver {
return &net.Resolver{PreferGo: true, Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
client, server := net.Pipe()
go func() {
defer server.Close()
// net.Resolver uses TCP framing when its connection is not a PacketConn.
var size uint16
if err := binary.Read(server, binary.BigEndian, &size); err != nil {
return
}
packet := make([]byte, size)
if _, err := io.ReadFull(server, packet); err != nil {
return
}
var msg dnsmessage.Message
if err := msg.Unpack(packet); err != nil {
return
}
msg.Header.Response = true
msg.Header.RecursionAvailable = true
for _, question := range msg.Questions {
if question.Type != dnsmessage.TypeA {
continue
}
for _, ip := range ips {
msg.Answers = append(msg.Answers, dnsmessage.Resource{
Header: dnsmessage.ResourceHeader{Name: question.Name, Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET},
Body: &dnsmessage.AResource{A: [4]byte(net.ParseIP(ip).To4())},
})
}
}
packet, err := msg.Pack()
if err != nil {
return
}
response := binary.BigEndian.AppendUint16(nil, uint16(len(packet)))
_, _ = server.Write(append(response, packet...))
}()
return client, nil
}}
}
// udpDNSTestServer starts a UDP server on loopback that answers A queries with the
// given IPs, and returns its listen address (host:port).
func udpDNSTestServer(t *testing.T, ips []string) string {
t.Helper()
conn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP("127.0.0.1")})
require.NoError(t, err)
t.Cleanup(func() { conn.Close() })
go func() {
buf := make([]byte, 512)
for {
n, addr, err := conn.ReadFromUDP(buf)
if err != nil {
return
}
var msg dnsmessage.Message
if err := msg.Unpack(buf[:n]); err != nil {
continue
}
msg.Header.Response = true
msg.Header.RecursionAvailable = true
for _, question := range msg.Questions {
if question.Type != dnsmessage.TypeA {
continue
}
for _, ip := range ips {
msg.Answers = append(msg.Answers, dnsmessage.Resource{
Header: dnsmessage.ResourceHeader{Name: question.Name, Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET},
Body: &dnsmessage.AResource{A: [4]byte(net.ParseIP(ip).To4())},
})
}
}
packet, err := msg.Pack()
if err != nil {
continue
}
_, _ = conn.WriteToUDP(packet, addr)
}
}()
return conn.LocalAddr().String()
}
func TestMonitorDNS(t *testing.T) {
t.Run("success", func(t *testing.T) {
responseUs, err := monitorDNS(context.Background(), "localhost", "")
require.NoError(t, err)
assert.GreaterOrEqual(t, responseUs, int64(0))
})
t.Run("lookup failure", func(t *testing.T) {
responseUs, err := monitorDNS(context.Background(), "", "")
assert.Equal(t, int64(-1), responseUs)
require.Error(t, err)
})
t.Run("custom server", func(t *testing.T) {
serverAddr := udpDNSTestServer(t, []string{"192.0.2.10"})
responseUs, err := monitorDNS(context.Background(), "example.test.", serverAddr)
require.NoError(t, err)
assert.GreaterOrEqual(t, responseUs, int64(0))
})
t.Run("custom server without port defaults to 53", func(t *testing.T) {
resolver := dnsResolverForServer("127.0.0.1")
conn, err := resolver.Dial(context.Background(), "udp", "")
require.NoError(t, err)
defer conn.Close()
assert.Equal(t, "127.0.0.1:53", conn.RemoteAddr().String())
})
t.Run("custom server unreachable", func(t *testing.T) {
responseUs, err := monitorDNS(context.Background(), "example.test.", "127.0.0.1:1")
assert.Equal(t, int64(-1), responseUs)
require.Error(t, err)
})
}
func TestMonitorManagerCancelsActiveProbe(t *testing.T) {
for _, action := range []string{"stop", "delete", "upsert", "sync replace", "sync remove"} {
t.Run(action, func(t *testing.T) {
started := make(chan struct{})
canceled := make(chan struct{})
release := make(chan struct{})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
close(started)
select {
case <-r.Context().Done():
close(canceled)
case <-release:
}
}))
defer server.Close()
defer close(release)
pm := newMonitorManager()
defer pm.Stop()
cfg := monitor.Config{ID: "test", Protocol: "http", Target: server.URL, Interval: 3600}
task := newMonitorTask(cfg)
// Seed history to ensure a canceled RunNow does not return an old result.
task.history.addSampleLocked(monitorSample{responseUs: 123, timestamp: time.Now()})
pm.monitors[cfg.ID] = task
done := make(chan *monitor.Result, 1)
go func() {
result, _ := pm.UpsertMonitor(cfg, true)
done <- result
}()
select {
case <-started:
case <-time.After(time.Second):
t.Fatal("probe did not start")
}
updated := cfg
updated.Interval--
switch action {
case "stop":
pm.Stop()
case "delete":
pm.DeleteMonitor(cfg.ID)
case "upsert":
_, err := pm.UpsertMonitor(updated, false)
require.NoError(t, err)
case "sync replace":
pm.SyncMonitors([]monitor.Config{updated})
case "sync remove":
pm.SyncMonitors(nil)
}
select {
case <-canceled:
case <-time.After(time.Second):
t.Fatal("active HTTP request was not canceled")
}
select {
case result := <-done:
assert.Nil(t, result)
case <-time.After(time.Second):
t.Fatal("RunNow did not return after cancellation")
}
task.history.mu.Lock()
assert.Len(t, task.history.samples, 1, "cancellation must not record packet loss")
task.history.mu.Unlock()
})
}
}
func TestMonitorResolutionCancellation(t *testing.T) {
for _, protocol := range []string{"tcp", "dns", "icmp"} {
t.Run(protocol, func(t *testing.T) {
started := make(chan struct{}, 1)
original := net.DefaultResolver
net.DefaultResolver = &net.Resolver{PreferGo: true, Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
select {
case started <- struct{}{}:
default:
}
<-ctx.Done()
return nil, ctx.Err()
}}
defer func() { net.DefaultResolver = original }()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
done := make(chan error, 1)
go func() {
var err error
switch protocol {
case "tcp":
_, err = monitorTCP(ctx, "monitor-cancellation.invalid.", 80)
case "dns":
_, err = monitorDNS(ctx, "monitor-cancellation.invalid.", "")
case "icmp":
_, err = monitorICMP(ctx, "monitor-cancellation.invalid.")
}
done <- err
}()
select {
case <-started:
case <-time.After(time.Second):
t.Fatal("lookup did not start")
}
cancel()
select {
case err := <-done:
require.Error(t, err)
case <-time.After(time.Second):
t.Fatal("lookup did not cancel")
}
})
}
}
func TestMonitorProbeTimeoutRecordsLoss(t *testing.T) {
release := make(chan struct{})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
select {
case <-r.Context().Done():
case <-release:
}
}))
defer server.Close()
defer close(release)
pm := newMonitorManager()
pm.probe = networkMonitorProbe(&http.Client{Timeout: 20 * time.Millisecond})
task := newMonitorTask(monitor.Config{ID: "timeout", Protocol: "http", Target: server.URL})
defer task.cancel()
result := task.runProbe(pm.probe)
require.NotNil(t, result)
assert.Equal(t, 100.0, result.PacketLoss)
assert.Equal(t, 100.0, result.PacketLoss1h)
require.Len(t, task.history.samples, 1)
assert.Equal(t, int64(-1), task.history.samples[0].responseUs)
assert.NoError(t, task.ctx.Err(), "a probe timeout must not cancel the task")
}

View File

@@ -5,7 +5,9 @@ import (
"errors" "errors"
"fmt" "fmt"
"log/slog" "log/slog"
"os"
"path" "path"
"path/filepath"
"runtime" "runtime"
"strconv" "strconv"
"strings" "strings"
@@ -32,6 +34,8 @@ type SensorConfig struct {
isBlacklist bool isBlacklist bool
hasWildcards bool hasWildcards bool
skipCollection bool skipCollection bool
skipGPU bool
sensorShadow string
firstRun bool firstRun bool
} }
@@ -41,13 +45,14 @@ func (a *Agent) newSensorConfig() *SensorConfig {
sensorsEnvVal, sensorsSet := utils.GetEnv("SENSORS") sensorsEnvVal, sensorsSet := utils.GetEnv("SENSORS")
skipCollection := sensorsSet && sensorsEnvVal == "" skipCollection := sensorsSet && sensorsEnvVal == ""
sensorsTimeout, _ := utils.GetEnv("SENSORS_TIMEOUT") sensorsTimeout, _ := utils.GetEnv("SENSORS_TIMEOUT")
skipGPU, _ := utils.GetEnv("SKIP_GPU")
return a.newSensorConfigWithEnv(primarySensor, sysSensors, sensorsEnvVal, sensorsTimeout, skipCollection) return a.newSensorConfigWithEnv(primarySensor, sysSensors, sensorsEnvVal, sensorsTimeout, skipCollection, skipGPU == "true")
} }
// newSensorConfigWithEnv creates a SensorConfig with the provided environment variables // newSensorConfigWithEnv creates a SensorConfig with the provided environment variables
// sensorsSet indicates if the SENSORS environment variable was explicitly set (even to empty string) // sensorsSet indicates if the SENSORS environment variable was explicitly set (even to empty string)
func (a *Agent) newSensorConfigWithEnv(primarySensor, sysSensors, sensorsEnvVal, sensorsTimeout string, skipCollection bool) *SensorConfig { func (a *Agent) newSensorConfigWithEnv(primarySensor, sysSensors, sensorsEnvVal, sensorsTimeout string, skipCollection, skipGPU bool) *SensorConfig {
timeout := 2 * time.Second timeout := 2 * time.Second
if sensorsTimeout != "" { if sensorsTimeout != "" {
if d, err := time.ParseDuration(sensorsTimeout); err == nil { if d, err := time.ParseDuration(sensorsTimeout); err == nil {
@@ -62,6 +67,7 @@ func (a *Agent) newSensorConfigWithEnv(primarySensor, sysSensors, sensorsEnvVal,
primarySensor: primarySensor, primarySensor: primarySensor,
timeout: timeout, timeout: timeout,
skipCollection: skipCollection, skipCollection: skipCollection,
skipGPU: skipGPU,
firstRun: true, firstRun: true,
sensors: make(map[string]struct{}), sensors: make(map[string]struct{}),
} }
@@ -73,6 +79,19 @@ func (a *Agent) newSensorConfigWithEnv(primarySensor, sysSensors, sensorsEnvVal,
common.EnvKey, common.EnvMap{common.HostSysEnvKey: sysSensors}, common.EnvKey, common.EnvMap{common.HostSysEnvKey: sysSensors},
) )
} }
if skipGPU && runtime.GOOS == "linux" {
// gopsutil reads every temp*_input before results can be filtered, so
// point it at a shadow tree built from the effective sysfs root instead.
if shadow, err := buildNonGpuSysShadow(effectiveSysRoot(config.context)); err == nil {
slog.Info("SKIP_GPU enabled, using non-GPU sensor sysfs shadow", "path", shadow)
config.sensorShadow = shadow
config.context = context.WithValue(config.context,
common.EnvKey, common.EnvMap{common.HostSysEnvKey: shadow},
)
} else {
slog.Warn("SKIP_GPU sensor shadow unavailable, falling back to post-read filtering", "err", err)
}
}
// handle blacklist // handle blacklist
if strings.HasPrefix(sensorsEnvVal, "-") { if strings.HasPrefix(sensorsEnvVal, "-") {
@@ -149,6 +168,9 @@ func (a *Agent) updateTemperatures(systemStats *system.Stats) {
if !isValidSensor(sensorName, a.sensorConfig) { if !isValidSensor(sensorName, a.sensorConfig) {
continue continue
} }
if a.sensorConfig.skipGPU && isGpuSensorKey(sensorName) {
continue
}
// set dashboard temperature // set dashboard temperature
switch a.sensorConfig.primarySensor { switch a.sensorConfig.primarySensor {
case "": case "":
@@ -245,3 +267,102 @@ func scaleTemperature(temp float64) float64 {
} }
return scaled100 return scaled100
} }
// effectiveSysRoot mirrors gopsutil's HostSys lookup, which lives in its
// internal package: context override, then HOST_SYS env, then /sys.
func effectiveSysRoot(ctx context.Context) string {
if envMap, ok := ctx.Value(common.EnvKey).(common.EnvMap); ok {
if v := envMap[common.HostSysEnvKey]; v != "" {
return v
}
}
if v := os.Getenv("HOST_SYS"); v != "" {
return v
}
return "/sys"
}
func (config *SensorConfig) cleanupSensorShadow() {
if config.sensorShadow == "" {
return
}
if err := os.RemoveAll(config.sensorShadow); err != nil {
slog.Warn("Error removing sensor sysfs shadow", "path", config.sensorShadow, "err", err)
return
}
config.sensorShadow = ""
}
func (a *Agent) cleanupSensorShadow() {
if a.sensorConfig != nil {
a.sensorConfig.cleanupSensorShadow()
}
}
func isGpuThermalZone(zoneType string) bool {
zoneType = strings.ToLower(strings.TrimSpace(zoneType))
return isGpuChipName(zoneType) || strings.Contains(zoneType, "gpu")
}
// buildNonGpuSysShadow links non-GPU sensor directories into a temp dir. Only
// static chip names and thermal-zone types are read; no sensor values are touched.
func buildNonGpuSysShadow(sysRoot string) (string, error) {
shadow, err := os.MkdirTemp("", "beszel-sensors-*")
if err != nil {
return "", err
}
shadowHwmon := filepath.Join(shadow, "class", "hwmon")
if err := os.MkdirAll(shadowHwmon, 0o755); err != nil {
os.RemoveAll(shadow)
return "", err
}
entries, err := os.ReadDir(filepath.Join(sysRoot, "class", "hwmon"))
if err != nil && !os.IsNotExist(err) {
os.RemoveAll(shadow)
return "", err
}
for _, entry := range entries {
chipDir := filepath.Join(sysRoot, "class", "hwmon", entry.Name())
// Some hwmon devices expose name under device/ (gopsutil's CentOS fallback).
name, ok := utils.ReadStringFileOK(filepath.Join(chipDir, "name"))
if !ok {
name, ok = utils.ReadStringFileOK(filepath.Join(chipDir, "device", "name"))
}
if !ok || isGpuChipName(name) {
continue
}
if err := os.Symlink(chipDir, filepath.Join(shadowHwmon, entry.Name())); err != nil {
os.RemoveAll(shadow)
return "", err
}
}
thermalEntries, err := os.ReadDir(filepath.Join(sysRoot, "class", "thermal"))
if err != nil {
if os.IsNotExist(err) {
return shadow, nil
}
os.RemoveAll(shadow)
return "", err
}
shadowThermal := filepath.Join(shadow, "class", "thermal")
if err := os.MkdirAll(shadowThermal, 0o755); err != nil {
os.RemoveAll(shadow)
return "", err
}
for _, entry := range thermalEntries {
if !strings.HasPrefix(entry.Name(), "thermal_zone") {
continue
}
zoneDir := filepath.Join(sysRoot, "class", "thermal", entry.Name())
zoneType, ok := utils.ReadStringFileOK(filepath.Join(zoneDir, "type"))
if !ok || isGpuThermalZone(zoneType) {
continue
}
if err := os.Symlink(zoneDir, filepath.Join(shadowThermal, entry.Name())); err != nil {
os.RemoveAll(shadow)
return "", err
}
}
return shadow, nil
}

View File

@@ -5,6 +5,8 @@ package agent
import ( import (
"context" "context"
"fmt" "fmt"
"os"
"path/filepath"
"testing" "testing"
"time" "time"
@@ -328,7 +330,7 @@ func TestNewSensorConfigWithEnv(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
result := agent.newSensorConfigWithEnv(tt.primarySensor, tt.sysSensors, tt.sensors, tt.sensorsTimeout, tt.skipCollection) result := agent.newSensorConfigWithEnv(tt.primarySensor, tt.sysSensors, tt.sensors, tt.sensorsTimeout, tt.skipCollection, false)
// Check primary sensor // Check primary sensor
assert.Equal(t, tt.expectedConfig.primarySensor, result.primarySensor) assert.Equal(t, tt.expectedConfig.primarySensor, result.primarySensor)
@@ -620,3 +622,143 @@ func TestUpdateTemperaturesSkipsOnTimeout(t *testing.T) {
assert.Equal(t, 0.0, agent.systemInfo.DashboardTemp) assert.Equal(t, 0.0, agent.systemInfo.DashboardTemp)
assert.Equal(t, map[string]float64{}, stats.Temperatures) assert.Equal(t, map[string]float64{}, stats.Temperatures)
} }
func TestIsGpuSensorKey(t *testing.T) {
for _, key := range []string{"xe", "XE_temp1", "amdgpu_edge", "NVIDIA"} {
assert.True(t, isGpuSensorKey(key), key)
}
for _, key := range []string{"coretemp_core_0", "acpitz", "xen_temp", "myxe", ""} {
assert.False(t, isGpuSensorKey(key), key)
}
}
func TestSkipGpuSensorShadow(t *testing.T) {
sysRoot := t.TempDir()
writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0", "name"), "coretemp\n")
writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0", "temp1_input"), "55000\n")
writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon1", "name"), "xe\n")
writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon1", "temp1_input"), "48000\n")
writeFile(t, filepath.Join(sysRoot, "class", "thermal", "thermal_zone0", "type"), "cpu-thermal\n")
writeFile(t, filepath.Join(sysRoot, "class", "thermal", "thermal_zone0", "temp"), "55000\n")
writeFile(t, filepath.Join(sysRoot, "class", "thermal", "thermal_zone1", "type"), "gpu\n")
writeFile(t, filepath.Join(sysRoot, "class", "thermal", "thermal_zone1", "temp"), "48000\n")
shadow, err := buildNonGpuSysShadow(sysRoot)
require.NoError(t, err)
t.Cleanup(func() { os.RemoveAll(shadow) })
assert.FileExists(t, filepath.Join(shadow, "class", "hwmon", "hwmon0", "temp1_input"))
assert.NoFileExists(t, filepath.Join(shadow, "class", "hwmon", "hwmon1"))
assert.FileExists(t, filepath.Join(shadow, "class", "thermal", "thermal_zone0", "temp"))
assert.NoFileExists(t, filepath.Join(shadow, "class", "thermal", "thermal_zone1"))
}
func TestSkipGpuSensorShadowDeviceName(t *testing.T) {
sysRoot := t.TempDir()
writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0", "device", "name"), "coretemp\n")
writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0", "device", "temp1_input"), "55000\n")
writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon1", "device", "name"), "xe\n")
writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon1", "device", "temp1_input"), "48000\n")
shadow, err := buildNonGpuSysShadow(sysRoot)
require.NoError(t, err)
t.Cleanup(func() { os.RemoveAll(shadow) })
assert.FileExists(t, filepath.Join(shadow, "class", "hwmon", "hwmon0", "device", "temp1_input"))
assert.NoFileExists(t, filepath.Join(shadow, "class", "hwmon", "hwmon1"))
}
func TestSkipGpuSensorShadowKeepsThermalZonesWithoutNonGpuHwmon(t *testing.T) {
sysRoot := t.TempDir()
writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0", "name"), "xe\n")
writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0", "temp1_input"), "48000\n")
writeFile(t, filepath.Join(sysRoot, "class", "thermal", "thermal_zone0", "type"), "cpu-thermal\n")
writeFile(t, filepath.Join(sysRoot, "class", "thermal", "thermal_zone0", "temp"), "55000\n")
writeFile(t, filepath.Join(sysRoot, "class", "thermal", "thermal_zone1", "type"), "gpu\n")
writeFile(t, filepath.Join(sysRoot, "class", "thermal", "thermal_zone1", "temp"), "48000\n")
shadow, err := buildNonGpuSysShadow(sysRoot)
require.NoError(t, err)
t.Cleanup(func() { os.RemoveAll(shadow) })
hwmonTemps, err := filepath.Glob(filepath.Join(shadow, "class", "hwmon", "hwmon*", "temp*_input"))
require.NoError(t, err)
assert.Empty(t, hwmonTemps)
assert.FileExists(t, filepath.Join(shadow, "class", "thermal", "thermal_zone0", "temp"))
assert.NoFileExists(t, filepath.Join(shadow, "class", "thermal", "thermal_zone1"))
}
func TestNewSensorConfigSkipGpuWiresShadow(t *testing.T) {
t.Setenv("SKIP_GPU", "true")
agent := &Agent{}
config := agent.newSensorConfig()
assert.True(t, config.skipGPU)
envMap, ok := config.context.Value(common.EnvKey).(common.EnvMap)
require.True(t, ok, "SKIP_GPU should point the sensor context at a sysfs shadow")
shadow, ok := envMap[common.HostSysEnvKey]
require.True(t, ok)
assert.DirExists(t, filepath.Join(shadow, "class", "hwmon"))
assert.Equal(t, shadow, config.sensorShadow)
config.cleanupSensorShadow()
assert.NoDirExists(t, shadow)
assert.Empty(t, config.sensorShadow)
}
func TestSkipGpuShadowUsesSysSensorsRoot(t *testing.T) {
sysRoot := t.TempDir()
writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0", "name"), "coretemp\n")
writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0", "temp1_input"), "55000\n")
writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon1", "name"), "xe\n")
writeFile(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon1", "temp1_input"), "48000\n")
agent := &Agent{}
config := agent.newSensorConfigWithEnv("", sysRoot, "", "", false, true)
t.Cleanup(config.cleanupSensorShadow)
envMap, ok := config.context.Value(common.EnvKey).(common.EnvMap)
require.True(t, ok, "SKIP_GPU should point the sensor context at a sysfs shadow")
shadow, ok := envMap[common.HostSysEnvKey]
require.True(t, ok)
require.NotEqual(t, sysRoot, shadow, "shadow must not be the SYS_SENSORS tree itself")
target, err := os.Readlink(filepath.Join(shadow, "class", "hwmon", "hwmon0"))
require.NoError(t, err)
assert.Equal(t, filepath.Join(sysRoot, "class", "hwmon", "hwmon0"), target)
assert.NoFileExists(t, filepath.Join(shadow, "class", "hwmon", "hwmon1"))
}
func TestUpdateTemperaturesSkipGpu(t *testing.T) {
originalGetSensorTemps := getSensorTemps
t.Cleanup(func() {
getSensorTemps = originalGetSensorTemps
})
getSensorTemps = func(ctx context.Context) ([]sensors.TemperatureStat, error) {
return []sensors.TemperatureStat{
{SensorKey: "coretemp_core_0", Temperature: 55},
{SensorKey: "XE", Temperature: 48},
}, nil
}
newAgent := func(skipGPU bool) *Agent {
agent := &Agent{
systemInfo: system.Info{},
sensorConfig: &SensorConfig{
context: context.Background(),
timeout: 2 * time.Second,
sensors: map[string]struct{}{},
skipGPU: skipGPU,
},
}
return agent
}
stats := &system.Stats{}
newAgent(true).updateTemperatures(stats)
assert.Equal(t, map[string]float64{"coretemp_core_0": 55}, stats.Temperatures)
stats = &system.Stats{}
newAgent(false).updateTemperatures(stats)
assert.Len(t, stats.Temperatures, 2)
}

View File

@@ -931,9 +931,6 @@ func (sm *SmartManager) parseSmartForSata(output []byte, deviceType string) (boo
if parsed, ok := smart.ParseSmartRawValueString(attr.Raw.String); ok { if parsed, ok := smart.ParseSmartRawValueString(attr.Raw.String); ok {
rawValue = parsed rawValue = parsed
} }
if smartData.SmartStatus == "PASSED" && rawValue > 0 && (attr.ID == 5 || attr.ID == 197 || attr.ID == 198) {
smartData.SmartStatus = "WARNING"
}
smartAttr := &smart.SmartAttribute{ smartAttr := &smart.SmartAttribute{
ID: attr.ID, ID: attr.ID,
Name: attr.Name, Name: attr.Name,

View File

@@ -7,7 +7,6 @@ import (
"fmt" "fmt"
"os" "os"
"path/filepath" "path/filepath"
"strconv"
"testing" "testing"
"github.com/henrygd/beszel/internal/entities/smart" "github.com/henrygd/beszel/internal/entities/smart"
@@ -90,27 +89,6 @@ func TestParseSmartForSata(t *testing.T) {
} }
} }
func TestParseSmartForSataWarnsForCriticalAttributes(t *testing.T) {
for _, attrID := range []int{5, 197, 198} {
t.Run("attribute "+strconv.Itoa(attrID), func(t *testing.T) {
jsonPayload := []byte(fmt.Sprintf(`{
"smartctl": {"exit_status": 0},
"device": {"name": "/dev/sda", "type": "sat"},
"model_name": "Example",
"serial_number": "WARNING%d",
"smart_status": {"passed": true},
"temperature": {"current": 30},
"ata_smart_attributes": {"table": [{"id": %d, "raw": {"value": 1, "string": "1"}}]}
}`, attrID, attrID))
sm := &SmartManager{SmartDataMap: make(map[string]*smart.SmartData)}
hasData, _ := sm.parseSmartForSata(jsonPayload, "")
require.True(t, hasData)
assert.Equal(t, "WARNING", sm.SmartDataMap[fmt.Sprintf("WARNING%d", attrID)].SmartStatus)
})
}
}
func TestParseSmartForSataPreservesFailedAndUnknownStatus(t *testing.T) { func TestParseSmartForSataPreservesFailedAndUnknownStatus(t *testing.T) {
for _, test := range []struct { for _, test := range []struct {
name string name string

View File

@@ -54,12 +54,18 @@ type poolBackend struct {
kernelStatsFn func() ([]zfs.PoolKernelStat, error) // procfs pool state/I/O source kernelStatsFn func() ([]zfs.PoolKernelStat, error) // procfs pool state/I/O source
poolStatusesFn func() ([]zfs.PoolStatus, error) // scrub/vdev detail source poolStatusesFn func() ([]zfs.PoolStatus, error) // scrub/vdev detail source
poolData []zfs.PoolStat // cached pool inventory (TTL below) // Utility-backed caches below are refreshed in the background after the
lastPoolStats time.Time // first collection, so cacheMu guards them against those goroutines.
kernelSamples map[string]poolKernelSample cacheMu sync.Mutex
poolData []zfs.PoolStat // cached pool inventory (TTL below)
lastPoolStats time.Time
poolRefreshing bool
datasetUsage map[string]zfsDatasetUsage // mountpoint -> usage datasetUsage map[string]zfsDatasetUsage // mountpoint -> usage
lastUsageRefresh time.Time lastUsageRefresh time.Time
usageRefreshing bool
kernelSamples map[string]poolKernelSample
// Detail data (pools, vdevs, scrub, datasets) is cached and refreshed on // Detail data (pools, vdevs, scrub, datasets) is cached and refreshed on
// an interval. Accessed from handler goroutines, so it is mutex-protected. // an interval. Accessed from handler goroutines, so it is mutex-protected.
@@ -80,7 +86,7 @@ func newZfsBackend() *poolBackend {
return &poolBackend{ return &poolBackend{
name: "zfs", name: "zfs",
poolStatsFn: optionalPoolSource(zfs.PoolStats), poolStatsFn: optionalPoolSource(zfs.PoolStats),
datasetsFn: zfs.Datasets, datasetsFn: optionalPoolSource(zfs.Datasets),
kernelStatsFn: optionalPoolSource(zfs.PoolKernelStats), kernelStatsFn: optionalPoolSource(zfs.PoolKernelStats),
poolStatusesFn: optionalPoolSource(zfs.PoolStatuses), poolStatusesFn: optionalPoolSource(zfs.PoolStatuses),
} }
@@ -177,21 +183,42 @@ func (b *poolBackend) updateBackendStats(systemStats *system.Stats) {
} }
// poolStats returns the cached pool inventory, calling its collector at most // poolStats returns the cached pool inventory, calling its collector at most
// every poolStatsRefreshInterval. On failure the previous inventory is // every poolStatsRefreshInterval. Only the first collection blocks; later
// retained and the refresh is retried on the next cadence. // refreshes run in the background because utilities like `zpool list` can hang
// for seconds on busy hosts, which would otherwise delay the hub's stats
// response. On failure the previous inventory is retained and the refresh is
// retried on the next cadence.
func (b *poolBackend) poolStats() []zfs.PoolStat { func (b *poolBackend) poolStats() []zfs.PoolStat {
if b.lastPoolStats.IsZero() || time.Since(b.lastPoolStats) >= poolStatsRefreshInterval { b.cacheMu.Lock()
pools, err := b.poolStatsFn() defer b.cacheMu.Unlock()
if err != nil { if b.poolRefreshing || (!b.lastPoolStats.IsZero() && time.Since(b.lastPoolStats) < poolStatsRefreshInterval) {
slog.Debug("Storage pool stats unavailable", "backend", b.name, "err", err) return b.poolData
} else {
b.poolData = pools
}
b.lastPoolStats = time.Now()
} }
if b.lastPoolStats.IsZero() {
b.storePoolStats(b.poolStatsFn())
return b.poolData
}
b.poolRefreshing = true
go func() {
pools, err := b.poolStatsFn()
b.cacheMu.Lock()
defer b.cacheMu.Unlock()
b.poolRefreshing = false
b.storePoolStats(pools, err)
}()
return b.poolData return b.poolData
} }
// storePoolStats records a pool inventory result. Callers must hold cacheMu.
func (b *poolBackend) storePoolStats(pools []zfs.PoolStat, err error) {
if err != nil {
slog.Debug("Storage pool stats unavailable", "backend", b.name, "err", err)
} else {
b.poolData = pools
}
b.lastPoolStats = time.Now()
}
// kernelStats reads cumulative pool counters and converts them to per-second // kernelStats reads cumulative pool counters and converts them to per-second
// rates. Counter decreases indicate a pool export/import and reset the // rates. Counter decreases indicate a pool export/import and reset the
// baseline instead of producing an underflow spike. // baseline instead of producing an underflow spike.
@@ -225,12 +252,33 @@ func (b *poolBackend) kernelStats() (map[string]zfs.PoolKernelStat, map[string]z
} }
// refreshDatasetUsage re-runs `zfs list` when the refresh window has elapsed // refreshDatasetUsage re-runs `zfs list` when the refresh window has elapsed
// and rebuilds the mountpoint-keyed usage map. // and returns the mountpoint-keyed usage map. Like poolStats, only the first
func (b *poolBackend) refreshDatasetUsage() { // collection blocks and later refreshes run in the background.
if !b.lastUsageRefresh.IsZero() && time.Since(b.lastUsageRefresh) < datasetUsageRefreshInterval { func (b *poolBackend) refreshDatasetUsage() map[string]zfsDatasetUsage {
return b.cacheMu.Lock()
defer b.cacheMu.Unlock()
if b.usageRefreshing || (!b.lastUsageRefresh.IsZero() && time.Since(b.lastUsageRefresh) < datasetUsageRefreshInterval) {
return b.datasetUsage
} }
datasets, err := b.datasets() if b.lastUsageRefresh.IsZero() {
b.storeDatasetUsage(b.datasets())
return b.datasetUsage
}
b.usageRefreshing = true
go func() {
datasets, err := b.datasets()
b.cacheMu.Lock()
defer b.cacheMu.Unlock()
b.usageRefreshing = false
b.storeDatasetUsage(datasets, err)
}()
return b.datasetUsage
}
// storeDatasetUsage rebuilds the usage map from a dataset listing. The map is
// replaced rather than mutated so returned references stay safe to read.
// Callers must hold cacheMu.
func (b *poolBackend) storeDatasetUsage(datasets []zfs.Dataset, err error) {
if err != nil { if err != nil {
slog.Debug("Storage pool dataset usage unavailable", "backend", b.name, "err", err) slog.Debug("Storage pool dataset usage unavailable", "backend", b.name, "err", err)
} else { } else {
@@ -251,8 +299,7 @@ func (b *poolBackend) refreshDatasetUsage() {
func (m *StoragePoolManager) DatasetUsage() map[string]zfsDatasetUsage { func (m *StoragePoolManager) DatasetUsage() map[string]zfsDatasetUsage {
for _, backend := range m.backends { for _, backend := range m.backends {
if backend.name == "zfs" { if backend.name == "zfs" {
backend.refreshDatasetUsage() return backend.refreshDatasetUsage()
return backend.datasetUsage
} }
} }
return nil return nil
@@ -442,7 +489,10 @@ func (m *StoragePoolManager) markDuplicateCharts(stats *system.Stats, filesystem
} }
} }
for _, backend := range m.backends { for _, backend := range m.backends {
for _, pool := range backend.poolData { backend.cacheMu.Lock()
pools := backend.poolData
backend.cacheMu.Unlock()
for _, pool := range pools {
sample := stats.ZfsPools[pool.Name] sample := stats.ZfsPools[pool.Name]
if sample == nil || pool.MountID == "" { if sample == nil || pool.MountID == "" {
continue continue

View File

@@ -323,6 +323,21 @@ func TestDatasetUsageRefreshOnErrorKeepsPrevious(t *testing.T) {
assert.Len(t, usage, 1, "previous usage should be retained on error") assert.Len(t, usage, 1, "previous usage should be retained on error")
} }
func TestDatasetUsageClearsAbsentBackend(t *testing.T) {
b := newZfsBackend()
b.datasetUsage = map[string]zfsDatasetUsage{"/tank": {used: 1, avail: 1}}
b.datasetsFn = optionalPoolSource(func() ([]zfs.Dataset, error) {
return nil, zfs.ErrNoZfs
})
datasets, err := b.datasets()
require.NoError(t, err, "an absent backend must not produce an error to log")
assert.Empty(t, datasets)
b.refreshDatasetUsage()
assert.Empty(t, b.datasetUsage)
assert.False(t, b.lastUsageRefresh.IsZero())
}
func TestGetDetailForceRefresh(t *testing.T) { func TestGetDetailForceRefresh(t *testing.T) {
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}} zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
poolCalls := 0 poolCalls := 0
@@ -503,3 +518,31 @@ func TestBtrfsPoolIdentities(t *testing.T) {
assert.Equal(t, first, zm.GetDetail(true).Pools[1].Name) assert.Equal(t, first, zm.GetDetail(true).Pools[1].Name)
assert.Equal(t, "renamed", zm.GetDetail(true).Pools[1].DisplayName) assert.Equal(t, "renamed", zm.GetDetail(true).Pools[1].DisplayName)
} }
func TestStaleUtilityCachesRefreshInBackground(t *testing.T) {
release := make(chan struct{})
b := &poolBackend{name: "zfs"}
b.poolStatsFn = func() ([]zfs.PoolStat, error) {
<-release
return []zfs.PoolStat{{Name: "new"}}, nil
}
b.datasetsFn = func() ([]zfs.Dataset, error) {
<-release
return []zfs.Dataset{{Name: "new", Mountpoint: "/new"}}, nil
}
b.poolData = []zfs.PoolStat{{Name: "old"}}
b.lastPoolStats = time.Now().Add(-2 * poolStatsRefreshInterval)
b.datasetUsage = map[string]zfsDatasetUsage{"/old": {}}
b.lastUsageRefresh = time.Now().Add(-2 * datasetUsageRefreshInterval)
// A hung utility must not block collection; cached data is served meanwhile.
for range 2 {
assert.Equal(t, "old", b.poolStats()[0].Name)
assert.Contains(t, b.refreshDatasetUsage(), "/old")
}
close(release)
require.Eventually(t, func() bool {
return b.poolStats()[0].Name == "new" && b.refreshDatasetUsage()["/new"] == zfsDatasetUsage{}
}, time.Second, time.Millisecond)
}

View File

@@ -70,6 +70,9 @@ type Dataset struct {
// PoolStats returns capacity and health for all pools on the system using // PoolStats returns capacity and health for all pools on the system using
// `zpool list`. Frequent health and I/O sampling uses PoolKernelStats instead. // `zpool list`. Frequent health and I/O sampling uses PoolKernelStats instead.
func PoolStats() ([]PoolStat, error) { func PoolStats() ([]PoolStat, error) {
if err := checkZfsDevice(); err != nil {
return nil, err
}
out, err := commandOutput("zpool", "list", "-Hp", "-o", "name,size,alloc,free,health") out, err := commandOutput("zpool", "list", "-Hp", "-o", "name,size,alloc,free,health")
if err != nil { if err != nil {
var exitErr *exec.ExitError var exitErr *exec.ExitError
@@ -84,6 +87,9 @@ func PoolStats() ([]PoolStat, error) {
// Datasets returns all datasets on the system with usage and mountpoint // Datasets returns all datasets on the system with usage and mountpoint
// information using `zfs list` (recursive by default). // information using `zfs list` (recursive by default).
func Datasets() ([]Dataset, error) { func Datasets() ([]Dataset, error) {
if err := checkZfsDevice(); err != nil {
return nil, err
}
out, err := commandOutput("zfs", "list", "-Hp", "-o", "name,used,avail,mountpoint") out, err := commandOutput("zfs", "list", "-Hp", "-o", "name,used,avail,mountpoint")
if err != nil { if err != nil {
return nil, fmt.Errorf("zfs list: %w", err) return nil, fmt.Errorf("zfs list: %w", err)

View File

@@ -13,7 +13,10 @@ import (
"strings" "strings"
) )
var procZfsPath = "/proc/spl/kstat/zfs" var (
procZfsPath = "/proc/spl/kstat/zfs"
devZfsPath = "/dev/zfs"
)
func ARCSize() (uint64, error) { func ARCSize() (uint64, error) {
file, err := os.Open(filepath.Join(procZfsPath, "arcstats")) file, err := os.Open(filepath.Join(procZfsPath, "arcstats"))
@@ -40,6 +43,19 @@ func ARCSize() (uint64, error) {
return 0, fmt.Errorf("size field not found in arcstats") return 0, fmt.Errorf("size field not found in arcstats")
} }
// checkZfsDevice lets containers without /dev/zfs fail fast instead of
// waiting for ZFS utility commands to time out.
func checkZfsDevice() error {
_, err := os.Stat(devZfsPath)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return ErrNoZfs
}
return err
}
return nil
}
// PoolKernelStats reads pool state and cumulative I/O counters directly from // PoolKernelStats reads pool state and cumulative I/O counters directly from
// procfs. These kstats are the same interfaces used by node_exporter's Linux // procfs. These kstats are the same interfaces used by node_exporter's Linux
// ZFS collector and avoid keeping a `zpool iostat` subprocess alive. // ZFS collector and avoid keeping a `zpool iostat` subprocess alive.

View File

@@ -88,3 +88,66 @@ func TestReadObjsetIORequiresAllCounters(t *testing.T) {
_, _, err := readObjsetIO(path) _, _, err := readObjsetIO(path)
require.Error(t, err) require.Error(t, err)
} }
func TestCollectorsSkipCommandsWhenDevZfsMissing(t *testing.T) {
root := t.TempDir()
oldDevZfsPath := devZfsPath
devZfsPath = filepath.Join(root, "missing")
t.Cleanup(func() { devZfsPath = oldDevZfsPath })
oldCommandOutput := commandOutput
commandOutput = func(name string, args ...string) ([]byte, error) {
t.Fatalf("unexpected %s call with %v", name, args)
return nil, nil
}
t.Cleanup(func() { commandOutput = oldCommandOutput })
_, err := PoolStats()
assert.ErrorIs(t, err, ErrNoZfs)
_, err = Datasets()
assert.ErrorIs(t, err, ErrNoZfs)
}
func TestDatasetsDelegatesWhenDevZfsPresent(t *testing.T) {
oldDevZfsPath := devZfsPath
devZfsPath = filepath.Join(t.TempDir(), "zfs")
require.NoError(t, os.WriteFile(devZfsPath, nil, 0o644))
t.Cleanup(func() { devZfsPath = oldDevZfsPath })
oldCommandOutput := commandOutput
commandOutput = func(name string, args ...string) ([]byte, error) {
assert.Equal(t, "zfs", name)
assert.Equal(t, []string{"list", "-Hp", "-o", "name,used,avail,mountpoint"}, args)
return []byte("tank\t50\t50\t/tank\n"), nil
}
t.Cleanup(func() { commandOutput = oldCommandOutput })
datasets, err := Datasets()
require.NoError(t, err)
assert.Equal(t, []Dataset{{Name: "tank", Used: 50, Avail: 50, Mountpoint: "/tank"}}, datasets)
}
func TestPoolStatsDelegatesToZpoolWhenDevZfsPresent(t *testing.T) {
root := t.TempDir()
devFile := filepath.Join(root, "zfs")
require.NoError(t, os.WriteFile(devFile, []byte(""), 0o644))
oldDevZfsPath := devZfsPath
devZfsPath = devFile
t.Cleanup(func() { devZfsPath = oldDevZfsPath })
oldCommandOutput := commandOutput
called := false
commandOutput = func(name string, args ...string) ([]byte, error) {
called = true
assert.Equal(t, "zpool", name)
assert.Equal(t, []string{"list", "-Hp", "-o", "name,size,alloc,free,health"}, args)
return []byte("tank\t100\t50\t50\tONLINE\n"), nil
}
t.Cleanup(func() { commandOutput = oldCommandOutput })
pools, err := PoolStats()
require.NoError(t, err)
assert.True(t, called)
assert.Equal(t, []PoolStat{{Name: "tank", Size: 100, Alloc: 50, Free: 50, Health: "ONLINE"}}, pools)
}

View File

@@ -0,0 +1,9 @@
//go:build !linux
package zfs
// The /dev/zfs probe is Linux-specific. Other platforms detect availability
// through the ZFS utilities themselves.
func checkZfsDevice() error {
return nil
}

View File

@@ -0,0 +1,33 @@
//go:build testing && !linux
package zfs
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestCollectorsUseUtilitiesOnNonLinux(t *testing.T) {
oldCommandOutput := commandOutput
commandOutput = func(name string, args ...string) ([]byte, error) {
switch name {
case "zpool":
return []byte("tank\t100\t50\t50\tONLINE\n"), nil
case "zfs":
return []byte("tank\t50\t50\t/tank\n"), nil
default:
t.Fatalf("unexpected command %s", name)
return nil, nil
}
}
t.Cleanup(func() { commandOutput = oldCommandOutput })
pools, err := PoolStats()
require.NoError(t, err)
assert.Equal(t, []PoolStat{{Name: "tank", Size: 100, Alloc: 50, Free: 50, Health: "ONLINE"}}, pools)
datasets, err := Datasets()
require.NoError(t, err)
assert.Equal(t, []Dataset{{Name: "tank", Used: 50, Avail: 50, Mountpoint: "/tank"}}, datasets)
}

View File

@@ -6,7 +6,7 @@ import "github.com/blang/semver"
const ( const (
// Version is the current version of the application. // Version is the current version of the application.
Version = "0.19.0" Version = "0.20.0"
// AppName is the name of the application. // AppName is the name of the application.
AppName = "beszel" AppName = "beszel"
) )
@@ -19,3 +19,6 @@ var MinVersionAgentResponse = semver.MustParse("0.13.0")
// MinVersionZfsData is the minimum agent version that supports ZFS detail requests. // MinVersionZfsData is the minimum agent version that supports ZFS detail requests.
var MinVersionZfsData = semver.MustParse("0.18.9") var MinVersionZfsData = semver.MustParse("0.18.9")
// MinVersionNetworkMonitors is the minimum agent version that supports network monitor sync.
var MinVersionNetworkMonitors = semver.MustParse("0.20.0")

38
go.mod
View File

@@ -5,22 +5,25 @@ go 1.27.1
require ( require (
github.com/blang/semver v3.5.1+incompatible github.com/blang/semver v3.5.1+incompatible
github.com/coreos/go-systemd/v22 v22.7.0 github.com/coreos/go-systemd/v22 v22.7.0
github.com/distribution/reference v0.6.0
github.com/ebitengine/purego v0.11.0 github.com/ebitengine/purego v0.11.0
github.com/fxamacker/cbor/v2 v2.9.3 github.com/fxamacker/cbor/v2 v2.9.4
github.com/gliderlabs/ssh v0.3.8 github.com/gliderlabs/ssh v0.3.8
github.com/lxzan/gws v1.10.1 github.com/lxzan/gws v1.10.2
github.com/nicholas-fedor/shoutrrr v0.20.0 github.com/nicholas-fedor/shoutrrr v0.21.0
github.com/opencontainers/go-digest v1.0.0
github.com/pocketbase/dbx v1.12.0 github.com/pocketbase/dbx v1.12.0
github.com/pocketbase/pocketbase v0.40.2 github.com/pocketbase/pocketbase v0.40.4
github.com/shirou/gopsutil/v4 v4.26.8 github.com/shirou/gopsutil/v4 v4.26.8
github.com/spf13/cast v1.10.0 github.com/spf13/cast v1.10.0
github.com/spf13/cobra v1.10.2 github.com/spf13/cobra v1.10.2
github.com/spf13/pflag v1.0.10 github.com/spf13/pflag v1.0.10
github.com/stretchr/testify v1.12.1 github.com/stretchr/testify v1.12.1
golang.org/x/crypto v0.56.0 golang.org/x/crypto v0.57.0
golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba
golang.org/x/net v0.58.0 golang.org/x/net v0.59.0
golang.org/x/sys v0.47.0 golang.org/x/oauth2 v0.37.0
golang.org/x/sys v0.48.0
gopkg.in/yaml.v3 v3.0.1 gopkg.in/yaml.v3 v3.0.1
howett.net/plist v1.0.1 howett.net/plist v1.0.1
) )
@@ -30,7 +33,7 @@ require (
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/disintegration/imaging v1.6.2 // indirect github.com/disintegration/imaging v1.6.2 // indirect
github.com/domodwyer/mailyak/v3 v3.6.2 // indirect github.com/domodwyer/mailyak/v3 v3.6.2 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect github.com/dustin/go-humanize v1.1.0 // indirect
github.com/eclipse/paho.golang v0.23.0 // indirect github.com/eclipse/paho.golang v0.23.0 // indirect
github.com/fatih/color v1.19.0 // indirect github.com/fatih/color v1.19.0 // indirect
github.com/fsnotify/fsnotify v1.10.1 // indirect github.com/fsnotify/fsnotify v1.10.1 // indirect
@@ -49,18 +52,23 @@ require (
github.com/mattn/go-isatty v0.0.24 // indirect github.com/mattn/go-isatty v0.0.24 // indirect
github.com/ncruces/go-strftime v1.0.0 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/pocketbase/ozzo-validation/v4 v4.3.0 // indirect github.com/pocketbase/ozzo-validation/v4 v4.3.0 // indirect
github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6 // indirect github.com/power-devops/perfstat v0.0.0-20260916203055-22a1a467d9f0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/tklauser/go-sysconf v0.4.0 // indirect github.com/tklauser/go-sysconf v0.4.0 // indirect
github.com/tklauser/numcpus v0.12.0 // indirect github.com/tklauser/numcpus v0.12.0 // indirect
github.com/x448/float16 v0.8.4 // indirect github.com/x448/float16 v0.8.4 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect
golang.org/x/image v0.45.0 // indirect golang.org/x/image v0.46.0 // indirect
golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/mod v0.41.0 // indirect
golang.org/x/sync v0.22.0 // indirect golang.org/x/sync v0.23.0 // indirect
golang.org/x/term v0.45.0 // indirect golang.org/x/term v0.46.0 // indirect
golang.org/x/text v0.41.0 // indirect golang.org/x/text v0.42.0 // indirect
golang.org/x/tools v0.50.0 // indirect
mellium.im/reader v0.1.0 // indirect
mellium.im/sasl v0.3.2 // indirect
mellium.im/xmlstream v0.15.4 // indirect
mellium.im/xmpp v0.23.0 // indirect
modernc.org/libc v1.74.4 // indirect modernc.org/libc v1.74.4 // indirect
modernc.org/mathutil v1.7.1 // indirect modernc.org/mathutil v1.7.1 // indirect
modernc.org/memory v1.12.1 // indirect modernc.org/memory v1.12.1 // indirect

84
go.sum
View File

@@ -15,10 +15,12 @@ github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6N
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/domodwyer/mailyak/v3 v3.6.2 h1:x3tGMsyFhTCaxp6ycgR0FE/bu5QiNp+hetUuCOBXMn8= github.com/domodwyer/mailyak/v3 v3.6.2 h1:x3tGMsyFhTCaxp6ycgR0FE/bu5QiNp+hetUuCOBXMn8=
github.com/domodwyer/mailyak/v3 v3.6.2/go.mod h1:lOm/u9CyCVWHeaAmHIdF4RiKVxKUT/H5XX10lIKAL6c= github.com/domodwyer/mailyak/v3 v3.6.2/go.mod h1:lOm/u9CyCVWHeaAmHIdF4RiKVxKUT/H5XX10lIKAL6c=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI=
github.com/ebitengine/purego v0.11.0 h1:jhp/D+Nyv7UUW8HAcmcjt2N2rYrYi9m3SL21k0Ua/NI= github.com/ebitengine/purego v0.11.0 h1:jhp/D+Nyv7UUW8HAcmcjt2N2rYrYi9m3SL21k0Ua/NI=
github.com/ebitengine/purego v0.11.0/go.mod h1:DCHPP08djqhNSoTfImcnHYQRZmd0qhakvrozqaEYhGQ= github.com/ebitengine/purego v0.11.0/go.mod h1:DCHPP08djqhNSoTfImcnHYQRZmd0qhakvrozqaEYhGQ=
github.com/eclipse/paho.golang v0.23.0 h1:KHgl2wz6EJo7cMBmkuhpt7C576vP+kpPv7jjvSyR6Mk= github.com/eclipse/paho.golang v0.23.0 h1:KHgl2wz6EJo7cMBmkuhpt7C576vP+kpPv7jjvSyR6Mk=
@@ -29,8 +31,8 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
github.com/fxamacker/cbor/v2 v2.9.3 h1:oQBnFATpNdY8gJHTndDDv5Xl4QqNaz51G5LLEPhng3Q= github.com/fxamacker/cbor/v2 v2.9.4 h1:xwjVlxEMR3S605oUlgBjKLTTeGFciYPGYCtF/35LKGo=
github.com/fxamacker/cbor/v2 v2.9.3/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/fxamacker/cbor/v2 v2.9.4/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
github.com/gabriel-vasile/mimetype v1.4.15 h1:05iP/CYtZ/w455R/KZM6rZ5ieAdh99UPtd+d3YzLmaI= github.com/gabriel-vasile/mimetype v1.4.15 h1:05iP/CYtZ/w455R/KZM6rZ5ieAdh99UPtd+d3YzLmaI=
github.com/gabriel-vasile/mimetype v1.4.15/go.mod h1:azpTcoLcDZRNgFou5j+APrqQx9HqVPWa6ijYQIIVswQ= github.com/gabriel-vasile/mimetype v1.4.15/go.mod h1:azpTcoLcDZRNgFou5j+APrqQx9HqVPWa6ijYQIIVswQ=
github.com/ganigeorgiev/fexpr v0.6.0 h1:Fza3O/QMBKEudUvxV862qe6GjxM60GJjjKytdp+VQus= github.com/ganigeorgiev/fexpr v0.6.0 h1:Fza3O/QMBKEudUvxV862qe6GjxM60GJjjKytdp+VQus=
@@ -75,29 +77,31 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/lufia/plan9stats v0.0.0-20260802145828-341c2f0c90b5 h1:eveIIGn4BGM3qknO74omf6HYr30/exH+eVUTuAgwjZ0= github.com/lufia/plan9stats v0.0.0-20260802145828-341c2f0c90b5 h1:eveIIGn4BGM3qknO74omf6HYr30/exH+eVUTuAgwjZ0=
github.com/lufia/plan9stats v0.0.0-20260802145828-341c2f0c90b5/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= github.com/lufia/plan9stats v0.0.0-20260802145828-341c2f0c90b5/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
github.com/lxzan/gws v1.10.1 h1:1xG+tDOV0lgDeVPf0wNT74u3cn0K3LpcavRrTPTrMwQ= github.com/lxzan/gws v1.10.2 h1:htReTvcY89iMk1ScVtUbk6J96kIZWaafj6r/lasK/NA=
github.com/lxzan/gws v1.10.1/go.mod h1:gXHSCPmTGryWJ4icuqy8Yho32E4YIMHH0fkDRYJRbdc= github.com/lxzan/gws v1.10.2/go.mod h1:gXHSCPmTGryWJ4icuqy8Yho32E4YIMHH0fkDRYJRbdc=
github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY= github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY=
github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
github.com/nicholas-fedor/shoutrrr v0.20.0 h1:hMAxIYlfAeZ1FcTDgU0kUOvVXUsOirWo8IWlnzGLkac= github.com/nicholas-fedor/shoutrrr v0.21.0 h1:as/mEwdaZMijCVu0FkTUEXashhvC3Y7C5g9dsXMcmQc=
github.com/nicholas-fedor/shoutrrr v0.20.0/go.mod h1:hgde37yNWCXh8+N6WemyDRMNYLOFTf326GsBx8Z7CFA= github.com/nicholas-fedor/shoutrrr v0.21.0/go.mod h1:dgg4kJv9K0tLXBH/1TXiSibNbM2hcd4SK6xb0sglyU4=
github.com/onsi/ginkgo/v2 v2.32.1 h1:6tlvcDm/3sE8lGJbZ4+d4mO3RLy24/tQWOFzVSQNIfw= github.com/onsi/ginkgo/v2 v2.32.2 h1:2o6vyFvR6snrJWgRVztC+OwuqqPEMI1UzYl2s2iU7Cg=
github.com/onsi/ginkgo/v2 v2.32.1/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= github.com/onsi/ginkgo/v2 v2.32.2/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44=
github.com/onsi/gomega v1.43.0 h1:VlG/1FxqNxhSO+lq/OHBNaaqwiBK/mO8JbVkX9Y+FeU= github.com/onsi/gomega v1.43.0 h1:VlG/1FxqNxhSO+lq/OHBNaaqwiBK/mO8JbVkX9Y+FeU=
github.com/onsi/gomega v1.43.0/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg= github.com/onsi/gomega v1.43.0/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pocketbase/dbx v1.12.0 h1:/oLErM+A0b4xI0PWTGPqSDVjzix48PqI/bng2l0PzoA= github.com/pocketbase/dbx v1.12.0 h1:/oLErM+A0b4xI0PWTGPqSDVjzix48PqI/bng2l0PzoA=
github.com/pocketbase/dbx v1.12.0/go.mod h1:xXRCIAKTHMgUCyCKZm55pUOdvFziJjQfXaWKhu2vhMs= github.com/pocketbase/dbx v1.12.0/go.mod h1:xXRCIAKTHMgUCyCKZm55pUOdvFziJjQfXaWKhu2vhMs=
github.com/pocketbase/ozzo-validation/v4 v4.3.0 h1:uKBDVma7bZqgR2a6AwE+k9hkuDFfiZMpBHQdZ1z3iQs= github.com/pocketbase/ozzo-validation/v4 v4.3.0 h1:uKBDVma7bZqgR2a6AwE+k9hkuDFfiZMpBHQdZ1z3iQs=
github.com/pocketbase/ozzo-validation/v4 v4.3.0/go.mod h1:6XNjSTw/Jb2F8LOkKO3oyzIWExbrGiYoS4uVxVwz90g= github.com/pocketbase/ozzo-validation/v4 v4.3.0/go.mod h1:6XNjSTw/Jb2F8LOkKO3oyzIWExbrGiYoS4uVxVwz90g=
github.com/pocketbase/pocketbase v0.40.2 h1:7gTqvt3bmilkphyZZ1QNhX19g3BXHqT7ynDyU81RVT4= github.com/pocketbase/pocketbase v0.40.4 h1:0SvSUreR3NhUMCs9LchE59oEG53efZ3cKiMGyAGBN9U=
github.com/pocketbase/pocketbase v0.40.2/go.mod h1:jc3YuyToy+ZXM4CeO7uSCN/htgR8yv+tjSE3eJZ8eh8= github.com/pocketbase/pocketbase v0.40.4/go.mod h1:2mU+80FLiY1fb13WZRg8Xx/lKg4nTjgKVJMigyRH6k0=
github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6 h1:jL3a8soXdzuTCcRnKhOmtcsVOObdDTFf4O2B403HPRU= github.com/power-devops/perfstat v0.0.0-20260916203055-22a1a467d9f0 h1:XA01Vk/wv9YikCi1V51yRzIHPMT5of9+cMpZoZvDn/M=
github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/power-devops/perfstat v0.0.0-20260916203055-22a1a467d9f0/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
@@ -132,37 +136,37 @@ go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa h1:QSyA8ishJCyT21kER9KwNt0b7BM3iRK4x9QXhjN5Fdk= golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba h1:Ck8QetSgk912qxWLMCKxd0in+aiyBQyDSMae6e/xmpU=
golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa/go.mod h1:zeBbvyFKDaLwa7CH/zI8KXt7gTl14SF7sO08Pl5jBCM= golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba/go.mod h1:50RgIsmK7OwqzTTeqcSXQW8SswW0o8fRcDxmqGluJ8E=
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0= golang.org/x/image v0.46.0 h1:b1+oYj0Jbp6K5MDT4i4/eZpYlk3V8SJhhDKh6LBHAyQ=
golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4= golang.org/x/image v0.46.0/go.mod h1:3B3W05VGVQyuXucLINLjXKrqISASfi4Xj+iCVkLMwew=
golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs= golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c=
golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE= golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98=
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58=
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE=
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= golang.org/x/tools v0.50.0 h1:c2ifzfcuY7L90lZ2aKd8S4K2NpASF08SZx9ZuJkHmSU=
golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0=
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -172,6 +176,14 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
howett.net/plist v1.0.1 h1:37GdZ8tP09Q35o9ych3ehygcsL+HqKSwzctveSlarvM= howett.net/plist v1.0.1 h1:37GdZ8tP09Q35o9ych3ehygcsL+HqKSwzctveSlarvM=
howett.net/plist v1.0.1/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g= howett.net/plist v1.0.1/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g=
mellium.im/reader v0.1.0 h1:UUEMev16gdvaxxZC7fC08j7IzuDKh310nB6BlwnxTww=
mellium.im/reader v0.1.0/go.mod h1:F+X5HXpkIfJ9EE1zHQG9lM/hO946iYAmU7xjg5dsQHI=
mellium.im/sasl v0.3.2 h1:PT6Xp7ccn9XaXAnJ03FcEjmAn7kK1x7aoXV6F+Vmrl0=
mellium.im/sasl v0.3.2/go.mod h1:NKXDi1zkr+BlMHLQjY3ofYuU4KSPFxknb8mfEu6SveY=
mellium.im/xmlstream v0.15.4 h1:gLKxcWl4rLMUpKgtzrTBvr4OexPeO/edYus+uK3F6ZI=
mellium.im/xmlstream v0.15.4/go.mod h1:yXaCW2++fmVO4L9piKVkyLDqnCmictVYF7FDQW8prb4=
mellium.im/xmpp v0.23.0 h1:rvKvOvMdIURCLaAWEJN8J0QpO3AJYCDKjxLsqtTPSjY=
mellium.im/xmpp v0.23.0/go.mod h1:GHDKlKKQe0LNmD9YqExyxnFEEBiz84KGqnfiA2VNzb8=
modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI= modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI=
modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU=

View File

@@ -20,10 +20,11 @@ type hubLike interface {
} }
type AlertManager struct { type AlertManager struct {
hub hubLike hub hubLike
stopOnce sync.Once stopOnce sync.Once
pendingAlerts sync.Map pendingAlerts sync.Map
alertsCache *AlertsCache alertsCache *AlertsCache
networkMonitors *networkMonitorCache
} }
type AlertMessageData struct { type AlertMessageData struct {
@@ -107,8 +108,9 @@ var supportsTitle = map[string]struct{}{
// NewAlertManager creates a new AlertManager instance. // NewAlertManager creates a new AlertManager instance.
func NewAlertManager(app hubLike) *AlertManager { func NewAlertManager(app hubLike) *AlertManager {
am := &AlertManager{ am := &AlertManager{
hub: app, hub: app,
alertsCache: NewAlertsCache(app), alertsCache: NewAlertsCache(app),
networkMonitors: newNetworkMonitorCache(app),
} }
am.bindEvents() am.bindEvents()
return am return am
@@ -116,6 +118,7 @@ func NewAlertManager(app hubLike) *AlertManager {
// Bind events to the alerts collection lifecycle // Bind events to the alerts collection lifecycle
func (am *AlertManager) bindEvents() { func (am *AlertManager) bindEvents() {
am.bindNetworkMonitorAlertEvents()
am.hub.OnRecordAfterUpdateSuccess("alerts").BindFunc(updateHistoryOnAlertUpdate) am.hub.OnRecordAfterUpdateSuccess("alerts").BindFunc(updateHistoryOnAlertUpdate)
am.hub.OnRecordAfterDeleteSuccess("alerts").BindFunc(resolveHistoryOnAlertDelete) am.hub.OnRecordAfterDeleteSuccess("alerts").BindFunc(resolveHistoryOnAlertDelete)
am.hub.OnRecordAfterUpdateSuccess("smart_devices").BindFunc(am.handleSmartDeviceAlert) am.hub.OnRecordAfterUpdateSuccess("smart_devices").BindFunc(am.handleSmartDeviceAlert)

View File

@@ -29,6 +29,13 @@ func UpsertUserAlerts(e *core.RequestEvent) error {
return e.BadRequestError("Bad data", err) return e.BadRequestError("Bad data", err)
} }
if reqData.Name == alertNameNetworkMonitorLoss {
if reqData.Value < 0 || reqData.Value >= 100 {
return e.BadRequestError("Monitor loss threshold must be at least 0 and below 100", nil)
}
reqData.Min = 0
}
alertsCollection, err := e.App.FindCachedCollectionByNameOrId("alerts") alertsCollection, err := e.App.FindCachedCollectionByNameOrId("alerts")
if err != nil { if err != nil {
return err return err

View File

@@ -11,6 +11,7 @@ import (
"strings" "strings"
"sync/atomic" "sync/atomic"
"testing" "testing"
"testing/synctest"
beszelTests "github.com/henrygd/beszel/internal/tests" beszelTests "github.com/henrygd/beszel/internal/tests"
pbTests "github.com/pocketbase/pocketbase/tests" pbTests "github.com/pocketbase/pocketbase/tests"
@@ -533,6 +534,20 @@ func TestSendTestNotification(t *testing.T) {
for _, url := range []string{localURL, "smtp://user:pass@127.0.0.1/?fromAddress=sender@example.com&toAddresses=recipient@example.com", "mqtt://127.0.0.1/topic"} { for _, url := range []string{localURL, "smtp://user:pass@127.0.0.1/?fromAddress=sender@example.com&toAddresses=recipient@example.com", "mqtt://127.0.0.1/topic"} {
scenarios = append(scenarios, beszelTests.ApiScenario{ scenarios = append(scenarios, beszelTests.ApiScenario{
BeforeTestFunc: func(tb testing.TB, _ *pbTests.TestApp, e *core.ServeEvent) {
if !strings.HasPrefix(url, "mqtt://") {
return
}
// Keep the real MQTT rejection path, but advance its library's
// fixed timeout using virtual time instead of waiting 10 seconds.
e.Router.BindFunc(func(re *core.RequestEvent) error {
var err error
synctest.Test(tb.(*testing.T), func(t *testing.T) {
err = re.Next()
})
return err
})
},
Name: "readonly cannot send to " + url, Name: "readonly cannot send to " + url,
Method: http.MethodPost, Method: http.MethodPost,
URL: "/api/beszel/test-notification", URL: "/api/beszel/test-notification",

View File

@@ -1,6 +1,7 @@
package alerts package alerts
import ( import (
"sync"
"time" "time"
"github.com/pocketbase/dbx" "github.com/pocketbase/dbx"
@@ -18,6 +19,9 @@ type CachedAlertData struct {
Triggered bool Triggered bool
Min uint8 Min uint8
PendingSince time.Time PendingSince time.Time
// Immutable after publication; decoded only when the alert record changes.
MonitorStates map[string]string
MonitorStatesValid bool
// Created types.DateTime // Created types.DateTime
} }
@@ -30,11 +34,18 @@ func (a *CachedAlertData) PopulateFromRecord(record *core.Record) {
a.Triggered = record.GetBool("triggered") a.Triggered = record.GetBool("triggered")
a.Min = uint8(record.GetInt("min")) a.Min = uint8(record.GetInt("min"))
a.PendingSince = record.GetDateTime("pending_since").Time() a.PendingSince = record.GetDateTime("pending_since").Time()
if a.Name == alertNameNetworkMonitorLoss {
var state networkMonitorAlertState
a.MonitorStatesValid = record.UnmarshalJSONField("state", &state) == nil
a.MonitorStates = state.Monitors
}
// a.Created = record.GetDateTime("created") // a.Created = record.GetDateTime("created")
} }
// AlertsCache provides an in-memory cache for system alerts. // AlertsCache provides an in-memory cache for system alerts.
type AlertsCache struct { type AlertsCache struct {
// Serialize lazy loads with updates so a late load cannot replace newer state.
loadMu sync.Mutex
app core.App app core.App
store *store.Store[string, *store.Store[string, CachedAlertData]] store *store.Store[string, *store.Store[string, CachedAlertData]]
populated bool populated bool
@@ -69,6 +80,8 @@ func (c *AlertsCache) bindEvents() *AlertsCache {
// PopulateFromDB clears current entries and loads all alerts from the database into the cache. // PopulateFromDB clears current entries and loads all alerts from the database into the cache.
func (c *AlertsCache) PopulateFromDB(force bool) error { func (c *AlertsCache) PopulateFromDB(force bool) error {
c.loadMu.Lock()
defer c.loadMu.Unlock()
if !force && c.populated { if !force && c.populated {
return nil return nil
} }
@@ -78,7 +91,7 @@ func (c *AlertsCache) PopulateFromDB(force bool) error {
} }
c.store.RemoveAll() c.store.RemoveAll()
for _, record := range records { for _, record := range records {
c.Update(record) c.update(record)
} }
c.populated = true c.populated = true
return nil return nil
@@ -86,6 +99,12 @@ func (c *AlertsCache) PopulateFromDB(force bool) error {
// Update adds or updates an alert record in the cache. // Update adds or updates an alert record in the cache.
func (c *AlertsCache) Update(record *core.Record) { func (c *AlertsCache) Update(record *core.Record) {
c.loadMu.Lock()
defer c.loadMu.Unlock()
c.update(record)
}
func (c *AlertsCache) update(record *core.Record) {
systemID := record.GetString("system") systemID := record.GetString("system")
if systemID == "" { if systemID == "" {
return return
@@ -102,6 +121,8 @@ func (c *AlertsCache) Update(record *core.Record) {
// Delete removes an alert record from the cache. // Delete removes an alert record from the cache.
func (c *AlertsCache) Delete(record *core.Record) { func (c *AlertsCache) Delete(record *core.Record) {
c.loadMu.Lock()
defer c.loadMu.Unlock()
systemID := record.GetString("system") systemID := record.GetString("system")
if systemID == "" { if systemID == "" {
return return
@@ -115,18 +136,23 @@ func (c *AlertsCache) Delete(record *core.Record) {
func (c *AlertsCache) GetSystemAlerts(systemID string) []CachedAlertData { func (c *AlertsCache) GetSystemAlerts(systemID string) []CachedAlertData {
systemStore, ok := c.store.GetOk(systemID) systemStore, ok := c.store.GetOk(systemID)
if !ok { if !ok {
// Populate cache for this system c.loadMu.Lock()
records, err := c.app.FindAllRecords("alerts", dbx.NewExp("system={:system}", dbx.Params{"system": systemID})) defer c.loadMu.Unlock()
if err != nil { systemStore, ok = c.store.GetOk(systemID)
return nil if !ok {
// Populate cache for this system
records, err := c.app.FindAllRecords("alerts", dbx.NewExp("system={:system}", dbx.Params{"system": systemID}))
if err != nil {
return nil
}
systemStore = store.New(map[string]CachedAlertData{})
for _, record := range records {
var ca CachedAlertData
ca.PopulateFromRecord(record)
systemStore.Set(record.Id, ca)
}
c.store.Set(systemID, systemStore)
} }
systemStore = store.New(map[string]CachedAlertData{})
for _, record := range records {
var ca CachedAlertData
ca.PopulateFromRecord(record)
systemStore.Set(record.Id, ca)
}
c.store.Set(systemID, systemStore)
} }
all := systemStore.GetAll() all := systemStore.GetAll()
alerts := make([]CachedAlertData, 0, len(all)) alerts := make([]CachedAlertData, 0, len(all))

View File

@@ -9,6 +9,12 @@ import (
// On triggered alert record delete, set matching alert history record to resolved // On triggered alert record delete, set matching alert history record to resolved
func resolveHistoryOnAlertDelete(e *core.RecordEvent) error { func resolveHistoryOnAlertDelete(e *core.RecordEvent) error {
if e.Record.GetString("name") == alertNameNetworkMonitorLoss {
if err := resolveNetworkMonitorHistory(e.App, e.Record.Id); err != nil {
return err
}
return e.Next()
}
if !e.Record.GetBool("triggered") { if !e.Record.GetBool("triggered") {
return e.Next() return e.Next()
} }
@@ -18,6 +24,10 @@ func resolveHistoryOnAlertDelete(e *core.RecordEvent) error {
// On alert record update, update alert history record // On alert record update, update alert history record
func updateHistoryOnAlertUpdate(e *core.RecordEvent) error { func updateHistoryOnAlertUpdate(e *core.RecordEvent) error {
// Network monitor incidents have separate history entries per monitor.
if e.Record.GetString("name") == alertNameNetworkMonitorLoss {
return e.Next()
}
original := e.Record.Original() original := e.Record.Original()
new := e.Record new := e.Record

View File

@@ -0,0 +1,269 @@
package alerts
import (
"database/sql"
"errors"
"fmt"
"math"
"net"
"strconv"
"time"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/core"
)
const alertNameNetworkMonitorLoss = "NetworkMonitorLoss"
// networkMonitorAlertState is this alert type's persisted runtime state.
// Monitor IDs map to their open history entries independently of history retention.
type networkMonitorAlertState struct {
Monitors map[string]string `json:"monitors"`
}
func (am *AlertManager) bindNetworkMonitorAlertEvents() {
// Hidden fields are still writable through the record API unless protected.
protectState := func(e *core.RecordRequestEvent) error {
e.Record.Set("state", e.Record.Original().Get("state"))
oldName, newName := e.Record.Original().GetString("name"), e.Record.GetString("name")
if oldName != "" && (oldName == alertNameNetworkMonitorLoss || newName == alertNameNetworkMonitorLoss) &&
(oldName != newName || e.Record.GetString("system") != e.Record.Original().GetString("system")) {
return e.BadRequestError("Delete and recreate the alert to change its type or system", nil)
}
if e.Record.GetString("name") == alertNameNetworkMonitorLoss {
if !e.HasSuperuserAuth() && (e.Auth == nil || !userHasSystem(e.App, e.Auth.Id, e.Record.GetString("system"))) {
return e.ForbiddenError("You do not have access to this system", nil)
}
e.Record.Set("triggered", e.Record.Original().GetBool("triggered"))
value := e.Record.GetFloat("value")
if math.IsNaN(value) || math.IsInf(value, 0) || value < 0 || value >= 100 {
return e.BadRequestError("Monitor loss threshold must be at least 0 and below 100", nil)
}
e.Record.Set("min", 0)
}
return e.Next()
}
am.hub.OnRecordCreateRequest("alerts").BindFunc(protectState)
am.hub.OnRecordUpdateRequest("alerts").BindFunc(protectState)
cleanup := func(e *core.RecordEvent) error {
if err := e.Next(); err != nil {
return err
}
return am.evaluateNetworkMonitorAlerts(e.App, e.Record.GetString("system"), nil)
}
am.hub.OnRecordAfterDeleteSuccess("network_monitors").BindFunc(cleanup)
am.hub.OnRecordAfterUpdateSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error {
if e.Record.GetBool("enabled") || !e.Record.Original().GetBool("enabled") {
return e.Next()
}
return cleanup(e)
})
}
// HandleNetworkMonitorAlerts runs after the full monitoring transaction commits,
// using its exact payload (dashboard requests can replace the cached payload).
// Omitted results and disconnected systems never imply recovery.
func (am *AlertManager) HandleNetworkMonitorAlerts(systemRecord *core.Record, results map[string]monitor.Result) error {
if systemRecord.GetString("status") != "up" {
return nil
}
alerts := am.alertsCache.GetAlertsByName(systemRecord.Id, alertNameNetworkMonitorLoss)
if len(alerts) == 0 {
return nil
}
monitors, err := am.networkMonitors.get(systemRecord.Id)
if err != nil {
return err
}
if !networkMonitorTransitionPending(alerts, monitors, results, time.Now()) {
return nil
}
// The cache only predicts a transition. Reload and recheck under the DB
// transaction before persisting, including current system/monitor status.
return am.evaluateNetworkMonitorAlerts(am.hub, systemRecord.Id, results)
}
// networkMonitorTransitionPending does no IO and never mutates cached maps.
func networkMonitorTransitionPending(alerts []CachedAlertData, monitors map[string]int, results map[string]monitor.Result, now time.Time) bool {
for _, alert := range alerts {
if !alert.MonitorStatesValid || alert.Triggered != (len(alert.MonitorStates) > 0) {
return true
}
for id := range alert.MonitorStates {
if _, enabled := monitors[id]; !enabled {
return true
}
}
for id, result := range results {
interval, enabled := monitors[id]
if !enabled || !monitorResultReady(result, interval, now) {
continue
}
_, active := alert.MonitorStates[id]
if (result.PacketLoss1h > alert.Value) != active {
return true
}
}
}
return false
}
func (am *AlertManager) evaluateNetworkMonitorAlerts(app core.App, systemID string, results map[string]monitor.Result) error {
var messages []AlertMessageData
err := app.RunInTransaction(func(tx core.App) error {
// Read configuration inside the transaction so concurrent threshold changes,
// disabling, and evaluations cannot overwrite each other's incident state.
alerts, err := tx.FindAllRecords("alerts", dbx.HashExp{"system": systemID, "name": alertNameNetworkMonitorLoss})
if err != nil || len(alerts) == 0 {
return err
}
system, err := tx.FindRecordById("systems", systemID)
if errors.Is(err, sql.ErrNoRows) {
// System deletion cascades to its alerts.
return nil
}
if err != nil {
return err
}
monitors, err := tx.FindAllRecords("network_monitors", dbx.HashExp{"system": systemID, "enabled": true})
if err != nil {
return err
}
enabled := make(map[string]*core.Record, len(monitors))
for _, m := range monitors {
enabled[m.Id] = m
}
now := time.Now()
for _, alert := range alerts {
var state networkMonitorAlertState
if err := alert.UnmarshalJSONField("state", &state); err != nil {
return err
}
states := state.Monitors
if states == nil {
states = map[string]string{}
}
changed := false
// Removing or disabling a monitor closes its incident silently.
for id, historyID := range states {
if _, ok := enabled[id]; !ok {
if err := resolveMonitorIncident(tx, historyID, now); err != nil {
return err
}
delete(states, id)
changed = true
}
}
if system.GetString("status") == "up" {
for _, m := range monitors {
result, ok := results[m.Id]
if !ok || !monitorResultReady(result, m.GetInt("interval"), now) {
continue
}
historyID, active := states[m.Id]
triggered := result.PacketLoss1h > alert.GetFloat("value")
if triggered == active {
continue
}
label := m.GetString("target")
if m.GetString("protocol") == "tcp" {
label = net.JoinHostPort(label, strconv.Itoa(m.GetInt("port")))
}
if triggered {
collection, err := tx.FindCachedCollectionByNameOrId("alerts_history")
if err != nil {
return err
}
history := core.NewRecord(collection)
history.Load(map[string]any{
"alert_id": alert.Id, "user": alert.GetString("user"), "system": systemID,
"name": alertNameNetworkMonitorLoss, "monitor_name": label, "value": result.PacketLoss1h,
})
if err := tx.Save(history); err != nil {
return err
}
states[m.Id] = history.Id
} else {
if err := resolveMonitorIncident(tx, historyID, now); err != nil {
return err
}
delete(states, m.Id)
}
changed = true
state, comparison := "loss", "exceeds"
if !triggered {
state, comparison = "recovered", "is at or below"
}
messages = append(messages, AlertMessageData{
UserID: alert.GetString("user"), SystemID: systemID,
Title: fmt.Sprintf("Network monitor %s on %s: %s", state, system.GetString("name"), label),
Message: fmt.Sprintf("%s on %s: loss over the past hour is %.2f%%, which %s the %.2f%% threshold.", label, system.GetString("name"), result.PacketLoss1h, comparison, alert.GetFloat("value")),
Link: am.hub.MakeLink("system", systemID), LinkText: "View " + system.GetString("name"),
})
}
}
if changed || alert.GetBool("triggered") != (len(states) > 0) {
alert.Set("state", networkMonitorAlertState{Monitors: states})
alert.Set("triggered", len(states) > 0)
if err := tx.Save(alert); err != nil {
return err
}
}
}
return nil
})
if err != nil {
return err
}
// Match other alert types: persist transitions before delivery, and respect
// the user's existing notification destinations and quiet hours.
for _, message := range messages {
if err := am.SendAlert(message); err != nil {
app.Logger().Error("Failed to send network monitor alert", "err", err)
}
}
return nil
}
func monitorResultReady(result monitor.Result, interval int, now time.Time) bool {
// Three completed attempts provide a short warm-up, including after an agent
// restart.
if result.SampleCount < 3 || result.LastProbeAt <= 0 || math.IsNaN(result.PacketLoss1h) || math.IsInf(result.PacketLoss1h, 0) || result.PacketLoss1h < 0 || result.PacketLoss1h > 100 {
return false
}
// Never interpret an empty one-hour window as zero loss.
maxAge := min(time.Hour, max(3*time.Duration(interval)*time.Second, 3*time.Minute))
age := now.Sub(time.UnixMilli(result.LastProbeAt))
return age >= -time.Minute && age <= maxAge
}
func resolveMonitorIncident(app core.App, id string, now time.Time) error {
record, err := app.FindRecordById("alerts_history", id)
if errors.Is(err, sql.ErrNoRows) {
// History can be purged independently.
return nil
}
if err != nil {
return err
}
if !record.GetDateTime("resolved").IsZero() {
return nil
}
record.Set("resolved", now.UTC())
return app.Save(record)
}
func resolveNetworkMonitorHistory(app core.App, alertID string) error {
records, err := app.FindAllRecords("alerts_history", dbx.HashExp{"alert_id": alertID, "resolved": ""})
if err != nil {
return err
}
for _, record := range records {
record.Set("resolved", time.Now().UTC())
if err := app.Save(record); err != nil {
return err
}
}
return nil
}

View File

@@ -0,0 +1,513 @@
//go:build testing
package alerts_test
import (
"sync"
"sync/atomic"
"testing"
"time"
"github.com/henrygd/beszel/internal/alerts"
"github.com/henrygd/beszel/internal/entities/monitor"
beszelTests "github.com/henrygd/beszel/internal/tests"
"github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/core"
pbTests "github.com/pocketbase/pocketbase/tests"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func networkAlertSetup(t *testing.T) (*beszelTests.TestHub, *core.Record, *core.Record, []*core.Record) {
t.Helper()
hub, system, alert := systemdTestSetup(t, false)
t.Cleanup(hub.Cleanup)
alert.Set("name", "NetworkMonitorLoss")
alert.Set("value", 5)
require.NoError(t, hub.Save(alert))
var monitors []*core.Record
for _, name := range []string{"gateway", "website"} {
record, err := beszelTests.CreateRecord(hub, "network_monitors", map[string]any{
"system": system.Id, "target": name + ".example.com", "protocol": "icmp", "interval": 60, "enabled": true,
})
require.NoError(t, err)
monitors = append(monitors, record)
}
// Avoid starting a system update worker in tests.
_, err := hub.DB().Update("systems", dbx.Params{"status": "up"}, dbx.HashExp{"id": system.Id}).Execute()
require.NoError(t, err)
system.Set("status", "up")
return hub, system, alert, monitors
}
func monitorResult(loss float64) monitor.Result {
return monitor.Result{LastProbeAt: time.Now().UnixMilli(), SampleCount: 60, PacketLoss1h: loss}
}
func TestNetworkMonitorAlertIndependentIncidents(t *testing.T) {
hub, system, alert, monitors := networkAlertSetup(t)
am := alerts.NewTestAlertManagerWithoutWorker(hub)
count := hub.TestMailer.TotalSend()
results := map[string]monitor.Result{monitors[0].Id: monitorResult(10), monitors[1].Id: monitorResult(0)}
check := func(active bool, open, sent int) {
t.Helper()
record, err := hub.FindRecordById("alerts", alert.Id)
require.NoError(t, err)
assert.Equal(t, active, record.GetBool("triggered"))
total, err := hub.CountRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id, "resolved": ""})
require.NoError(t, err)
assert.EqualValues(t, open, total)
assert.Equal(t, count+sent, hub.TestMailer.TotalSend())
}
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
check(true, 1, 1)
message := hub.TestMailer.Messages()[count]
assert.Contains(t, message.Text, "gateway.example.com")
assert.Contains(t, message.Text, "10.00%")
assert.Contains(t, message.Text, "5.00%")
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
check(true, 1, 1)
// Persisted monitor state prevents duplicate notifications after a hub restart.
am = alerts.NewTestAlertManagerWithoutWorker(hub)
results[monitors[1].Id] = monitorResult(20)
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
check(true, 2, 2)
results[monitors[0].Id] = monitorResult(5) // Equality is a recovery.
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
check(true, 1, 3)
results[monitors[1].Id] = monitorResult(0)
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
check(false, 0, 4)
histories, err := hub.FindAllRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id})
require.NoError(t, err)
require.Len(t, histories, 2)
for _, history := range histories {
assert.NotEmpty(t, history.GetString("monitor_name"))
}
}
func TestNetworkMonitorAlertTargetLabel(t *testing.T) {
for _, tc := range []struct {
protocol, target, label string
port int
}{
{"icmp", "gateway.example.com", "gateway.example.com", 0},
{"http", "https://example.com/health", "https://example.com/health", 0},
{"tcp", "example.com", "example.com:8443", 8443},
{"tcp", "2001:db8::1", "[2001:db8::1]:443", 443},
} {
t.Run(tc.label, func(t *testing.T) {
hub, system, alert, monitors := networkAlertSetup(t)
m := monitors[0]
m.Set("protocol", tc.protocol)
m.Set("target", tc.target)
m.Set("port", tc.port)
require.NoError(t, hub.Save(m))
am := alerts.NewTestAlertManagerWithoutWorker(hub)
require.NoError(t, am.HandleNetworkMonitorAlerts(system, map[string]monitor.Result{m.Id: monitorResult(10)}))
histories, err := hub.FindAllRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id})
require.NoError(t, err)
require.Len(t, histories, 1)
assert.Equal(t, tc.label, histories[0].GetString("monitor_name"))
assert.Contains(t, hub.TestMailer.Messages()[hub.TestMailer.TotalSend()-1].Text, tc.label)
})
}
}
func TestNetworkMonitorAlertIgnoresUnknownResults(t *testing.T) {
for _, scenario := range []string{"missing", "stale", "warmup", "no probes", "down", "paused", "future", "expired hourly window"} {
t.Run(scenario, func(t *testing.T) {
hub, system, alert, monitors := networkAlertSetup(t)
am := alerts.NewTestAlertManagerWithoutWorker(hub)
apply := func(loss float64) {
result := monitorResult(loss)
results := map[string]monitor.Result{monitors[0].Id: result}
switch scenario {
case "missing":
results = nil
case "stale":
result.LastProbeAt = time.Now().Add(-10 * time.Minute).UnixMilli()
results[monitors[0].Id] = result
case "expired hourly window":
monitors[0].Set("interval", 3600)
require.NoError(t, hub.Save(monitors[0]))
result.LastProbeAt = time.Now().Add(-2 * time.Hour).UnixMilli()
results[monitors[0].Id] = result
case "future":
result.LastProbeAt = time.Now().Add(time.Hour).UnixMilli()
results[monitors[0].Id] = result
case "warmup":
result.SampleCount = 2
results[monitors[0].Id] = result
case "no probes":
result.SampleCount = 0
results[monitors[0].Id] = result
case "down":
_, err := hub.DB().Update("systems", dbx.Params{"status": scenario}, dbx.HashExp{"id": system.Id}).Execute()
require.NoError(t, err)
case "paused":
record, err := hub.FindRecordById("systems", system.Id)
require.NoError(t, err)
record.Set("status", "paused")
require.NoError(t, hub.Save(record))
}
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
}
count := hub.TestMailer.TotalSend()
apply(100)
assert.Equal(t, count, hub.TestMailer.TotalSend())
_, err := hub.DB().Update("systems", dbx.Params{"status": "up"}, dbx.HashExp{"id": system.Id}).Execute()
require.NoError(t, err)
require.NoError(t, am.HandleNetworkMonitorAlerts(system, map[string]monitor.Result{monitors[0].Id: monitorResult(10)}))
apply(0)
assert.Equal(t, count+1, hub.TestMailer.TotalSend(), "unknown data must not recover an incident")
record, err := hub.FindRecordById("alerts", alert.Id)
require.NoError(t, err)
assert.True(t, record.GetBool("triggered"))
})
}
}
func TestNetworkMonitorAlertCleanup(t *testing.T) {
for _, scenario := range []string{"disable monitor", "delete monitor", "disable alert", "purge history", "delete system"} {
t.Run(scenario, func(t *testing.T) {
hub, system, alert, monitors := networkAlertSetup(t)
am := alerts.NewTestAlertManagerWithoutWorker(hub)
results := map[string]monitor.Result{monitors[0].Id: monitorResult(10), monitors[1].Id: monitorResult(20)}
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
count := hub.TestMailer.TotalSend()
switch scenario {
case "disable monitor":
monitors[0].Set("enabled", false)
require.NoError(t, hub.Save(monitors[0]))
case "delete monitor":
require.NoError(t, hub.Delete(monitors[0]))
case "disable alert":
require.NoError(t, hub.Delete(alert))
case "delete system":
require.NoError(t, hub.Delete(system))
case "purge history":
history, err := hub.FindAllRecords("alerts_history")
require.NoError(t, err)
for _, record := range history {
require.NoError(t, hub.Delete(record))
}
}
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
assert.Equal(t, count, hub.TestMailer.TotalSend())
open, err := hub.CountRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id, "resolved": ""})
require.NoError(t, err)
if scenario == "disable monitor" || scenario == "delete monitor" {
assert.EqualValues(t, 1, open)
record, err := hub.FindRecordById("alerts", alert.Id)
require.NoError(t, err)
assert.True(t, record.GetBool("triggered"))
require.NoError(t, hub.Delete(monitors[1]))
record, err = hub.FindRecordById("alerts", alert.Id)
require.NoError(t, err)
assert.False(t, record.GetBool("triggered"))
} else {
assert.Zero(t, open)
}
})
}
}
func TestNetworkMonitorAlertPerUserThresholds(t *testing.T) {
hub, system, alert, monitors := networkAlertSetup(t)
user, err := beszelTests.CreateUser(hub, "monitor2@example.com", "password")
require.NoError(t, err)
other, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{"name": "NetworkMonitorLoss", "system": system.Id, "user": user.Id, "value": 20})
require.NoError(t, err)
am := alerts.NewTestAlertManagerWithoutWorker(hub)
results := map[string]monitor.Result{monitors[0].Id: monitorResult(10)}
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
other, err = hub.FindRecordById("alerts", other.Id)
require.NoError(t, err)
assert.False(t, other.GetBool("triggered"))
// Editing the threshold re-evaluates on the next batch, without losing state.
alert, err = hub.FindRecordById("alerts", alert.Id)
require.NoError(t, err)
alert.Set("value", 15)
require.NoError(t, hub.Save(alert))
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
alert, err = hub.FindRecordById("alerts", alert.Id)
require.NoError(t, err)
assert.False(t, alert.GetBool("triggered"))
}
func TestNetworkMonitorAlertAPI(t *testing.T) {
for _, tc := range []struct {
name string
value float64
direct, denied, patch bool
status int
}{
{name: "zero threshold", value: 0, status: 200},
{name: "fractional threshold", value: 5.5, status: 200},
{name: "negative threshold", value: -1, status: 400},
{name: "unreachable threshold", value: 100, status: 400},
{name: "bulk inaccessible system", value: 5, denied: true, status: 200},
{name: "direct inaccessible system", value: 5, direct: true, denied: true, status: 403},
{name: "direct invalid threshold", value: -1, direct: true, status: 400},
{name: "direct private state", value: 5, direct: true, status: 200},
{name: "patch preserves state", value: 10, direct: true, patch: true, status: 200},
} {
t.Run(tc.name, func(t *testing.T) {
hub, user := beszelTests.GetHubWithUser(t)
defer hub.Cleanup()
owner := user.Id
if tc.denied {
other, err := beszelTests.CreateUser(hub, "other@example.com", "password")
require.NoError(t, err)
owner = other.Id
}
systems, err := beszelTests.CreateSystems(hub, 1, owner, "paused")
require.NoError(t, err)
token, err := user.NewAuthToken()
require.NoError(t, err)
body := map[string]any{"name": "NetworkMonitorLoss", "value": tc.value, "min": 60, "systems": []string{systems[0].Id}, "overwrite": true}
url, method := "/api/beszel/user-alerts", "POST"
if tc.direct {
url = "/api/collections/alerts/records"
body["system"], body["user"] = systems[0].Id, user.Id
body["state"], body["triggered"] = map[string]any{"monitors": map[string]string{"fake": "fake"}}, true
}
if tc.patch {
alert, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{"name": "NetworkMonitorLoss", "system": systems[0].Id, "user": user.Id, "value": 5, "triggered": true, "state": map[string]any{"monitors": map[string]string{"real": "history"}}})
require.NoError(t, err)
url += "/" + alert.Id
method = "PATCH"
body["triggered"] = false
}
content := `"success":true`
if tc.direct {
content = `"name":"NetworkMonitorLoss"`
}
if tc.status == 400 {
content = `"status":400`
}
if tc.status == 403 {
content = `"status":403`
}
scenario := beszelTests.ApiScenario{
Name: tc.name, Method: method, URL: url, Body: jsonReader(body),
Headers: map[string]string{"Authorization": token}, ExpectedStatus: tc.status, ExpectedContent: []string{content},
TestAppFactory: func(testing.TB) *pbTests.TestApp { return hub.TestApp },
}
scenario.Test(t)
records, err := hub.FindAllRecords("alerts")
require.NoError(t, err)
if tc.status != 200 || tc.denied {
assert.Empty(t, records)
return
}
require.Len(t, records, 1)
assert.Equal(t, tc.value, records[0].GetFloat("value"))
assert.Zero(t, records[0].GetInt("min"))
state := struct {
Monitors map[string]string `json:"monitors"`
}{}
require.NoError(t, records[0].UnmarshalJSONField("state", &state))
states := state.Monitors
if tc.patch {
assert.Equal(t, map[string]string{"real": "history"}, states)
assert.True(t, records[0].GetBool("triggered"))
} else {
assert.Empty(t, states)
assert.False(t, records[0].GetBool("triggered"))
}
})
}
}
type monitorCountingHub struct {
*beszelTests.TestHub
transactions atomic.Int64
beforeTransaction func()
}
func (h *monitorCountingHub) RunInTransaction(fn func(core.App) error) error {
h.transactions.Add(1)
if h.beforeTransaction != nil {
h.beforeTransaction()
}
return h.App.RunInTransaction(fn)
}
// Count actual SQL on both DB connections, including queries through record APIs.
func monitorSQLCounter(t *testing.T, app core.App) *atomic.Int64 {
t.Helper()
count := &atomic.Int64{}
for _, builder := range []dbx.Builder{app.ConcurrentDB(), app.NonconcurrentDB()} {
db := builder.(*dbx.DB)
old := db.LogFunc
db.LogFunc = func(string, ...any) { count.Add(1) }
t.Cleanup(func() { db.LogFunc = old })
}
return count
}
func TestNetworkMonitorAlertSteadyStateNoDatabaseWork(t *testing.T) {
hub, system, alert, monitors := networkAlertSetup(t)
counted := &monitorCountingHub{TestHub: hub}
am := alerts.NewTestAlertManagerWithoutWorker(counted)
sql := monitorSQLCounter(t, hub)
results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}
evaluate := func() { t.Helper(); require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) }
noWork := func() {
t.Helper()
sql.Store(0)
counted.transactions.Store(0)
for range 100 {
evaluate()
}
assert.Zero(t, sql.Load(), "steady state must not issue SQL")
assert.Zero(t, counted.transactions.Load(), "steady state must not open transactions")
}
// One-time lazy loads are permitted, including on hub restart.
evaluate()
assert.Positive(t, sql.Load())
noWork()
// Realtime metric saves invoke record hooks but must not invalidate config.
fresh, err := hub.FindRecordById("network_monitors", monitors[0].Id)
require.NoError(t, err)
monitors[0] = fresh
monitors[0].Set("loss1h", 0)
monitors[0].Set("res", 100)
require.NoError(t, hub.Save(monitors[0]))
noWork()
results[monitors[0].Id] = monitorResult(10)
evaluate()
assert.Positive(t, sql.Load(), "transitions must still be persisted")
assert.EqualValues(t, 1, counted.transactions.Load())
noWork()
// Missing and stale observations must not enter the transaction either.
results = nil
noWork()
results = map[string]monitor.Result{monitors[0].Id: {SampleCount: 60, LastProbeAt: time.Now().Add(-10 * time.Minute).UnixMilli()}}
noWork()
results[monitors[0].Id] = monitorResult(0)
evaluate()
noWork()
require.NoError(t, hub.Delete(alert))
noWork()
}
func TestNetworkMonitorAlertConfigCacheInvalidation(t *testing.T) {
hub, system, alert, monitors := networkAlertSetup(t)
am := alerts.NewTestAlertManagerWithoutWorker(hub)
results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
count := hub.TestMailer.TotalSend()
// Widening the interval makes this observation fresh. A stale interval cache
// would miss the failure indefinitely, even though results keep arriving.
result := monitorResult(10)
result.LastProbeAt = time.Now().Add(-4 * time.Minute).UnixMilli()
results[monitors[0].Id] = result
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
assert.Equal(t, count, hub.TestMailer.TotalSend())
monitors[0].Set("interval", 120)
require.NoError(t, hub.Save(monitors[0]))
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
assert.Equal(t, count+1, hub.TestMailer.TotalSend())
// Disable, then re-enable the same ID: its new failure must be detected.
monitors[0].Set("enabled", false)
require.NoError(t, hub.Save(monitors[0]))
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
monitors[0].Set("enabled", true)
require.NoError(t, hub.Save(monitors[0]))
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
assert.Equal(t, count+2, hub.TestMailer.TotalSend())
// A new monitor must also become eligible without restarting the hub.
created, err := beszelTests.CreateRecord(hub, "network_monitors", map[string]any{
"system": system.Id, "name": "new", "target": "new.example.com", "protocol": "icmp", "interval": 60, "enabled": true,
})
require.NoError(t, err)
results[created.Id] = monitorResult(10)
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
assert.Equal(t, count+3, hub.TestMailer.TotalSend())
// Threshold changes refresh cached config and preserve the active incidents.
alert, err = hub.FindRecordById("alerts", alert.Id)
require.NoError(t, err)
alert.Set("value", 15)
require.NoError(t, hub.Save(alert))
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
assert.Equal(t, count+5, hub.TestMailer.TotalSend())
}
func TestNetworkMonitorAlertRevalidatesCandidate(t *testing.T) {
for _, change := range []string{"threshold", "disable alert", "disable monitor", "down"} {
t.Run(change, func(t *testing.T) {
hub, system, alert, monitors := networkAlertSetup(t)
counted := &monitorCountingHub{TestHub: hub}
am := alerts.NewTestAlertManagerWithoutWorker(counted)
results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
count := hub.TestMailer.TotalSend()
// Change the DB after the cache predicts a transition, before its transaction.
counted.beforeTransaction = func() {
counted.beforeTransaction = nil
switch change {
case "threshold":
alert.Set("value", 20)
require.NoError(t, hub.Save(alert))
case "disable alert":
require.NoError(t, hub.Delete(alert))
case "disable monitor":
monitors[0].Set("enabled", false)
require.NoError(t, hub.Save(monitors[0]))
case "down":
_, err := hub.DB().Update("systems", dbx.Params{"status": "down"}, dbx.HashExp{"id": system.Id}).Execute()
require.NoError(t, err)
}
}
results[monitors[0].Id] = monitorResult(10)
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
assert.EqualValues(t, 1, counted.transactions.Load())
assert.Equal(t, count, hub.TestMailer.TotalSend())
histories, err := hub.CountRecords("alerts_history")
require.NoError(t, err)
assert.Zero(t, histories)
})
}
}
func TestNetworkMonitorAlertConcurrentEvaluations(t *testing.T) {
hub, system, _, monitors := networkAlertSetup(t)
am := alerts.NewTestAlertManagerWithoutWorker(hub)
results := map[string]monitor.Result{monitors[0].Id: monitorResult(10)}
count := hub.TestMailer.TotalSend()
var wg sync.WaitGroup
errs := make(chan error, 8)
for range 8 {
wg.Go(func() { errs <- am.HandleNetworkMonitorAlerts(system, results) })
}
wg.Wait()
close(errs)
for err := range errs {
require.NoError(t, err)
}
assert.Equal(t, count+1, hub.TestMailer.TotalSend())
}
func TestNetworkMonitorAlertCacheAfterRollback(t *testing.T) {
hub, system, _, monitors := networkAlertSetup(t)
am := alerts.NewTestAlertManagerWithoutWorker(hub)
results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
_, err := hub.DB().NewQuery(`CREATE TRIGGER fail_alert BEFORE UPDATE ON alerts BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute()
require.NoError(t, err)
count := hub.TestMailer.TotalSend()
results[monitors[0].Id] = monitorResult(10)
require.Error(t, am.HandleNetworkMonitorAlerts(system, results))
assert.Equal(t, count, hub.TestMailer.TotalSend())
histories, err := hub.CountRecords("alerts_history")
require.NoError(t, err)
assert.Zero(t, histories)
_, err = hub.DB().NewQuery("DROP TRIGGER fail_alert").Execute()
require.NoError(t, err)
// A failed transition must not be published to the cache and mask the retry.
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
assert.Equal(t, count+1, hub.TestMailer.TotalSend())
}

View File

@@ -47,7 +47,7 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
return nil return nil
} }
alerts := am.alertsCache.GetAlertsExcludingNames(systemRecord.Id, "Status", alertNameSystemdFailed, containerAlertName) alerts := am.alertsCache.GetAlertsExcludingNames(systemRecord.Id, "Status", alertNameSystemdFailed, containerAlertName, alertNameNetworkMonitorLoss)
if len(alerts) == 0 { if len(alerts) == 0 {
return nil return nil
} }

View File

@@ -11,8 +11,9 @@ import (
func NewTestAlertManagerWithoutWorker(app hubLike) *AlertManager { func NewTestAlertManagerWithoutWorker(app hubLike) *AlertManager {
return &AlertManager{ return &AlertManager{
hub: app, hub: app,
alertsCache: NewAlertsCache(app), alertsCache: NewAlertsCache(app),
networkMonitors: newNetworkMonitorCache(app),
} }
} }

View File

@@ -0,0 +1,79 @@
package alerts
import (
"sync"
"github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/core"
)
// networkMonitorCache keeps just the enabled monitor IDs and probe intervals
// needed for the alert fast path. Names and targets are read only on transitions.
// Returned maps are immutable; configuration changes invalidate the whole entry.
type networkMonitorCache struct {
app core.App
mu sync.RWMutex
systems map[string]map[string]int
}
func newNetworkMonitorCache(app core.App) *networkMonitorCache {
c := &networkMonitorCache{app: app, systems: make(map[string]map[string]int)}
invalidate := func(e *core.RecordEvent) error {
c.invalidate(e.Record.GetString("system"))
return e.Next()
}
app.OnRecordAfterCreateSuccess("network_monitors").BindFunc(invalidate)
app.OnRecordAfterDeleteSuccess("network_monitors").BindFunc(invalidate)
app.OnRecordAfterUpdateSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error {
old := e.Record.Original()
// Realtime metric saves also invoke this hook. They must not evict config.
if old.GetString("system") != e.Record.GetString("system") ||
old.GetBool("enabled") != e.Record.GetBool("enabled") ||
old.GetInt("interval") != e.Record.GetInt("interval") {
c.invalidate(old.GetString("system"))
c.invalidate(e.Record.GetString("system"))
}
return e.Next()
})
app.OnRecordAfterDeleteSuccess("systems").BindFunc(func(e *core.RecordEvent) error {
c.invalidate(e.Record.Id)
return e.Next()
})
return c
}
func (c *networkMonitorCache) invalidate(systemID string) {
c.mu.Lock()
delete(c.systems, systemID)
c.mu.Unlock()
}
func (c *networkMonitorCache) get(systemID string) (map[string]int, error) {
c.mu.RLock()
monitors, ok := c.systems[systemID]
c.mu.RUnlock()
if ok {
return monitors, nil
}
c.mu.Lock()
defer c.mu.Unlock()
if monitors, ok := c.systems[systemID]; ok {
return monitors, nil
}
// Keep the lock through the load so a concurrent config change cannot be
// invalidated first and then overwritten by the older query result.
var rows []struct {
ID string `db:"id"`
Interval int `db:"interval"`
}
if err := c.app.DB().Select("id", "interval").From("network_monitors").
Where(dbx.HashExp{"system": systemID, "enabled": true}).All(&rows); err != nil {
return nil, err
}
monitors = make(map[string]int, len(rows))
for _, row := range rows {
monitors[row.ID] = row.Interval
}
c.systems[systemID] = monitors
return monitors, nil
}

View File

@@ -11,6 +11,7 @@ import (
"strings" "strings"
"sync/atomic" "sync/atomic"
"testing" "testing"
"testing/synctest"
"github.com/nicholas-fedor/shoutrrr/pkg/types" "github.com/nicholas-fedor/shoutrrr/pkg/types"
"golang.org/x/net/dns/dnsmessage" "golang.org/x/net/dns/dnsmessage"
@@ -178,39 +179,45 @@ func TestPublicNotificationTCP(t *testing.T) {
} { } {
t.Run(rawURL, func(t *testing.T) { t.Run(rawURL, func(t *testing.T) {
t.Parallel() t.Parallel()
// MQTT waits for a fixed library timeout even after a dial failure.
// Virtual time preserves the full send/cleanup path without that delay.
t.Run("internal destination", func(t *testing.T) { t.Run("internal destination", func(t *testing.T) {
err := sendPublicNotification(strings.ReplaceAll(rawURL, "HOST", "127.0.0.1"), "test") synctest.Test(t, func(t *testing.T) {
if !errors.Is(err, errInternalDestination) { err := sendPublicNotification(strings.ReplaceAll(rawURL, "HOST", "127.0.0.1"), "test")
t.Fatalf("expected blocked destination, got %v", err) if !errors.Is(err, errInternalDestination) {
} t.Fatalf("expected blocked destination, got %v", err)
}
})
}) })
t.Run("public destination uses injected dialer", func(t *testing.T) { t.Run("public destination uses injected dialer", func(t *testing.T) {
var calls atomic.Int32 synctest.Test(t, func(t *testing.T) {
stopped := errors.New("test dial stopped") var calls atomic.Int32
service, err := newPublicNotificationService(strings.ReplaceAll(rawURL, "HOST", "8.8.8.8"), types.SenderOptions{ stopped := errors.New("test dial stopped")
DialContext: func(ctx context.Context, network, address string) (net.Conn, error) { service, err := newPublicNotificationService(strings.ReplaceAll(rawURL, "HOST", "8.8.8.8"), types.SenderOptions{
calls.Add(1) DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
if network != "tcp" || !strings.HasPrefix(address, "8.8.8.8:") { calls.Add(1)
t.Errorf("unexpected dial: %s %s", network, address) if network != "tcp" || !strings.HasPrefix(address, "8.8.8.8:") {
} t.Errorf("unexpected dial: %s %s", network, address)
if err := checkNotificationAddress(address); err != nil { }
t.Error(err) if err := checkNotificationAddress(address); err != nil {
} t.Error(err)
return nil, stopped }
}, return nil, stopped
},
})
if err != nil {
t.Fatal(err)
}
if closer, ok := service.(io.Closer); ok {
defer closer.Close()
}
if err := service.Send("test", &types.Params{}); err == nil {
t.Fatal("expected dial failure")
}
if calls.Load() == 0 {
t.Fatal("custom dialer was not used")
}
}) })
if err != nil {
t.Fatal(err)
}
if closer, ok := service.(io.Closer); ok {
defer closer.Close()
}
if err := service.Send("test", &types.Params{}); err == nil {
t.Fatal("expected dial failure")
}
if calls.Load() == 0 {
t.Fatal("custom dialer was not used")
}
}) })
}) })
} }

View File

@@ -1,9 +1,11 @@
package main package main
import ( import (
"errors"
"fmt" "fmt"
"log" "log"
"os" "os"
"runtime"
"strings" "strings"
"github.com/henrygd/beszel" "github.com/henrygd/beszel"
@@ -14,6 +16,12 @@ import (
"golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh"
) )
type noKeyProvidedError struct{}
func (noKeyProvidedError) Error() string {
return "no key provided: must set -key flag, KEY env var, or KEY_FILE env var. Use 'beszel-agent help' for usage"
}
// cli options // cli options
type cmdOptions struct { type cmdOptions struct {
key string // key is the public key(s) for SSH authentication. key string // key is the public key(s) for SSH authentication.
@@ -124,7 +132,7 @@ func (opts *cmdOptions) loadPublicKeys() ([]ssh.PublicKey, error) {
// Try key file // Try key file
keyFile, ok := utils.GetEnv("KEY_FILE") keyFile, ok := utils.GetEnv("KEY_FILE")
if !ok { if !ok {
return nil, fmt.Errorf("no key provided: must set -key flag, KEY env var, or KEY_FILE env var. Use 'beszel-agent help' for usage") return nil, noKeyProvidedError{}
} }
pubKey, err := os.ReadFile(keyFile) pubKey, err := os.ReadFile(keyFile)
@@ -138,6 +146,14 @@ func (opts *cmdOptions) getAddress() string {
return agent.GetAddress(opts.listen) return agent.GetAddress(opts.listen)
} }
func isBenignStartupError(err error, goos string) bool {
if goos != "windows" {
return false
}
var noKeyErr noKeyProvidedError
return errors.As(err, &noKeyErr)
}
// handleFingerprint handles the "fingerprint" command with subcommands "view" and "reset". // handleFingerprint handles the "fingerprint" command with subcommands "view" and "reset".
func handleFingerprint() { func handleFingerprint() {
subCmd := "" subCmd := ""
@@ -182,6 +198,12 @@ func main() {
var err error var err error
serverConfig.Keys, err = opts.loadPublicKeys() serverConfig.Keys, err = opts.loadPublicKeys()
if err != nil { if err != nil {
if isBenignStartupError(err, runtime.GOOS) {
// WinGet launches the executable without configuration during validation.
// Exit successfully in that case while retaining the error on other platforms.
log.Print("Failed to load public keys:", err)
return
}
log.Fatal("Failed to load public keys:", err) log.Fatal("Failed to load public keys:", err)
} }

View File

@@ -2,6 +2,7 @@ package main
import ( import (
"crypto/ed25519" "crypto/ed25519"
"errors"
"os" "os"
"path/filepath" "path/filepath"
"testing" "testing"
@@ -187,6 +188,26 @@ func TestLoadPublicKeys(t *testing.T) {
} }
} }
func TestIsBenignStartupError(t *testing.T) {
tests := []struct {
name string
err error
goos string
want bool
}{
{name: "missing key on windows", err: noKeyProvidedError{}, goos: "windows", want: true},
{name: "wrapped missing key on windows", err: errors.Join(errors.New("startup failed"), noKeyProvidedError{}), goos: "windows", want: true},
{name: "missing key on linux", err: noKeyProvidedError{}, goos: "linux", want: false},
{name: "different error on windows", err: errors.New("invalid key"), goos: "windows", want: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, isBenignStartupError(tt.err, tt.goos))
})
}
}
func TestGetNetwork(t *testing.T) { func TestGetNetwork(t *testing.T) {
tests := []struct { tests := []struct {
name string name string

View File

@@ -24,6 +24,8 @@ const (
GetSystemdInfo GetSystemdInfo
// Request ZFS detail data from agent // Request ZFS detail data from agent
GetZfsData GetZfsData
// Sync network monitor configuration to agent
SyncNetworkMonitors
// Add new actions here... // Add new actions here...
) )

View File

@@ -12,7 +12,7 @@ RUN apk add --no-cache ca-certificates && update-ca-certificates
# Build # Build
ARG TARGETOS TARGETARCH ARG TARGETOS TARGETARCH
RUN CGO_ENABLED=0 GOGC=75 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags "-w -s" -o /agent ./internal/cmd/agent RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags "-w -s" -o /agent ./internal/cmd/agent
RUN rm -rf /tmp/* RUN rm -rf /tmp/*

View File

@@ -10,14 +10,14 @@ COPY . ./
# Build # Build
ARG TARGETOS TARGETARCH ARG TARGETOS TARGETARCH
RUN CGO_ENABLED=0 GOGC=75 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags "-w -s" -o /agent ./internal/cmd/agent RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags "-w -s" -o /agent ./internal/cmd/agent
RUN rm -rf /tmp/* RUN rm -rf /tmp/*
# -------------------------- # --------------------------
# Final image: default scratch-based agent # Final image: default scratch-based agent
# -------------------------- # --------------------------
FROM alpine:3.23 FROM alpine:3.24
COPY --from=builder /agent /agent COPY --from=builder /agent /agent
# AMD GPU name lookup (used by agent on Linux when /usr/share/libdrm/amdgpu.ids is read) # AMD GPU name lookup (used by agent on Linux when /usr/share/libdrm/amdgpu.ids is read)
@@ -28,4 +28,4 @@ RUN apk add --no-cache smartmontools zfs
# Ensure data persistence across container recreations # Ensure data persistence across container recreations
VOLUME ["/var/lib/beszel-agent"] VOLUME ["/var/lib/beszel-agent"]
ENTRYPOINT ["/agent"] ENTRYPOINT ["/agent"]

View File

@@ -10,13 +10,13 @@ COPY . ./
# Build # Build
ARG TARGETOS TARGETARCH ARG TARGETOS TARGETARCH
RUN CGO_ENABLED=0 GOGC=75 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags "-w -s" -o /agent ./internal/cmd/agent RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags "-w -s" -o /agent ./internal/cmd/agent
# -------------------------- # --------------------------
# Final image # Final image
# Note: must cap_add: [CAP_PERFMON] and mount /dev/dri/ as volume # Note: must cap_add: [CAP_PERFMON] and mount /dev/dri/ as volume
# -------------------------- # --------------------------
FROM alpine:3.23 FROM alpine:3.24
COPY --from=builder /agent /agent COPY --from=builder /agent /agent

View File

@@ -10,7 +10,7 @@ COPY . ./
# Build # Build
ARG TARGETOS TARGETARCH ARG TARGETOS TARGETARCH
RUN CGO_ENABLED=0 GOGC=75 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -tags glibc -ldflags "-w -s" -o /agent ./internal/cmd/agent RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -tags glibc -ldflags "-w -s" -o /agent ./internal/cmd/agent
# -------------------------- # --------------------------
# Smartmontools builder stage # Smartmontools builder stage

View File

@@ -17,7 +17,7 @@ RUN set -eux; \
if [ "$TARGETARCH" = "arm" ] && [ -n "$TARGETVARIANT" ]; then \ if [ "$TARGETARCH" = "arm" ] && [ -n "$TARGETVARIANT" ]; then \
export GOARM="${TARGETVARIANT#v}"; \ export GOARM="${TARGETVARIANT#v}"; \
fi; \ fi; \
CGO_ENABLED=0 GOGC=75 GOOS=$TARGETOS GOARCH=$TARGETARCH \ CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH \
go build -tags glibc -ldflags "-w -s" -o /agent ./internal/cmd/agent go build -tags glibc -ldflags "-w -s" -o /agent ./internal/cmd/agent
# -------------------------- # --------------------------
@@ -70,7 +70,9 @@ RUN set -eux; \
# -------------------------- # --------------------------
FROM --platform=$TARGETPLATFORM debian:bookworm-slim AS zfsutils-builder FROM --platform=$TARGETPLATFORM debian:bookworm-slim AS zfsutils-builder
RUN apt-get update && apt-get install -y --no-install-recommends \ # zfsutils-linux is distributed in Debian's contrib component.
RUN sed -i 's/Components: main/Components: main contrib/' /etc/apt/sources.list.d/debian.sources \
&& apt-get update && apt-get install -y --no-install-recommends \
zfsutils-linux \ zfsutils-linux \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*

View File

@@ -17,7 +17,7 @@ RUN update-ca-certificates
# Build # Build
ARG TARGETOS TARGETARCH ARG TARGETOS TARGETARCH
RUN CGO_ENABLED=0 GOGC=75 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags "-w -s" -o /beszel ./internal/cmd/hub RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags "-w -s" -o /beszel ./internal/cmd/hub
# ? ------------------------- # ? -------------------------
FROM scratch FROM scratch
@@ -31,4 +31,4 @@ VOLUME ["/beszel_data"]
EXPOSE 8090 EXPOSE 8090
ENTRYPOINT [ "/beszel" ] ENTRYPOINT [ "/beszel" ]
CMD ["serve", "--http=0.0.0.0:8090"] CMD ["serve", "--http=0.0.0.0:8090"]

View File

@@ -186,11 +186,12 @@ type Stats struct {
NetworkRecv float64 `json:"nr,omitzero" cbor:"4,keyasint,omitzero"` // deprecated 0.18.3 (MB) - keep field for old agents/records NetworkRecv float64 `json:"nr,omitzero" cbor:"4,keyasint,omitzero"` // deprecated 0.18.3 (MB) - keep field for old agents/records
Bandwidth [2]uint64 `json:"b,omitzero" cbor:"9,keyasint,omitzero"` // [sent bytes, recv bytes] Bandwidth [2]uint64 `json:"b,omitzero" cbor:"9,keyasint,omitzero"` // [sent bytes, recv bytes]
Health DockerHealth `json:"-" cbor:"5,keyasint"` Health DockerHealth `json:"-" cbor:"5,keyasint"`
Status string `json:"-" cbor:"6,keyasint"` Status string `json:"-" cbor:"6,keyasint"`
Id string `json:"-" cbor:"7,keyasint"` Id string `json:"-" cbor:"7,keyasint"`
Image string `json:"-" cbor:"8,keyasint"` Image string `json:"-" cbor:"8,keyasint"`
Ports string `json:"-" cbor:"10,keyasint"` Ports string `json:"-" cbor:"10,keyasint"`
UpdateAvailable bool `json:"u,omitzero" cbor:"11,keyasint,omitzero"`
// PrevCpu [2]uint64 `json:"-"` // PrevCpu [2]uint64 `json:"-"`
CpuSystem uint64 `json:"-"` CpuSystem uint64 `json:"-"`
CpuContainer uint64 `json:"-"` CpuContainer uint64 `json:"-"`

View File

@@ -0,0 +1,114 @@
package monitor
import "time"
// MaxProbeTimeout is the longest agent probe timeout (currently HTTP).
// Hub requests that run a probe must allow this time in addition to transport overhead.
const MaxProbeTimeout = 10 * time.Second
type SyncAction uint8
const (
// SyncActionReplace indicates a full sync where the provided configs should replace all existing monitors for the system.
SyncActionReplace SyncAction = iota
// SyncActionUpsert indicates an incremental sync where the provided config should be added or updated.
SyncActionUpsert
// SyncActionDelete indicates an incremental sync where the provided config should be removed.
SyncActionDelete
)
// Config defines a network monitor task sent from hub to agent.
type Config struct {
// ID is the stable network_monitors record ID generated by the hub.
ID string `cbor:"0,keyasint"`
Target string `cbor:"1,keyasint"`
Protocol string `cbor:"2,keyasint"` // "icmp", "tcp", "http", or "dns"
Port uint16 `cbor:"3,keyasint,omitempty"`
Interval uint16 `cbor:"4,keyasint"` // seconds
// Server is the DNS server to query (host or host:port, default port 53).
// Only used when Protocol is "dns"; empty means use the system resolver.
Server string `cbor:"5,keyasint,omitempty"`
}
// CertInfo holds details of the leaf TLS certificate presented by a target.
type CertInfo struct {
// Expires is the certificate's NotAfter Unix timestamp in milliseconds.
Expires int64 `cbor:"0,keyasint" json:"expires"`
Issuer string `cbor:"1,keyasint,omitempty" json:"issuer,omitempty"`
}
// SyncRequest defines an incremental or full monitor sync request sent to the agent.
type SyncRequest struct {
Action SyncAction `cbor:"0,keyasint"`
Config Config `cbor:"1,keyasint,omitempty"`
Configs []Config `cbor:"2,keyasint,omitempty"`
RunNow bool `cbor:"3,keyasint,omitempty"`
}
// SyncResponse returns the immediate result for an upsert when requested.
type SyncResponse struct {
Result Result `cbor:"0,keyasint,omitempty"`
}
// Result holds aggregated monitor results for a single target.
//
// 0: avg response in microseconds
//
// 1: 1h average response in microseconds
//
// 2: min response in microseconds
//
// 3: 1h min response in microseconds
//
// 4: max response in microseconds
//
// 5: 1h max response in microseconds
//
// 6: packet loss percentage (0-100)
//
// 7: 1h packet loss percentage (0-100)
type Result struct {
AvgResponse int64 `cbor:"0,keyasint,omitempty"`
AvgResponse1h int64 `cbor:"1,keyasint,omitempty"`
MinResponse int64 `cbor:"2,keyasint,omitempty"`
MinResponse1h int64 `cbor:"3,keyasint,omitempty"`
MaxResponse int64 `cbor:"4,keyasint,omitempty"`
MaxResponse1h int64 `cbor:"5,keyasint,omitempty"`
PacketLoss float64 `cbor:"6,keyasint,omitempty"`
PacketLoss1h float64 `cbor:"7,keyasint,omitempty"`
// LastProbeAt is the latest completed probe's Unix timestamp in milliseconds.
LastProbeAt int64 `cbor:"8,keyasint"`
// SampleCount includes all completed probes since this monitor started.
// Used for alert warm-up even when the interval is longer than 20 minutes.
SampleCount int64 `cbor:"9,keyasint,omitempty"`
// Counts and sum cover the current response window (or latest-sample
// fallback), not the hourly window or lifetime SampleCount.
TotalCount int64 `cbor:"10,keyasint"`
SuccessCount int64 `cbor:"11,keyasint"`
ResponseSum int64 `cbor:"12,keyasint"`
// Cert is set for HTTPS targets when a certificate check has new info the hub has not stored yet.
Cert *CertInfo `cbor:"13,keyasint,omitempty"`
}
// Stats holds response times in microseconds and packet loss percentage (0-100).
type Stats struct {
ResAvg float64 `json:"res_avg" db:"-"` // Derived for display; not stored.
ResMin float64 `json:"res_min" db:"res_min"`
ResMax float64 `json:"res_max" db:"res_max"`
Loss float64 `json:"loss" db:"-"` // Derived for display; not stored.
TotalCount int64 `json:"-" db:"total_count"`
SuccessCount int64 `json:"-" db:"success_count"`
ResponseSum int64 `json:"-" db:"res_sum"`
}
func (s Stats) FromResult(result Result) Stats {
return Stats{
ResAvg: float64(result.AvgResponse),
ResMin: float64(result.MinResponse),
ResMax: float64(result.MaxResponse),
Loss: result.PacketLoss,
TotalCount: result.TotalCount,
SuccessCount: result.SuccessCount,
ResponseSum: result.ResponseSum,
}
}

View File

@@ -7,6 +7,7 @@ import (
"time" "time"
"github.com/henrygd/beszel/internal/entities/container" "github.com/henrygd/beszel/internal/entities/container"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/entities/systemd" "github.com/henrygd/beszel/internal/entities/systemd"
) )
@@ -210,5 +211,6 @@ type CombinedData struct {
Details *Details `cbor:"4,keyasint,omitempty"` Details *Details `cbor:"4,keyasint,omitempty"`
// SystemdServicesUpdated distinguishes a fresh empty snapshot from a response // SystemdServicesUpdated distinguishes a fresh empty snapshot from a response
// that omitted systemd data (for example, a short-cache dashboard request). // that omitted systemd data (for example, a short-cache dashboard request).
SystemdServicesUpdated bool `json:"systemdUpdated,omitempty" cbor:"5,keyasint,omitempty"` SystemdServicesUpdated bool `json:"systemdUpdated,omitempty" cbor:"5,keyasint,omitempty"`
Monitors map[string]monitor.Result `cbor:"6,keyasint"`
} }

View File

@@ -978,11 +978,18 @@ func TestAgentWebSocketIntegration(t *testing.T) {
} }
} }
// Verify system status // A connected WebSocket does not mean the hub has finished verifying
updatedSystemRecord, err := testApp.FindRecordById("systems", systemRecord.Id) // the agent and updating the system. Wait for the database state rather
require.NoError(t, err) // than assuming that work completes within a fixed sleep under load.
status := updatedSystemRecord.GetString("status") var status string
assert.Equal(t, tc.expectSystemStatus, status, "System status should match expected value") require.EventuallyWithT(t, func(c *assert.CollectT) {
updatedSystemRecord, err := testApp.FindRecordById("systems", systemRecord.Id)
if !assert.NoError(c, err) {
return
}
status = updatedSystemRecord.GetString("status")
assert.Equal(c, tc.expectSystemStatus, status, "System status should match expected value")
}, 5*time.Second, 20*time.Millisecond)
t.Logf("%s - System status: %s, Fingerprint: %s", tc.description, status, finalFingerprint) t.Logf("%s - System status: %s, Fingerprint: %s", tc.description, status, finalFingerprint)
}) })
@@ -1142,42 +1149,43 @@ func TestMultipleSystemsWithSameUniversalToken(t *testing.T) {
// Verify system creation/reuse behavior // Verify system creation/reuse behavior
if tc.expectConnection { if tc.expectConnection {
// Count systems after connection expectedSystemsAfter := systemsBeforeCount
systemsAfter, err := testApp.FindRecordsByFilter("systems", "users ~ {:userId}", "", -1, 0, map[string]any{"userId": userRecord.Id})
require.NoError(t, err)
systemsAfterCount := len(systemsAfter)
if tc.expectNewSystem { if tc.expectNewSystem {
// Should have created a new system expectedSystemsAfter++
systemCount++ systemCount++
assert.Equal(t, systemsBeforeCount+1, systemsAfterCount, "Should have created a new system")
assert.Equal(t, systemCount, systemsAfterCount, "Total system count should match expected")
} else {
// Should have reused existing system
assert.Equal(t, systemsBeforeCount, systemsAfterCount, "Should not have created a new system")
assert.Equal(t, systemCount, systemsAfterCount, "Total system count should remain the same")
} }
time.Sleep(20 * time.Millisecond) // WebSocket connection precedes the hub's asynchronous system
// setup. Re-read all database state until setup is complete.
var systemId, status string
require.EventuallyWithT(t, func(c *assert.CollectT) {
systemsAfter, err := testApp.FindRecordsByFilter("systems", "users ~ {:userId}", "", -1, 0, map[string]any{"userId": userRecord.Id})
if !assert.NoError(c, err) {
return
}
assert.Len(c, systemsAfter, expectedSystemsAfter, "System creation/reuse should match expected behavior")
assert.Len(c, systemsAfter, systemCount, "Total system count should match expected")
// Verify that a fingerprint record exists for this fingerprint fingerprints, err := testApp.FindRecordsByFilter("fingerprints", "token = {:token} && fingerprint = {:fingerprint}", "", -1, 0, map[string]any{
fingerprints, err := testApp.FindRecordsByFilter("fingerprints", "token = {:token} && fingerprint = {:fingerprint}", "", -1, 0, map[string]any{ "token": universalToken,
"token": universalToken, "fingerprint": tc.agentFingerprint,
"fingerprint": tc.agentFingerprint, })
}) if !assert.NoError(c, err) || !assert.Len(c, fingerprints, 1, "Should have exactly one fingerprint record for this token+fingerprint combination") {
require.NoError(t, err) return
require.Len(t, fingerprints, 1, "Should have exactly one fingerprint record for this token+fingerprint combination") }
fingerprint := fingerprints[0] fingerprint := fingerprints[0]
assert.Equal(t, universalToken, fingerprint.GetString("token"), "Fingerprint should have the universal token") assert.Equal(c, universalToken, fingerprint.GetString("token"), "Fingerprint should have the universal token")
assert.Equal(t, tc.agentFingerprint, fingerprint.GetString("fingerprint"), "Fingerprint should match agent's fingerprint") assert.Equal(c, tc.agentFingerprint, fingerprint.GetString("fingerprint"), "Fingerprint should match agent's fingerprint")
// Verify system status systemId = fingerprint.GetString("system")
systemId := fingerprint.GetString("system") system, err := testApp.FindRecordById("systems", systemId)
system, err := testApp.FindRecordById("systems", systemId) if !assert.NoError(c, err) {
require.NoError(t, err) return
status := system.GetString("status") }
assert.Equal(t, tc.expectSystemStatus, status, "System status should match expected value") status = system.GetString("status")
assert.Equal(c, tc.expectSystemStatus, status, "System status should match expected value")
}, 5*time.Second, 20*time.Millisecond)
t.Logf("%s - System ID: %s, Status: %s, New System: %v", tc.description, systemId, status, tc.expectNewSystem) t.Logf("%s - System ID: %s, Status: %s, New System: %v", tc.description, systemId, status, tc.expectNewSystem)
} }

View File

@@ -2,7 +2,11 @@ package hub
import ( import (
"context" "context"
"fmt"
"log/slog"
"net"
"net/http" "net/http"
"net/netip"
"regexp" "regexp"
"strings" "strings"
"time" "time"
@@ -78,12 +82,81 @@ func (h *Hub) registerMiddlewares(se *core.ServeEvent) {
} }
// authenticate with trusted header // authenticate with trusted header
if trustedHeader, _ := utils.GetEnv("TRUSTED_AUTH_HEADER"); trustedHeader != "" { if trustedHeader, _ := utils.GetEnv("TRUSTED_AUTH_HEADER"); trustedHeader != "" {
// only honor the header from these peers, if set
trustedProxies, restricted := parseTrustedProxies()
se.Router.BindFunc(func(e *core.RequestEvent) error { se.Router.BindFunc(func(e *core.RequestEvent) error {
if restricted && !isTrustedProxy(trustedProxies, e.Request.RemoteAddr) {
return e.Next()
}
return authorizeRequestWithEmail(e, e.Request.Header.Get(trustedHeader)) return authorizeRequestWithEmail(e, e.Request.Header.Get(trustedHeader))
}) })
} }
} }
// parseTrustedProxies reads TRUSTED_PROXY_IPS (comma-separated IPs or CIDRs).
// restricted is false when the variable is unset or empty, meaning the trusted
// header is accepted from any peer. Invalid entries are skipped with a warning,
// so a typo narrows the allowlist rather than widening it.
func parseTrustedProxies() (prefixes []netip.Prefix, restricted bool) {
value, _ := utils.GetEnv("TRUSTED_PROXY_IPS")
if value == "" {
return nil, false
}
for entry := range strings.SplitSeq(value, ",") {
entry = strings.TrimSpace(entry)
if entry == "" {
continue
}
if prefix, err := parseProxyPrefix(entry); err == nil {
prefixes = append(prefixes, prefix)
} else {
slog.Warn("Ignoring invalid TRUSTED_PROXY_IPS entry", "entry", entry)
}
}
return prefixes, true
}
// parseProxyPrefix parses an IP or CIDR into a masked prefix. IPv4-mapped IPv6
// entries are converted to IPv4 so they match IPv4 peers.
func parseProxyPrefix(entry string) (netip.Prefix, error) {
prefix, err := netip.ParsePrefix(entry)
if err != nil {
addr, err := netip.ParseAddr(entry)
if err != nil {
return netip.Prefix{}, err
}
addr = addr.Unmap()
return netip.PrefixFrom(addr, addr.BitLen()), nil
}
if prefix.Addr().Is4In6() {
if prefix.Bits() < 96 {
return netip.Prefix{}, fmt.Errorf("%s covers more than the IPv4-mapped range", entry)
}
prefix = netip.PrefixFrom(prefix.Addr().Unmap(), prefix.Bits()-96)
}
return prefix.Masked(), nil
}
// isTrustedProxy reports whether the peer address of a request (host:port) is
// within one of the prefixes.
func isTrustedProxy(prefixes []netip.Prefix, remoteAddr string) bool {
host, _, err := net.SplitHostPort(remoteAddr)
if err != nil {
host = remoteAddr
}
addr, err := netip.ParseAddr(host)
if err != nil {
return false
}
addr = addr.Unmap().WithZone("")
for _, prefix := range prefixes {
if prefix.Contains(addr) {
return true
}
}
return false
}
// registerApiRoutes registers custom API routes // registerApiRoutes registers custom API routes
func (h *Hub) registerApiRoutes(se *core.ServeEvent) error { func (h *Hub) registerApiRoutes(se *core.ServeEvent) error {
// auth protected routes // auth protected routes

View File

@@ -6,12 +6,16 @@ import (
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
"net/http/httptest"
"sort"
"testing" "testing"
"time"
beszelTests "github.com/henrygd/beszel/internal/tests" beszelTests "github.com/henrygd/beszel/internal/tests"
"github.com/henrygd/beszel/internal/migrations" "github.com/henrygd/beszel/internal/migrations"
"github.com/pocketbase/dbx" "github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/apis"
"github.com/pocketbase/pocketbase/core" "github.com/pocketbase/pocketbase/core"
pbTests "github.com/pocketbase/pocketbase/tests" pbTests "github.com/pocketbase/pocketbase/tests"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
@@ -26,6 +30,59 @@ func jsonReader(v any) io.Reader {
return bytes.NewReader(data) return bytes.NewReader(data)
} }
type gatedReader struct {
data []byte
started chan struct{}
release chan struct{}
offset int
}
func (r *gatedReader) Read(p []byte) (int, error) {
if r.offset == 0 {
close(r.started)
<-r.release
}
if r.offset >= len(r.data) {
return 0, io.EOF
}
n := copy(p, r.data[r.offset:])
r.offset += n
return n, nil
}
func firstUserTestMux(t *testing.T) (*beszelTests.TestHub, http.Handler) {
t.Helper()
hub, err := beszelTests.NewTestHub(t.TempDir())
require.NoError(t, err)
_ = hub.StartHub()
router, err := apis.NewRouter(hub.TestApp)
require.NoError(t, err)
serveEvent := &core.ServeEvent{App: hub.TestApp, Router: router}
var handler http.Handler
err = hub.TestApp.OnServe().Trigger(serveEvent, func(e *core.ServeEvent) error {
var buildErr error
handler, buildErr = e.Router.BuildMux()
return buildErr
})
require.NoError(t, err)
require.NotNil(t, handler)
return hub, handler
}
func postFirstUser(handler http.Handler, email string) *httptest.ResponseRecorder {
body, _ := json.Marshal(map[string]string{
"email": email,
"password": "password123",
})
req := httptest.NewRequest(http.MethodPost, "/api/beszel/create-user", bytes.NewReader(body))
req.Header.Set("Content-Type", "application/json")
recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, req)
return recorder
}
func TestApiRoutesAuthentication(t *testing.T) { func TestApiRoutesAuthentication(t *testing.T) {
hub, user := beszelTests.GetHubWithUser(t) hub, user := beszelTests.GetHubWithUser(t)
defer hub.Cleanup() defer hub.Cleanup()
@@ -789,6 +846,87 @@ func TestFirstUserCreation(t *testing.T) {
}) })
} }
func TestFirstUserBootstrapAtomicity(t *testing.T) {
t.Run("concurrent complete requests produce exactly one winner", func(t *testing.T) {
hub, handler := firstUserTestMux(t)
defer hub.Cleanup()
start := make(chan struct{})
statuses := make(chan int, 2)
for _, email := range []string{"first@example.com", "second@example.com"} {
go func(email string) {
<-start
statuses <- postFirstUser(handler, email).Code
}(email)
}
close(start)
got := []int{<-statuses, <-statuses}
sort.Ints(got)
require.Equal(t, []int{http.StatusOK, http.StatusForbidden}, got)
users, err := hub.FindAllRecords("users")
require.NoError(t, err)
require.Len(t, users, 1)
superusers, err := hub.FindAllRecords(core.CollectionNameSuperusers)
require.NoError(t, err)
require.Len(t, superusers, 1)
require.NotEqual(t, migrations.TempAdminEmail, superusers[0].Email())
})
t.Run("partial body cannot retain stale bootstrap authorization", func(t *testing.T) {
hub, handler := firstUserTestMux(t)
defer hub.Cleanup()
body, err := json.Marshal(map[string]string{
"email": "parked@example.com",
"password": "password123",
})
require.NoError(t, err)
gated := &gatedReader{
data: body,
started: make(chan struct{}),
release: make(chan struct{}),
}
parkedRequest := httptest.NewRequest(http.MethodPost, "/api/beszel/create-user", gated)
parkedRequest.Header.Set("Content-Type", "application/json")
parkedRecorder := httptest.NewRecorder()
parkedDone := make(chan struct{})
go func() {
handler.ServeHTTP(parkedRecorder, parkedRequest)
close(parkedDone)
}()
select {
case <-gated.started:
case <-time.After(2 * time.Second):
t.Fatal("parked request did not begin reading its body")
}
operatorRecorder := postFirstUser(handler, "operator@example.com")
require.Equal(t, http.StatusOK, operatorRecorder.Code)
lateRecorder := postFirstUser(handler, "late@example.com")
require.Equal(t, http.StatusForbidden, lateRecorder.Code)
close(gated.release)
select {
case <-parkedDone:
case <-time.After(2 * time.Second):
t.Fatal("parked request did not finish")
}
require.Equal(t, http.StatusForbidden, parkedRecorder.Code)
users, err := hub.FindAllRecords("users")
require.NoError(t, err)
require.Len(t, users, 1)
require.Equal(t, "operator@example.com", users[0].Email())
superusers, err := hub.FindAllRecords(core.CollectionNameSuperusers)
require.NoError(t, err)
require.Len(t, superusers, 1)
require.Equal(t, "operator@example.com", superusers[0].Email())
})
}
func TestCreateUserEndpointAvailability(t *testing.T) { func TestCreateUserEndpointAvailability(t *testing.T) {
t.Run("CreateUserEndpoint available when no users exist", func(t *testing.T) { t.Run("CreateUserEndpoint available when no users exist", func(t *testing.T) {
hub, _ := beszelTests.NewTestHub(t.TempDir()) hub, _ := beszelTests.NewTestHub(t.TempDir())
@@ -969,6 +1107,79 @@ func TestTrustedHeaderMiddleware(t *testing.T) {
} }
} }
func TestTrustedHeaderProxyAllowlist(t *testing.T) {
var hubs []*beszelTests.TestHub
defer func() {
for _, hub := range hubs {
hub.Cleanup()
}
}()
testAppFactory := func(t testing.TB) *pbTests.TestApp {
hub, _ := beszelTests.NewTestHub(t.TempDir())
hubs = append(hubs, hub)
hub.StartHub()
return hub.TestApp
}
// httptest requests arrive from 192.0.2.1:1234
testCases := []struct {
name string
proxies string
expectedStatus int
expectedContent []string
}{
{
name: "peer inside an allowed range",
proxies: "10.0.0.0/8, 192.0.2.0/24",
expectedStatus: 200,
expectedContent: []string{"\"key\":", "\"v\":"},
},
{
name: "peer is the listed address",
proxies: "192.0.2.1",
expectedStatus: 200,
expectedContent: []string{"\"key\":", "\"v\":"},
},
{
name: "peer outside the allowlist",
proxies: "10.0.0.0/8",
expectedStatus: 401,
expectedContent: []string{"requires valid"},
},
{
name: "allowlist with no valid entry",
proxies: "proxy.internal",
expectedStatus: 401,
expectedContent: []string{"requires valid"},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
t.Setenv("TRUSTED_AUTH_HEADER", "X-Beszel-Trusted")
t.Setenv("TRUSTED_PROXY_IPS", tc.proxies)
scenario := beszelTests.ApiScenario{
Name: "GET /getkey - with trusted header",
Method: http.MethodGet,
URL: "/api/beszel/getkey",
Headers: map[string]string{
"X-Beszel-Trusted": "user@test.com",
},
ExpectedStatus: tc.expectedStatus,
ExpectedContent: tc.expectedContent,
TestAppFactory: testAppFactory,
BeforeTestFunc: func(t testing.TB, app *pbTests.TestApp, e *core.ServeEvent) {
beszelTests.CreateUser(app, "user@test.com", "password123")
},
}
scenario.Test(t)
})
}
}
func TestUpdateEndpoint(t *testing.T) { func TestUpdateEndpoint(t *testing.T) {
t.Setenv("CHECK_UPDATES", "true") t.Setenv("CHECK_UPDATES", "true")

View File

@@ -78,7 +78,7 @@ func setCollectionAuthSettings(app core.App) error {
return err return err
} }
if err := applyCollectionRules(app, []string{"containers", "container_stats", "system_stats", "systemd_services"}, collectionRules{ if err := applyCollectionRules(app, []string{"containers", "container_stats", "system_stats", "systemd_services", "network_monitor_stats"}, collectionRules{
list: &systemScopedReadRule, list: &systemScopedReadRule,
}); err != nil { }); err != nil {
return err return err
@@ -108,6 +108,16 @@ func setCollectionAuthSettings(app core.App) error {
return err return err
} }
if err := applyCollectionRules(app, []string{"network_monitors"}, collectionRules{
list: &systemScopedReadRule,
view: &systemScopedReadRule,
create: &systemScopedWriteRule,
update: &systemScopedWriteRule,
delete: &systemScopedWriteRule,
}); err != nil {
return err
}
if err := applyCollectionRules(app, []string{"system_details"}, collectionRules{ if err := applyCollectionRules(app, []string{"system_details"}, collectionRules{
list: &systemScopedReadRule, list: &systemScopedReadRule,
view: &systemScopedReadRule, view: &systemScopedReadRule,

View File

@@ -106,9 +106,12 @@ func (h *Hub) StartHub() error {
// TODO: move to users package // TODO: move to users package
// handle default values for user / user_settings creation // handle default values for user / user_settings creation
h.App.OnRecordAuthWithOAuth2Request("users").BindFunc(h.um.InitializeOAuthUserRole)
h.App.OnRecordCreate("users").BindFunc(h.um.InitializeUserRole) h.App.OnRecordCreate("users").BindFunc(h.um.InitializeUserRole)
h.App.OnRecordCreate("user_settings").BindFunc(h.um.InitializeUserSettings) h.App.OnRecordCreate("user_settings").BindFunc(h.um.InitializeUserSettings)
bindNetworkMonitorsEvents(h)
pb, ok := h.App.(*pocketbase.PocketBase) pb, ok := h.App.(*pocketbase.PocketBase)
if !ok { if !ok {
return errors.New("not a pocketbase app") return errors.New("not a pocketbase app")
@@ -122,6 +125,8 @@ func (h *Hub) initialize(app core.App) error {
settings := app.Settings() settings := app.Settings()
// batch requests (for alerts) // batch requests (for alerts)
settings.Batch.Enabled = true settings.Batch.Enabled = true
settings.Batch.MaxRequests = 100
settings.Batch.MaxBodySize = 1 << 20 // 1 MiB
// set URL if APP_URL env is set // set URL if APP_URL env is set
if appURL, isSet := utils.GetEnv("APP_URL"); isSet { if appURL, isSet := utils.GetEnv("APP_URL"); isSet {
h.appURL = appURL h.appURL = appURL

View File

@@ -0,0 +1,171 @@
package hub
import (
"strconv"
"time"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/hub/systems"
"github.com/pocketbase/pocketbase/core"
"github.com/pocketbase/pocketbase/tools/types"
)
// generateMonitorID creates a stable hash ID for a monitor based on its configuration and the system it belongs to.
func generateMonitorID(systemId string, config monitor.Config) string {
args := []string{systemId, config.Target, config.Protocol}
// only use port for TCP monitors, since for other protocols it's not relevant as standalone value
if config.Protocol == "tcp" {
args = append(args, strconv.FormatUint(uint64(config.Port), 10))
}
// only use server for DNS monitors, so the same target queried via different servers gets distinct monitors
if config.Protocol == "dns" {
args = append(args, config.Server)
}
return systems.MakeStableHashId(args...)
}
// bindNetworkMonitorsEvents keeps monitor records and agent monitor state in sync.
func bindNetworkMonitorsEvents(hub *Hub) {
// on create, make sure the id is set to a stable hash
hub.OnRecordCreate("network_monitors").BindFunc(func(e *core.RecordEvent) error {
systemID := e.Record.GetString("system")
config := monitorConfigFromRecord(e.Record)
id := generateMonitorID(systemID, *config)
e.Record.Set("id", id)
return e.Next()
})
// sync monitor to agent on creation and persist the first result immediately when available
hub.OnRecordAfterCreateSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error {
err := e.Next()
if err != nil {
return err
}
if !e.Record.GetBool("enabled") {
return nil
}
// If connected, run the monitor immediately. Paused systems may be absent
// from the manager; their monitors will sync when they reconnect.
system, err := hub.sm.GetSystem(e.Record.GetString("system"))
if err == nil && system.Status == "up" {
go hub.upsertNetworkMonitor(e.Record, true)
}
return nil
})
// On API update requests, if the monitor config changed in a way that requires a new ID, create a new
// record with the new ID and delete the old one. Otherwise, just update the existing monitor on the agent.
hub.OnRecordUpdateRequest("network_monitors").BindFunc(func(e *core.RecordRequestEvent) error {
systemID := e.Record.GetString("system")
protocol := e.Record.GetString("protocol")
// only tcp uses port - set other protocols port to zero
if protocol != "tcp" {
e.Record.Set("port", 0)
}
// only dns uses server - clear it for other protocols
if protocol != "dns" {
e.Record.Set("server", "")
}
ID := generateMonitorID(systemID, *monitorConfigFromRecord(e.Record))
if ID != e.Record.Id {
newRecord := copyMonitorToNewRecord(e.Record, ID)
if err := e.App.Save(newRecord); err != nil {
return err
}
if err := e.App.Delete(e.Record); err != nil {
return err
}
return nil
}
err := e.Next()
if err != nil {
return err
}
if e.Record.GetBool("enabled") {
// if the monitor is enabled, sync the updated config to the agent now
runNow := !e.Record.Original().GetBool("enabled")
err = hub.upsertNetworkMonitor(e.Record, runNow)
} else {
// if the monitor is paused, remove it from the agent
err = hub.deleteNetworkMonitor(e.Record)
}
if err != nil {
hub.Logger().Warn("failed to sync updated monitor", "system", systemID, "monitor", e.Record.Id, "err", err)
}
return nil
})
// sync monitor to agent on delete
hub.OnRecordAfterDeleteSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error {
if err := hub.deleteNetworkMonitor(e.Record); err != nil {
hub.Logger().Warn("failed to delete monitor on agent", "system", e.Record.GetString("system"), "monitor", e.Record.Id, "err", err)
}
return e.Next()
})
}
// monitorConfigFromRecord builds a monitor config from a network_monitors record.
func monitorConfigFromRecord(record *core.Record) *monitor.Config {
return &monitor.Config{
ID: record.Id,
Target: record.GetString("target"),
Protocol: record.GetString("protocol"),
Port: uint16(record.GetInt("port")),
Interval: uint16(record.GetInt("interval")),
Server: record.GetString("server"),
}
}
// setMonitorResultFields stores the latest monitor result values on the record.
func setMonitorResultFields(record *core.Record, result monitor.Result) {
nowString := time.Now().UTC().Format(types.DefaultDateLayout)
record.Set("res", result.AvgResponse)
record.Set("resAvg1h", result.AvgResponse1h)
record.Set("resMin1h", result.MinResponse1h)
record.Set("resMax1h", result.MaxResponse1h)
record.Set("loss1h", result.PacketLoss1h)
if result.Cert != nil {
record.Set("certInfo", result.Cert)
}
record.Set("updated", nowString)
}
// copyMonitorToNewRecord creates a new record with the same field values as the old one.
// This is used when the monitor config changes in a way that requires a new ID, so we need
// to create a new record with the new ID and delete the old one.
func copyMonitorToNewRecord(oldRecord *core.Record, newID string) *core.Record {
collection := oldRecord.Collection()
newRecord := core.NewRecord(collection)
newRecord.Id = newID
fields := []string{"system", "target", "protocol", "port", "server", "interval", "enabled"}
for _, field := range fields {
newRecord.Set(field, oldRecord.Get(field))
}
return newRecord
}
// upsertNetworkMonitor creates or updates the record's monitor on the target system. If runNow
// is true, it will also trigger an immediate monitor run and update the record with the result.
func (h *Hub) upsertNetworkMonitor(record *core.Record, runNow bool) error {
systemID := record.GetString("system")
system, err := h.sm.GetSystem(systemID)
if err != nil {
return err
}
result, err := system.UpsertNetworkMonitor(*monitorConfigFromRecord(record), runNow)
if err != nil || result == nil {
return err
}
setMonitorResultFields(record, *result)
return h.App.SaveNoValidate(record)
}
// deleteNetworkMonitor removes the record's monitor from the target system.
func (h *Hub) deleteNetworkMonitor(record *core.Record) error {
systemID := record.GetString("system")
system, err := h.sm.GetSystem(systemID)
if err != nil {
return err
}
return system.DeleteNetworkMonitor(record.Id)
}

View File

@@ -0,0 +1,262 @@
package hub
import (
"bytes"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/pocketbase/pocketbase/apis"
"github.com/pocketbase/pocketbase/core"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestCreateNetworkMonitorsOnPausedSystem(t *testing.T) {
for _, batch := range []bool{false, true} {
name := "single"
if batch {
name = "batch"
}
t.Run(name, func(t *testing.T) {
hub, testApp, err := createTestHub(t)
require.NoError(t, err)
defer cleanupTestHub(hub, testApp)
bindNetworkMonitorsEvents(hub)
user, err := createTestUser(hub)
require.NoError(t, err)
system, err := createTestRecord(hub, "systems", map[string]any{
"name": "Paused", "host": "localhost", "port": "45876",
"status": "paused", "users": []string{user.Id},
})
require.NoError(t, err)
// Paused systems are not loaded into the manager at startup.
_, err = hub.sm.GetSystem(system.Id)
require.Error(t, err)
payload := func(target string) map[string]any {
return map[string]any{
"system": system.Id, "target": target, "protocol": "icmp",
"interval": 60, "enabled": true,
}
}
url := "/api/collections/network_monitors/records"
var body any = payload("1.1.1.1")
count := 1
if batch {
body = map[string]any{"requests": []map[string]any{
{"method": "POST", "url": url, "body": payload("1.1.1.1")},
{"method": "POST", "url": url, "body": payload("8.8.8.8")},
}}
url = "/api/batch"
count = 2
}
data, err := json.Marshal(body)
require.NoError(t, err)
token, err := user.NewAuthToken()
require.NoError(t, err)
router, err := apis.NewRouter(hub)
require.NoError(t, err)
handler, err := router.BuildMux()
require.NoError(t, err)
request := httptest.NewRequest(http.MethodPost, url, bytes.NewReader(data))
request.Header.Set("Content-Type", "application/json")
request.Header.Set("Authorization", token)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
assert.Equal(t, http.StatusOK, response.Code, response.Body.String())
records, err := hub.FindAllRecords("network_monitors")
require.NoError(t, err)
require.Len(t, records, count)
for _, record := range records {
assert.Equal(t, system.Id, record.GetString("system"))
assert.True(t, record.GetBool("enabled"))
}
})
}
}
func TestGenerateMonitorID(t *testing.T) {
tests := []struct {
name string
systemID string
config monitor.Config
expected string
}{
{
name: "HTTP monitor on example.com",
systemID: "sys123",
config: monitor.Config{
Protocol: "http",
Target: "example.com",
Port: 0,
Interval: 60,
},
expected: "a20a5827",
},
{
name: "HTTP monitor on example.com with different port",
systemID: "sys123",
config: monitor.Config{
Protocol: "http",
Target: "example.com",
Port: 8080,
Interval: 60,
},
expected: "a20a5827",
},
{
name: "HTTP monitor on example.com with different system ID",
systemID: "sys1234",
config: monitor.Config{
Protocol: "http",
Target: "example.com",
Port: 80,
Interval: 60,
},
expected: "ab602ae7",
},
{
name: "Same monitor, different interval",
systemID: "sys1234",
config: monitor.Config{
Protocol: "http",
Target: "example.com",
Port: 80,
Interval: 120,
},
expected: "ab602ae7",
},
{
name: "ICMP monitor on 1.1.1.1",
systemID: "sys456",
config: monitor.Config{
Protocol: "icmp",
Target: "1.1.1.1",
Port: 0,
Interval: 10,
},
expected: "6d13a4a4",
}, {
name: "ICMP monitor on 1.1.1.1 with different system ID",
systemID: "sys4567",
config: monitor.Config{
Protocol: "icmp",
Target: "1.1.1.1",
Port: 0,
Interval: 10,
},
expected: "ddd6c81",
},
{
name: "TCP monitor on example.com with port 443",
systemID: "sys789",
config: monitor.Config{
Protocol: "tcp",
Target: "example.com",
Port: 443,
Interval: 30,
},
expected: "677b991",
},
{
name: "TCP monitor on example.com with port 8443",
systemID: "sys789",
config: monitor.Config{
Protocol: "tcp",
Target: "example.com",
Port: 8443,
Interval: 30,
},
expected: "84167969",
},
{
name: "DNS monitor on example.com with server 1.1.1.1",
systemID: "sys999",
config: monitor.Config{
Protocol: "dns",
Target: "example.com",
Server: "1.1.1.1",
Interval: 30,
},
expected: "2175898b",
},
{
name: "DNS monitor on example.com with different server",
systemID: "sys999",
config: monitor.Config{
Protocol: "dns",
Target: "example.com",
Server: "8.8.8.8",
Interval: 30,
},
expected: "ebcd8b33",
},
{
name: "DNS monitor on example.com with no server (system resolver)",
systemID: "sys999",
config: monitor.Config{
Protocol: "dns",
Target: "example.com",
Server: "",
Interval: 30,
},
expected: "19476a7",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := generateMonitorID(tt.systemID, tt.config)
assert.Equal(t, tt.expected, got, "generateMonitorID() = %v, want %v", got, tt.expected)
})
}
}
func TestCopyMonitorToNewRecordDropsResultFields(t *testing.T) {
hub, testApp, err := createTestHub(t)
require.NoError(t, err)
defer cleanupTestHub(hub, testApp)
collection, err := hub.FindCachedCollectionByNameOrId("network_monitors")
require.NoError(t, err)
assert.Nil(t, collection.Fields.GetByName("name"))
oldRecord := core.NewRecord(collection)
oldRecord.Load(map[string]any{
"system": "sys123",
"target": "https://example.com",
"protocol": "http",
"port": 443,
"server": "1.1.1.1",
"interval": 60,
"enabled": true,
"res": 1200,
"resAvg1h": 1300,
"resMin1h": 900,
"resMax1h": 1600,
"loss1h": 5,
"certInfo": map[string]any{"expires": 1800000000000},
"updated": "2026-04-29 12:00:00.000Z",
})
newRecord := copyMonitorToNewRecord(oldRecord, "next12345")
assert.Equal(t, "next12345", newRecord.Id)
assert.Equal(t, "https://example.com", newRecord.GetString("target"))
assert.Equal(t, "http", newRecord.GetString("protocol"))
assert.Equal(t, 443, newRecord.GetInt("port"))
assert.Equal(t, "1.1.1.1", newRecord.GetString("server"))
assert.True(t, newRecord.GetBool("enabled"))
assert.Contains(t, []string{"", "null"}, newRecord.GetString("certInfo"))
assert.Zero(t, newRecord.GetFloat("res"))
assert.Zero(t, newRecord.GetFloat("resAvg1h"))
assert.Zero(t, newRecord.GetFloat("resMin1h"))
assert.Zero(t, newRecord.GetFloat("resMax1h"))
assert.Zero(t, newRecord.GetFloat("loss1h"))
assert.Equal(t, "", newRecord.GetString("updated"))
}

View File

@@ -0,0 +1,46 @@
//go:build testing
package systems
import (
"testing"
"github.com/henrygd/beszel/internal/entities/container"
"github.com/pocketbase/dbx"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestCreateContainerRecordsPersistsImageUpdateAvailability(t *testing.T) {
_, app := newTestSystemWithHub(t)
const (
systemID = "system123"
containerID = "abcdef123456"
image = "nginx:latest"
)
data := &container.Stats{
Id: containerID,
Name: "web",
Image: image,
UpdateAvailable: true,
}
require.NoError(t, createContainerRecords(app, []*container.Stats{data}, systemID))
var record struct {
Image string `db:"image"`
UpdateAvailable bool `db:"updatable"`
}
require.NoError(t, app.DB().Select("image", "updatable").From("containers").
Where(dbx.HashExp{"id": containerID}).One(&record))
assert.Equal(t, image, record.Image)
assert.True(t, record.UpdateAvailable)
data.UpdateAvailable = false
require.NoError(t, createContainerRecords(app, []*container.Stats{data}, systemID))
require.NoError(t, app.DB().Select("image", "updatable").From("containers").
Where(dbx.HashExp{"id": containerID}).One(&record))
assert.Equal(t, image, record.Image)
assert.False(t, record.UpdateAvailable)
}

View File

@@ -0,0 +1,159 @@
//go:build testing
package systems
import (
"context"
"crypto/ed25519"
"crypto/rand"
"net"
"sync/atomic"
"testing"
"time"
"github.com/fxamacker/cbor/v2"
"github.com/henrygd/beszel/internal/common"
"github.com/henrygd/beszel/internal/entities/monitor"
esystem "github.com/henrygd/beszel/internal/entities/system"
"github.com/henrygd/beszel/internal/hub/expirymap"
"github.com/pocketbase/pocketbase/core"
"github.com/stretchr/testify/require"
"golang.org/x/crypto/ssh"
)
func TestSSHNetworkMonitorReconnectSync(t *testing.T) {
sys, app := newTestSystemWithHub(t)
sys.manager.zfsFetchMap = expirymap.New[zfsFetchState](time.Hour)
t.Cleanup(sys.manager.zfsFetchMap.StopCleaner)
sys.ctx = context.Background()
sys.Status = up
_, key, err := ed25519.GenerateKey(rand.Reader)
require.NoError(t, err)
signer, err := ssh.NewSignerFromKey(key)
require.NoError(t, err)
config := &ssh.ServerConfig{NoClientAuth: true, ServerVersion: "SSH-2.0-beszel_0.20.0"}
config.AddHostKey(signer)
listener, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
t.Cleanup(func() { _ = listener.Close() })
sys.Host, sys.Port, err = net.SplitHostPort(listener.Addr().String())
require.NoError(t, err)
sys.manager.sshConfig = &ssh.ClientConfig{User: "test", HostKeyCallback: ssh.InsecureIgnoreHostKey(), Timeout: time.Second}
t.Cleanup(sys.closeSSHConnection)
requests := make(chan monitor.SyncRequest, 10)
var failSync atomic.Bool
go func() {
for {
conn, err := listener.Accept()
if err != nil {
return
}
go func() {
server, channels, reqs, err := ssh.NewServerConn(conn, config)
if err != nil {
_ = conn.Close()
return
}
defer server.Close()
go ssh.DiscardRequests(reqs)
for channel := range channels {
ch, reqs, err := channel.Accept()
if err != nil {
return
}
go func() {
defer ch.Close()
for req := range reqs {
if req.Type != "shell" {
_ = req.Reply(false, nil)
continue
}
_ = req.Reply(true, nil)
var request common.HubRequest[cbor.RawMessage]
if cbor.NewDecoder(ch).Decode(&request) != nil {
return
}
response := common.AgentResponse{}
switch request.Action {
case common.GetData:
response.SystemData = &esystem.CombinedData{}
case common.SyncNetworkMonitors:
var syncReq monitor.SyncRequest
if cbor.Unmarshal(request.Data, &syncReq) != nil {
return
}
requests <- syncReq
if failSync.Load() {
response.Error = "test sync failure"
} else {
response.Data, _ = cbor.Marshal(monitor.SyncResponse{})
}
}
_ = cbor.NewEncoder(ch).Encode(response)
_, _ = ch.SendRequest("exit-status", false, ssh.Marshal(struct{ Status uint32 }{0}))
return
}
}()
}
}()
}
}()
collection, err := app.FindCachedCollectionByNameOrId("network_monitors")
require.NoError(t, err)
probe := core.NewRecord(collection)
probe.Load(map[string]any{"system": sys.Id, "target": "localhost", "protocol": "tcp", "port": 80, "interval": 60, "enabled": true})
require.NoError(t, app.SaveNoValidate(probe))
fetch := func() {
t.Helper()
_, err := sys.fetchDataFromAgent(common.DataRequestOptions{})
require.NoError(t, err, "monitor sync failure must not fail stats fetching")
}
receive := func() monitor.SyncRequest {
t.Helper()
select {
case req := <-requests:
require.Equal(t, monitor.SyncActionReplace, req.Action)
return req
case <-time.After(time.Second):
t.Fatal("missing full monitor sync")
return monitor.SyncRequest{}
}
}
fetch()
require.Equal(t, probe.Id, receive().Configs[0].ID)
require.False(t, sys.monitorsNeedSync.Load())
fetch()
require.Empty(t, requests, "steady-state fetch must not resync")
// Simulate loss of the agent process/connection and its in-memory monitors.
require.NoError(t, sys.client.Load().Close())
fetch()
require.Equal(t, probe.Id, receive().Configs[0].ID)
require.False(t, sys.monitorsNeedSync.Load())
// Failed replacements are retried on the next successful stats fetch.
require.NoError(t, sys.client.Load().Close())
failSync.Store(true)
fetch()
receive()
require.True(t, sys.monitorsNeedSync.Load())
failSync.Store(false)
fetch()
receive()
require.False(t, sys.monitorsNeedSync.Load())
probe.Set("enabled", false)
require.NoError(t, app.SaveNoValidate(probe))
require.NoError(t, sys.client.Load().Close())
fetch()
require.Empty(t, receive().Configs, "empty replacement must clear stale monitors")
}
func TestPendingNetworkMonitorSyncQueryFailure(t *testing.T) {
sys, app := newTestSystemWithHub(t)
_, err := app.DB().NewQuery("DROP TABLE network_monitors").Execute()
require.NoError(t, err)
sys.monitorsNeedSync.Store(true)
sys.syncPendingNetworkMonitors()
require.True(t, sys.monitorsNeedSync.Load())
}

View File

@@ -0,0 +1,270 @@
//go:build testing
package systems
import (
"fmt"
"testing"
"time"
"github.com/fxamacker/cbor/v2"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/entities/system"
"github.com/pocketbase/pocketbase/core"
"github.com/pocketbase/pocketbase/tools/subscriptions"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestNetworkMonitorProbePruning(t *testing.T) {
for _, tc := range []struct {
name string
monitors map[string]monitor.Result
fail bool
want map[string]int64
}{
{"nil report", nil, false, map[string]int64{"monitor1": 1000, "monitor2": 1000}},
{"empty report", map[string]monitor.Result{}, false, map[string]int64{}},
{"removed monitor", map[string]monitor.Result{"monitor1": {LastProbeAt: 1000}}, false, map[string]int64{"monitor1": 1000}},
{"rolled back report", map[string]monitor.Result{}, true, map[string]int64{"monitor1": 1000, "monitor2": 1000}},
} {
t.Run(tc.name, func(t *testing.T) {
sys, app := newTestSystemWithHub(t)
sys.lastSavedMonitorProbe = map[string]int64{"monitor1": 1000, "monitor2": 1000}
// Preserve the distinction between nil and empty across the agent transport.
encoded, err := cbor.Marshal(system.CombinedData{Monitors: tc.monitors})
require.NoError(t, err)
var data system.CombinedData
require.NoError(t, cbor.Unmarshal(encoded, &data))
if tc.fail {
_, err = app.DB().NewQuery(`CREATE TRIGGER fail_system_update BEFORE UPDATE ON systems BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute()
require.NoError(t, err)
}
_, err = sys.createRecords(&data)
if tc.fail {
require.Error(t, err)
} else {
require.NoError(t, err)
}
assert.Equal(t, tc.want, sys.lastSavedMonitorProbe)
})
}
}
func TestNetworkMonitorStatsFreshness(t *testing.T) {
for _, realtime := range []bool{false, true} {
name := "sql"
if realtime {
name = "realtime"
}
t.Run(name, func(t *testing.T) {
sys, app := newTestSystemWithHub(t)
if realtime {
client := subscriptions.NewDefaultClient()
client.Subscribe("network_monitors/*")
app.SubscriptionsBroker().Register(client)
t.Cleanup(func() { app.SubscriptionsBroker().Unregister(client.Id()) })
}
col, err := app.FindCachedCollectionByNameOrId("network_monitors")
require.NoError(t, err)
for _, id := range []string{"monitor1", "monitor2"} {
record := core.NewRecord(col)
record.Id = id
record.Set("system", sys.Id)
require.NoError(t, app.SaveNoValidate(record))
}
data := &system.CombinedData{Monitors: map[string]monitor.Result{
"monitor1": {LastProbeAt: 1000, AvgResponse: 20, TotalCount: 6, SuccessCount: 6, ResponseSum: 123},
"monitor2": {LastProbeAt: 1000, PacketLoss: 100, TotalCount: 1},
}}
count := func(want int64) {
t.Helper()
got, err := app.CountRecords("network_monitor_stats")
require.NoError(t, err)
assert.Equal(t, want, got)
}
save := func() {
t.Helper()
_, err := sys.createRecords(data)
require.NoError(t, err)
}
save()
count(2)
stored, err := app.FindAllRecords("network_monitor_stats")
require.NoError(t, err)
for _, record := range stored {
result := data.Monitors[record.GetString("monitor")]
assert.EqualValues(t, result.TotalCount, record.GetInt("total_count"))
assert.EqualValues(t, result.SuccessCount, record.GetInt("success_count"))
assert.EqualValues(t, result.ResponseSum, record.GetInt("res_sum"))
}
// A resume can overlap the scheduled update with the same probe.
errs := make(chan error, 4)
for range 4 {
go func() {
_, err := sys.createRecords(data)
errs <- err
}()
}
for range 4 {
require.NoError(t, <-errs)
}
count(2)
// A rolling hourly value can change without a new probe.
result := data.Monitors["monitor1"]
result.AvgResponse1h = 42
data.Monitors["monitor1"] = result
save()
count(2)
record, err := app.FindRecordById("network_monitors", "monitor1")
require.NoError(t, err)
assert.Equal(t, 42, record.GetInt("resAvg1h"))
// Identical response values and failed probes still count as new measurements.
for id, result := range data.Monitors {
result.LastProbeAt = 301000
data.Monitors[id] = result
}
save()
count(4)
// A failed individual insert must remain retryable, even if others commit.
_, err = app.DB().NewQuery(`CREATE TRIGGER fail_monitor_insert BEFORE INSERT ON network_monitor_stats WHEN NEW.monitor = 'monitor1' BEGIN SELECT RAISE(ABORT, 'test insert failure'); END`).Execute()
require.NoError(t, err)
for id, result := range data.Monitors {
result.LastProbeAt = 601000
data.Monitors[id] = result
}
save()
count(5)
assert.Equal(t, int64(301000), sys.lastSavedMonitorProbe["monitor1"])
assert.Equal(t, int64(601000), sys.lastSavedMonitorProbe["monitor2"])
_, err = app.DB().NewQuery("DROP TRIGGER fail_monitor_insert").Execute()
require.NoError(t, err)
save()
count(6)
// Failure after inserting stats rolls back the whole transaction and its markers.
_, err = app.DB().NewQuery(`CREATE TRIGGER fail_system_update BEFORE UPDATE ON systems BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute()
require.NoError(t, err)
result = data.Monitors["monitor1"]
result.LastProbeAt = 901000
data.Monitors["monitor1"] = result
_, err = sys.createRecords(data)
require.Error(t, err)
count(6)
assert.Equal(t, int64(601000), sys.lastSavedMonitorProbe["monitor1"])
_, err = app.DB().NewQuery("DROP TRIGGER fail_system_update").Execute()
require.NoError(t, err)
save()
count(7)
// Clock rollback is a new probe identity, not a reason to stall writes.
result.LastProbeAt = 500
data.Monitors["monitor1"] = result
save()
count(8)
// Recreated systems intentionally accept the first result without restoring state.
sys = &System{Id: sys.Id, manager: sys.manager}
save()
count(10)
})
}
}
// Observes the committed DB through the hub, not the transaction's app.
type monitorAlertHub struct {
stubHub
handle func(*core.Record, map[string]monitor.Result) error
}
func (h monitorAlertHub) HandleNetworkMonitorAlerts(record *core.Record, results map[string]monitor.Result) error {
return h.handle(record, results)
}
func TestNetworkMonitorAlertsAfterCommit(t *testing.T) {
for _, realtime := range []bool{false, true} {
t.Run(fmt.Sprint(realtime), func(t *testing.T) {
sys, app := newTestSystemWithHub(t)
if realtime {
client := subscriptions.NewDefaultClient()
client.Subscribe("network_monitors/*")
app.SubscriptionsBroker().Register(client)
}
collection, err := app.FindCachedCollectionByNameOrId("network_monitors")
require.NoError(t, err)
record := core.NewRecord(collection)
record.Set("system", sys.Id)
require.NoError(t, app.SaveNoValidate(record))
called := 0
result := monitor.Result{LastProbeAt: time.Now().UnixMilli(), SampleCount: 3, PacketLoss1h: 10}
sys.manager.hub = monitorAlertHub{stubHub: stubHub{app}, handle: func(systemRecord *core.Record, results map[string]monitor.Result) error {
called++
assert.Equal(t, sys.Id, systemRecord.Id)
assert.Equal(t, result, results[record.Id])
saved, err := app.FindRecordById("network_monitors", record.Id)
require.NoError(t, err)
assert.Equal(t, 10.0, saved.GetFloat("loss1h"))
return nil
}}
data := &system.CombinedData{Monitors: map[string]monitor.Result{record.Id: result}}
_, err = sys.createRecords(data)
require.NoError(t, err)
assert.Equal(t, 1, called)
// A transaction that fails after writing monitor stats must not notify.
_, err = app.DB().NewQuery(`CREATE TRIGGER fail_system BEFORE UPDATE ON systems BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute()
require.NoError(t, err)
_, err = sys.createRecords(data)
require.Error(t, err)
assert.Equal(t, 1, called)
})
}
}
func TestNetworkMonitorCertPersistence(t *testing.T) {
for _, realtime := range []bool{false, true} {
name := "sql"
if realtime {
name = "realtime"
}
t.Run(name, func(t *testing.T) {
sys, app := newTestSystemWithHub(t)
if realtime {
client := subscriptions.NewDefaultClient()
client.Subscribe("network_monitors/*")
app.SubscriptionsBroker().Register(client)
t.Cleanup(func() { app.SubscriptionsBroker().Unregister(client.Id()) })
}
col, err := app.FindCachedCollectionByNameOrId("network_monitors")
require.NoError(t, err)
record := core.NewRecord(col)
record.Id = "monitor1"
record.Set("system", sys.Id)
require.NoError(t, app.SaveNoValidate(record))
storedCert := func() monitor.CertInfo {
t.Helper()
record, err := app.FindRecordById("network_monitors", "monitor1")
require.NoError(t, err)
var cert monitor.CertInfo
require.NoError(t, record.UnmarshalJSONField("certInfo", &cert))
return cert
}
cert := &monitor.CertInfo{Expires: 1_800_000_000_000, Issuer: "Test CA"}
_, err = sys.createRecords(&system.CombinedData{Monitors: map[string]monitor.Result{
"monitor1": {LastProbeAt: 1000, Cert: cert},
}})
require.NoError(t, err)
assert.Equal(t, *cert, storedCert())
// Results without cert info keep the stored certificate.
_, err = sys.createRecords(&system.CombinedData{Monitors: map[string]monitor.Result{
"monitor1": {LastProbeAt: 2000},
}})
require.NoError(t, err)
assert.Equal(t, *cert, storedCert())
})
}
}

View File

@@ -0,0 +1,184 @@
//go:build testing
package systems
import (
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/blang/semver"
"github.com/fxamacker/cbor/v2"
"github.com/henrygd/beszel/internal/common"
"github.com/henrygd/beszel/internal/entities/monitor"
esystem "github.com/henrygd/beszel/internal/entities/system"
"github.com/henrygd/beszel/internal/hub/ws"
"github.com/lxzan/gws"
"github.com/pocketbase/pocketbase/core"
"github.com/stretchr/testify/require"
)
type monitorSyncClient struct {
gws.BuiltinEventHandler
requests chan common.HubRequest[monitor.SyncRequest]
failSync atomic.Bool
}
func (c *monitorSyncClient) OnMessage(conn *gws.Conn, message *gws.Message) {
defer message.Close()
var req common.HubRequest[cbor.RawMessage]
if err := cbor.Unmarshal(message.Bytes(), &req); err != nil {
return
}
resp := common.AgentResponse{Id: req.Id}
if req.Action == common.GetData {
resp.SystemData = &esystem.CombinedData{}
} else {
var data monitor.SyncRequest
if err := cbor.Unmarshal(req.Data, &data); err != nil {
return
}
c.requests <- common.HubRequest[monitor.SyncRequest]{Id: req.Id, Action: req.Action, Data: data}
if c.failSync.Load() {
resp.Error = "test sync failure"
} else {
resp.Data, _ = cbor.Marshal(monitor.SyncResponse{})
}
}
response, _ := cbor.Marshal(resp)
_ = conn.WriteMessage(gws.OpcodeBinary, response)
}
// Avoid the production delayed disconnect notification; these tests explicitly
// remove each connection from the manager before reconnecting.
type monitorSyncServer struct{ ws.Handler }
func (*monitorSyncServer) OnClose(*gws.Conn, error) {}
func TestNetworkMonitorSyncSkipsOlderAgents(t *testing.T) {
for _, version := range []string{"0.0.0", "0.18.0", "0.19.0"} {
t.Run(version, func(t *testing.T) {
// No transport: attempting to send any request would fail.
sys := &System{agentVersion: semver.MustParse(version)}
require.NoError(t, sys.SyncNetworkMonitors(nil))
result, err := sys.UpsertNetworkMonitor(monitor.Config{ID: "test"}, true)
require.NoError(t, err)
require.Nil(t, result)
require.NoError(t, sys.DeleteNetworkMonitor("test"))
})
}
}
func TestNetworkMonitorReconnectSync(t *testing.T) {
for _, change := range []string{"delete", "disable", "retry"} {
t.Run(change, func(t *testing.T) {
sys, app := newTestSystemWithHub(t)
record, err := app.FindRecordById("systems", sys.Id)
require.NoError(t, err)
// Suppress unrelated system-stat requests while exercising reconnects.
record.Set("status", paused)
require.NoError(t, app.SaveNoValidate(record))
collection, err := app.FindCachedCollectionByNameOrId("network_monitors")
require.NoError(t, err)
probe := core.NewRecord(collection)
probe.Load(map[string]any{
"system": sys.Id, "target": "localhost", "protocol": "tcp",
"port": 80, "interval": 60, "enabled": true,
})
require.NoError(t, app.SaveNoValidate(probe))
sm := NewSystemManager(stubHub{app})
t.Cleanup(func() {
sm.cancel()
_ = sm.RemoveSystem(sys.Id)
sm.smartFetchMap.StopCleaner()
sm.zfsFetchMap.StopCleaner()
})
version := semver.MustParse("0.20.0")
connections := make(chan *ws.WsConn, 1)
upgrader := gws.NewUpgrader(&monitorSyncServer{}, nil)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
conn, err := upgrader.Upgrade(w, r)
if err != nil {
t.Error(err)
return
}
wsConn := ws.NewWsConnection(conn, version)
conn.Session().Store("wsConn", wsConn)
connections <- wsConn
conn.ReadLoop()
}))
t.Cleanup(server.Close)
client := &monitorSyncClient{requests: make(chan common.HubRequest[monitor.SyncRequest], 2)}
connect := func() monitor.SyncRequest {
t.Helper()
conn, _, err := gws.NewClient(client, &gws.ClientOption{Addr: "ws" + strings.TrimPrefix(server.URL, "http")})
require.NoError(t, err)
t.Cleanup(func() { _ = conn.NetConn().Close() })
go conn.ReadLoop()
select {
case wsConn := <-connections:
require.NoError(t, sm.AddWebSocketSystem(sys.Id, version, wsConn))
case <-time.After(3 * time.Second):
t.Fatal("websocket connection was not established")
}
select {
case req := <-client.requests:
require.Equal(t, common.SyncNetworkMonitors, req.Action)
require.Equal(t, monitor.SyncActionReplace, req.Data.Action)
return req.Data
case <-time.After(3 * time.Second):
t.Fatal("reconnected agent did not receive a monitor replacement")
return monitor.SyncRequest{}
}
}
client.failSync.Store(change == "retry")
initial := connect()
require.Len(t, initial.Configs, 1)
require.Equal(t, probe.Id, initial.Configs[0].ID)
if change == "retry" {
system, err := sm.GetSystem(sys.Id)
require.NoError(t, err)
require.Eventually(t, system.monitorsNeedSync.Load, time.Second, time.Millisecond)
// A second failed sync must not fail the stats fetch or clear pending state.
_, err = system.fetchDataFromAgent(common.DataRequestOptions{})
require.NoError(t, err)
require.True(t, system.monitorsNeedSync.Load())
require.Len(t, client.requests, 1)
<-client.requests
client.failSync.Store(false)
_, err = system.fetchDataFromAgent(common.DataRequestOptions{})
require.NoError(t, err)
require.False(t, system.monitorsNeedSync.Load())
require.Len(t, client.requests, 1)
retry := <-client.requests
require.Equal(t, monitor.SyncActionReplace, retry.Data.Action)
require.Equal(t, initial.Configs, retry.Data.Configs)
_, err = system.fetchDataFromAgent(common.DataRequestOptions{})
require.NoError(t, err)
require.Empty(t, client.requests, "successful sync must not repeat on every fetch")
return
}
require.NoError(t, sm.RemoveSystem(sys.Id))
if change == "delete" {
require.NoError(t, app.Delete(probe))
} else {
probe.Set("enabled", false)
require.NoError(t, app.SaveNoValidate(probe))
}
require.Empty(t, connect().Configs, "reconnect must clear the agent's previous probe")
})
}
}
func TestGetMonitorConfigsForSystemQueryError(t *testing.T) {
sys, app := newTestSystemWithHub(t)
_, err := app.DB().NewQuery("DROP TABLE network_monitors").Execute()
require.NoError(t, err)
_, err = sys.manager.GetMonitorConfigsForSystem(sys.Id)
require.Error(t, err, "a failed query must not be treated as an empty monitor set")
}

View File

@@ -0,0 +1,80 @@
package systems
import (
"context"
"fmt"
"time"
"github.com/henrygd/beszel"
"github.com/henrygd/beszel/internal/common"
"github.com/henrygd/beszel/internal/entities/monitor"
)
// syncPendingNetworkMonitors runs on WebSocket connect and after successful stats
// fetches. Failed syncs retry on the next update without taking the system down.
func (sys *System) syncPendingNetworkMonitors() {
if !sys.monitorsNeedSync.Swap(false) {
return
}
if err := sys.syncAllNetworkMonitors(); err != nil {
sys.monitorsNeedSync.Store(true)
sys.manager.hub.Logger().Warn("failed to sync monitors to agent", "system", sys.Id, "err", err)
}
}
func (sys *System) syncAllNetworkMonitors() error {
configs, err := sys.manager.GetMonitorConfigsForSystem(sys.Id)
if err != nil {
return fmt.Errorf("failed to load monitors: %w", err)
}
// An empty set must also replace probes retained across a disconnect.
return sys.SyncNetworkMonitors(configs)
}
// SyncNetworkMonitors sends monitor configurations to the agent.
func (sys *System) SyncNetworkMonitors(configs []monitor.Config) error {
_, err := sys.syncNetworkMonitors(monitor.SyncRequest{Action: monitor.SyncActionReplace, Configs: configs})
return err
}
// UpsertNetworkMonitor sends a single monitor configuration change to the agent.
func (sys *System) UpsertNetworkMonitor(config monitor.Config, runNow bool) (*monitor.Result, error) {
resp, err := sys.syncNetworkMonitors(monitor.SyncRequest{
Action: monitor.SyncActionUpsert,
Config: config,
RunNow: runNow,
})
if err != nil {
return nil, err
}
if resp.Result == (monitor.Result{}) {
return nil, nil
}
result := resp.Result
return &result, nil
}
// DeleteNetworkMonitor removes a single monitor task from the agent.
func (sys *System) DeleteNetworkMonitor(id string) error {
_, err := sys.syncNetworkMonitors(monitor.SyncRequest{
Action: monitor.SyncActionDelete,
Config: monitor.Config{ID: id},
})
return err
}
func (sys *System) syncNetworkMonitors(req monitor.SyncRequest) (monitor.SyncResponse, error) {
if sys.agentVersion.LT(beszel.MinVersionNetworkMonitors) {
return monitor.SyncResponse{}, nil
}
timeout := 5 * time.Second
if req.Action == monitor.SyncActionUpsert && req.RunNow {
// Allow the probe to finish, including a timeout result, while preserving
// the normal request budget for transport and response handling.
timeout += monitor.MaxProbeTimeout
}
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
var result monitor.SyncResponse
return result, sys.request(ctx, common.SyncNetworkMonitors, req, &result)
}

View File

@@ -9,6 +9,7 @@ import (
"math/rand" "math/rand"
"net" "net"
"strings" "strings"
"sync"
"sync/atomic" "sync/atomic"
"time" "time"
@@ -18,6 +19,7 @@ import (
"github.com/henrygd/beszel/internal/hub/ws" "github.com/henrygd/beszel/internal/hub/ws"
"github.com/henrygd/beszel/internal/entities/container" "github.com/henrygd/beszel/internal/entities/container"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/entities/smart" "github.com/henrygd/beszel/internal/entities/smart"
"github.com/henrygd/beszel/internal/entities/system" "github.com/henrygd/beszel/internal/entities/system"
"github.com/henrygd/beszel/internal/entities/systemd" "github.com/henrygd/beszel/internal/entities/systemd"
@@ -30,6 +32,8 @@ import (
"github.com/lxzan/gws" "github.com/lxzan/gws"
"github.com/pocketbase/dbx" "github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/core" "github.com/pocketbase/pocketbase/core"
"github.com/pocketbase/pocketbase/tools/security"
"github.com/pocketbase/pocketbase/tools/types"
"golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh"
) )
@@ -52,6 +56,13 @@ type System struct {
smartInterval time.Duration // Interval for periodic SMART data updates smartInterval time.Duration // Interval for periodic SMART data updates
zfsFetching atomic.Bool // True if ZFS pools are currently being fetched zfsFetching atomic.Bool // True if ZFS pools are currently being fetched
zfsInterval time.Duration // Interval for periodic ZFS detail data updates zfsInterval time.Duration // Interval for periodic ZFS detail data updates
// A fresh connection needs a full monitor configuration sync.
monitorsNeedSync atomic.Bool
// Serialize persistence from scheduled updates and resumes through commit.
recordsMu sync.Mutex
// Protected by recordsMu; realtime reads don't consume probes.
lastSavedMonitorProbe map[string]int64
} }
func (sm *SystemManager) NewSystem(systemId string) *System { func (sm *SystemManager) NewSystem(systemId string) *System {
@@ -211,11 +222,15 @@ func (sys *System) handlePaused() {
// createRecords updates the system record and adds system_stats and container_stats records // createRecords updates the system record and adds system_stats and container_stats records
func (sys *System) createRecords(data *system.CombinedData) (*core.Record, error) { func (sys *System) createRecords(data *system.CombinedData) (*core.Record, error) {
sys.recordsMu.Lock()
defer sys.recordsMu.Unlock()
systemRecord, err := sys.getRecord(sys.manager.hub) systemRecord, err := sys.getRecord(sys.manager.hub)
if err != nil { if err != nil {
return nil, err return nil, err
} }
hub := sys.manager.hub hub := sys.manager.hub
savedMonitorProbes := make(map[string]int64)
err = hub.RunInTransaction(func(txApp core.App) error { err = hub.RunInTransaction(func(txApp core.App) error {
// add system_stats record // add system_stats record
systemStatsCollection, err := txApp.FindCachedCollectionByNameOrId("system_stats") systemStatsCollection, err := txApp.FindCachedCollectionByNameOrId("system_stats")
@@ -264,6 +279,16 @@ func (sys *System) createRecords(data *system.CombinedData) (*core.Record, error
if err := createSystemDetailsRecord(txApp, data.Details, sys.Id); err != nil { if err := createSystemDetailsRecord(txApp, data.Details, sys.Id); err != nil {
return err return err
} }
// sync display name with hostname if enabled (details are fetched once per agent connection)
if syncNames, _ := utils.GetEnv("SYNC_SYSTEM_NAMES"); syncNames == "true" && data.Details.Hostname != "" {
systemRecord.Set("name", data.Details.Hostname)
}
}
if data.Monitors != nil {
if err := sys.updateNetworkMonitorsRecords(txApp, data.Monitors, savedMonitorProbes); err != nil {
return err
}
} }
if err := sys.syncZfsPoolHealth(txApp, data.Stats.ZfsPools); err != nil { if err := sys.syncZfsPoolHealth(txApp, data.Stats.ZfsPools); err != nil {
@@ -287,6 +312,29 @@ func (sys *System) createRecords(data *system.CombinedData) (*core.Record, error
return nil return nil
}) })
// Publish only successful inserts after the entire transaction commits.
if err == nil && len(savedMonitorProbes) > 0 {
if sys.lastSavedMonitorProbe == nil {
sys.lastSavedMonitorProbe = savedMonitorProbes
} else {
for id, timestamp := range savedMonitorProbes {
sys.lastSavedMonitorProbe[id] = timestamp
}
}
}
// A non-nil report includes cached results for all remaining monitors.
if err == nil && data.Monitors != nil {
for id := range sys.lastSavedMonitorProbe {
if _, exists := data.Monitors[id]; !exists {
delete(sys.lastSavedMonitorProbe, id)
}
}
}
if err == nil {
if alertErr := hub.HandleNetworkMonitorAlerts(systemRecord, data.Monitors); alertErr != nil {
hub.Logger().Error("Error handling network monitor alerts", "err", alertErr)
}
}
return systemRecord, err return systemRecord, err
} }
@@ -337,7 +385,7 @@ func createSystemdStatsRecords(app core.App, data []*systemd.Service, systemId s
} }
suffix := fmt.Sprintf("%d", i) suffix := fmt.Sprintf("%d", i)
valueStrings = append(valueStrings, fmt.Sprintf("({:id%[1]s}, {:system}, {:name%[1]s}, {:state%[1]s}, {:sub%[1]s}, {:cpu%[1]s}, {:cpuPeak%[1]s}, {:memory%[1]s}, {:memPeak%[1]s}, {:updated})", suffix)) valueStrings = append(valueStrings, fmt.Sprintf("({:id%[1]s}, {:system}, {:name%[1]s}, {:state%[1]s}, {:sub%[1]s}, {:cpu%[1]s}, {:cpuPeak%[1]s}, {:memory%[1]s}, {:memPeak%[1]s}, {:updated})", suffix))
params["id"+suffix] = makeStableHashId(systemId, service.Name) params["id"+suffix] = MakeStableHashId(systemId, service.Name)
params["name"+suffix] = service.Name params["name"+suffix] = service.Name
params["state"+suffix] = service.State params["state"+suffix] = service.State
params["sub"+suffix] = service.Sub params["sub"+suffix] = service.Sub
@@ -363,6 +411,120 @@ func createSystemdStatsRecords(app core.App, data []*systemd.Service, systemId s
return err return err
} }
func (sys *System) updateNetworkMonitorsRecords(app core.App, monitorResults map[string]monitor.Result, savedProbes map[string]int64) error {
if len(monitorResults) == 0 {
return nil
}
var err error
systemId := sys.Id
const monitorCollectionName = "network_monitors"
// If realtime updates are active, we save via PocketBase records to trigger realtime events.
// Otherwise we can do a more efficient direct update via SQL
realtimeActive := utils.RealtimeActiveForCollection(app, monitorCollectionName, func(filterQuery string) bool {
return !strings.Contains(filterQuery, "system") || strings.Contains(filterQuery, systemId)
})
now := time.Now().UTC()
nowMilli := now.UnixMilli()
nowString := now.Format(types.DefaultDateLayout)
var db dbx.Builder
var updateQuery *dbx.Query
if !realtimeActive {
db = app.DB()
monitorFields := []string{"res", "resMin1h", "resMax1h", "resAvg1h", "loss1h", "updated"}
setClauses := make([]string, len(monitorFields))
for i, f := range monitorFields {
setClauses[i] = fmt.Sprintf("%s={:%s}", f, f)
}
// Results omit certInfo unless it changed, so keep the stored value.
setClauses = append(setClauses, "certInfo=COALESCE({:certInfo}, certInfo)")
queryString := fmt.Sprintf("UPDATE %s SET %s WHERE id={:id}", monitorCollectionName, strings.Join(setClauses, ", "))
updateQuery = db.NewQuery(queryString)
}
// update network_monitors records
for id, result := range monitorResults {
monitorData := map[string]any{
"id": id,
"res": result.AvgResponse,
"resAvg1h": result.AvgResponse1h,
"resMin1h": result.MinResponse1h,
"resMax1h": result.MaxResponse1h,
"loss1h": result.PacketLoss1h,
"updated": nowString,
}
switch realtimeActive {
case true:
var record *core.Record
record, err = app.FindRecordById(monitorCollectionName, id)
if err == nil {
if result.Cert != nil {
monitorData["certInfo"] = result.Cert
}
record.Load(monitorData)
err = app.SaveNoValidate(record)
}
default:
monitorData["certInfo"] = nil
if result.Cert != nil {
var cert []byte
if cert, err = json.Marshal(result.Cert); err == nil {
monitorData["certInfo"] = string(cert)
}
}
if err == nil {
_, err = updateQuery.Bind(dbx.Params(monitorData)).Execute()
}
}
if err != nil {
app.Logger().Warn("Failed to update monitor", "system", systemId, "monitor", id, "err", err)
}
}
// handle stats collection — one record per monitor
const statsCollectionName = "network_monitor_stats"
var statsCollection *core.Collection
if realtimeActive {
statsCollection, _ = app.FindCachedCollectionByNameOrId(statsCollectionName)
}
for monitorId, result := range monitorResults {
// Compare identity, not ordering, so agent clock changes don't stall writes.
if result.LastProbeAt == sys.lastSavedMonitorProbe[monitorId] {
continue
}
statsRecordData := map[string]any{
"system": systemId,
"monitor": monitorId,
"type": "1m",
"created": nowMilli,
"res_min": result.MinResponse,
"res_max": result.MaxResponse,
"total_count": result.TotalCount,
"success_count": result.SuccessCount,
"res_sum": result.ResponseSum,
}
switch realtimeActive {
case true:
record := core.NewRecord(statsCollection)
record.Load(statsRecordData)
err = app.SaveNoValidate(record)
default:
statsRecordData["id"] = security.PseudorandomStringWithAlphabet(10, core.DefaultIdAlphabet)
_, err = db.Insert(statsCollectionName, dbx.Params(statsRecordData)).Execute()
}
if err != nil {
app.Logger().Error("Failed to update monitor stats", "system", systemId, "monitor", monitorId, "err", err)
} else {
savedProbes[monitorId] = result.LastProbeAt
}
}
return nil
}
// createContainerRecords creates container records // createContainerRecords creates container records
func createContainerRecords(app core.App, data []*container.Stats, systemId string) error { func createContainerRecords(app core.App, data []*container.Stats, systemId string) error {
if len(data) == 0 { if len(data) == 0 {
@@ -376,7 +538,7 @@ func createContainerRecords(app core.App, data []*container.Stats, systemId stri
valueStrings := make([]string, 0, len(data)) valueStrings := make([]string, 0, len(data))
for i, container := range data { for i, container := range data {
suffix := fmt.Sprintf("%d", i) suffix := fmt.Sprintf("%d", i)
valueStrings = append(valueStrings, fmt.Sprintf("({:id%[1]s}, {:system}, {:name%[1]s}, {:image%[1]s}, {:ports%[1]s}, {:status%[1]s}, {:health%[1]s}, {:cpu%[1]s}, {:memory%[1]s}, {:net%[1]s}, {:updated})", suffix)) valueStrings = append(valueStrings, fmt.Sprintf("({:id%[1]s}, {:system}, {:name%[1]s}, {:image%[1]s}, {:ports%[1]s}, {:status%[1]s}, {:health%[1]s}, {:cpu%[1]s}, {:memory%[1]s}, {:net%[1]s}, {:updateAvailable%[1]s}, {:updated})", suffix))
params["id"+suffix] = container.Id params["id"+suffix] = container.Id
params["name"+suffix] = container.Name params["name"+suffix] = container.Name
params["image"+suffix] = container.Image params["image"+suffix] = container.Image
@@ -390,9 +552,10 @@ func createContainerRecords(app core.App, data []*container.Stats, systemId stri
netBytes = uint64((container.NetworkSent + container.NetworkRecv) * 1024 * 1024) netBytes = uint64((container.NetworkSent + container.NetworkRecv) * 1024 * 1024)
} }
params["net"+suffix] = netBytes params["net"+suffix] = netBytes
params["updateAvailable"+suffix] = container.UpdateAvailable
} }
queryString := fmt.Sprintf( queryString := fmt.Sprintf(
"INSERT INTO containers (id, system, name, image, ports, status, health, cpu, memory, net, updated) VALUES %s ON CONFLICT(id) DO UPDATE SET system = excluded.system, name = excluded.name, image = excluded.image, ports = excluded.ports, status = excluded.status, health = excluded.health, cpu = excluded.cpu, memory = excluded.memory, net = excluded.net, updated = excluded.updated", "INSERT INTO containers (id, system, name, image, ports, status, health, cpu, memory, net, updatable, updated) VALUES %s ON CONFLICT(id) DO UPDATE SET system = excluded.system, name = excluded.name, image = excluded.image, ports = excluded.ports, status = excluded.status, health = excluded.health, cpu = excluded.cpu, memory = excluded.memory, net = excluded.net, updatable = excluded.updatable, updated = excluded.updated",
strings.Join(valueStrings, ","), strings.Join(valueStrings, ","),
) )
_, err := app.DB().NewQuery(queryString).Bind(params).Execute() _, err := app.DB().NewQuery(queryString).Bind(params).Execute()
@@ -488,7 +651,10 @@ func (sys *System) request(ctx context.Context, action common.WebSocketAction, r
err := sys.sshTransport.RequestWithRetry(ctx, action, req, dest, 1) err := sys.sshTransport.RequestWithRetry(ctx, action, req, dest, 1)
// Keep legacy SSH client/version fields in sync for other code paths. // Keep legacy SSH client/version fields in sync for other code paths.
if sys.sshTransport != nil { if sys.sshTransport != nil {
sys.client.Store(sys.sshTransport.GetClient()) client := sys.sshTransport.GetClient()
if previous := sys.client.Swap(client); client != nil && client != previous {
sys.monitorsNeedSync.Store(true)
}
sys.agentVersion = sys.sshTransport.GetAgentVersion() sys.agentVersion = sys.sshTransport.GetAgentVersion()
} }
return err return err
@@ -546,8 +712,14 @@ func (sys *System) fetchDataFromAgent(options common.DataRequestOptions) (*syste
if sys.WsConn != nil && sys.WsConn.IsConnected() { if sys.WsConn != nil && sys.WsConn.IsConnected() {
wsData, err := sys.fetchDataViaWebSocket(options) wsData, err := sys.fetchDataViaWebSocket(options)
if err == nil { if err == nil {
sys.syncPendingNetworkMonitors()
return wsData, nil return wsData, nil
} }
// A slow collection doesn't mean the connection is broken. Closing it
// would force the agent into a reconnect loop, so only report the error.
if errors.Is(err, context.DeadlineExceeded) {
return nil, err
}
// close the WebSocket connection if error and try SSH // close the WebSocket connection if error and try SSH
sys.closeWebSocketConnection() sys.closeWebSocketConnection()
} }
@@ -556,15 +728,23 @@ func (sys *System) fetchDataFromAgent(options common.DataRequestOptions) (*syste
if err != nil { if err != nil {
return nil, err return nil, err
} }
sys.syncPendingNetworkMonitors()
return sshData, nil return sshData, nil
} }
// wsDataRequestTimeout bounds how long to wait for stats over WebSocket. Agent
// collection can legitimately take several seconds (e.g. a slow `zpool list`),
// so this must be well above the request manager's 5s default.
var wsDataRequestTimeout = 30 * time.Second
func (sys *System) fetchDataViaWebSocket(options common.DataRequestOptions) (*system.CombinedData, error) { func (sys *System) fetchDataViaWebSocket(options common.DataRequestOptions) (*system.CombinedData, error) {
if sys.WsConn == nil || !sys.WsConn.IsConnected() { if sys.WsConn == nil || !sys.WsConn.IsConnected() {
return nil, errors.New("no websocket connection") return nil, errors.New("no websocket connection")
} }
ctx, cancel := context.WithTimeout(context.Background(), wsDataRequestTimeout)
defer cancel()
wsTransport := transport.NewWebSocketTransport(sys.WsConn) wsTransport := transport.NewWebSocketTransport(sys.WsConn)
err := wsTransport.Request(context.Background(), common.GetData, options, sys.data) err := wsTransport.Request(ctx, common.GetData, options, sys.data)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -621,7 +801,7 @@ func (sys *System) FetchZfsDataFromAgent(force bool) (*zfs.ZfsData, error) {
return &result, err return &result, err
} }
func makeStableHashId(strings ...string) string { func MakeStableHashId(strings ...string) string {
hash := fnv.New32a() hash := fnv.New32a()
for _, str := range strings { for _, str := range strings {
hash.Write([]byte(str)) hash.Write([]byte(str))
@@ -790,6 +970,7 @@ func (s *System) createSSHClient() error {
return err return err
} }
s.agentVersion, _ = extractAgentVersion(string(client.Conn.ServerVersion())) s.agentVersion, _ = extractAgentVersion(string(client.Conn.ServerVersion()))
s.monitorsNeedSync.Store(true)
s.manager.resetFailedSmartFetchState(s.Id) s.manager.resetFailedSmartFetchState(s.Id)
s.manager.resetFailedZfsFetchState(s.Id) s.manager.resetFailedZfsFetchState(s.Id)
return nil return nil

View File

@@ -9,6 +9,7 @@ import (
"github.com/henrygd/beszel/internal/hub/ws" "github.com/henrygd/beszel/internal/hub/ws"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/entities/system" "github.com/henrygd/beszel/internal/entities/system"
"github.com/henrygd/beszel/internal/hub/expirymap" "github.com/henrygd/beszel/internal/hub/expirymap"
@@ -17,6 +18,7 @@ import (
"github.com/henrygd/beszel" "github.com/henrygd/beszel"
"github.com/blang/semver" "github.com/blang/semver"
"github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/core" "github.com/pocketbase/pocketbase/core"
"github.com/pocketbase/pocketbase/tools/store" "github.com/pocketbase/pocketbase/tools/store"
"golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh"
@@ -62,6 +64,7 @@ type hubLike interface {
core.App core.App
GetSSHKey(dataDir string) (ssh.Signer, error) GetSSHKey(dataDir string) (ssh.Signer, error)
HandleSystemAlerts(systemRecord *core.Record, data *system.CombinedData) error HandleSystemAlerts(systemRecord *core.Record, data *system.CombinedData) error
HandleNetworkMonitorAlerts(systemRecord *core.Record, results map[string]monitor.Result) error
HandleStatusAlerts(status string, systemRecord *core.Record) error HandleStatusAlerts(status string, systemRecord *core.Record) error
HandleContainerAlerts(systemRecord *core.Record, data *system.CombinedData, fetchLogs func(containerID string) (string, error)) error HandleContainerAlerts(systemRecord *core.Record, data *system.CombinedData, fetchLogs func(containerID string) (string, error)) error
CancelPendingStatusAlerts(systemID string) CancelPendingStatusAlerts(systemID string)
@@ -187,7 +190,9 @@ func (sm *SystemManager) onRecordAfterCreateSuccess(e *core.RecordEvent) error {
// It clears system info when the status is changed to paused. // It clears system info when the status is changed to paused.
func (sm *SystemManager) onRecordUpdate(e *core.RecordEvent) error { func (sm *SystemManager) onRecordUpdate(e *core.RecordEvent) error {
if e.Record.GetString("status") == paused { if e.Record.GetString("status") == paused {
e.Record.Set("info", system.Info{}) var prevInfo system.Info
e.Record.UnmarshalJSONField("info", &prevInfo)
e.Record.Set("info", system.Info{AgentVersion: prevInfo.AgentVersion})
} }
return e.Next() return e.Next()
} }
@@ -214,11 +219,15 @@ func (sm *SystemManager) onRecordAfterUpdateSuccess(e *core.RecordEvent) error {
// Pause monitoring but keep system in manager for potential resume // Pause monitoring but keep system in manager for potential resume
system.closeSSHConnection() system.closeSSHConnection()
} }
_ = deactivateAlerts(e.App, e.Record.Id) _ = deactivateAlerts(e.App, e.Record.Id, false)
sm.hub.CancelPendingStatusAlerts(e.Record.Id) sm.hub.CancelPendingStatusAlerts(e.Record.Id)
sm.hub.CancelPendingContainerAlerts(e.Record.Id) sm.hub.CancelPendingContainerAlerts(e.Record.Id)
return e.Next() return e.Next()
case pending: case pending:
// Keep an active status alert until connectivity is confirmed. This lets
// pending -> up resolve it and send the recovery notification after a
// system address or other connection setting is changed.
_ = deactivateAlerts(e.App, e.Record.Id, true)
// Resume monitoring, preferring existing WebSocket connection // Resume monitoring, preferring existing WebSocket connection
if ok && system.WsConn != nil { if ok && system.WsConn != nil {
go system.update() go system.update()
@@ -228,7 +237,6 @@ func (sm *SystemManager) onRecordAfterUpdateSuccess(e *core.RecordEvent) error {
if err := sm.AddRecord(e.Record, nil); err != nil { if err := sm.AddRecord(e.Record, nil); err != nil {
e.App.Logger().Error("Error adding record", "err", err) e.App.Logger().Error("Error adding record", "err", err)
} }
_ = deactivateAlerts(e.App, e.Record.Id)
return e.Next() return e.Next()
case down: case down:
// Docker state is unknown while the system is unreachable. Do not let a // Docker state is unknown while the system is unreachable. Do not let a
@@ -251,8 +259,9 @@ func (sm *SystemManager) onRecordAfterUpdateSuccess(e *core.RecordEvent) error {
} }
} }
// Trigger status change alerts for up/down transitions // A connection-setting update moves a down system through pending before it
if (newStatus == down && prevStatus == up) || (newStatus == up && prevStatus == down) { // comes up, so recover active status alerts on any non-up -> up transition.
if (newStatus == down && prevStatus == up) || (newStatus == up && prevStatus != up) {
if err := sm.hub.HandleStatusAlerts(newStatus, e.Record); err != nil { if err := sm.hub.HandleStatusAlerts(newStatus, e.Record); err != nil {
e.App.Logger().Error("Error handling status alerts", "err", err) e.App.Logger().Error("Error handling status alerts", "err", err)
} }
@@ -346,10 +355,15 @@ func (sm *SystemManager) AddWebSocketSystem(systemId string, agentVersion semver
system := sm.NewSystem(systemId) system := sm.NewSystem(systemId)
system.WsConn = wsConn system.WsConn = wsConn
system.agentVersion = agentVersion system.agentVersion = agentVersion
system.monitorsNeedSync.Store(true)
if err := sm.AddRecord(systemRecord, system); err != nil { if err := sm.AddRecord(systemRecord, system); err != nil {
return err return err
} }
// Sync network monitors to the newly connected agent
go system.syncPendingNetworkMonitors()
return nil return nil
} }
@@ -362,6 +376,16 @@ func (sm *SystemManager) resetFailedSmartFetchState(systemID string) {
} }
} }
// GetMonitorConfigsForSystem returns all enabled monitor configs for a system.
func (sm *SystemManager) GetMonitorConfigsForSystem(systemID string) ([]monitor.Config, error) {
var configs []monitor.Config
err := sm.hub.DB().
NewQuery("SELECT id, target, protocol, port, interval FROM network_monitors WHERE system = {:system} AND enabled = true").
Bind(dbx.Params{"system": systemID}).
All(&configs)
return configs, err
}
// resetFailedZfsFetchState clears only failed ZFS cooldown entries so a fresh // resetFailedZfsFetchState clears only failed ZFS cooldown entries so a fresh
// agent reconnect retries ZFS discovery immediately after configuration changes. // agent reconnect retries ZFS discovery immediately after configuration changes.
func (sm *SystemManager) resetFailedZfsFetchState(systemID string) { func (sm *SystemManager) resetFailedZfsFetchState(systemID string) {
@@ -395,18 +419,23 @@ func (sm *SystemManager) createSSHClientConfig() error {
return nil return nil
} }
// deactivateAlerts finds all triggered alerts for a system and sets them to inactive. // deactivateAlerts finds triggered alerts for a system and sets them to inactive.
// This is called when a system is paused or goes offline to prevent continued alerts. // Status alerts can be preserved while connection changes are pending so that a
func deactivateAlerts(app core.App, systemID string) error { // confirmed recovery still produces an "up" notification.
// Monitor incidents remain open: a missing observation does not establish recovery.
func deactivateAlerts(app core.App, systemID string, preserveStatusAlert bool) error {
// Note: Direct SQL updates don't trigger SSE, so we use the PocketBase API // Note: Direct SQL updates don't trigger SSE, so we use the PocketBase API
// _, err := app.DB().NewQuery(fmt.Sprintf("UPDATE alerts SET triggered = false WHERE system = '%s'", systemID)).Execute() // _, err := app.DB().NewQuery(fmt.Sprintf("UPDATE alerts SET triggered = false WHERE system = '%s'", systemID)).Execute()
alerts, err := app.FindRecordsByFilter("alerts", fmt.Sprintf("system = '%s' && triggered = 1", systemID), "", -1, 0) alerts, err := app.FindRecordsByFilter("alerts", fmt.Sprintf("system = '%s' && triggered = 1 && name != 'NetworkMonitorLoss'", systemID), "", -1, 0)
if err != nil { if err != nil {
return err return err
} }
for _, alert := range alerts { for _, alert := range alerts {
if preserveStatusAlert && alert.GetString("name") == "Status" {
continue
}
alert.Set("triggered", false) alert.Set("triggered", false)
if err := app.SaveNoValidate(alert); err != nil { if err := app.SaveNoValidate(alert); err != nil {
return err return err

View File

@@ -6,6 +6,8 @@ import (
"time" "time"
"github.com/henrygd/beszel/internal/common" "github.com/henrygd/beszel/internal/common"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/entities/system"
"github.com/henrygd/beszel/internal/hub/utils" "github.com/henrygd/beszel/internal/hub/utils"
"github.com/pocketbase/dbx" "github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/apis" "github.com/pocketbase/pocketbase/apis"
@@ -165,7 +167,7 @@ func (sm *SystemManager) fetchRealtimeDataAndNotify() {
if err != nil { if err != nil {
return return
} }
bytes, err := json.Marshal(data) bytes, err := marshalRealtimeData(data)
if err == nil { if err == nil {
notify(sm.hub, system, fetch.subscription, bytes) notify(sm.hub, system, fetch.subscription, bytes)
} }
@@ -204,6 +206,22 @@ func (sm *SystemManager) finishRealtimeFetch(fetch realtimeFetch) {
} }
} }
// marshalRealtimeData marshals combined agent data for a realtime broadcast, converting
// the per-monitor results into the derived metric fields the frontend charts expect.
func marshalRealtimeData(data *system.CombinedData) ([]byte, error) {
if len(data.Monitors) == 0 {
return json.Marshal(data)
}
monitorStats := make(map[string]monitor.Stats, len(data.Monitors))
for id, result := range data.Monitors {
monitorStats[id] = monitor.Stats{}.FromResult(result)
}
return json.Marshal(struct {
*system.CombinedData
Monitors map[string]monitor.Stats `json:"Monitors"`
}{data, monitorStats})
}
// notify broadcasts realtime data to all clients subscribed to a specific subscription. // notify broadcasts realtime data to all clients subscribed to a specific subscription.
// Custom topics bypass collection rules, so check current access for every // Custom topics bypass collection rules, so check current access for every
// recipient, including clients whose authentication or membership was revoked. // recipient, including clients whose authentication or membership was revoked.

View File

@@ -77,7 +77,7 @@ func (sys *System) saveSmartDevices(smartData map[string]smart.SmartData, comple
currentIDs := make(map[string]struct{}, len(smartData)) currentIDs := make(map[string]struct{}, len(smartData))
for deviceKey := range smartData { for deviceKey := range smartData {
currentIDs[makeStableHashId(sys.Id, deviceKey)] = struct{}{} currentIDs[MakeStableHashId(sys.Id, deviceKey)] = struct{}{}
} }
err = hub.RunInTransaction(func(txApp core.App) error { err = hub.RunInTransaction(func(txApp core.App) error {
@@ -115,7 +115,7 @@ func (sys *System) saveSmartDevices(smartData map[string]smart.SmartData, comple
} }
func (sys *System) upsertSmartDeviceRecord(app core.App, collection *core.Collection, deviceKey string, device smart.SmartData) error { func (sys *System) upsertSmartDeviceRecord(app core.App, collection *core.Collection, deviceKey string, device smart.SmartData) error {
recordID := makeStableHashId(sys.Id, deviceKey) recordID := MakeStableHashId(sys.Id, deviceKey)
record, err := app.FindRecordById(collection, recordID) record, err := app.FindRecordById(collection, recordID)
if err != nil { if err != nil {

View File

@@ -7,6 +7,7 @@ import (
"testing" "testing"
"time" "time"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/entities/smart" "github.com/henrygd/beszel/internal/entities/smart"
esystem "github.com/henrygd/beszel/internal/entities/system" esystem "github.com/henrygd/beszel/internal/entities/system"
"github.com/henrygd/beszel/internal/hub/expirymap" "github.com/henrygd/beszel/internal/hub/expirymap"
@@ -28,7 +29,8 @@ func (stubHub) GetSSHKey(dataDir string) (ssh.Signer, error) { return nil, nil }
func (stubHub) HandleSystemAlerts(systemRecord *core.Record, data *esystem.CombinedData) error { func (stubHub) HandleSystemAlerts(systemRecord *core.Record, data *esystem.CombinedData) error {
return nil return nil
} }
func (stubHub) HandleStatusAlerts(status string, systemRecord *core.Record) error { return nil } func (stubHub) HandleNetworkMonitorAlerts(*core.Record, map[string]monitor.Result) error { return nil }
func (stubHub) HandleStatusAlerts(status string, systemRecord *core.Record) error { return nil }
func (stubHub) HandleContainerAlerts(systemRecord *core.Record, data *esystem.CombinedData, fetchLogs func(containerID string) (string, error)) error { func (stubHub) HandleContainerAlerts(systemRecord *core.Record, data *esystem.CombinedData, fetchLogs func(containerID string) (string, error)) error {
return nil return nil
} }
@@ -212,7 +214,7 @@ func TestSaveSmartDevices_IncompleteDataDoesNotRemoveDevices(t *testing.T) {
}, false)) }, false))
assert.Len(t, countSmartDeviceRecords(t, testApp, sys.Id), 2) assert.Len(t, countSmartDeviceRecords(t, testApp, sys.Id), 2)
recordA, err := testApp.FindRecordById("smart_devices", makeStableHashId(sys.Id, "AAA")) recordA, err := testApp.FindRecordById("smart_devices", MakeStableHashId(sys.Id, "AAA"))
require.NoError(t, err) require.NoError(t, err)
assert.EqualValues(t, 42, recordA.GetInt("temp")) assert.EqualValues(t, 42, recordA.GetInt("temp"))
} }

View File

@@ -0,0 +1,34 @@
//go:build testing
package systems
import (
"testing"
"github.com/henrygd/beszel/internal/entities/system"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestCreateRecordsSyncSystemNames(t *testing.T) {
for _, tc := range []struct {
name string
env string
hostname string
expected string
}{
{"disabled", "", "new-host", "test-system"},
{"enabled", "true", "new-host", "new-host"},
{"enabled with empty hostname", "true", "", "test-system"},
} {
t.Run(tc.name, func(t *testing.T) {
t.Setenv("SYNC_SYSTEM_NAMES", tc.env)
sys, app := newTestSystemWithHub(t)
_, err := sys.createRecords(&system.CombinedData{Details: &system.Details{Hostname: tc.hostname}})
require.NoError(t, err)
record, err := app.FindRecordById("systems", sys.Id)
require.NoError(t, err)
assert.Equal(t, tc.expected, record.GetString("name"))
})
}
}

View File

@@ -14,9 +14,9 @@ func TestGetSystemdServiceId(t *testing.T) {
serviceName := "nginx.service" serviceName := "nginx.service"
// Call multiple times and ensure same result // Call multiple times and ensure same result
id1 := makeStableHashId(systemId, serviceName) id1 := MakeStableHashId(systemId, serviceName)
id2 := makeStableHashId(systemId, serviceName) id2 := MakeStableHashId(systemId, serviceName)
id3 := makeStableHashId(systemId, serviceName) id3 := MakeStableHashId(systemId, serviceName)
assert.Equal(t, id1, id2) assert.Equal(t, id1, id2)
assert.Equal(t, id2, id3) assert.Equal(t, id2, id3)
@@ -29,10 +29,10 @@ func TestGetSystemdServiceId(t *testing.T) {
serviceName1 := "nginx.service" serviceName1 := "nginx.service"
serviceName2 := "apache.service" serviceName2 := "apache.service"
id1 := makeStableHashId(systemId1, serviceName1) id1 := MakeStableHashId(systemId1, serviceName1)
id2 := makeStableHashId(systemId2, serviceName1) id2 := MakeStableHashId(systemId2, serviceName1)
id3 := makeStableHashId(systemId1, serviceName2) id3 := MakeStableHashId(systemId1, serviceName2)
id4 := makeStableHashId(systemId2, serviceName2) id4 := MakeStableHashId(systemId2, serviceName2)
// All IDs should be different // All IDs should be different
assert.NotEqual(t, id1, id2) assert.NotEqual(t, id1, id2)
@@ -56,14 +56,14 @@ func TestGetSystemdServiceId(t *testing.T) {
} }
for _, tc := range testCases { for _, tc := range testCases {
id := makeStableHashId(tc.systemId, tc.serviceName) id := MakeStableHashId(tc.systemId, tc.serviceName)
// FNV-32 produces 8 hex characters // FNV-32 produces 8 hex characters
assert.Len(t, id, 8, "ID should be 8 characters for systemId='%s', serviceName='%s'", tc.systemId, tc.serviceName) assert.Len(t, id, 8, "ID should be 8 characters for systemId='%s', serviceName='%s'", tc.systemId, tc.serviceName)
} }
}) })
t.Run("hexadecimal output", func(t *testing.T) { t.Run("hexadecimal output", func(t *testing.T) {
id := makeStableHashId("test-system", "test-service") id := MakeStableHashId("test-system", "test-service")
assert.NotEmpty(t, id) assert.NotEmpty(t, id)
// Should only contain hexadecimal characters // Should only contain hexadecimal characters

View File

@@ -129,7 +129,7 @@ func (sys *System) saveZfsPools(zfsData *zfs.ZfsData) error {
} }
func (sys *System) upsertZfsPoolRecord(app core.App, collection *core.Collection, pool *zfs.PoolDetail) error { func (sys *System) upsertZfsPoolRecord(app core.App, collection *core.Collection, pool *zfs.PoolDetail) error {
recordID := makeStableHashId(sys.Id, pool.Name) recordID := MakeStableHashId(sys.Id, pool.Name)
record, err := app.FindRecordById(collection, recordID) record, err := app.FindRecordById(collection, recordID)
if err != nil { if err != nil {
@@ -171,7 +171,7 @@ func (sys *System) syncZfsPoolHealth(app core.App, pools map[string]*system.ZfsP
if pool == nil { if pool == nil {
continue continue
} }
recordID := makeStableHashId(sys.Id, name) recordID := MakeStableHashId(sys.Id, name)
record, err := app.FindRecordById(collection, recordID) record, err := app.FindRecordById(collection, recordID)
if err != nil { if err != nil {
if !errors.Is(err, sql.ErrNoRows) { if !errors.Is(err, sql.ErrNoRows) {

View File

@@ -135,14 +135,14 @@ func TestSavePartialBackendInventory(t *testing.T) {
{Name: healthyKey, Alloc: 10}, {Name: failedKey, Alloc: 10}, {Name: healthyKey, Alloc: 10}, {Name: failedKey, Alloc: 10},
}} }}
require.NoError(t, sys.saveZfsPools(initial)) require.NoError(t, sys.saveZfsPools(initial))
failedID := makeStableHashId(sys.Id, failedKey) failedID := MakeStableHashId(sys.Id, failedKey)
before, err := app.FindRecordById("zfs_pools", failedID) before, err := app.FindRecordById("zfs_pools", failedID)
require.NoError(t, err) require.NoError(t, err)
partial := &zfs.ZfsData{CompleteBackends: []string{healthy}, Pools: []*zfs.PoolDetail{ partial := &zfs.ZfsData{CompleteBackends: []string{healthy}, Pools: []*zfs.PoolDetail{
{Name: healthyKey, Alloc: 20}, {Name: failedKey, Alloc: 99}, {Name: healthyKey, Alloc: 20}, {Name: failedKey, Alloc: 99},
}} }}
assert.ErrorIs(t, sys.saveZfsPools(partial), errIncompleteZfsData) assert.ErrorIs(t, sys.saveZfsPools(partial), errIncompleteZfsData)
fresh, err := app.FindRecordById("zfs_pools", makeStableHashId(sys.Id, healthyKey)) fresh, err := app.FindRecordById("zfs_pools", MakeStableHashId(sys.Id, healthyKey))
require.NoError(t, err) require.NoError(t, err)
assert.EqualValues(t, 20, fresh.GetInt("alloc")) assert.EqualValues(t, 20, fresh.GetInt("alloc"))
cached, err := app.FindRecordById("zfs_pools", failedID) cached, err := app.FindRecordById("zfs_pools", failedID)
@@ -169,7 +169,7 @@ func TestSyncZfsPoolHealthWritesOnlyTransitions(t *testing.T) {
require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{ require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{
"tank": {Total: 100, Used: 25, Health: "ONLINE"}, "tank": {Total: 100, Used: 25, Health: "ONLINE"},
})) }))
record, err := app.FindRecordById(collection, makeStableHashId(sys.Id, "tank")) record, err := app.FindRecordById(collection, MakeStableHashId(sys.Id, "tank"))
require.NoError(t, err) require.NoError(t, err)
firstUpdated := record.GetDateTime("updated") firstUpdated := record.GetDateTime("updated")
assert.Equal(t, "ONLINE", record.GetString("health")) assert.Equal(t, "ONLINE", record.GetString("health"))
@@ -193,7 +193,7 @@ func TestSyncZfsPoolHealthWritesOnlyTransitions(t *testing.T) {
func TestZfsRawCapacityPersistence(t *testing.T) { func TestZfsRawCapacityPersistence(t *testing.T) {
sys, app := newTestSystemWithHub(t) sys, app := newTestSystemWithHub(t)
require.NoError(t, sys.saveZfsPools(&zfs.ZfsData{Complete: true, Pools: []*zfs.PoolDetail{{Name: "btrfs", Size: 200, Alloc: 10, Raw: true}}})) require.NoError(t, sys.saveZfsPools(&zfs.ZfsData{Complete: true, Pools: []*zfs.PoolDetail{{Name: "btrfs", Size: 200, Alloc: 10, Raw: true}}}))
record, err := app.FindRecordById("zfs_pools", makeStableHashId(sys.Id, "btrfs")) record, err := app.FindRecordById("zfs_pools", MakeStableHashId(sys.Id, "btrfs"))
require.NoError(t, err) require.NoError(t, err)
require.True(t, record.GetBool("raw")) require.True(t, record.GetBool("raw"))
require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{"btrfs": {Total: 1, Used: 0.25}})) require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{"btrfs": {Total: 1, Used: 0.25}}))
@@ -210,7 +210,7 @@ func TestBtrfsDisplayNameKeepsRecordIdentity(t *testing.T) {
key: {DisplayName: "tank", Health: "ONLINE"}, key: {DisplayName: "tank", Health: "ONLINE"},
"tank": {Health: "ONLINE"}, "tank": {Health: "ONLINE"},
})) }))
id := makeStableHashId(sys.Id, key) id := MakeStableHashId(sys.Id, key)
record, err := app.FindRecordById("zfs_pools", id) record, err := app.FindRecordById("zfs_pools", id)
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, "tank", record.GetString("display_name")) assert.Equal(t, "tank", record.GetString("display_name"))
@@ -225,6 +225,6 @@ func TestBtrfsDisplayNameKeepsRecordIdentity(t *testing.T) {
record, err = app.FindRecordById("zfs_pools", id) record, err = app.FindRecordById("zfs_pools", id)
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, "detail name", record.GetString("display_name")) assert.Equal(t, "detail name", record.GetString("display_name"))
_, err = app.FindRecordById("zfs_pools", makeStableHashId(sys.Id, "tank")) _, err = app.FindRecordById("zfs_pools", MakeStableHashId(sys.Id, "tank"))
require.NoError(t, err) require.NoError(t, err)
} }

View File

@@ -18,6 +18,38 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
func TestPauseSystemPreservesAgentVersion(t *testing.T) {
hub, user := tests.GetHubWithUser(t)
defer hub.Cleanup()
record, err := tests.CreateRecord(hub, "systems", map[string]any{
"name": "pause-info-test",
"host": "localhost",
"port": "33914",
"users": []string{user.Id},
})
require.NoError(t, err)
record.Set("info", system.Info{
AgentVersion: "0.20.0",
Cpu: 42.5,
MemPct: 60,
Uptime: 3600,
Services: []uint16{3, 1},
})
require.NoError(t, hub.Save(record))
record.Set("status", "paused")
require.NoError(t, hub.Save(record))
pausedRecord, err := hub.FindRecordById("systems", record.Id)
require.NoError(t, err)
assert.Equal(t, "paused", pausedRecord.GetString("status"))
var info system.Info
require.NoError(t, pausedRecord.UnmarshalJSONField("info", &info))
assert.Equal(t, system.Info{AgentVersion: "0.20.0"}, info)
}
func TestSystemManagerNew(t *testing.T) { func TestSystemManagerNew(t *testing.T) {
hub, err := tests.NewTestHub(t.TempDir()) hub, err := tests.NewTestHub(t.TempDir())
if err != nil { if err != nil {
@@ -133,6 +165,55 @@ func TestSystemManagerNew(t *testing.T) {
}) })
} }
func TestStatusAlertRecoveryAfterPendingTransition(t *testing.T) {
hub, user := tests.GetHubWithUser(t)
defer hub.Cleanup()
userSettings, err := hub.FindFirstRecordByFilter("user_settings", "user={:user}", map[string]any{"user": user.Id})
require.NoError(t, err)
userSettings.Set("settings", map[string]any{
"emails": []string{"test@example.com"},
"webhooks": []string{},
})
require.NoError(t, hub.Save(userSettings))
record, err := tests.CreateRecord(hub, "systems", map[string]any{
"name": "changed-address",
"host": "192.0.2.1",
"port": "33914",
"users": []string{user.Id},
})
require.NoError(t, err)
record.Set("status", "down")
require.NoError(t, hub.Save(record))
alert, err := tests.CreateRecord(hub, "alerts", map[string]any{
"name": "Status",
"system": record.Id,
"user": user.Id,
"min": 1,
"triggered": true,
})
require.NoError(t, err)
initialEmailCount := hub.TestMailer.TotalSend()
// The edit dialog temporarily moves the system through pending. The active
// status alert must remain active until the new connection is confirmed.
record.Set("host", "192.0.2.2")
record.Set("status", "pending")
require.NoError(t, hub.Save(record))
alert, err = hub.FindRecordById("alerts", alert.Id)
require.NoError(t, err)
assert.True(t, alert.GetBool("triggered"), "pending connection update should preserve the active status alert")
record.Set("status", "up")
require.NoError(t, hub.Save(record))
alert, err = hub.FindRecordById("alerts", alert.Id)
require.NoError(t, err)
assert.False(t, alert.GetBool("triggered"), "pending -> up should resolve the active status alert")
assert.Equal(t, initialEmailCount+1, hub.TestMailer.TotalSend(), "recovery should send an up notification")
}
func testOld(t *testing.T, hub *tests.TestHub) { func testOld(t *testing.T, hub *tests.TestHub) {
user, err := tests.CreateUser(hub, "test@testy.com", "testtesttest") user, err := tests.CreateUser(hub, "test@testy.com", "testtesttest")
require.NoError(t, err) require.NoError(t, err)

View File

@@ -0,0 +1,82 @@
//go:build testing
package systems
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/blang/semver"
"github.com/fxamacker/cbor/v2"
"github.com/henrygd/beszel/internal/common"
esystem "github.com/henrygd/beszel/internal/entities/system"
"github.com/henrygd/beszel/internal/hub/ws"
"github.com/lxzan/gws"
"github.com/stretchr/testify/require"
)
// slowDataClient answers GetData only after release is closed, simulating an
// agent whose collection outlasts the hub's request timeout.
type slowDataClient struct {
gws.BuiltinEventHandler
release chan struct{}
}
func (c *slowDataClient) OnMessage(conn *gws.Conn, message *gws.Message) {
defer message.Close()
var req common.HubRequest[cbor.RawMessage]
if err := cbor.Unmarshal(message.Bytes(), &req); err != nil || req.Action != common.GetData {
return
}
<-c.release
response, _ := cbor.Marshal(common.AgentResponse{Id: req.Id, SystemData: &esystem.CombinedData{}})
_ = conn.WriteMessage(gws.OpcodeBinary, response)
}
func TestFetchDataTimeoutKeepsWebSocketOpen(t *testing.T) {
originalTimeout := wsDataRequestTimeout
wsDataRequestTimeout = 50 * time.Millisecond
t.Cleanup(func() { wsDataRequestTimeout = originalTimeout })
connections := make(chan *ws.WsConn, 1)
upgrader := gws.NewUpgrader(&monitorSyncServer{}, nil)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
conn, err := upgrader.Upgrade(w, r)
if err != nil {
t.Error(err)
return
}
wsConn := ws.NewWsConnection(conn, semver.MustParse("0.20.0"))
conn.Session().Store("wsConn", wsConn)
connections <- wsConn
conn.ReadLoop()
}))
t.Cleanup(server.Close)
client := &slowDataClient{release: make(chan struct{})}
conn, _, err := gws.NewClient(client, &gws.ClientOption{Addr: "ws" + strings.TrimPrefix(server.URL, "http")})
require.NoError(t, err)
t.Cleanup(func() { _ = conn.NetConn().Close() })
go conn.ReadLoop()
var sys *System
select {
case wsConn := <-connections:
sys = &System{WsConn: wsConn}
case <-time.After(3 * time.Second):
t.Fatal("websocket connection was not established")
}
_, err = sys.fetchDataFromAgent(common.DataRequestOptions{})
require.ErrorIs(t, err, context.DeadlineExceeded)
require.True(t, sys.WsConn.IsConnected(), "a slow collection must not close the connection")
// The late response is discarded and the next request still succeeds.
close(client.release)
_, err = sys.fetchDataFromAgent(common.DataRequestOptions{})
require.NoError(t, err)
}

View File

@@ -0,0 +1,127 @@
//go:build testing
package hub
import (
"net/netip"
"os"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestParseTrustedProxies(t *testing.T) {
testCases := []struct {
name string
value string
prefixes []string
restricted bool
}{
{
name: "empty",
value: "",
restricted: false,
},
{
name: "blank",
value: " , ",
prefixes: nil,
restricted: true,
},
{
name: "single addresses become host prefixes",
value: "10.0.0.5, 2001:db8::1",
prefixes: []string{"10.0.0.5/32", "2001:db8::1/128"},
restricted: true,
},
{
name: "cidrs are masked",
value: "172.16.5.9/12,fd00::1/64",
prefixes: []string{"172.16.0.0/12", "fd00::/64"},
restricted: true,
},
{
name: "ipv4-mapped entries become ipv4",
value: "::ffff:10.0.0.5, ::ffff:10.0.0.0/104",
prefixes: []string{"10.0.0.5/32", "10.0.0.0/8"},
restricted: true,
},
{
name: "invalid entries are skipped, valid ones kept",
value: "proxy.internal, 10.0.0.0/8, 300.1.1.1, ::ffff:0.0.0.0/64",
prefixes: []string{"10.0.0.0/8"},
restricted: true,
},
{
name: "only invalid entries trust nobody",
value: "proxy.internal",
prefixes: nil,
restricted: true,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
t.Setenv("TRUSTED_PROXY_IPS", tc.value)
prefixes, restricted := parseTrustedProxies()
assert.Equal(t, tc.restricted, restricted)
var got []string
for _, p := range prefixes {
got = append(got, p.String())
}
assert.Equal(t, tc.prefixes, got)
})
}
t.Run("unset", func(t *testing.T) {
t.Setenv("TRUSTED_PROXY_IPS", "")
os.Unsetenv("TRUSTED_PROXY_IPS")
prefixes, restricted := parseTrustedProxies()
assert.False(t, restricted)
assert.Nil(t, prefixes)
})
t.Run("prefixed env var takes precedence", func(t *testing.T) {
t.Setenv("TRUSTED_PROXY_IPS", "10.0.0.0/8")
t.Setenv("BESZEL_HUB_TRUSTED_PROXY_IPS", "192.168.0.0/16")
prefixes, restricted := parseTrustedProxies()
assert.True(t, restricted)
require.Len(t, prefixes, 1)
assert.Equal(t, "192.168.0.0/16", prefixes[0].String())
})
}
func TestIsTrustedProxy(t *testing.T) {
prefixes := []netip.Prefix{
netip.MustParsePrefix("10.0.0.0/8"),
netip.MustParsePrefix("2001:db8::/32"),
netip.MustParsePrefix("fe80::/10"),
}
testCases := []struct {
name string
remoteAddr string
trusted bool
}{
{"ipv4 in prefix", "10.20.30.40:51234", true},
{"ipv4 outside prefix", "11.0.0.1:51234", false},
{"ipv6 in prefix", "[2001:db8:1::2]:443", true},
{"ipv6 outside prefix", "[2001:db9::1]:443", false},
{"ipv4-mapped ipv6 matches ipv4 prefix", "[::ffff:10.1.2.3]:80", true},
{"zone is ignored", "[fe80::1%eth0]:80", true},
{"no port", "10.1.2.3", true},
{"empty", "", false},
{"garbage", "not-an-address:80", false},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
assert.Equal(t, tc.trusted, isTrustedProxy(prefixes, tc.remoteAddr))
})
}
t.Run("empty allowlist trusts nobody", func(t *testing.T) {
assert.False(t, isTrustedProxy(nil, "10.0.0.1:1"))
})
}

View File

@@ -58,36 +58,38 @@ func Update(cmd *cobra.Command, _ []string) {
func restartService() { func restartService() {
// Check if we're running as a service by looking for systemd // Check if we're running as a service by looking for systemd
if _, err := exec.LookPath("systemctl"); err == nil { if _, err := exec.LookPath("systemctl"); err == nil {
// Check if beszel service exists and is active // install-hub.sh names the unit beszel-hub.service. beszel.service is
cmd := exec.Command("systemctl", "is-active", "beszel.service") // kept as a fallback for hand written units.
if err := cmd.Run(); err == nil { for _, unit := range []string{"beszel-hub.service", "beszel.service"} {
ghupdate.ColorPrint(ghupdate.ColorYellow, "Restarting beszel service...") if err := exec.Command("systemctl", "is-active", unit).Run(); err != nil {
restartCmd := exec.Command("systemctl", "restart", "beszel.service") continue
if err := restartCmd.Run(); err != nil {
ghupdate.ColorPrintf(ghupdate.ColorYellow, "Warning: Failed to restart service: %v\n", err)
ghupdate.ColorPrint(ghupdate.ColorYellow, "Please restart the service manually: sudo systemctl restart beszel")
} else {
ghupdate.ColorPrint(ghupdate.ColorGreen, "Service restarted successfully")
} }
reportRestart(exec.Command("systemctl", "restart", unit), "sudo systemctl restart "+unit)
return return
} }
} }
// Check for OpenRC (Alpine Linux) // Check for OpenRC (Alpine Linux)
if _, err := exec.LookPath("rc-service"); err == nil { if _, err := exec.LookPath("rc-service"); err == nil {
cmd := exec.Command("rc-service", "beszel", "status") for _, service := range []string{"beszel-hub", "beszel"} {
if err := cmd.Run(); err == nil { if err := exec.Command("rc-service", service, "status").Run(); err != nil {
ghupdate.ColorPrint(ghupdate.ColorYellow, "Restarting beszel service...") continue
restartCmd := exec.Command("rc-service", "beszel", "restart")
if err := restartCmd.Run(); err != nil {
ghupdate.ColorPrintf(ghupdate.ColorYellow, "Warning: Failed to restart service: %v\n", err)
ghupdate.ColorPrint(ghupdate.ColorYellow, "Please restart the service manually: sudo rc-service beszel restart")
} else {
ghupdate.ColorPrint(ghupdate.ColorGreen, "Service restarted successfully")
} }
reportRestart(exec.Command("rc-service", service, "restart"), "sudo rc-service "+service+" restart")
return return
} }
} }
ghupdate.ColorPrint(ghupdate.ColorYellow, "Service restart not attempted. If running as a service, restart manually.") ghupdate.ColorPrint(ghupdate.ColorYellow, "Service restart not attempted. If running as a service, restart manually.")
} }
// reportRestart runs the restart command and prints the result.
func reportRestart(cmd *exec.Cmd, manualCommand string) {
ghupdate.ColorPrint(ghupdate.ColorYellow, "Restarting beszel service...")
if err := cmd.Run(); err != nil {
ghupdate.ColorPrintf(ghupdate.ColorYellow, "Warning: Failed to restart service: %v\n", err)
ghupdate.ColorPrint(ghupdate.ColorYellow, "Please restart the service manually: "+manualCommand)
} else {
ghupdate.ColorPrint(ghupdate.ColorGreen, "Service restarted successfully")
}
}

View File

@@ -1,7 +1,11 @@
// Package utils provides utility functions for the hub. // Package utils provides utility functions for the hub.
package utils package utils
import "os" import (
"os"
"github.com/pocketbase/pocketbase/core"
)
// GetEnv retrieves an environment variable with a "BESZEL_HUB_" prefix, or falls back to the unprefixed key. // GetEnv retrieves an environment variable with a "BESZEL_HUB_" prefix, or falls back to the unprefixed key.
func GetEnv(key string) (value string, exists bool) { func GetEnv(key string) (value string, exists bool) {
@@ -10,3 +14,26 @@ func GetEnv(key string) (value string, exists bool) {
} }
return os.LookupEnv(key) return os.LookupEnv(key)
} }
// realtimeActiveForCollection checks if there are active WebSocket subscriptions for the given collection.
func RealtimeActiveForCollection(app core.App, collectionName string, validateFn func(filterQuery string) bool) bool {
broker := app.SubscriptionsBroker()
if broker.TotalClients() == 0 {
return false
}
for _, client := range broker.Clients() {
subs := client.Subscriptions(collectionName)
if len(subs) > 0 {
if validateFn == nil {
return true
}
for k := range subs {
filter := subs[k].Query["filter"]
if validateFn(filter) {
return true
}
}
}
}
return false
}

Some files were not shown because too many files have changed in this diff Show More