feat: add TLS certificate expiry check to HTTPS network monitors (#2401)

Co-authored-by: henrygd <hank@henrygd.me>

agent/network_monitor.go

@@ -15,6 +15,7 @@ type MonitorManager struct {
 	mu          sync.RWMutex
 	monitors    map[string]*monitorTask // keyed by monitor ID
 	probe       monitorProbe
+	certCheck   certChecker
 	resumeGuard monitorResumeGuard
 }
 
@@ -23,7 +24,7 @@ func newMonitorManager() *MonitorManager {
 }
 
 func newMonitorManagerWithProbe(probe monitorProbe) *MonitorManager {
-	return &MonitorManager{monitors: make(map[string]*monitorTask), probe: probe}
+	return &MonitorManager{monitors: make(map[string]*monitorTask), probe: probe, certCheck: checkCert}
 }
 
 // SyncMonitors replaces all monitor tasks with the given configs.
@@ -107,7 +108,7 @@ func (pm *MonitorManager) UpsertMonitor(config monitor.Config, runNow bool) (*mo
 		if !runNow {
 			return nil, nil
 		}
-		return task.runProbe(pm.probe), nil
+		return pm.runNow(task), nil
 	}
 	if exists {
 		task.cancel()
@@ -119,7 +120,7 @@ func (pm *MonitorManager) UpsertMonitor(config monitor.Config, runNow bool) (*mo
 	pm.mu.Unlock()
 
 	if runNow {
-		result := task.runProbe(pm.probe)
+		result := pm.runNow(task)
 		pm.startMonitor(task)
 		return result, nil
 	}
@@ -127,6 +128,19 @@ func (pm *MonitorManager) UpsertMonitor(config monitor.Config, runNow bool) (*mo
 	return nil, nil
 }
 
+// runNow runs a probe and any due certificate check concurrently, so the
+// response fits within the hub's single probe timeout budget.
+func (pm *MonitorManager) runNow(task *monitorTask) *monitor.Result {
+	var wg sync.WaitGroup
+	wg.Go(func() { task.refreshCert(pm.certCheck) })
+	result := task.runProbe(pm.probe)
+	wg.Wait()
+	if result != nil {
+		result.Cert = task.certInfo()
+	}
+	return result
+}
+
 // DeleteMonitor stops and removes a single monitor task.
 func (pm *MonitorManager) DeleteMonitor(id string) {
 	if id == "" {
@@ -158,6 +172,11 @@ func (pm *MonitorManager) GetResults(durationMs uint16) map[string]monitor.Resul
 		if !ok {
 			continue
 		}
+		// Only the default interval updates monitor records on the hub, so
+		// realtime requests must not consume the unsent certificate.
+		if durationMs == defaultDataCacheTimeMs {
+			result.Cert = task.takeUnsentCert()
+		}
 		results[task.config.ID] = result
 	}
 

agent/network_monitor_cert.go

@@ -0,0 +1,74 @@
+package agent
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"net"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/henrygd/beszel/internal/entities/monitor"
+)
+
+const (
+	certCheckInterval      = 24 * time.Hour
+	certCheckRetryInterval = time.Hour
+)
+
+// certChecker fetches the leaf certificate for an HTTPS target.
+type certChecker func(context.Context, string) (monitor.CertInfo, error)
+
+// certCheckEnabled reports whether a monitor's certificate is checked, which is
+// the case for every HTTP monitor with an https target.
+func certCheckEnabled(config monitor.Config) bool {
+	return config.Protocol == "http" && len(config.Target) > 8 && strings.EqualFold(config.Target[:8], "https://")
+}
+
+// checkCert reads the leaf certificate presented by an HTTPS target. The chain is
+// not verified, so expired or self-signed certificates are still reported.
+func checkCert(ctx context.Context, target string) (monitor.CertInfo, error) {
+	address, host, err := certAddress(target)
+	if err != nil {
+		return monitor.CertInfo{}, err
+	}
+	ctx, cancel := context.WithTimeout(ctx, monitor.MaxProbeTimeout)
+	defer cancel()
+	dialer := tls.Dialer{Config: &tls.Config{ServerName: host, InsecureSkipVerify: true}}
+	conn, err := dialer.DialContext(ctx, "tcp", address)
+	if err != nil {
+		return monitor.CertInfo{}, err
+	}
+	defer conn.Close()
+	certs := conn.(*tls.Conn).ConnectionState().PeerCertificates
+	if len(certs) == 0 {
+		return monitor.CertInfo{}, errors.New("no peer certificates")
+	}
+	leaf := certs[0]
+	return monitor.CertInfo{
+		Expires: leaf.NotAfter.UnixMilli(),
+		Issuer:  leaf.Issuer.CommonName,
+	}, nil
+}
+
+// certAddress returns the dial address and server name for an HTTPS URL.
+func certAddress(target string) (address, host string, err error) {
+	u, err := url.Parse(target)
+	if err != nil {
+		return "", "", err
+	}
+	if !strings.EqualFold(u.Scheme, "https") {
+		return "", "", fmt.Errorf("certificate check requires an https target: %s", target)
+	}
+	host = u.Hostname()
+	if host == "" {
+		return "", "", fmt.Errorf("missing host in target: %s", target)
+	}
+	port := u.Port()
+	if port == "" {
+		port = "443"
+	}
+	return net.JoinHostPort(host, port), host, nil
+}

agent/network_monitor_cert_test.go

@@ -0,0 +1,184 @@
+//go:build testing
+
+package agent
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"testing/synctest"
+	"time"
+
+	"github.com/henrygd/beszel/internal/entities/monitor"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCheckCertReadsUnverifiedLeaf(t *testing.T) {
+	server := httptest.NewTLSServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
+	defer server.Close()
+
+	// httptest uses a self-signed certificate, which must still be reported.
+	info, err := checkCert(context.Background(), server.URL)
+	require.NoError(t, err)
+	leaf := server.Certificate()
+	assert.Equal(t, leaf.NotAfter.UnixMilli(), info.Expires)
+	assert.Equal(t, leaf.Issuer.CommonName, info.Issuer)
+}
+
+func TestCertAddress(t *testing.T) {
+	tests := []struct {
+		target, address, host string
+		wantErr               bool
+	}{
+		{target: "https://example.com", address: "example.com:443", host: "example.com"},
+		{target: "https://example.com:8443/path?q=1", address: "example.com:8443", host: "example.com"},
+		{target: "HTTPS://[::1]:9443", address: "[::1]:9443", host: "::1"},
+		{target: "http://example.com", wantErr: true},
+		{target: "https://", wantErr: true},
+	}
+	for _, tt := range tests {
+		address, host, err := certAddress(tt.target)
+		if tt.wantErr {
+			assert.Error(t, err, tt.target)
+			continue
+		}
+		require.NoError(t, err, tt.target)
+		assert.Equal(t, tt.address, address)
+		assert.Equal(t, tt.host, host)
+	}
+}
+
+func TestRefreshCertCadence(t *testing.T) {
+	synctest.Test(t, func(t *testing.T) {
+		task := newMonitorTask(monitor.Config{ID: "test", Target: "https://example.test", Protocol: "http"})
+		defer task.cancel()
+		var calls int
+		var fail error
+		// Far enough out that the regular interval applies for the whole test.
+		expires := time.Now().Add(365 * 24 * time.Hour).UnixMilli()
+		check := func(context.Context, string) (monitor.CertInfo, error) {
+			calls++
+			if fail != nil {
+				return monitor.CertInfo{}, fail
+			}
+			return monitor.CertInfo{Expires: expires + int64(calls)}, nil
+		}
+
+		task.refreshCert(check)
+		require.NotNil(t, task.certInfo())
+		assert.Equal(t, expires+1, task.certInfo().Expires)
+
+		// Not due again until the check interval passes.
+		time.Sleep(certCheckInterval - time.Second)
+		task.refreshCert(check)
+		assert.Equal(t, 1, calls)
+		time.Sleep(time.Second)
+		task.refreshCert(check)
+		assert.Equal(t, 2, calls)
+
+		// Failures keep the last known certificate and retry sooner.
+		fail = errors.New("connection refused")
+		time.Sleep(certCheckInterval)
+		task.refreshCert(check)
+		assert.Equal(t, 3, calls)
+		assert.Equal(t, expires+2, task.certInfo().Expires)
+		time.Sleep(certCheckRetryInterval)
+		fail = nil
+		task.refreshCert(check)
+		assert.Equal(t, 4, calls)
+		assert.Equal(t, expires+4, task.certInfo().Expires)
+	})
+}
+
+func TestRefreshCertRetriesSoonerNearExpiry(t *testing.T) {
+	for _, tc := range []struct {
+		name     string
+		expires  time.Duration // relative to the check
+		interval time.Duration
+	}{
+		{"expired", -time.Hour, certCheckRetryInterval},
+		{"expires before next regular check", certCheckInterval - time.Minute, certCheckRetryInterval},
+		{"expires after next regular check", certCheckInterval + time.Minute, certCheckInterval},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			synctest.Test(t, func(t *testing.T) {
+				task := newMonitorTask(monitor.Config{ID: "test", Target: "https://example.test", Protocol: "http"})
+				defer task.cancel()
+				var calls int
+				check := func(context.Context, string) (monitor.CertInfo, error) {
+					calls++
+					return monitor.CertInfo{Expires: time.Now().Add(tc.expires).UnixMilli()}, nil
+				}
+				task.refreshCert(check)
+				time.Sleep(tc.interval - time.Second)
+				task.refreshCert(check)
+				assert.Equal(t, 1, calls)
+				time.Sleep(time.Second)
+				task.refreshCert(check)
+				assert.Equal(t, 2, calls)
+			})
+		})
+	}
+}
+
+func TestCertCheckEnabled(t *testing.T) {
+	tests := []struct {
+		protocol, target string
+		want             bool
+	}{
+		{"http", "https://example.com", true},
+		{"http", "HTTPS://example.com:8443/path", true},
+		{"http", "http://example.com", false},
+		{"http", "https://", false},
+		{"tcp", "https://example.com", false},
+		{"icmp", "example.com", false},
+	}
+	for _, tt := range tests {
+		assert.Equal(t, tt.want, certCheckEnabled(monitor.Config{Protocol: tt.protocol, Target: tt.target}), tt.protocol+" "+tt.target)
+	}
+}
+
+func TestRefreshCertSkipsNonHTTPS(t *testing.T) {
+	task := newMonitorTask(monitor.Config{ID: "test", Target: "http://example.test", Protocol: "http"})
+	defer task.cancel()
+	task.refreshCert(func(context.Context, string) (monitor.CertInfo, error) {
+		t.Fatal("certificate check must not run for non-https targets")
+		return monitor.CertInfo{}, nil
+	})
+	assert.Nil(t, task.certInfo())
+}
+
+func TestUpsertMonitorRunNowIncludesCert(t *testing.T) {
+	server := httptest.NewTLSServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
+	defer server.Close()
+
+	pm := newMonitorManagerWithProbe(func(context.Context, monitor.Config) (int64, error) { return 100, nil })
+	defer pm.Stop()
+	config := monitor.Config{ID: "cert", Target: server.URL, Protocol: "http", Interval: 60}
+	result, err := pm.UpsertMonitor(config, true)
+	require.NoError(t, err)
+	require.NotNil(t, result)
+	require.NotNil(t, result.Cert)
+	assert.Equal(t, server.Certificate().NotAfter.UnixMilli(), result.Cert.Expires)
+
+	// Realtime results never carry the certificate, and the default interval
+	// sends it only once per check.
+	assert.Nil(t, pm.GetResults(1000)["cert"].Cert)
+	results := pm.GetResults(defaultDataCacheTimeMs)
+	require.NotNil(t, results["cert"].Cert)
+	assert.Equal(t, result.Cert.Expires, results["cert"].Cert.Expires)
+	assert.Nil(t, pm.GetResults(defaultDataCacheTimeMs)["cert"].Cert)
+
+	// Changing the interval keeps the known certificate without resending it.
+	config.Interval = 30
+	_, err = pm.UpsertMonitor(config, false)
+	require.NoError(t, err)
+	pm.mu.RLock()
+	task := pm.monitors["cert"]
+	pm.mu.RUnlock()
+	assert.NotNil(t, task.certInfo())
+	assert.Nil(t, pm.GetResults(defaultDataCacheTimeMs)["cert"].Cert)
+}

agent/network_monitor_schedule.go

@@ -14,9 +14,12 @@ func (pm *MonitorManager) startMonitor(task *monitorTask) {
 	}
 	delay := getStagger(interval.Milliseconds())
 	slog.Debug("starting monitor task", "target", task.config.Target, "delay", delay, "interval", interval)
+	// Certificate checks piggyback on probe ticks, so they run at most once per
+	// probe interval after they become due.
 	go runMonitorSchedule(task.ctx, interval, delay, func() {
 		if _, allowed := task.resumeGuard.snapshot(); allowed {
 			task.runProbe(pm.probe)
+			task.refreshCert(pm.certCheck)
 		}
 	})
 }

agent/network_monitor_task.go

@@ -21,6 +21,12 @@ type monitorTask struct {
 	runMu          sync.Mutex
 	inflight       *monitorRun
 	lastFailureLog int64 // Unix nanoseconds
+
+	certMu        sync.Mutex
+	cert          *monitor.CertInfo
+	certUnsent    bool // cert has not been included in a stats result yet
+	certChecking  bool
+	nextCertCheck time.Time
 }
 
 type monitorRun struct {
@@ -45,6 +51,11 @@ func newMonitorTaskFromExisting(config monitor.Config, existing *monitorTask) *m
 	task := newMonitorTask(config)
 	if existing != nil {
 		task.history = existing.history.clone()
+		// Keep the last known certificate, but check again soon for the new config.
+		// The hub already stores it, so it is not marked unsent.
+		if config.Target == existing.config.Target {
+			task.cert = existing.certInfo()
+		}
 	}
 	return task
 }
@@ -107,6 +118,70 @@ func (task *monitorTask) runProbe(probe monitorProbe) *monitor.Result {
 	return copyMonitorResult(run.result)
 }
 
+// refreshCert checks the certificate of an HTTPS target when due. A failed
+// check keeps the last known certificate and retries sooner, as does a
+// certificate that expires before the next regular check, so renewals show up
+// quickly. Concurrent callers skip rather than wait, and no lock is held during
+// network I/O.
+func (task *monitorTask) refreshCert(check certChecker) {
+	if check == nil || !certCheckEnabled(task.config) {
+		return
+	}
+	task.certMu.Lock()
+	if task.certChecking || time.Now().Before(task.nextCertCheck) {
+		task.certMu.Unlock()
+		return
+	}
+	task.certChecking = true
+	task.certMu.Unlock()
+
+	info, err := check(task.ctx, task.config.Target)
+
+	task.certMu.Lock()
+	defer task.certMu.Unlock()
+	task.certChecking = false
+	if task.ctx.Err() != nil {
+		return
+	}
+	if err != nil {
+		task.nextCertCheck = time.Now().Add(certCheckRetryInterval)
+		slog.Warn("certificate check failed", "err", err, "target", task.config.Target)
+		return
+	}
+	task.cert = &info
+	task.certUnsent = true
+	now := time.Now()
+	interval := certCheckInterval
+	if time.UnixMilli(info.Expires).Before(now.Add(certCheckInterval)) {
+		interval = certCheckRetryInterval
+	}
+	task.nextCertCheck = now.Add(interval)
+}
+
+// certInfo returns a copy of the latest certificate info, or nil if unknown.
+func (task *monitorTask) certInfo() *monitor.CertInfo {
+	task.certMu.Lock()
+	defer task.certMu.Unlock()
+	if task.cert == nil {
+		return nil
+	}
+	cert := *task.cert
+	return &cert
+}
+
+// takeUnsentCert returns the latest certificate info once after each successful
+// check, so unchanged info is not resent with every stats result.
+func (task *monitorTask) takeUnsentCert() *monitor.CertInfo {
+	task.certMu.Lock()
+	defer task.certMu.Unlock()
+	if !task.certUnsent {
+		return nil
+	}
+	task.certUnsent = false
+	cert := *task.cert
+	return &cert
+}
+
 func copyMonitorResult(result *monitor.Result) *monitor.Result {
 	if result == nil {
 		return nil