0.0.1

subtract mem cache from container stats

.github/workflows/docker-images.yml

@@ -30,7 +30,7 @@ jobs:
         uses: oven-sh/setup-bun@v2
 
       - name: Install dependencies
-        run: bun install --cwd ./hub/site
+        run: bun install --no-save --cwd ./hub/site
 
       - name: Build site
         run: bun run --cwd ./hub/site build

.github/workflows/goreleaser-action.yml

@@ -21,7 +21,7 @@ jobs:
         uses: oven-sh/setup-bun@v2
 
       - name: Install dependencies
-        run: bun install --cwd ./hub/site
+        run: bun install --no-save --cwd ./hub/site
 
       - name: Build site
         run: bun run --cwd ./hub/site build

.gitignore

@@ -6,4 +6,5 @@ temp
 beszel
 beszel-agent
 beszel_data
+beszel_data*
 dist

agent/docker-compose.yml

@@ -5,8 +5,8 @@ services:
     restart: unless-stopped
     network_mode: host
     volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
+      - /var/run/docker.sock:/var/run/docker.sock:ro
     environment:
-      - PORT=45876
+      PORT: 45876
-      - KEY="ssh-ed25519 YOUR_PUBLIC_KEY"
+      KEY: 'ssh-ed25519 YOUR_PUBLIC_KEY'
-      # - FILESYSTEM=/dev/sda1 # set to the correct filesystem for disk I/O stats
+      # FILESYSTEM: /dev/sda1 # set to the correct filesystem for disk I/O stats

agent/main.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"log"
@@ -23,11 +22,21 @@ import (
 	psutilNet "github.com/shirou/gopsutil/v4/net"
 )
 
-var Version = "0.0.1-alpha.3"
+var Version = "0.0.1"
 
 var containerCpuMap = make(map[string][2]uint64)
 var containerCpuMutex = &sync.Mutex{}
 
+var sem = make(chan struct{}, 15)
+
+func acquireSemaphore() {
+	sem <- struct{}{}
+}
+
+func releaseSemaphore() {
+	<-sem
+}
+
 var diskIoStats = DiskIoStats{
 	Read:       0,
 	Write:      0,
@@ -44,59 +53,20 @@ var netIoStats = NetIoStats{
 
 // client for docker engine api
 var client = &http.Client{
-	Timeout: time.Second * 5,
+	Timeout: time.Second,
 	Transport: &http.Transport{
 		Dial: func(proto, addr string) (net.Conn, error) {
 			return net.Dial("unix", "/var/run/docker.sock")
 		},
 		ForceAttemptHTTP2:   false,
-		MaxIdleConns:        100,
-		MaxIdleConnsPerHost: 100,
 		IdleConnTimeout:     90 * time.Second,
+		DisableCompression:  true,
+		MaxIdleConnsPerHost: 50,
 		DisableKeepAlives:   false,
 	},
 }
 
-type SystemData struct {
+func getSystemStats() (*SystemInfo, *SystemStats) {
-	Stats      SystemStats      `json:"stats"`
-	Info       SystemInfo       `json:"info"`
-	Containers []ContainerStats `json:"container"`
-}
-
-type SystemInfo struct {
-	Cores    int    `json:"c"`
-	Threads  int    `json:"t"`
-	CpuModel string `json:"m"`
-	// Os       string  `json:"o"`
-	Uptime  uint64  `json:"u"`
-	Cpu     float64 `json:"cpu"`
-	MemPct  float64 `json:"mp"`
-	DiskPct float64 `json:"dp"`
-}
-
-type SystemStats struct {
-	Cpu          float64 `json:"cpu"`
-	Mem          float64 `json:"m"`
-	MemUsed      float64 `json:"mu"`
-	MemPct       float64 `json:"mp"`
-	MemBuffCache float64 `json:"mb"`
-	Disk         float64 `json:"d"`
-	DiskUsed     float64 `json:"du"`
-	DiskPct      float64 `json:"dp"`
-	DiskRead     float64 `json:"dr"`
-	DiskWrite    float64 `json:"dw"`
-	NetworkSent  float64 `json:"ns"`
-	NetworkRecv  float64 `json:"nr"`
-}
-
-type ContainerStats struct {
-	Name string  `json:"n"`
-	Cpu  float64 `json:"c"`
-	Mem  float64 `json:"m"`
-	// MemPct float64 `json:"mp"`
-}
-
-func getSystemStats() (SystemInfo, SystemStats) {
 	c, _ := cpu.Percent(0, false)
 	v, _ := mem.VirtualMemory()
 	d, _ := disk.Usage("/")
@@ -105,7 +75,7 @@ func getSystemStats() (SystemInfo, SystemStats) {
 	memPct := twoDecimals(v.UsedPercent)
 	diskPct := twoDecimals(d.UsedPercent)
 
-	systemStats := SystemStats{
+	systemStats := &SystemStats{
 		Cpu:          cpuPct,
 		Mem:          bytesToGigabytes(v.Total),
 		MemUsed:      bytesToGigabytes(v.Used),
@@ -116,7 +86,7 @@ func getSystemStats() (SystemInfo, SystemStats) {
 		DiskPct:      diskPct,
 	}
 
-	systemInfo := SystemInfo{
+	systemInfo := &SystemInfo{
 		Cpu:     cpuPct,
 		MemPct:  memPct,
 		DiskPct: diskPct,
@@ -182,54 +152,63 @@ func getSystemStats() (SystemInfo, SystemStats) {
 
 }
 
-func getDockerStats() ([]ContainerStats, error) {
+func getDockerStats() ([]*ContainerStats, error) {
 	resp, err := client.Get("http://localhost/containers/json")
 	if err != nil {
-		return []ContainerStats{}, err
+		return []*ContainerStats{}, err
 	}
 	defer resp.Body.Close()
 
-	var containers []Container
+	var containers []*Container
 	if err := json.NewDecoder(resp.Body).Decode(&containers); err != nil {
 		panic(err)
 	}
 
+	containerStats := make([]*ContainerStats, 0, len(containers))
+
+	// store valid ids to clean up old container ids from map
+	validIds := make(map[string]struct{}, len(containers))
+
 	var wg sync.WaitGroup
-	var containerStats []ContainerStats
 
 	for _, ctr := range containers {
+		ctr.IdShort = ctr.ID[:12]
+		validIds[ctr.IdShort] = struct{}{}
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
 			cstats, err := getContainerStats(ctr)
 			if err != nil {
-				log.Printf("Error getting container stats: %+v\n", err)
+				// retry once
-				return
+				cstats, err = getContainerStats(ctr)
+				if err != nil {
+					log.Printf("Error getting container stats: %+v\n", err)
+					return
+				}
 			}
 			containerStats = append(containerStats, cstats)
 		}()
 	}
 
-	// clean up old containers from map
+	wg.Wait()
-	validNames := make(map[string]struct{}, len(containers))
+
-	for _, ctr := range containers {
+	for id := range containerCpuMap {
-		validNames[ctr.Names[0][1:]] = struct{}{}
+		if _, exists := validIds[id]; !exists {
-	}
+			// log.Printf("Removing container cpu map entry: %+v\n", id)
-	for name := range containerCpuMap {
+			delete(containerCpuMap, id)
-		if _, exists := validNames[name]; !exists {
-			delete(containerCpuMap, name)
 		}
 	}
 
-	wg.Wait()
-
 	return containerStats, nil
 }
 
-func getContainerStats(ctr Container) (ContainerStats, error) {
+func getContainerStats(ctr *Container) (*ContainerStats, error) {
-	resp, err := client.Get("http://localhost/containers/" + ctr.ID + "/stats?stream=0&one-shot=1")
+	// use semaphore to limit concurrency
+	acquireSemaphore()
+	defer releaseSemaphore()
+	resp, err := client.Get("http://localhost/containers/" + ctr.IdShort + "/stats?stream=0&one-shot=1")
 	if err != nil {
-		return ContainerStats{}, err
+		return &ContainerStats{}, err
 	}
 	defer resp.Body.Close()
 
@@ -240,26 +219,30 @@ func getContainerStats(ctr Container) (ContainerStats, error) {
 
 	name := ctr.Names[0][1:]
 
-	// memory
+	// memory (https://docs.docker.com/reference/cli/docker/container/stats/)
-	usedMemory := statsJson.MemoryStats.Usage - statsJson.MemoryStats.Cache
+	memCache := statsJson.MemoryStats.Stats["inactive_file"]
+	if memCache == 0 {
+		memCache = statsJson.MemoryStats.Stats["cache"]
+	}
+	usedMemory := statsJson.MemoryStats.Usage - memCache
 	// pctMemory := float64(usedMemory) / float64(statsJson.MemoryStats.Limit) * 100
 
 	// cpu
 	// add default values to containerCpu if it doesn't exist
 	containerCpuMutex.Lock()
 	defer containerCpuMutex.Unlock()
-	if _, ok := containerCpuMap[name]; !ok {
+	if _, ok := containerCpuMap[ctr.IdShort]; !ok {
-		containerCpuMap[name] = [2]uint64{0, 0}
+		containerCpuMap[ctr.IdShort] = [2]uint64{0, 0}
 	}
-	cpuDelta := statsJson.CPUStats.CPUUsage.TotalUsage - containerCpuMap[name][0]
+	cpuDelta := statsJson.CPUStats.CPUUsage.TotalUsage - containerCpuMap[ctr.IdShort][0]
-	systemDelta := statsJson.CPUStats.SystemUsage - containerCpuMap[name][1]
+	systemDelta := statsJson.CPUStats.SystemUsage - containerCpuMap[ctr.IdShort][1]
 	cpuPct := float64(cpuDelta) / float64(systemDelta) * 100
 	if cpuPct > 100 {
-		return ContainerStats{}, errors.New("cpu pct is greater than 100")
+		return &ContainerStats{}, fmt.Errorf("%s cpu pct greater than 100: %+v", name, cpuPct)
 	}
-	containerCpuMap[name] = [2]uint64{statsJson.CPUStats.CPUUsage.TotalUsage, statsJson.CPUStats.SystemUsage}
+	containerCpuMap[ctr.IdShort] = [2]uint64{statsJson.CPUStats.CPUUsage.TotalUsage, statsJson.CPUStats.SystemUsage}
 
-	cStats := ContainerStats{
+	cStats := &ContainerStats{
 		Name: name,
 		Cpu:  twoDecimals(cpuPct),
 		Mem:  bytesToMegabytes(float64(usedMemory)),
@@ -268,12 +251,12 @@ func getContainerStats(ctr Container) (ContainerStats, error) {
 	return cStats, nil
 }
 
-func gatherStats() SystemData {
+func gatherStats() *SystemData {
 	systemInfo, systemStats := getSystemStats()
-	stats := SystemData{
+	stats := &SystemData{
 		Stats:      systemStats,
 		Info:       systemInfo,
-		Containers: []ContainerStats{},
+		Containers: []*ContainerStats{},
 	}
 	containerStats, err := getDockerStats()
 	if err == nil {
@@ -320,8 +303,7 @@ func main() {
 	if pubKeyEnv, exists := os.LookupEnv("KEY"); exists {
 		pubKey = []byte(pubKeyEnv)
 	} else {
-		pubKey = []byte("ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJgPK8kpPOwPFIq6BIa7Bu/xwrjt5VRQCz3az3Glt4jp")
+		log.Fatal("KEY environment variable is not set")
-		// log.Fatal("KEY environment variable is not set")
 	}
 
 	if filesystem, exists := os.LookupEnv("FILESYSTEM"); exists {

agent/types.go

@@ -2,8 +2,48 @@ package main
 
 import "time"
 
+type SystemData struct {
+	Stats      *SystemStats      `json:"stats"`
+	Info       *SystemInfo       `json:"info"`
+	Containers []*ContainerStats `json:"container"`
+}
+
+type SystemInfo struct {
+	Cores    int    `json:"c"`
+	Threads  int    `json:"t"`
+	CpuModel string `json:"m"`
+	// Os       string  `json:"o"`
+	Uptime  uint64  `json:"u"`
+	Cpu     float64 `json:"cpu"`
+	MemPct  float64 `json:"mp"`
+	DiskPct float64 `json:"dp"`
+}
+
+type SystemStats struct {
+	Cpu          float64 `json:"cpu"`
+	Mem          float64 `json:"m"`
+	MemUsed      float64 `json:"mu"`
+	MemPct       float64 `json:"mp"`
+	MemBuffCache float64 `json:"mb"`
+	Disk         float64 `json:"d"`
+	DiskUsed     float64 `json:"du"`
+	DiskPct      float64 `json:"dp"`
+	DiskRead     float64 `json:"dr"`
+	DiskWrite    float64 `json:"dw"`
+	NetworkSent  float64 `json:"ns"`
+	NetworkRecv  float64 `json:"nr"`
+}
+
+type ContainerStats struct {
+	Name string  `json:"n"`
+	Cpu  float64 `json:"c"`
+	Mem  float64 `json:"m"`
+	// MemPct float64 `json:"mp"`
+}
+
 type Container struct {
 	ID      string `json:"Id"`
+	IdShort string
 	Names   []string
 	Image   string
 	ImageID string

hub/alerts.go

@@ -0,0 +1,142 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/mail"
+
+	"github.com/pocketbase/dbx"
+	"github.com/pocketbase/pocketbase/models"
+	"github.com/pocketbase/pocketbase/tools/mailer"
+	"github.com/pocketbase/pocketbase/tools/types"
+)
+
+func handleSystemAlerts(newStatus string, newRecord *models.Record, oldRecord *models.Record) {
+	alertRecords, err := app.Dao().FindRecordsByExpr("alerts",
+		dbx.NewExp("system = {:system}", dbx.Params{"system": oldRecord.Get("id")}),
+	)
+	if err != nil || len(alertRecords) == 0 {
+		// log.Println("no alerts found for system")
+		return
+	}
+	// log.Println("found alerts", len(alertRecords))
+	var systemInfo *SystemInfo
+	for _, alertRecord := range alertRecords {
+		name := alertRecord.Get("name").(string)
+		switch name {
+		case "Status":
+			handleStatusAlerts(newStatus, oldRecord, alertRecord)
+		case "CPU", "Memory", "Disk":
+			if newStatus != "up" {
+				continue
+			}
+			if systemInfo == nil {
+				systemInfo = getSystemInfo(newRecord)
+			}
+			if name == "CPU" {
+				handleSlidingValueAlert(newRecord, alertRecord, name, systemInfo.Cpu)
+			} else if name == "Memory" {
+				handleSlidingValueAlert(newRecord, alertRecord, name, systemInfo.MemPct)
+			} else if name == "Disk" {
+				handleSlidingValueAlert(newRecord, alertRecord, name, systemInfo.DiskPct)
+			}
+		}
+	}
+}
+
+func getSystemInfo(record *models.Record) *SystemInfo {
+	var SystemInfo SystemInfo
+	json.Unmarshal([]byte(record.Get("info").(types.JsonRaw)), &SystemInfo)
+	return &SystemInfo
+}
+
+func handleSlidingValueAlert(newRecord *models.Record, alertRecord *models.Record, name string, curValue float64) {
+	triggered := alertRecord.Get("triggered").(bool)
+	threshold := alertRecord.Get("value").(float64)
+	// fmt.Println(name, curValue, "threshold", threshold, "triggered", triggered)
+	var subject string
+	var body string
+	if !triggered && curValue > threshold {
+		alertRecord.Set("triggered", true)
+		systemName := newRecord.Get("name").(string)
+		subject = fmt.Sprintf("%s usage threshold exceeded on %s", name, systemName)
+		body = fmt.Sprintf("%s usage on %s is %.1f%%.\n\n- Beszel", name, systemName, curValue)
+	} else if triggered && curValue <= threshold {
+		alertRecord.Set("triggered", false)
+		systemName := newRecord.Get("name").(string)
+		subject = fmt.Sprintf("%s usage returned below threshold on %s", name, systemName)
+		body = fmt.Sprintf("%s usage on %s is below threshold at %.1f%%.\n\n%s\n\n- Beszel", name, systemName, curValue, app.Settings().Meta.AppUrl+"/system/"+systemName)
+	} else {
+		// fmt.Println(name, "not triggered")
+		return
+	}
+	if err := app.Dao().SaveRecord(alertRecord); err != nil {
+		// app.Logger().Error("failed to save alert record", "err", err.Error())
+		return
+	}
+	// expand the user relation and send the alert
+	if errs := app.Dao().ExpandRecord(alertRecord, []string{"user"}, nil); len(errs) > 0 {
+		// app.Logger().Error("failed to expand user relation", "errs", errs)
+		return
+	}
+	if user := alertRecord.ExpandedOne("user"); user != nil {
+		sendAlert(EmailData{
+			to:   user.Get("email").(string),
+			subj: subject,
+			body: body,
+		})
+	}
+}
+
+func handleStatusAlerts(newStatus string, oldRecord *models.Record, alertRecord *models.Record) error {
+	var alertStatus string
+	switch newStatus {
+	case "up":
+		if oldRecord.Get("status") == "down" {
+			alertStatus = "up"
+		}
+	case "down":
+		if oldRecord.Get("status") == "up" {
+			alertStatus = "down"
+		}
+	}
+	if alertStatus == "" {
+		return nil
+	}
+	// expand the user relation
+	if errs := app.Dao().ExpandRecord(alertRecord, []string{"user"}, nil); len(errs) > 0 {
+		return fmt.Errorf("failed to expand: %v", errs)
+	}
+	user := alertRecord.ExpandedOne("user")
+	if user == nil {
+		return nil
+	}
+	emoji := "\U0001F534"
+	if alertStatus == "up" {
+		emoji = "\u2705"
+	}
+	// send alert
+	systemName := oldRecord.Get("name").(string)
+	sendAlert(EmailData{
+		to:   user.Get("email").(string),
+		subj: fmt.Sprintf("Connection to %s is %s %v", systemName, alertStatus, emoji),
+		body: fmt.Sprintf("Connection to %s is %s\n\n- Beszel", systemName, alertStatus),
+	})
+	return nil
+}
+
+func sendAlert(data EmailData) {
+	// fmt.Println("sending alert", "to", data.to, "subj", data.subj, "body", data.body)
+	message := &mailer.Message{
+		From: mail.Address{
+			Address: app.Settings().Meta.SenderAddress,
+			Name:    app.Settings().Meta.SenderName,
+		},
+		To:      []mail.Address{{Address: data.to}},
+		Subject: data.subj,
+		Text:    data.body,
+	}
+	if err := app.NewMailClient().Send(message); err != nil {
+		app.Logger().Error("Failed to send alert: ", "err", err.Error())
+	}
+}

hub/main.go

@@ -12,29 +12,29 @@ import (
 	"log"
 	"net/http"
 	"net/http/httputil"
-	"net/mail"
 	"net/url"
 	"os"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/labstack/echo/v5"
-	"github.com/pocketbase/dbx"
 	"github.com/pocketbase/pocketbase"
 	"github.com/pocketbase/pocketbase/apis"
 	"github.com/pocketbase/pocketbase/core"
 	"github.com/pocketbase/pocketbase/models"
 	"github.com/pocketbase/pocketbase/plugins/migratecmd"
 	"github.com/pocketbase/pocketbase/tools/cron"
-	"github.com/pocketbase/pocketbase/tools/mailer"
+	"github.com/pocketbase/pocketbase/tools/types"
 	"github.com/spf13/cobra"
 	"golang.org/x/crypto/ssh"
 )
 
-var Version = "0.0.1-alpha.3"
+var Version = "0.0.1"
 
 var app *pocketbase.PocketBase
-var serverConnections = make(map[string]Server)
+var serverConnections = make(map[string]*Server)
+var serverConnectionsLock = sync.Mutex{}
 
 func main() {
 	app = pocketbase.NewWithConfig(pocketbase.Config{
@@ -175,7 +175,7 @@ func main() {
 		}
 
 		// alerts
-		handleStatusAlerts(newStatus, oldRecord)
+		handleSystemAlerts(newStatus, newRecord, oldRecord)
 		return nil
 	})
 
@@ -209,49 +209,53 @@ func startSystemUpdateTicker() {
 }
 
 func updateSystems() {
-	// handle max of 1/3 + 1 servers at a time
-	numServers := len(serverConnections)/3 + 1
-	// find systems that are not paused and updated more than 58 seconds ago
-	fiftyEightSecondsAgo := time.Now().UTC().Add(-58 * time.Second).Format("2006-01-02 15:04:05")
 	records, err := app.Dao().FindRecordsByFilter(
-		"2hz5ncl8tizk5nx", // collection
+		"2hz5ncl8tizk5nx",    // collection
-		"status != 'paused' && updated < {:updated}", // filter
+		"status != 'paused'", // filter
-		"updated",  // sort
+		"updated",            // sort
-		numServers, // limit
+		-1,                   // limit
-		0,          // offset
+		0,                    // offset
-		dbx.Params{"updated": fiftyEightSecondsAgo},
 	)
-	if err != nil {
+	// log.Println("records", len(records))
-		app.Logger().Error("Failed to query systems: ", "err", err.Error())
+	if err != nil || len(records) == 0 {
+		// app.Logger().Error("Failed to query systems")
 		return
 	}
-	for _, record := range records {
+	fiftySecondsAgo := time.Now().UTC().Add(-50 * time.Second)
-		updateSystem(record)
+	batchSize := len(records)/4 + 1
+	for i := 0; i < batchSize; i++ {
+		if records[i].Get("updated").(types.DateTime).Time().After(fiftySecondsAgo) {
+			break
+		}
+		// log.Println("updating", records[i].Get(("name")))
+		go updateSystem(records[i])
 	}
 }
 
 func updateSystem(record *models.Record) {
-	var server Server
+	var server *Server
 	// check if server connection data exists
 	if _, ok := serverConnections[record.Id]; ok {
 		server = serverConnections[record.Id]
 	} else {
 		// create server connection struct
-		server = Server{
+		server = &Server{
 			Host: record.Get("host").(string),
 			Port: record.Get("port").(string),
 		}
-		client, err := getServerConnection(&server)
+		client, err := getServerConnection(server)
 		if err != nil {
 			app.Logger().Error("Failed to connect:", "err", err.Error(), "server", server.Host, "port", server.Port)
 			updateServerStatus(record, "down")
 			return
 		}
 		server.Client = client
+		serverConnectionsLock.Lock()
 		serverConnections[record.Id] = server
+		serverConnectionsLock.Unlock()
 	}
 	// get server stats from agent
-	systemData, err := requestJson(&server)
+	systemData, err := requestJson(server)
 	if err != nil {
 		if err.Error() == "retry" {
 			// if previous connection was closed, try again
@@ -312,6 +316,8 @@ func deleteServerConnection(record *models.Record) {
 		if serverConnections[record.Id].Client != nil {
 			serverConnections[record.Id].Client.Close()
 		}
+		serverConnectionsLock.Lock()
+		defer serverConnectionsLock.Unlock()
 		delete(serverConnections, record.Id)
 	}
 }
@@ -378,69 +384,6 @@ func requestJson(server *Server) (SystemData, error) {
 	return systemData, nil
 }
 
-func sendAlert(data EmailData) {
-	message := &mailer.Message{
-		From: mail.Address{
-			Address: app.Settings().Meta.SenderAddress,
-			Name:    app.Settings().Meta.SenderName,
-		},
-		To:      []mail.Address{{Address: data.to}},
-		Subject: data.subj,
-		Text:    data.body,
-	}
-	if err := app.NewMailClient().Send(message); err != nil {
-		app.Logger().Error("Failed to send alert: ", "err", err.Error())
-	}
-}
-
-func handleStatusAlerts(newStatus string, oldRecord *models.Record) error {
-	var alertStatus string
-	switch newStatus {
-	case "up":
-		if oldRecord.Get("status") == "down" {
-			alertStatus = "up"
-		}
-	case "down":
-		if oldRecord.Get("status") == "up" {
-			alertStatus = "down"
-		}
-	}
-	if alertStatus == "" {
-		return nil
-	}
-	alerts, err := app.Dao().FindRecordsByFilter("alerts", "name = 'status' && system = {:system}", "-created", -1, 0, dbx.Params{
-		"system": oldRecord.Get("id")})
-	if err != nil {
-		log.Println("failed to get users", "err", err.Error())
-		return nil
-	}
-	if len(alerts) == 0 {
-		return nil
-	}
-	// expand the user relation
-	if errs := app.Dao().ExpandRecords(alerts, []string{"user"}, nil); len(errs) > 0 {
-		return fmt.Errorf("failed to expand: %v", errs)
-	}
-	systemName := oldRecord.Get("name").(string)
-	emoji := "\U0001F534"
-	if alertStatus == "up" {
-		emoji = "\u2705"
-	}
-	for _, alert := range alerts {
-		user := alert.ExpandedOne("user")
-		if user == nil {
-			continue
-		}
-		// send alert
-		sendAlert(EmailData{
-			to:   user.Get("email").(string),
-			subj: fmt.Sprintf("Connection to %s is %s %v", systemName, alertStatus, emoji),
-			body: fmt.Sprintf("Connection to %s is %s\n\n- Beszel", systemName, alertStatus),
-		})
-	}
-	return nil
-}
-
 func getSSHKey() ([]byte, error) {
 	dataDir := app.DataDir()
 	// check if the key pair already exists

hub/migrations/1720568457_collections_snapshot.go

@@ -15,7 +15,7 @@ func init() {
 			{
 				"id": "2hz5ncl8tizk5nx",
 				"created": "2024-07-07 16:08:20.979Z",
-				"updated": "2024-07-17 15:27:00.429Z",
+				"updated": "2024-07-22 19:39:17.434Z",
 				"name": "systems",
 				"type": "base",
 				"system": false,
@@ -102,7 +102,7 @@ func init() {
 						"unique": false,
 						"options": {
 							"collectionId": "_pb_users_auth_",
-							"cascadeDelete": false,
+							"cascadeDelete": true,
 							"minSelect": null,
 							"maxSelect": null,
 							"displayFields": null
@@ -250,7 +250,7 @@ func init() {
 			{
 				"id": "_pb_users_auth_",
 				"created": "2024-07-14 16:25:18.226Z",
-				"updated": "2024-07-20 00:55:02.071Z",
+				"updated": "2024-07-22 20:10:20.670Z",
 				"name": "users",
 				"type": "auth",
 				"system": false,
@@ -304,7 +304,7 @@ func init() {
 				"options": {
 					"allowEmailAuth": true,
 					"allowOAuth2Auth": true,
-					"allowUsernameAuth": true,
+					"allowUsernameAuth": false,
 					"exceptEmailDomains": null,
 					"manageRule": null,
 					"minPasswordLength": 8,
@@ -316,7 +316,7 @@ func init() {
 			{
 				"id": "elngm8x1l60zi2v",
 				"created": "2024-07-15 01:16:04.044Z",
-				"updated": "2024-07-15 22:44:12.297Z",
+				"updated": "2024-07-22 19:13:16.498Z",
 				"name": "alerts",
 				"type": "base",
 				"system": false,
@@ -364,16 +364,43 @@ func init() {
 						"options": {
 							"maxSelect": 1,
 							"values": [
-								"status"
+								"Status",
+								"CPU",
+								"Memory",
+								"Disk"
 							]
 						}
+					},
+					{
+						"system": false,
+						"id": "o2ablxvn",
+						"name": "value",
+						"type": "number",
+						"required": false,
+						"presentable": false,
+						"unique": false,
+						"options": {
+							"min": null,
+							"max": null,
+							"noDecimal": false
+						}
+					},
+					{
+						"system": false,
+						"id": "6hgdf6hs",
+						"name": "triggered",
+						"type": "bool",
+						"required": false,
+						"presentable": false,
+						"unique": false,
+						"options": {}
 					}
 				],
 				"indexes": [],
 				"listRule": "@request.auth.id != \"\" && user.id = @request.auth.id",
 				"viewRule": "",
 				"createRule": "@request.auth.id != \"\" && user.id = @request.auth.id",
-				"updateRule": null,
+				"updateRule": "@request.auth.id != \"\" && user.id = @request.auth.id",
 				"deleteRule": "@request.auth.id != \"\" && user.id = @request.auth.id",
 				"options": {}
 			}

hub/site/package.json

@@ -17,6 +17,7 @@
     "@radix-ui/react-label": "^2.1.0",
     "@radix-ui/react-select": "^2.1.1",
     "@radix-ui/react-separator": "^1.1.0",
+    "@radix-ui/react-slider": "^1.2.0",
     "@radix-ui/react-slot": "^1.1.0",
     "@radix-ui/react-switch": "^1.1.0",
     "@radix-ui/react-toast": "^1.2.1",

hub/site/src/components/add-system.tsx

@@ -27,16 +27,16 @@ export function AddSystemButton() {
 	function copyDockerCompose(port: string) {
 		copyToClipboard(`services:
   beszel-agent:
-    image: 'henrygd/beszel-agent'
+    image: "henrygd/beszel-agent"
-    container_name: 'beszel-agent'
+    container_name: "beszel-agent"
     restart: unless-stopped
     network_mode: host
-		volumes:
+    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
+      - /var/run/docker.sock:/var/run/docker.sock:ro
     environment:
-		  - PORT=${port}
+      PORT: ${port}
-      - KEY="${publicKey}"
+      KEY: "${publicKey}"
-			# - FILESYSTEM=/dev/sda1 # set to the correct filesystem for disk I/O stats`)
+      # FILESYSTEM: /dev/sda1 # set to the correct filesystem for disk I/O stats`)
 	}
 
 	useEffect(() => {

hub/site/src/components/charts/bandwidth-chart.tsx

@@ -47,7 +47,13 @@ export default function BandwidthChart({
 				<YAxis
 					className="tracking-tighter"
 					width={75}
-					domain={[0, (max: number) => (max < 0.4 ? 0.4 : Math.ceil(max))]}
+					domain={[0, (max: number) => (max <= 0.4 ? 0.4 : Math.ceil(max))]}
+					tickFormatter={(value) => {
+						if (value >= 100) {
+							return value.toFixed(0)
+						}
+						return value.toFixed((value * 100) % 1 === 0 ? 1 : 2)
+					}}
 					tickLine={false}
 					axisLine={false}
 					unit={' MB/s'}
@@ -66,7 +72,8 @@ export default function BandwidthChart({
 					tickFormatter={hourWithMinutes}
 				/>
 				<ChartTooltip
-					// cursor={false}
+					animationEasing="ease-out"
+					animationDuration={150}
 					content={
 						<ChartTooltipContent
 							unit=" MB/s"

hub/site/src/components/charts/container-cpu-chart.tsx

@@ -69,7 +69,7 @@ export default function ContainerCpuChart({
 			>
 				<CartesianGrid vertical={false} />
 				<YAxis
-					domain={[0, (max: number) => Math.max(Math.ceil(max), 0.4)]}
+					// domain={[0, (max: number) => Math.max(Math.ceil(max), 0.4)]}
 					width={47}
 					tickLine={false}
 					axisLine={false}

hub/site/src/components/charts/disk-chart.tsx

@@ -77,7 +77,8 @@ export default function DiskChart({
 					tickFormatter={hourWithMinutes}
 				/>
 				<ChartTooltip
-					// cursor={false}
+					animationEasing="ease-out"
+					animationDuration={150}
 					content={
 						<ChartTooltipContent
 							unit=" GB"

hub/site/src/components/charts/disk-io-chart.tsx

@@ -47,7 +47,7 @@ export default function DiskIoChart({
 				<YAxis
 					className="tracking-tighter"
 					width={75}
-					domain={[0, (max: number) => (max < 0.4 ? 0.4 : Math.ceil(max))]}
+					domain={[0, (max: number) => (max <= 0.4 ? 0.4 : Math.ceil(max))]}
 					tickFormatter={(value) => {
 						if (value >= 100) {
 							return value.toFixed(0)
@@ -72,7 +72,8 @@ export default function DiskIoChart({
 					tickFormatter={hourWithMinutes}
 				/>
 				<ChartTooltip
-					// cursor={false}
+					animationEasing="ease-out"
+					animationDuration={150}
 					content={
 						<ChartTooltipContent
 							unit=" MB/s"

hub/site/src/components/login/forgot-pass-form.tsx

@@ -86,9 +86,12 @@ export default function ForgotPassword() {
 					<DialogHeader>
 						<DialogTitle>Command line instructions</DialogTitle>
 					</DialogHeader>
-					<p className="text-primary/70 text-[0.95em]">
+					<p className="text-primary/70 text-[0.95em] leading-relaxed">
-						Use the following command to reset
+						If you've lost the password to your admin account, you may reset it using the following
-						your password:
+						command.
+					</p>
+					<p className="text-primary/70 text-[0.95em] leading-relaxed">
+						Then log into the backend and reset your user account password in the users table.
 					</p>
 					<code className="bg-muted rounded-sm py-0.5 px-2.5 mr-auto text-sm">
 						beszel admin update youremail@example.com newpassword

hub/site/src/components/systems-table/systems-table.tsx

@@ -304,7 +304,7 @@ export default function SystemsTable() {
 															? 'auto'
 															: cell.column.getSize(),
 												}}
-												className={'overflow-hidden relative py-3'}
+												className={'overflow-hidden relative py-2.5'}
 											>
 												{flexRender(cell.column.columnDef.cell, cell.getContext())}
 											</TableCell>

hub/site/src/components/table-alerts.tsx

@@ -13,9 +13,18 @@ import { cn, isAdmin } from '@/lib/utils'
 import { Button } from '@/components/ui/button'
 import { Switch } from '@/components/ui/switch'
 import { AlertRecord, SystemRecord } from '@/types'
-import { useMemo, useState } from 'react'
+import { lazy, Suspense, useMemo, useState } from 'react'
 import { toast } from './ui/use-toast'
 
+const Slider = lazy(() => import('./ui/slider'))
+
+const failedUpdateToast = () =>
+	toast({
+		title: 'Failed to update alert',
+		description: 'Please check logs for more details.',
+		variant: 'destructive',
+	})
+
 export default function AlertsButton({ system }: { system: SystemRecord }) {
 	const alerts = useStore($alerts)
 
@@ -38,7 +47,7 @@ export default function AlertsButton({ system }: { system: SystemRecord }) {
 					/>
 				</Button>
 			</DialogTrigger>
-			<DialogContent>
+			<DialogContent className="max-h-full overflow-auto">
 				<DialogHeader>
 					<DialogTitle className="mb-1">Alerts for {system.name}</DialogTitle>
 					<DialogDescription>
@@ -54,38 +63,57 @@ export default function AlertsButton({ system }: { system: SystemRecord }) {
 								to ensure alerts are delivered.{' '}
 							</span>
 						)}
-						Webhook delivery and more alert options will be added in the future.
 					</DialogDescription>
 				</DialogHeader>
-				<Alert system={system} alerts={systemAlerts} />
+				<div className="grid gap-3">
+					<AlertStatus system={system} alerts={systemAlerts} />
+					<AlertWithSlider
+						system={system}
+						alerts={systemAlerts}
+						name="CPU"
+						title="CPU Usage"
+						description="Triggers when CPU usage exceeds a threshold."
+					/>
+					<AlertWithSlider
+						system={system}
+						alerts={systemAlerts}
+						name="Memory"
+						title="Memory Usage"
+						description="Triggers when memory usage exceeds a threshold."
+					/>
+					<AlertWithSlider
+						system={system}
+						alerts={systemAlerts}
+						name="Disk"
+						title="Disk Usage"
+						description="Triggers when disk usage exceeds a threshold."
+					/>
+				</div>
 			</DialogContent>
 		</Dialog>
 	)
 }
 
-function Alert({ system, alerts }: { system: SystemRecord; alerts: AlertRecord[] }) {
+function AlertStatus({ system, alerts }: { system: SystemRecord; alerts: AlertRecord[] }) {
 	const [pendingChange, setPendingChange] = useState(false)
 
 	const alert = useMemo(() => {
-		return alerts.find((alert) => alert.name === 'status')
+		return alerts.find((alert) => alert.name === 'Status')
 	}, [alerts])
 
 	return (
 		<label
-			htmlFor="status"
+			htmlFor="alert-status"
-			className="space-y-2 flex flex-row items-center justify-between rounded-lg border p-4 cursor-pointer"
+			className="flex flex-row items-center justify-between gap-4 rounded-lg border p-4 cursor-pointer"
 		>
-			<div className="grid gap-0.5 select-none">
+			<div className="grid gap-1 select-none">
-				<p className="font-medium text-base">System status</p>
+				<p className="font-semibold">System Status</p>
-				<span
+				<span className="block text-sm text-foreground opacity-80">
-					id=":r3m:-form-item-description"
-					className="block text-[0.8rem] text-foreground opacity-80"
-				>
 					Triggers when status switches between up and down.
 				</span>
 			</div>
 			<Switch
-				id="status"
+				id="alert-status"
 				className={cn('transition-opacity', pendingChange && 'opacity-40')}
 				checked={!!alert}
 				value={!!alert ? 'on' : 'off'}
@@ -101,15 +129,11 @@ function Alert({ system, alerts }: { system: SystemRecord; alerts: AlertRecord[]
 							pb.collection('alerts').create({
 								system: system.id,
 								user: pb.authStore.model!.id,
-								name: 'status',
+								name: 'Status',
 							})
 						}
 					} catch (e) {
-						toast({
+						failedUpdateToast()
-							title: 'Failed to update alert',
-							description: 'Please check logs for more details.',
-							variant: 'destructive',
-						})
 					} finally {
 						setPendingChange(false)
 					}
@@ -118,3 +142,93 @@ function Alert({ system, alerts }: { system: SystemRecord; alerts: AlertRecord[]
 		</label>
 	)
 }
+
+function AlertWithSlider({
+	system,
+	alerts,
+	name,
+	title,
+	description,
+}: {
+	system: SystemRecord
+	alerts: AlertRecord[]
+	name: string
+	title: string
+	description: string
+}) {
+	const [pendingChange, setPendingChange] = useState(false)
+	const [liveValue, setLiveValue] = useState(50)
+
+	const alert = useMemo(() => {
+		const alert = alerts.find((alert) => alert.name === name)
+		if (alert) {
+			setLiveValue(alert.value)
+		}
+		return alert
+	}, [alerts])
+
+	return (
+		<div className="rounded-lg border">
+			<label
+				htmlFor={`alert-${name}`}
+				className={cn('flex flex-row items-center justify-between gap-4 cursor-pointer p-4', {
+					'pb-0': !!alert,
+				})}
+			>
+				<div className="grid gap-1 select-none">
+					<p className="font-semibold">{title}</p>
+					<span className="block text-sm text-foreground opacity-80">{description}</span>
+				</div>
+				<Switch
+					id={`alert-${name}`}
+					className={cn('transition-opacity', pendingChange && 'opacity-40')}
+					checked={!!alert}
+					value={!!alert ? 'on' : 'off'}
+					onCheckedChange={async (active) => {
+						if (pendingChange) {
+							return
+						}
+						setPendingChange(true)
+						try {
+							if (!active && alert) {
+								await pb.collection('alerts').delete(alert.id)
+							} else if (active) {
+								pb.collection('alerts').create({
+									system: system.id,
+									user: pb.authStore.model!.id,
+									name,
+									value: liveValue,
+								})
+							}
+						} catch (e) {
+							failedUpdateToast()
+						} finally {
+							setPendingChange(false)
+						}
+					}}
+				/>
+			</label>
+			{alert && (
+				<div className="flex mt-2 mb-3 gap-3 px-4">
+					<Suspense>
+						<Slider
+							defaultValue={[liveValue]}
+							onValueCommit={(val) => {
+								pb.collection('alerts').update(alert.id, {
+									value: val[0],
+								})
+							}}
+							onValueChange={(val) => {
+								setLiveValue(val[0])
+							}}
+							min={10}
+							max={99}
+							// step={1}
+						/>
+					</Suspense>
+					<span className="tabular-nums tracking-tighter text-[.92em]">{liveValue}%</span>
+				</div>
+			)}
+		</div>
+	)
+}

hub/site/src/components/ui/slider.tsx

@@ -0,0 +1,23 @@
+import * as React from 'react'
+import * as SliderPrimitive from '@radix-ui/react-slider'
+
+import { cn } from '@/lib/utils'
+
+const Slider = React.forwardRef<
+	React.ElementRef<typeof SliderPrimitive.Root>,
+	React.ComponentPropsWithoutRef<typeof SliderPrimitive.Root>
+>(({ className, ...props }, ref) => (
+	<SliderPrimitive.Root
+		ref={ref}
+		className={cn('relative flex w-full touch-none select-none items-center', className)}
+		{...props}
+	>
+		<SliderPrimitive.Track className="relative h-2 w-full grow overflow-hidden rounded-full bg-secondary">
+			<SliderPrimitive.Range className="absolute h-full bg-primary" />
+		</SliderPrimitive.Track>
+		<SliderPrimitive.Thumb className="block h-5 w-5 rounded-full border-2 border-primary bg-background ring-offset-background transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50" />
+	</SliderPrimitive.Root>
+))
+Slider.displayName = SliderPrimitive.Root.displayName
+
+export default Slider

hub/site/src/lib/utils.ts

@@ -26,18 +26,26 @@ export async function copyToClipboard(content: string) {
 		})
 	}
 }
-
+const verifyAuth = () => {
-export const updateSystemList = () => {
+	pb.collection('users')
-	pb.collection<SystemRecord>('systems')
+		.authRefresh()
-		.getFullList({ sort: '+name' })
+		.catch(() => {
-		.then((records) => {
+			pb.authStore.clear()
-			$systems.set(records)
 		})
 }
 
+export const updateSystemList = async () => {
+	try {
+		const records = await pb.collection<SystemRecord>('systems').getFullList({ sort: '+name' })
+		$systems.set(records)
+	} catch (e) {
+		verifyAuth()
+	}
+}
+
 export const updateAlerts = () => {
 	pb.collection('alerts')
-		.getFullList<AlertRecord>({ fields: 'id,name,system' })
+		.getFullList<AlertRecord>({ fields: 'id,name,system,value' })
 		.then((records) => {
 			$alerts.set(records)
 		})

readme.md

@@ -1,28 +1,22 @@
-# Beszel \*WIP\*
+# Beszel
 
 A lightweight server resource monitoring hub with historical data, docker stats, and alerts.
 
-<!-- <table width="100%">
+[![Docker Image Size (tag)](https://img.shields.io/docker/image-size/henrygd/beszel-agent/0.0.1-alpha.9?logo=docker&label=agent%20image%20size)](https://hub.docker.com/r/henrygd/beszel-agent)
-  <tbody>
+[![Docker Image Size (tag)](https://img.shields.io/docker/image-size/henrygd/beszel/0.0.1-alpha.9?logo=docker&label=hub%20image%20size)](https://hub.docker.com/r/henrygd/beszel)
-    <tr>
-      <td width="50%"><img src="https://henrygd-assets.b-cdn.net/social-image-server/before-capture.png" alt="example of turso.tech/pricing link which is missing an og:image as of may 11 2024"/></td>
-      <td width="50%"><img src="https://henrygd-assets.b-cdn.net/social-image-server/after-capture.webp" alt="example of turso.tech/pricing link using an image generated by the server as it's og:image"/></td>
-    </tr>
-  </tbody>
-</table> -->
 
-<!-- ## Features
+![Screenshot of the hub](https://henrygd-assets.b-cdn.net/beszel/screenshot.png)
+
+## Features
 
 - **Lightweight**: Much smaller and less demanding than leading solutions.
-- **Historical data**: Stats are available for up to 30 days.
 - **Docker stats**: CPU and memory usage history for each container.
-- **Alerts**: Lorem ipsum dolor sit amet, consectetur adipiscing elit.
+- **Alerts**: Configurable alerts for CPU, memory, and disk usage, and system status.
-- **Simple**: Lorem ipsum dolor sit amet, consectetur adipiscing elit.
 - **Multi-user**: Each user has their own systems. Admins can share systems across users.
-- **Secure**: Lorem ipsum dolor sit amet, consectetur adipiscing elit.
+- **Simple**: Easy setup and doesn't require anything to be publicly available online.
-- **Oauth / OIDC**: Supports many OAuth2 providers and password auth can be disabled.
+- **OAuth / OIDC**: Supports many OAuth2 providers. Password auth can be disabled.
-- **Automated backups**: Automatically back up your data to S3-compatible storage.
+- **Automatic backups**: Save and restore your data to / from disk or S3-compatible storage.
-- **Open source**: MIT license and no paywalled features. -->
+- **REST API**: Use your metrics in your own scripts and applications.
 
 ## Introduction
 
@@ -32,14 +26,11 @@ The hub is a web application that provides a dashboard to view and manage your c
 
 The agent runs on each system you want to monitor. It creates a minimal SSH server through which it communicates system metrics to the hub.
 
-[![Docker Image Size (tag)](https://img.shields.io/docker/image-size/henrygd/beszel-agent/0.0.1-alpha.3?logo=docker&label=agent%20image%20size)](https://hub.docker.com/r/henrygd/beszel-agent)
-[![Docker Image Size (tag)](https://img.shields.io/docker/image-size/henrygd/beszel/0.0.1-alpha.3?logo=docker&label=hub%20image%20size)](https://hub.docker.com/r/henrygd/beszel)
-
 ## Getting started
 
 If using the binary instead of docker, ignore 4-5 and run the agent using the binary instead.
 
-1. Start the hub (see [Installation](#installation)). The binary command is `beszel serve`.
+1. Start the hub (see [installation](#installation)). The binary command is `beszel serve`.
 2. Open http://localhost:8090 and create an admin user.
 3. Click "Add system." Enter the name and host of the system you want to monitor.
 4. Click "Copy docker compose" to copy the agent's docker-compose.yml file to your clipboard.
@@ -104,8 +95,9 @@ Use `beszel update` and `beszel-agent update` to update to the latest version.
 
 Beszel supports OpenID Connect and many OAuth2 authentication providers (see list below). To enable, do the following:
 
-1. Create an OAuth2 application using your provider of choice. The redirect / callback URL should be `<your-beszel-url>/api/oauth2-redirect`.
+1. Make sure your "Application URL" is set correctly in the PocketBase settings.
-2. When you have the client ID and secret, go to the "Auth providers" page and enable your provider.
+2. Create an OAuth2 application using your provider of choice. The redirect / callback URL should be `<your-beszel-url>/api/oauth2-redirect`.
+3. When you have the client ID and secret, go to the "Auth providers" page and enable your provider.
 
 <details>
   <summary>Supported provider list</summary>
@@ -144,7 +136,25 @@ The hub and agent communicate over SSH, so they don't need to be exposed to the
 
 When the hub is started for the first time, it generates an ED25519 key pair.
 
-The agent's SSH server is configured to accept connections only using this key. It does not provide a pty or accept any input, so it is not possible to execute commands on the agent even if your private key is compromised.
+The agent's SSH server is configured to accept connections only using this key. It does not provide a pseudo-terminal or accept input, so it's not possible to execute commands on the agent even if your private key is compromised.
+
+## User roles
+
+### Admin
+
+Assumed to have an admin account in PocketBase, so links to backups, SMTP settings, etc., are shown in the hub.
+
+The first user created automatically becomes an admin and can log into PocketBase.
+
+Please note that changing a user's role will not create a PocketBase admin account for them. If you want to do that, go to Settings > Admins in PocketBase and add them there.
+
+### User
+
+Can create their own systems and alerts. Links to PocketBase settings are not shown in the hub.
+
+### Read only
+
+Cannot create systems, but can view any system that has been shared with them by an admin. Can create alerts.
 
 ## FAQ / Troubleshooting
 
@@ -164,8 +174,17 @@ If it's not set, the agent will try to find the filesystem mounted on `/` and us
 - Run `lsblk` and choose an option under "NAME"
 - Run `sudo fdisk -l` and choose an option under "Device"
 
+### Docker containers are not populating reliably
+
+Try upgrading your docker version on the agent system. I had this issue on a machine running version 24. It was fixed by upgrading to version 27.
+
 ### Month / week records are not populating reliably
 
 Records for longer time periods are made by averaging stats from the shorter time periods. They require the agent to be running uninterrupted for long enough to get a full set of data.
 
 If you pause / unpause the agent for longer than one minute, the data will be incomplete and the timing for the current interval will reset.
+
+<!--
+## Support
+
+My country, the USA, and many others, are actively involved in the genocide of the Palestinian people. I would greatly appreciate any effort you could make to pressure your government to stop enabling this violence. -->