beszel-ipv6/agent/storage_pool.go

package agent

import (
	"errors"
	"log/slog"
	"os/exec"
	"strings"
	"sync"
	"time"

	"github.com/henrygd/beszel/agent/btrfs"
	"github.com/henrygd/beszel/agent/zfs"
	"github.com/henrygd/beszel/internal/entities/system"
	zfsentity "github.com/henrygd/beszel/internal/entities/zfs"
)

// zfsDatasetUsage holds usage values for a ZFS dataset mountpoint.
type zfsDatasetUsage struct {
	used  uint64
	avail uint64
}

// datasetUsageRefreshInterval controls how often `zfs list` is re-run for the
// mountpoint usage map. Dataset inventory changes rarely.
const datasetUsageRefreshInterval = 5 * time.Minute

// poolStatsRefreshInterval controls how often `zpool list` is re-run for pool
// capacity. Health and I/O are read from procfs on Linux, so the utility only
// needs to refresh slow-moving space accounting.
const poolStatsRefreshInterval = time.Minute

// btrfsFilesystems is the btrfs source; overridable in tests.
var btrfsFilesystems = btrfs.Filesystems

type poolKernelSample struct {
	nread  uint64
	nwrite uint64
	at     time.Time
}

// StoragePoolManager combines independent backend inventories. Metrics and
// dataset usage require the agent lock; GetDetail is safe for concurrent calls.
type StoragePoolManager struct {
	backends       []*poolBackend
	detailInterval time.Duration
}

// poolBackend owns one backend's collectors and caches. Collector functions
// are immutable after construction and may run concurrently for metrics/details.
type poolBackend struct {
	name           string
	poolStatsFn    func() ([]zfs.PoolStat, error)       // capacity/health source
	datasetsFn     func() ([]zfs.Dataset, error)        // dataset inventory source
	kernelStatsFn  func() ([]zfs.PoolKernelStat, error) // procfs pool state/I/O source
	poolStatusesFn func() ([]zfs.PoolStatus, error)     // scrub/vdev detail source

	poolData      []zfs.PoolStat // cached pool inventory (TTL below)
	lastPoolStats time.Time
	kernelSamples map[string]poolKernelSample

	datasetUsage     map[string]zfsDatasetUsage // mountpoint -> usage
	lastUsageRefresh time.Time

	// Detail data (pools, vdevs, scrub, datasets) is cached and refreshed on
	// an interval. Accessed from handler goroutines, so it is mutex-protected.
	detailMu          sync.Mutex
	detail            *zfsentity.ZfsData
	lastDetailRefresh time.Time
	detailFailed      bool
}

func newStoragePoolManager() *StoragePoolManager {
	return &StoragePoolManager{
		backends:       []*poolBackend{newZfsBackend(), newBtrfsBackend()},
		detailInterval: time.Hour,
	}
}

func newZfsBackend() *poolBackend {
	return &poolBackend{
		name:           "zfs",
		poolStatsFn:    optionalPoolSource(zfs.PoolStats),
		datasetsFn:     optionalPoolSource(zfs.Datasets),
		kernelStatsFn:  optionalPoolSource(zfs.PoolKernelStats),
		poolStatusesFn: optionalPoolSource(zfs.PoolStatuses),
	}
}

func newBtrfsBackend() *poolBackend {
	return &poolBackend{
		name:           "btrfs",
		poolStatsFn:    btrfsSource(btrfsPoolStats),
		kernelStatsFn:  btrfsSource(btrfsKernelStats),
		poolStatusesFn: btrfsSource(btrfsPoolStatuses),
	}
}

// datasets is optional: only backends that expose datasets provide a collector.
func (b *poolBackend) datasets() ([]zfs.Dataset, error) {
	if b.datasetsFn == nil {
		return nil, nil
	}
	return b.datasetsFn()
}

// A missing utility/interface is a successfully observed absent backend.
func optionalPoolSource[T any](source func() ([]T, error)) func() ([]T, error) {
	return func() ([]T, error) {
		items, err := source()
		if errors.Is(err, zfs.ErrNoZfs) || errors.Is(err, exec.ErrNotFound) || errors.Is(err, errors.ErrUnsupported) {
			return nil, nil
		}
		return items, err
	}
}

func btrfsSource[T any](convert func(btrfs.Filesystem) T) func() ([]T, error) {
	return func() ([]T, error) {
		filesystems, err := optionalPoolSource(btrfsFilesystems)()
		if err != nil {
			return nil, err
		}
		items := make([]T, 0, len(filesystems))
		for _, fs := range filesystems {
			items = append(items, convert(fs))
		}
		return items, nil
	}
}

// Update refreshes systemStats.ZfsPools with the latest pool data. I/O
// throughput and health come from inexpensive kernel kstats on Linux. Pool
// capacity and dataset usage come from separately cached utility calls. The
// pool map is empty when both backends are absent.
func (m *StoragePoolManager) Update(systemStats *system.Stats) {
	// Rebuild the combined map so successful pool removals clear old samples.
	systemStats.ZfsPools = nil
	for _, backend := range m.backends {
		backend.updateBackendStats(systemStats)
	}
}

func (b *poolBackend) updateBackendStats(systemStats *system.Stats) {
	pools := b.poolStats()
	if len(pools) == 0 {
		b.kernelSamples = nil
		return
	}

	kernelStats, ioRates := b.kernelStats()

	if systemStats.ZfsPools == nil {
		systemStats.ZfsPools = make(map[string]*system.ZfsPool, len(pools))
	}
	for i := range pools {
		pool := &pools[i]
		// Full precision, matching the dataset values below; the frontend
		// formats any magnitude.
		stats := &system.ZfsPool{
			DisplayName: pool.DisplayName,
			Raw:         pool.Raw,
			Total:       float64(pool.Size) / (1024 * 1024 * 1024),
			Used:        float64(pool.Alloc) / (1024 * 1024 * 1024),
			Health:      pool.Health,
		}
		if kernel, exists := kernelStats[pool.Name]; exists && kernel.Health != "" {
			stats.Health = kernel.Health
		}
		if io, exists := ioRates[pool.Name]; exists {
			stats.ReadBytes = io.NRead
			stats.WriteBytes = io.NWrite
		}
		slog.Debug("Storage pool sample", "backend", b.name, "pool", pool.Name, "health", stats.Health, "used_gb", stats.Used, "read_bps", stats.ReadBytes, "write_bps", stats.WriteBytes)
		systemStats.ZfsPools[pool.Name] = stats
	}

}

// poolStats returns the cached pool inventory, calling its collector at most
// every poolStatsRefreshInterval. On failure the previous inventory is
// retained and the refresh is retried on the next cadence.
func (b *poolBackend) poolStats() []zfs.PoolStat {
	if b.lastPoolStats.IsZero() || time.Since(b.lastPoolStats) >= poolStatsRefreshInterval {
		pools, err := b.poolStatsFn()
		if err != nil {
			slog.Debug("Storage pool stats unavailable", "backend", b.name, "err", err)
		} else {
			b.poolData = pools
		}
		b.lastPoolStats = time.Now()
	}
	return b.poolData
}

// kernelStats reads cumulative pool counters and converts them to per-second
// rates. Counter decreases indicate a pool export/import and reset the
// baseline instead of producing an underflow spike.
func (b *poolBackend) kernelStats() (map[string]zfs.PoolKernelStat, map[string]zfs.PoolIoStats) {
	if b.kernelStatsFn == nil {
		return nil, nil
	}
	stats, err := b.kernelStatsFn()
	if err != nil {
		slog.Debug("Storage pool kernel stats unavailable", "backend", b.name, "err", err)
		return nil, nil
	}
	now := time.Now()
	byName := make(map[string]zfs.PoolKernelStat, len(stats))
	rates := make(map[string]zfs.PoolIoStats, len(stats))
	nextSamples := make(map[string]poolKernelSample, len(stats))
	for _, stat := range stats {
		byName[stat.Name] = stat
		if previous, ok := b.kernelSamples[stat.Name]; ok && now.After(previous.at) &&
			stat.NRead >= previous.nread && stat.NWrite >= previous.nwrite {
			seconds := now.Sub(previous.at).Seconds()
			rates[stat.Name] = zfs.PoolIoStats{
				NRead:  uint64(float64(stat.NRead-previous.nread) / seconds),
				NWrite: uint64(float64(stat.NWrite-previous.nwrite) / seconds),
			}
		}
		nextSamples[stat.Name] = poolKernelSample{nread: stat.NRead, nwrite: stat.NWrite, at: now}
	}
	b.kernelSamples = nextSamples
	return byName, rates
}

// refreshDatasetUsage re-runs `zfs list` when the refresh window has elapsed
// and rebuilds the mountpoint-keyed usage map.
func (b *poolBackend) refreshDatasetUsage() {
	if !b.lastUsageRefresh.IsZero() && time.Since(b.lastUsageRefresh) < datasetUsageRefreshInterval {
		return
	}
	datasets, err := b.datasets()
	if err != nil {
		slog.Debug("Storage pool dataset usage unavailable", "backend", b.name, "err", err)
	} else {
		usage := make(map[string]zfsDatasetUsage, len(datasets))
		for _, ds := range datasets {
			if ds.Mountpoint != "" && ds.Mountpoint != "-" {
				usage[ds.Mountpoint] = zfsDatasetUsage{used: ds.Used, avail: ds.Avail}
			}
		}
		b.datasetUsage = usage
	}
	b.lastUsageRefresh = time.Now()
}

// DatasetUsage returns ZFS dataset usage keyed by mountpoint, refreshed at
// most every datasetUsageRefreshInterval. On failure the previous map is
// retained and a debug log is emitted.
func (m *StoragePoolManager) DatasetUsage() map[string]zfsDatasetUsage {
	for _, backend := range m.backends {
		if backend.name == "zfs" {
			backend.refreshDatasetUsage()
			return backend.datasetUsage
		}
	}
	return nil
}

// GetDetail combines backend snapshots, identifying successful inventories so
// the hub can accept partial updates without deleting failed backend records.
func (m *StoragePoolManager) GetDetail(force bool) *zfsentity.ZfsData {
	data := &zfsentity.ZfsData{Complete: true}
	for _, backend := range m.backends {
		snapshot := backend.getBackendDetail(force, m.detailInterval)
		data.Pools = append(data.Pools, snapshot.Pools...)
		if snapshot.Complete {
			data.CompleteBackends = append(data.CompleteBackends, backend.name)
		} else {
			data.Complete = false
		}
	}
	return data
}

func (b *poolBackend) getBackendDetail(force bool, interval time.Duration) *zfsentity.ZfsData {
	b.detailMu.Lock()
	defer b.detailMu.Unlock()

	if force || b.detailFailed || b.detail == nil || time.Since(b.lastDetailRefresh) >= interval {
		if data, err := b.collectDetail(b.detail); err != nil {
			b.detailFailed = true
			slog.Debug("Storage pool detail collection failed", "backend", b.name, "err", err)
			if b.detail == nil {
				return &zfsentity.ZfsData{}
			}
			return &zfsentity.ZfsData{Pools: b.detail.Pools}
		} else {
			b.detailFailed = false
			b.detail = data
			b.lastDetailRefresh = time.Now()
		}
	}
	if b.detail == nil {
		return &zfsentity.ZfsData{}
	}
	return b.detail
}

// collectDetail builds a ZfsData payload from the current system state.
func (b *poolBackend) collectDetail(previous *zfsentity.ZfsData) (*zfsentity.ZfsData, error) {
	pools, err := b.poolStatsFn()
	if err != nil {
		return nil, err
	}
	if len(pools) == 0 {
		return &zfsentity.ZfsData{Pools: []*zfsentity.PoolDetail{}, Complete: true}, nil
	}

	statuses, statusErr := b.poolStatusesFn()
	if statusErr != nil {
		slog.Debug("Storage pool status unavailable", "backend", b.name, "err", statusErr)
	}
	datasets, datasetsErr := b.datasets()
	if datasetsErr != nil {
		slog.Debug("Storage pool datasets unavailable", "backend", b.name, "err", datasetsErr)
	}

	statusByPool := make(map[string]zfs.PoolStatus, len(statuses))
	for _, st := range statuses {
		statusByPool[st.Name] = st
	}

	previousByPool := make(map[string]*zfsentity.PoolDetail)
	if previous != nil {
		for _, pool := range previous.Pools {
			if pool != nil {
				previousByPool[pool.Name] = pool
			}
		}
	}

	data := &zfsentity.ZfsData{Pools: make([]*zfsentity.PoolDetail, 0, len(pools)), Complete: true}
	for i := range pools {
		p := &pools[i]
		detail := &zfsentity.PoolDetail{
			DisplayName: p.DisplayName,
			Raw:         p.Raw,
			Name:        p.Name,
			Health:      p.Health,
			Size:        p.Size,
			Alloc:       p.Alloc,
			Free:        p.Free,
		}
		if st, ok := statusByPool[p.Name]; statusErr == nil && ok {
			if st.Scrub.State != "" && st.Scrub.State != "NONE" {
				detail.Scrub = &zfsentity.Scrub{
					State:    st.Scrub.State,
					Progress: st.Scrub.Progress,
					Errors:   st.Scrub.Errors,
				}
			}
			for _, v := range st.Vdevs {
				detail.Vdevs = append(detail.Vdevs, &zfsentity.Vdev{
					Name:         v.Name,
					State:        v.State,
					ReadErrs:     v.ReadErrs,
					WriteErrs:    v.WriteErrs,
					ChecksumErrs: v.ChecksumErrs,
				})
			}
		} else {
			if cached := previousByPool[p.Name]; cached != nil {
				detail.Scrub = cached.Scrub
				detail.Vdevs = cached.Vdevs
			}
		}
		if datasetsErr == nil {
			foundDataset := false
			for _, ds := range datasets {
				if poolOfDataset(ds.Name) == p.Name {
					foundDataset = true
					detail.Datasets = append(detail.Datasets, &zfsentity.Dataset{
						Name:       ds.Name,
						Used:       ds.Used,
						Avail:      ds.Avail,
						Mountpoint: ds.Mountpoint,
					})
				}
			}
			if !foundDataset {
				if cached := previousByPool[p.Name]; cached != nil {
					detail.Datasets = cached.Datasets
				}
			}
		} else if cached := previousByPool[p.Name]; cached != nil {
			detail.Datasets = cached.Datasets
		}
		data.Pools = append(data.Pools, detail)
	}
	return data, nil
}

// poolOfDataset returns the pool name for a dataset name (everything before
// the first '/'). Datasets without a separator belong to a pool of the same
// name.
func poolOfDataset(name string) string {
	if idx := strings.IndexByte(name, '/'); idx >= 0 {
		return name[:idx]
	}
	return name
}

// ZfsMountpoints returns the set of mountpoints backed by ZFS datasets.
func (m *StoragePoolManager) ZfsMountpoints() map[string]bool {
	usage := m.DatasetUsage()
	mountpoints := make(map[string]bool, len(usage))
	for mountpoint := range usage {
		mountpoints[mountpoint] = true
	}
	return mountpoints
}

func btrfsPoolStats(fs btrfs.Filesystem) zfs.PoolStat {
	return zfs.PoolStat{MountID: fs.MountID, IODevice: fs.IODevice, Raw: fs.Raw, DisplayName: fs.Name, Name: "b:" + fs.UUID, Size: fs.Size, Alloc: fs.Alloc, Free: fs.Size - min(fs.Alloc, fs.Size), Health: fs.Health}
}

func btrfsKernelStats(fs btrfs.Filesystem) zfs.PoolKernelStat {
	return zfs.PoolKernelStat{Name: "b:" + fs.UUID, Health: fs.Health, NRead: fs.NRead, NWrite: fs.NWrite}
}

func btrfsPoolStatuses(fs btrfs.Filesystem) zfs.PoolStatus {
	status := zfs.PoolStatus{Name: "b:" + fs.UUID, State: fs.Health, Scrub: zfs.ScrubStatus{State: "NONE"}}
	for _, dev := range fs.Devices {
		status.Vdevs = append(status.Vdevs, zfs.VdevStatus{
			Name: dev.Name, State: dev.State,
			ReadErrs: dev.ReadErrs, WriteErrs: dev.WriteErrs, ChecksumErrs: dev.CorruptionErrs,
		})
	}
	return status
}

// markDuplicateCharts leaves pool telemetry and detail intact, but tells the
// hub which charts already have a filesystem equivalent. Only exact kernel
// filesystem and I/O-device matches qualify; labels are never used.
func (m *StoragePoolManager) markDuplicateCharts(stats *system.Stats, filesystems map[string]*system.FsStats, mountID func(string) string) {
	identities := make(map[string]string, len(filesystems))
	for device, fs := range filesystems {
		if fs.DiskTotal > 0 {
			identities[device] = mountID(fs.Mountpoint)
		}
	}
	for _, backend := range m.backends {
		for _, pool := range backend.poolData {
			sample := stats.ZfsPools[pool.Name]
			if sample == nil || pool.MountID == "" {
				continue
			}
			for device, identity := range identities {
				if identity != pool.MountID {
					continue
				}
				// Raw physical usage is not equivalent to a filesystem usage chart.
				sample.HideUsage = !pool.Raw
				if pool.IODevice != "" && pool.IODevice == device {
					sample.HideIO = true
				}
			}
		}
	}
}