fix(agent): fix QNAP MD RAID arrays incorrectly reported as FAILED (#2065)

agent/mdraid_linux.go

@@ -17,15 +17,17 @@ import (
 var mdraidSysfsRoot = "/sys"
 
 type mdraidHealth struct {
-	level         string
+	level          string
-	arrayState    string
+	arrayState     string
-	degraded      uint64
+	degraded       uint64
-	raidDisks     uint64
+	faultyDisks    uint64
-	syncAction    string
+	populatedDisks uint64
-	syncCompleted string
+	raidDisks      uint64
-	syncSpeed     string
+	syncAction     string
-	mismatchCnt   uint64
+	syncCompleted  string
-	capacity      uint64
+	syncSpeed      string
+	mismatchCnt    uint64
+	capacity       uint64
 }
 
 // scanMdraidDevices discovers Linux md arrays exposed in sysfs.
@@ -92,6 +94,9 @@ func (sm *SmartManager) collectMdraidHealth(deviceInfo *DeviceInfo) (bool, error
 	if health.degraded > 0 {
 		attrs = append(attrs, &smart.SmartAttribute{Name: "Degraded", RawValue: health.degraded})
 	}
+	if health.faultyDisks > 0 {
+		attrs = append(attrs, &smart.SmartAttribute{Name: "FaultyDisks", RawValue: health.faultyDisks})
+	}
 	if health.syncAction != "" {
 		attrs = append(attrs, &smart.SmartAttribute{Name: "SyncAction", RawString: health.syncAction})
 	}
@@ -152,6 +157,7 @@ func readMdraidHealth(blockName string) (mdraidHealth, bool) {
 	if val, ok := utils.ReadUintFile(filepath.Join(mdDir, "degraded")); ok {
 		out.degraded = val
 	}
+	out.faultyDisks, out.populatedDisks = countMdraidMemberStates(blockName, mdraidSysfsRoot)
 	if val, ok := utils.ReadUintFile(filepath.Join(mdDir, "mismatch_cnt")); ok {
 		out.mismatchCnt = val
 	}
@@ -177,7 +183,19 @@ func mdraidSmartStatus(health mdraidHealth) string {
 	case "resync", "recover", "reshape":
 		return "WARNING"
 	}
+	// Use actual faulty member count rather than the degraded counter, which
+	// equals raid_disks minus active_disks. On QNAP systems raid_disks may be
+	// set to a large value (e.g. 32) while only a few slots are ever used,
+	// making degraded misleadingly large despite zero failed disks.
+	if health.faultyDisks > 0 {
+		return "FAILED"
+	}
 	if health.degraded > 0 {
+		if isSparseSlotDegraded(health) {
+			// A sysfs snapshot cannot distinguish reserved slots from a removed
+			// member on sparse arrays, so report the ambiguity as a warning.
+			return "WARNING"
+		}
 		return "FAILED"
 	}
 	if health.mismatchCnt > 0 {
@@ -196,6 +214,43 @@ func mdraidSmartStatus(health mdraidHealth) string {
 	return "UNKNOWN"
 }
 
+// countMdraidMemberStates reads member device directories under
+// block/<name>/md and returns how many are explicitly marked "faulty", plus
+// how many are populated at all (regardless of state). populatedDisks lets
+// callers distinguish RAID slots that were never used (QNAP reserves far
+// more raid_disks than it ever populates) from members that went missing.
+func countMdraidMemberStates(blockName, root string) (faultyDisks, populatedDisks uint64) {
+	devDir := filepath.Join(root, "block", blockName, "md")
+	entries, err := os.ReadDir(devDir)
+	if err != nil {
+		return 0, 0
+	}
+	for _, ent := range entries {
+		if !strings.HasPrefix(ent.Name(), "dev-") {
+			continue
+		}
+		populatedDisks++
+		statePath := filepath.Join(devDir, ent.Name(), "state")
+		state := utils.ReadStringFile(statePath)
+		if strings.Contains(state, "faulty") {
+			faultyDisks++
+		}
+	}
+	return faultyDisks, populatedDisks
+}
+
+// isSparseSlotDegraded reports whether a non-zero "degraded" count may be
+// explained by RAID slots that were never populated. QNAP configures system
+// arrays with raid_disks set to a large fixed maximum (e.g. 32) far beyond the
+// handful of slots it ever populates, so sparse slots outnumber populated ones.
+func isSparseSlotDegraded(health mdraidHealth) bool {
+	if health.populatedDisks == 0 || health.raidDisks <= health.populatedDisks {
+		return false
+	}
+	sparseSlots := health.raidDisks - health.populatedDisks
+	return sparseSlots > health.populatedDisks
+}
+
 // isMdraidBlockName matches /dev/mdN-style block device names.
 func isMdraidBlockName(name string) bool {
 	if !strings.HasPrefix(name, "md") {

agent/mdraid_linux_test.go

@@ -40,6 +40,15 @@ func TestMdraidMockSysfsScanAndCollect(t *testing.T) {
 	write(filepath.Join(mdDir, "sync_completed"), "10%\n")
 	write(filepath.Join(mdDir, "sync_speed"), "100M\n")
 	write(filepath.Join(mdDir, "mismatch_cnt"), "0\n")
+
+	// Simulate two healthy member devices (no faulty state).
+	for _, dev := range []string{"dev-sda", "dev-sdb"} {
+		devPath := filepath.Join(mdDir, dev)
+		if err := os.MkdirAll(devPath, 0o755); err != nil {
+			t.Fatal(err)
+		}
+		write(filepath.Join(devPath, "state"), "in_sync\n")
+	}
 	write(filepath.Join(queueDir, "logical_block_size"), "512\n")
 	write(filepath.Join(tmp, "block", "md0", "size"), "2048\n")
 
@@ -81,15 +90,77 @@ func TestMdraidMockSysfsScanAndCollect(t *testing.T) {
 	}
 }
 
+func TestCountMdraidMemberStates(t *testing.T) {
+	tmp := t.TempDir()
+
+	write := func(path, content string) {
+		t.Helper()
+		if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	mdDir := filepath.Join(tmp, "block", "md0", "md")
+
+	// No dev-* entries: zero faulty, zero populated.
+	if faulty, populated := countMdraidMemberStates("md0", tmp); faulty != 0 || populated != 0 {
+		t.Fatalf("no members: got (faulty=%d populated=%d), want (0,0)", faulty, populated)
+	}
+
+	// Two healthy members.
+	write(filepath.Join(mdDir, "dev-sda", "state"), "in_sync\n")
+	write(filepath.Join(mdDir, "dev-sdb", "state"), "in_sync\n")
+	if faulty, populated := countMdraidMemberStates("md0", tmp); faulty != 0 || populated != 2 {
+		t.Fatalf("all in_sync: got (faulty=%d populated=%d), want (0,2)", faulty, populated)
+	}
+
+	// One faulty member.
+	write(filepath.Join(mdDir, "dev-sdb", "state"), "faulty\n")
+	if faulty, populated := countMdraidMemberStates("md0", tmp); faulty != 1 || populated != 2 {
+		t.Fatalf("one faulty: got (faulty=%d populated=%d), want (1,2)", faulty, populated)
+	}
+
+	// QNAP-style: 28 degraded slots but no dev-* entries for them, 4 in_sync.
+	write(filepath.Join(mdDir, "dev-sdb", "state"), "in_sync\n")
+	write(filepath.Join(mdDir, "dev-sdc", "state"), "in_sync\n")
+	write(filepath.Join(mdDir, "dev-sdd", "state"), "in_sync\n")
+	if faulty, populated := countMdraidMemberStates("md0", tmp); faulty != 0 || populated != 4 {
+		t.Fatalf("qnap sparse: got (faulty=%d populated=%d), want (0,4)", faulty, populated)
+	}
+}
+
 func TestMdraidSmartStatus(t *testing.T) {
 	if got := mdraidSmartStatus(mdraidHealth{arrayState: "inactive"}); got != "FAILED" {
 		t.Fatalf("mdraidSmartStatus(inactive) = %q, want FAILED", got)
 	}
-	if got := mdraidSmartStatus(mdraidHealth{arrayState: "active", degraded: 1, syncAction: "recover"}); got != "WARNING" {
+	if got := mdraidSmartStatus(mdraidHealth{arrayState: "active", degraded: 1, faultyDisks: 1, syncAction: "recover"}); got != "WARNING" {
 		t.Fatalf("mdraidSmartStatus(degraded+recover) = %q, want WARNING", got)
 	}
-	if got := mdraidSmartStatus(mdraidHealth{arrayState: "active", degraded: 1}); got != "FAILED" {
+	if got := mdraidSmartStatus(mdraidHealth{arrayState: "active", degraded: 1, faultyDisks: 1}); got != "FAILED" {
-		t.Fatalf("mdraidSmartStatus(degraded) = %q, want FAILED", got)
+		t.Fatalf("mdraidSmartStatus(degraded+faulty) = %q, want FAILED", got)
+	}
+	// QNAP-style: raid_disks=32 but only 4 populated; degraded=28 but no faulty devices.
+	if got := mdraidSmartStatus(mdraidHealth{arrayState: "clean", degraded: 28, faultyDisks: 0, raidDisks: 32, populatedDisks: 4}); got != "WARNING" {
+		t.Fatalf("mdraidSmartStatus(qnap sparse) = %q, want WARNING", got)
+	}
+	// A member disappearing from the same sparse array is indistinguishable
+	// from another reserved slot, so it must not be reported as healthy.
+	if got := mdraidSmartStatus(mdraidHealth{arrayState: "clean", degraded: 29, faultyDisks: 0, raidDisks: 32, populatedDisks: 3}); got != "WARNING" {
+		t.Fatalf("mdraidSmartStatus(qnap sparse missing member) = %q, want WARNING", got)
+	}
+	// A genuinely missing member (removed dev-* entry, not just an unpopulated
+	// QNAP reserve slot) must still fail: raid_disks=4, only 3 populated, all
+	// of them in_sync, so faultyDisks==0 but degraded==1.
+	if got := mdraidSmartStatus(mdraidHealth{arrayState: "clean", degraded: 1, faultyDisks: 0, raidDisks: 4, populatedDisks: 3}); got != "FAILED" {
+		t.Fatalf("mdraidSmartStatus(missing member) = %q, want FAILED", got)
+	}
+	// Degraded with no member-state info at all (e.g. sysfs read failed) must
+	// still fail rather than being silently treated as a sparse QNAP array.
+	if got := mdraidSmartStatus(mdraidHealth{arrayState: "clean", degraded: 1, faultyDisks: 0, raidDisks: 4, populatedDisks: 0}); got != "FAILED" {
+		t.Fatalf("mdraidSmartStatus(degraded, no member info) = %q, want FAILED", got)
 	}
 	if got := mdraidSmartStatus(mdraidHealth{arrayState: "active", syncAction: "recover"}); got != "WARNING" {
 		t.Fatalf("mdraidSmartStatus(recover) = %q, want WARNING", got)

supplemental/helm/beszel-agent/Chart.yaml

@@ -2,9 +2,9 @@ apiVersion: v1
 description: Installs beszel-agent in kubernetes
 home: https://github.com/henrygd/beszel/tree/main/supplemental/helm/beszel-agent
 name: beszel-agent
-appVersion: "0.18.7"
+appVersion: "0.18.8"
 # Bump this version when publishing chart changes.
-version: 0.1.4
+version: 0.1.5
 sources:
   - https://github.com/henrygd/beszel/tree/main/supplemental/helm/beszel-agent
   - https://www.beszel.dev/

supplemental/helm/beszel-agent/README.md

@@ -80,7 +80,7 @@ Essential parameters to configure:
 | `secret.sshKey` | `ssh-key` | Key name in the secret for the SSH public key |
 | `secret.tokenKey` | `token` | Key name in the secret for the authentication token |
 | `image.repository` | `henrygd/beszel-agent` | Container image |
-| `image.tag` | Chart AppVersion (0.18.7) | Image version |
+| `image.tag` | Chart AppVersion (0.18.8) | Image version |
 | `hostNetwork` | `false` | Use host network for network monitoring |
 | `tolerations` | Allows all taints | Tolerations for running on tainted nodes |
 
@@ -385,7 +385,7 @@ helm upgrade beszel-agent ./beszel-agent \
 
 # Change image version
 helm upgrade beszel-agent ./beszel-agent \
-  --set image.tag="0.18.7"
+  --set image.tag="0.18.8"
 ```
 
 ### Restart All Agents
@@ -522,7 +522,7 @@ kubectl get secret beszel-agent -o jsonpath='{.data.ssh-key}' | base64 -d
 ## Chart Information
 
 - **Chart Version**: 0.1.0
-- **App Version**: 0.18.7
+- **App Version**: 0.18.8
 - **Kubernetes Version**: 1.19+
 - **Maintainer**: cloudwithdan (nikoloskid@pm.me)
 

supplemental/helm/beszel-hub/Chart.yaml

@@ -2,9 +2,9 @@ apiVersion: v1
 description: Installs beszel-hub in kubernetes
 home: https://github.com/henrygd/beszel/tree/main/supplemental/helm/beszel-hub
 name: beszel-hub
-appVersion: "0.18.7"
+appVersion: "0.18.8"
 # Bump this version when publishing chart changes.
-version: 0.1.4
+version: 0.1.5
 sources:
   - https://github.com/henrygd/beszel/tree/main/supplemental/helm/beszel-hub
   - https://www.beszel.dev/

supplemental/helm/beszel-hub/README.md

@@ -47,7 +47,7 @@ Key configuration options in `values.yaml`:
 |-----------|---------|-------------|
 | `replicaCount` | `1` | Number of Beszel Hub replicas |
 | `image.repository` | `henrygd/beszel` | Container image repository |
-| `image.tag` | Chart AppVersion (0.18.7) | Container image tag |
+| `image.tag` | Chart AppVersion (0.18.8) | Container image tag |
 | `image.pullPolicy` | `IfNotPresent` | Image pull policy |
 | `service.port` | `8090` | Service port |
 | `persistentVolumeClaim.enabled` | `true` | Enable persistent volume |
@@ -169,7 +169,7 @@ tolerations:
 ```yaml
 replicaCount: 3
 image:
-  tag: "0.18.7"
+  tag: "0.18.8"
 service:
   type: LoadBalancer
 ingress:
@@ -330,7 +330,7 @@ By default, Beszel Hub uses a PersistentVolumeClaim for data storage. Ensure you
 ## Chart Information
 
 - **Chart Version**: 0.1.0
-- **App Version**: 0.18.7
+- **App Version**: 0.18.8
 - **Kubernetes Version**: 1.19+
 - **Maintainer**: cloudwithdan (nikoloskid@pm.me)