[Feature] Basic systemd service monitoring (#1153)

* basic systemd service monitoring

* update to work after /internal rename

* monitor systemd service cpu and memory usage

---------

Co-authored-by: henrygd <hank@henrygd.me>

internal/site/src/components/charts/systemd-services-table.tsx

@@ -0,0 +1,195 @@
+import { SystemdService } from "@/types";
+import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
+import { Trans } from "@lingui/react/macro";
+import { memo, useMemo, useState } from "react";
+import { Button } from "@/components/ui/button";
+import { ChevronDownIcon, ChevronsUpDownIcon, XIcon } from "lucide-react";
+import { cn } from "@/lib/utils";
+import { Input } from "@/components/ui/input";
+
+interface SystemdServicesTableProps {
+    services: SystemdService[];
+}
+
+type SortKey = "name" | "status" | "cpu" | "mem";
+
+const statusPriority: { [key: string]: number } = {
+    failed: 1,
+    activating: 2,
+    active: 3,
+    deactivating: 4,
+    inactive: 5,
+};
+
+const getStatusColor = (status: string) => {
+    switch (status) {
+        case 'active':
+            return 'text-green-500';
+        case 'failed':
+            return 'text-red-500';
+        case 'activating':
+        case 'reloading':
+            return 'text-blue-500';
+        case 'inactive':
+        case 'deactivating':
+            return 'text-gray-500';
+        default:
+            return '';
+    }
+};
+
+const getStatusDotColor = (status: string) => {
+    switch (status) {
+        case 'active':
+            return 'bg-green-500';
+        case 'failed':
+            return 'bg-red-500';
+        case 'activating':
+        case 'reloading':
+            return 'bg-blue-500';
+        case 'inactive':
+        case 'deactivating':
+            return 'bg-gray-500';
+        default:
+            return 'bg-gray-400';
+    }
+};
+
+export default memo(function SystemdServicesTable({ services }: SystemdServicesTableProps) {
+    const [isExpanded, setIsExpanded] = useState(false);
+    const [filter, setFilter] = useState("");
+    const [sortKey, setSortKey] = useState<SortKey>("status");
+    const [sortAsc, setSortAsc] = useState(true);
+
+    const handleSort = (key: SortKey) => {
+        if (sortKey === key) {
+            setSortAsc(!sortAsc);
+        } else {
+            setSortKey(key);
+            setSortAsc(true);
+        }
+    };
+
+    const sortedServices = useMemo(() => {
+        return [...services].sort((a, b) => {
+            let compare = 0;
+            switch (sortKey) {
+                case "name":
+                    compare = a.n.localeCompare(b.n);
+                    break;
+                case "status":
+                    const priorityA = statusPriority[a.s] || 99;
+                    const priorityB = statusPriority[b.s] || 99;
+                    compare = priorityA - priorityB;
+                    if (compare === 0) {
+                        compare = a.n.localeCompare(b.n);
+                    }
+                    break;
+                case "cpu":
+                    compare = (a.c ?? 0) - (b.c ?? 0);
+                    break;
+                case "mem":
+                    compare = (a.m ?? 0) - (b.m ?? 0);
+                    break;
+            }
+            return sortAsc ? compare : -compare;
+        });
+    }, [services, sortKey, sortAsc]);
+
+    const failedServices = useMemo(() => sortedServices.filter(s => s.s === 'failed'), [sortedServices]);
+    const activeServicesCount = useMemo(() => services.filter(s => s.s === 'active').length, [services]);
+
+    const filteredServices = useMemo(() => {
+        if (!filter) {
+            return sortedServices;
+        }
+        return sortedServices.filter(service => service.n.toLowerCase().includes(filter.toLowerCase()));
+    }, [sortedServices, filter]);
+
+    const servicesToShow = isExpanded ? filteredServices : failedServices;
+
+    const summary = (
+        <span className="text-sm text-muted-foreground ml-2">
+            ({failedServices.length} <Trans>failed</Trans>, {activeServicesCount} <Trans>active</Trans>)
+        </span>
+    );
+
+    const SortableHeader = ({ sortKey: key, children }: { sortKey: SortKey, children: React.ReactNode }) => (
+        <TableHead onClick={() => handleSort(key)} className="cursor-pointer">
+            <div className="flex items-center gap-1">
+                {children}
+                <ChevronsUpDownIcon className="h-3 w-3" />
+            </div>
+        </TableHead>
+    );
+
+    return (
+        <div>
+            <div className="flex justify-between items-center mb-2">
+                <h3 className="text-lg font-semibold">
+                    <Trans>Systemd Services</Trans> {summary}
+                </h3>
+                {isExpanded && (
+                    <div className="relative max-w-xs">
+                        <Input
+                            placeholder="Filter services..."
+                            value={filter}
+                            onChange={(e) => setFilter(e.target.value)}
+                            className="ps-4 pe-8"
+                        />
+                        {filter && (
+                            <Button
+                                type="button"
+                                variant="ghost"
+                                size="icon"
+                                aria-label="Clear"
+                                className="absolute right-1 top-1/2 -translate-y-1/2 h-7 w-7 text-gray-500 hover:text-gray-900 dark:text-gray-400 dark:hover:text-gray-100"
+                                onClick={() => setFilter("")}
+                            >
+                                <XIcon className="h-4 w-4" />
+                            </Button>
+                        )}
+                    </div>
+                )}
+            </div>
+            <div className="rounded-md border">
+                <Table>
+                    <TableHeader>
+                        <TableRow>
+                            <SortableHeader sortKey="name"><Trans>Service</Trans></SortableHeader>
+                            <SortableHeader sortKey="status"><Trans>Status</Trans></SortableHeader>
+                            <SortableHeader sortKey="cpu"><Trans>CPU Usage</Trans></SortableHeader>
+                            <SortableHeader sortKey="mem"><Trans>Memory</Trans></SortableHeader>
+                        </TableRow>
+                    </TableHeader>
+                    <TableBody>
+                        {servicesToShow.map((service) => (
+                            <TableRow key={service.n}>
+                                <TableCell className="font-medium">{service.n}</TableCell>
+                                <TableCell className={cn("flex items-center gap-2", getStatusColor(service.s))}>
+                                    <span className={cn("h-2 w-2 rounded-full", getStatusDotColor(service.s))} />
+                                    {service.s}
+                                </TableCell>
+                                <TableCell>{(service.c ?? 0).toFixed(2)}%</TableCell>
+                                <TableCell>{(service.m ?? 0).toFixed(2)} MB</TableCell>
+                            </TableRow>
+                        ))}
+                    </TableBody>
+                </Table>
+            </div>
+            {failedServices.length === 0 && !isExpanded && (
+                <div className="text-center py-4 text-muted-foreground">
+                    <Trans>No failed services.</Trans>
+                </div>
+            )}
+            <div className="flex justify-center mt-2">
+                <Button variant="ghost" onClick={() => setIsExpanded(!isExpanded)} className="flex items-center gap-1">
+                    <span>
+                        {isExpanded ? <Trans>Show less</Trans> : <Trans>Show all</Trans>} ({services.length})
+                    </span>
+                    <ChevronDownIcon className={cn("h-4 w-4 transition-transform", isExpanded && "rotate-180")} />
+                </Button>
+            </div>
+        </div>
+    );
+})

internal/site/src/components/routes/system.tsx

@@ -49,7 +49,7 @@ import {
 	toFixedFloat,
 	useBrowserStorage,
 } from "@/lib/utils"
-import type { ChartData, ChartTimes, ContainerStatsRecord, GPUData, SystemRecord, SystemStatsRecord } from "@/types"
+import type { ChartData, ChartTimes, ContainerStatsRecord, GPUData, SystemRecord, SystemStatsRecord, SystemdStatsRecord } from "@/types"
 import ChartTimeSelect from "../charts/chart-time-select"
 import { $router, navigate } from "../router"
 import Spinner from "../spinner"
@@ -62,6 +62,7 @@ import { Separator } from "../ui/separator"
 import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "../ui/tooltip"
 import NetworkSheet from "./system/network-sheet"
 import LineChartDefault from "../charts/line-chart"
+import SystemdServicesTable from "../charts/systemd-services-table"
 
 type ChartTimeData = {
 	time: number
@@ -95,7 +96,7 @@ function getTimeData(chartTime: ChartTimes, lastCreated: number) {
 }
 
 // add empty values between records to make gaps if interval is too large
-function addEmptyValues<T extends SystemStatsRecord | ContainerStatsRecord>(
+function addValues<T extends SystemStatsRecord | ContainerStatsRecord | SystemdStatsRecord>(
 	prevRecords: T[],
 	newRecords: T[],
 	expectedInterval: number
@@ -119,7 +120,7 @@ function addEmptyValues<T extends SystemStatsRecord | ContainerStatsRecord>(
 	return modifiedRecords
 }
 
-async function getStats<T extends SystemStatsRecord | ContainerStatsRecord>(
+async function getStats<T extends SystemStatsRecord | ContainerStatsRecord | SystemdStatsRecord>(
 	collection: string,
 	system: SystemRecord,
 	chartTime: ChartTimes
@@ -153,6 +154,7 @@ export default memo(function SystemDetail({ name }: { name: string }) {
 	const [grid, setGrid] = useBrowserStorage("grid", true)
 	const [system, setSystem] = useState({} as SystemRecord)
 	const [systemStats, setSystemStats] = useState([] as SystemStatsRecord[])
+	const [systemdStats, setSystemdStats] = useState([] as SystemdStatsRecord[])
 	const [containerData, setContainerData] = useState([] as ChartData["containerData"])
 	const netCardRef = useRef<HTMLDivElement>(null)
 	const persistChartTime = useRef(false)
@@ -171,6 +173,7 @@ export default memo(function SystemDetail({ name }: { name: string }) {
 			}
 			persistChartTime.current = false
 			setSystemStats([])
+			setSystemdStats([])
 			setContainerData([])
 			setContainerFilterBar(null)
 			$containerFilter.set("")
@@ -235,7 +238,8 @@ export default memo(function SystemDetail({ name }: { name: string }) {
 		Promise.allSettled([
 			getStats<SystemStatsRecord>("system_stats", system, chartTime),
 			getStats<ContainerStatsRecord>("container_stats", system, chartTime),
-		]).then(([systemStats, containerStats]) => {
+			getStats<SystemdStatsRecord>("systemd_stats", system, chartTime),
+		]).then(([systemStats, containerStats, systemdStats]) => {
 			// loading: false
 			setChartLoading(false)
 
@@ -244,18 +248,29 @@ export default memo(function SystemDetail({ name }: { name: string }) {
 			const ss_cache_key = `${system.id}_${chartTime}_system_stats`
 			let systemData = (cache.get(ss_cache_key) || []) as SystemStatsRecord[]
 			if (systemStats.status === "fulfilled" && systemStats.value.length) {
-				systemData = systemData.concat(addEmptyValues(systemData, systemStats.value, expectedInterval))
+				systemData = systemData.concat(addValues(systemData, systemStats.value, expectedInterval))
 				if (systemData.length > 120) {
 					systemData = systemData.slice(-100)
 				}
 				cache.set(ss_cache_key, systemData)
 			}
 			setSystemStats(systemData)
+			// make new systemd stats
+			const sds_cache_key = `${system.id}_${chartTime}_systemd_stats`
+			let systemdData = (cache.get(sds_cache_key) || []) as SystemdStatsRecord[]
+			if (systemdStats.status === "fulfilled" && systemdStats.value.length) {
+				systemdData = systemdData.concat(addValues(systemdData, systemdStats.value, expectedInterval))
+				if (systemdData.length > 120) {
+					systemdData = systemdData.slice(-100)
+				}
+				cache.set(sds_cache_key, systemdData)
+			}
+			setSystemdStats(systemdData)
 			// make new container stats
 			const cs_cache_key = `${system.id}_${chartTime}_container_stats`
 			let containerData = (cache.get(cs_cache_key) || []) as ContainerStatsRecord[]
 			if (containerStats.status === "fulfilled" && containerStats.value.length) {
-				containerData = containerData.concat(addEmptyValues(containerData, containerStats.value, expectedInterval))
+				containerData = containerData.concat(addValues(containerData, containerStats.value, expectedInterval))
 				if (containerData.length > 120) {
 					containerData = containerData.slice(-100)
 				}
@@ -847,6 +862,18 @@ export default memo(function SystemDetail({ name }: { name: string }) {
 					</div>
 				)}
 
+				{/* systemd services table */}
+				{(systemdStats.at(-1)?.stats?.length ?? 0) > 0 && (
+					<Card className="col-span-full">
+						<CardHeader className="pb-5 pt-4 gap-1 relative max-sm:py-3 max-sm:px-4">
+							<CardTitle className="text-xl sm:text-2xl"><Trans>Systemd Services</Trans></CardTitle>
+						</CardHeader>
+						<div className="px-4 pb-4">
+							<SystemdServicesTable services={systemdStats.at(-1)!.stats!} />
+						</div>
+					</Card>
+				)}
+
 				{/* extra filesystem charts */}
 				{Object.keys(systemStats.at(-1)?.stats.efs ?? {}).length > 0 && (
 					<div className="grid xl:grid-cols-2 gap-4">

internal/site/src/components/systems-table/systems-table-columns.tsx

@@ -4,12 +4,15 @@ import { useStore } from "@nanostores/react"
 import { getPagePath } from "@nanostores/router"
 import type { CellContext, ColumnDef, HeaderContext } from "@tanstack/react-table"
 import type { ClassValue } from "clsx"
+import type { SystemRecord, SystemStats, SystemStatsRecord, SystemdService, SystemdStatsRecord  } from "@/types"
 import {
 	ArrowUpDownIcon,
 	ChevronRightSquareIcon,
+	CheckIcon,
 	CopyIcon,
 	CpuIcon,
 	HardDriveIcon,
+	ListChecks,
 	MemoryStickIcon,
 	MoreHorizontalIcon,
 	PauseCircleIcon,
@@ -32,7 +35,6 @@ import {
 	getMeterState,
 	parseSemVer,
 } from "@/lib/utils"
-import type { SystemRecord } from "@/types"
 import { SystemDialog } from "../add-system"
 import AlertButton from "../alerts/alert-button"
 import { $router, Link } from "../router"
@@ -300,6 +302,15 @@ export default function SystemsTableColumns(viewMode: "table" | "grid"): ColumnD
 				)
 			},
 		},
+		{
+			id: "systemd",
+			name: () => t`Services`,
+			size: 50,
+			Icon: ListChecks,
+			hideSort: true,
+			header: sortableHeader,
+			cell: ({ row }) => <SystemdCell systemId={row.original.id} />,
+		},
 		{
 			id: "actions",
 			// @ts-expect-error
@@ -363,6 +374,46 @@ export function IndicatorDot({ system, className }: { system: SystemRecord; clas
 	)
 }
 
+const SystemdCell = ({ systemId }: { systemId: string }) => {
+	const [stats, setStats] = useState<SystemdService[] | null>(null);
+
+	useEffect(() => {
+		const fetchStats = async () => {
+			try {
+				const record = await pb.collection("systemd_stats").getFirstListItem<SystemdStatsRecord>(`system="${systemId}"`, {
+					sort: "-created",
+				});
+				setStats(record.stats);
+			} catch (error) {
+				// Handle case where no stats are found
+				setStats(null);
+			}
+		};
+
+		fetchStats();
+	}, [systemId]);
+
+	if (!stats) {
+		return <span className="text-muted-foreground">-</span>;
+	}
+
+	const failed = stats.filter(s => s.s === 'failed').length;
+
+	if (failed > 0) {
+		return (
+			<div className="tabular-nums text-red-500">
+				{failed}
+			</div>
+		);
+	}
+
+	return (
+		<div className="text-green-500 flex items-center justify-center">
+			<CheckIcon className="size-4" />
+		</div>
+	);
+};
+
 export const ActionsButton = memo(({ system }: { system: SystemRecord }) => {
 	const [deleteOpen, setDeleteOpen] = useState(false)
 	const [editOpen, setEditOpen] = useState(false)

internal/site/src/locales/en/en.po

@@ -75,6 +75,10 @@ msgstr "5 min"
 msgid "Actions"
 msgstr "Actions"
 
+#: src/components/charts/systemd-services-table.tsx
+msgid "active"
+msgstr "active"
+
 #: src/components/alerts-history-columns.tsx
 #: src/components/routes/settings/alerts-history-data-table.tsx
 msgid "Active"
@@ -343,6 +347,7 @@ msgstr "Copy YAML"
 msgid "CPU"
 msgstr "CPU"
 
+#: src/components/charts/systemd-services-table.tsx
 #: src/components/routes/system.tsx
 #: src/components/routes/system.tsx
 #: src/lib/alerts.ts
@@ -523,6 +528,10 @@ msgstr "Export your current systems configuration."
 msgid "Fahrenheit (°F)"
 msgstr "Fahrenheit (°F)"
 
+#: src/components/charts/systemd-services-table.tsx
+msgid "failed"
+msgstr "failed"
+
 #: src/lib/api.ts
 msgid "Failed to authenticate"
 msgstr "Failed to authenticate"
@@ -680,6 +689,7 @@ msgstr "Manual setup instructions"
 msgid "Max 1 min"
 msgstr "Max 1 min"
 
+#: src/components/charts/systemd-services-table.tsx
 #: src/components/systems-table/systems-table-columns.tsx
 msgid "Memory"
 msgstr "Memory"
@@ -718,6 +728,10 @@ msgstr "Network traffic of public interfaces"
 msgid "Network unit"
 msgstr "Network unit"
 
+#: src/components/charts/systemd-services-table.tsx
+msgid "No failed services."
+msgstr "No failed services."
+
 #: src/components/command-palette.tsx
 msgid "No results found."
 msgstr "No results found."
@@ -926,6 +940,14 @@ msgstr "See <0>notification settings</0> to configure how you receive alerts."
 msgid "Sent"
 msgstr "Sent"
 
+#: src/components/charts/systemd-services-table.tsx
+msgid "Service"
+msgstr "Service"
+
+#: src/components/systems-table/systems-table-columns.tsx
+msgid "Services"
+msgstr "Services"
+
 #: src/components/routes/settings/general.tsx
 msgid "Set percentage thresholds for meter colors."
 msgstr "Set percentage thresholds for meter colors."
@@ -941,6 +963,14 @@ msgstr "Settings"
 msgid "Settings saved"
 msgstr "Settings saved"
 
+#: src/components/charts/systemd-services-table.tsx
+msgid "Show all"
+msgstr "Show all"
+
+#: src/components/charts/systemd-services-table.tsx
+msgid "Show less"
+msgstr "Show less"
+
 #: src/components/login/auth-form.tsx
 msgid "Sign in"
 msgstr "Sign in"
@@ -958,6 +988,7 @@ msgstr "Sort By"
 msgid "State"
 msgstr "State"
 
+#: src/components/charts/systemd-services-table.tsx
 #: src/components/systems-table/systems-table.tsx
 #: src/lib/alerts.ts
 msgid "Status"
@@ -982,6 +1013,11 @@ msgstr "System"
 msgid "System load averages over time"
 msgstr "System load averages over time"
 
+#: src/components/charts/systemd-services-table.tsx
+#: src/components/routes/system.tsx
+msgid "Systemd Services"
+msgstr "Systemd Services"
+
 #: src/components/navbar.tsx
 msgid "Systems"
 msgstr "Systems"

internal/site/src/locales/is/is.po

@@ -80,6 +80,10 @@ msgstr ""
 msgid "Actions"
 msgstr "Aðgerðir"
 
+#: src/components/charts/systemd-services-table.tsx
+msgid "active"
+msgstr ""
+
 #: src/components/alerts-history-columns.tsx
 #: src/components/routes/settings/alerts-history-data-table.tsx
 msgid "Active"
@@ -348,6 +352,7 @@ msgstr ""
 msgid "CPU"
 msgstr "Örgjörvi"
 
+#: src/components/charts/systemd-services-table.tsx
 #: src/components/routes/system.tsx
 #: src/components/routes/system.tsx
 #: src/lib/alerts.ts
@@ -528,6 +533,10 @@ msgstr ""
 msgid "Fahrenheit (°F)"
 msgstr ""
 
+#: src/components/charts/systemd-services-table.tsx
+msgid "failed"
+msgstr ""
+
 #: src/lib/api.ts
 msgid "Failed to authenticate"
 msgstr "Villa í auðkenningu"
@@ -685,6 +694,7 @@ msgstr ""
 msgid "Max 1 min"
 msgstr "Mest 1 mínúta"
 
+#: src/components/charts/systemd-services-table.tsx
 #: src/components/systems-table/systems-table-columns.tsx
 msgid "Memory"
 msgstr "Minni"
@@ -723,6 +733,10 @@ msgstr ""
 msgid "Network unit"
 msgstr ""
 
+#: src/components/charts/systemd-services-table.tsx
+msgid "No failed services."
+msgstr ""
+
 #: src/components/command-palette.tsx
 msgid "No results found."
 msgstr "Engar niðurstöður fundust."
@@ -931,6 +945,14 @@ msgstr ""
 msgid "Sent"
 msgstr "Sent"
 
+#: src/components/charts/systemd-services-table.tsx
+msgid "Service"
+msgstr ""
+
+#: src/components/systems-table/systems-table-columns.tsx
+msgid "Services"
+msgstr ""
+
 #: src/components/routes/settings/general.tsx
 msgid "Set percentage thresholds for meter colors."
 msgstr "Stilltu prósentuþröskuld fyrir mælaliti."
@@ -946,6 +968,14 @@ msgstr "Stillingar"
 msgid "Settings saved"
 msgstr "Stillingar vistaðar"
 
+#: src/components/charts/systemd-services-table.tsx
+msgid "Show all"
+msgstr ""
+
+#: src/components/charts/systemd-services-table.tsx
+msgid "Show less"
+msgstr ""
+
 #: src/components/login/auth-form.tsx
 msgid "Sign in"
 msgstr "Innskrá"
@@ -963,6 +993,7 @@ msgstr "Raða eftir"
 msgid "State"
 msgstr ""
 
+#: src/components/charts/systemd-services-table.tsx
 #: src/components/systems-table/systems-table.tsx
 #: src/lib/alerts.ts
 msgid "Status"
@@ -987,6 +1018,11 @@ msgstr "Kerfi"
 msgid "System load averages over time"
 msgstr ""
 
+#: src/components/charts/systemd-services-table.tsx
+#: src/components/routes/system.tsx
+msgid "Systemd Services"
+msgstr ""
+
 #: src/components/navbar.tsx
 msgid "Systems"
 msgstr "Kerfi"

internal/site/src/types.d.ts

@@ -147,6 +147,17 @@ export interface SystemStats {
 	ni?: Record<string, [number, number, number, number]>
 }
 
+export interface SystemdService {
+	/** name */
+	n: string
+	/** status */
+	s: string
+	/** cpu percent */
+	c: number
+	/** memory used (mb) */
+	m: number
+}
+
 export interface GPUData {
 	/** name */
 	n: string
@@ -185,6 +196,12 @@ export interface ContainerStatsRecord extends RecordModel {
 	created: string | number
 }
 
+export interface SystemdStatsRecord extends RecordModel {
+	system: string
+	stats: SystemdService[]
+	created: string | number
+}
+
 interface ContainerStats {
 	/** name */
 	n: string