[Feature] Add detailed CPU metrics (User, System, IOWait, Steal) with per-core monitoring (#1356)

* Add user, system io wait

* add per cpu core

* add total

internal/entities/system/system.go

@@ -11,7 +11,12 @@ import (
 type Stats struct {
 	Cpu            float64             `json:"cpu" cbor:"0,keyasint"`
 	MaxCpu         float64             `json:"cpum,omitempty" cbor:"1,keyasint,omitempty"`
-	Mem            float64             `json:"m" cbor:"2,keyasint"`
+	CpuUser        float64                `json:"cpuu,omitempty" cbor:"33,keyasint,omitempty"`
+	CpuSystem      float64                `json:"cpus,omitempty" cbor:"34,keyasint,omitempty"`
+	CpuIowait      float64                `json:"cpui,omitempty" cbor:"35,keyasint,omitempty"`
+	CpuSteal       float64                `json:"cpust,omitempty" cbor:"36,keyasint,omitempty"`
+	CpuCores       map[string][4]float64  `json:"cpuc,omitempty" cbor:"37,keyasint,omitempty"` // [user, system, iowait, steal] per core
+	Mem            float64                `json:"m" cbor:"2,keyasint"`
 	MemUsed        float64             `json:"mu" cbor:"3,keyasint"`
 	MemPct         float64             `json:"mp" cbor:"4,keyasint"`
 	MemBuffCache   float64             `json:"mb" cbor:"5,keyasint"`

internal/site/src/components/charts/hooks.ts

@@ -118,6 +118,28 @@ export function useNetworkInterfaces(interfaces: SystemStats["ni"]) {
 				dataKey: ({ stats }: SystemStatsRecord) => stats?.ni?.[key]?.[index],
 				color: `hsl(${220 + (((sortedKeys.indexOf(key) * 360) / sortedKeys.length) % 360)}, 70%, 50%)`,
 
+				opacity: 0.3,
+			}))
+		},
+	}
+}
+
+// Assures consistent colors for CPU cores
+export function useCpuCores(cores: SystemStats["cpuc"]) {
+	const keys = Object.keys(cores ?? {})
+	// Sort cores by name (cpu0, cpu1, cpu2, etc.)
+	const sortedKeys = keys.sort((a, b) => {
+		const numA = Number.parseInt(a.replace("cpu", ""))
+		const numB = Number.parseInt(b.replace("cpu", ""))
+		return numA - numB
+	})
+	return {
+		length: sortedKeys.length,
+		data: (index = 0) => {
+			return sortedKeys.map((key) => ({
+				label: key,
+				dataKey: ({ stats }: SystemStatsRecord) => stats?.cpuc?.[key]?.[index],
+				color: `hsl(${(((sortedKeys.indexOf(key) * 360) / sortedKeys.length) % 360)}, 70%, 50%)`,
 				opacity: 0.3,
 			}))
 		},

internal/site/src/components/routes/system.tsx

@@ -73,6 +73,7 @@ import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from ".
 import { Separator } from "../ui/separator"
 import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "../ui/tooltip"
 import NetworkSheet from "./system/network-sheet"
+import CpuCoresSheet from "./system/cpu-cores-sheet"
 import LineChartDefault from "../charts/line-chart"
 
 
@@ -585,18 +586,49 @@ export default memo(function SystemDetail({ id }: { id: string }) {
 						grid={grid}
 						title={t`CPU Usage`}
 						description={t`Average system-wide CPU utilization`}
-						cornerEl={maxValSelect}
+						cornerEl={
+							<>
+								{maxValSelect}
+								<CpuCoresSheet chartData={chartData} dataEmpty={dataEmpty} grid={grid} maxValues={maxValues} />
+							</>
+						}
+						legend={true}
 					>
 						<AreaChartDefault
 							chartData={chartData}
 							maxToggled={maxValues}
+							legend={true}
 							dataPoints={[
 								{
-									label: t`CPU Usage`,
+									label: t`Total`,
 									dataKey: ({ stats }) => (showMax ? stats?.cpum : stats?.cpu),
 									color: 1,
 									opacity: 0.4,
 								},
+								{
+									label: t`User`,
+									dataKey: ({ stats }) => stats?.cpuu,
+									color: 2,
+									opacity: 0.3,
+								},
+								{
+									label: t`System`,
+									dataKey: ({ stats }) => stats?.cpus,
+									color: 3,
+									opacity: 0.3,
+								},
+								{
+									label: t`IOWait`,
+									dataKey: ({ stats }) => stats?.cpui,
+									color: 4,
+									opacity: 0.3,
+								},
+								{
+									label: t`Steal`,
+									dataKey: ({ stats }) => stats?.cpust,
+									color: 5,
+									opacity: 0.3,
+								},
 							]}
 							tickFormatter={(val) => `${toFixedFloat(val, 2)}%`}
 							contentFormatter={({ value }) => `${decimalString(value)}%`}

internal/site/src/components/routes/system/cpu-cores-sheet.tsx

@@ -0,0 +1,119 @@
+import { t } from "@lingui/core/macro"
+import { MoreHorizontalIcon } from "lucide-react"
+import { memo, useRef, useState } from "react"
+import AreaChartDefault from "@/components/charts/area-chart"
+import ChartTimeSelect from "@/components/charts/chart-time-select"
+import { Button } from "@/components/ui/button"
+import { Sheet, SheetContent, SheetTrigger } from "@/components/ui/sheet"
+import { DialogTitle } from "@/components/ui/dialog"
+import { decimalString, toFixedFloat } from "@/lib/utils"
+import type { ChartData, SystemStatsRecord } from "@/types"
+import { ChartCard } from "../system"
+
+export default memo(function CpuCoresSheet({
+	chartData,
+	dataEmpty,
+	grid,
+	maxValues,
+}: {
+	chartData: ChartData
+	dataEmpty: boolean
+	grid: boolean
+	maxValues: boolean
+}) {
+	const [cpuCoresOpen, setCpuCoresOpen] = useState(false)
+	const hasOpened = useRef(false)
+
+	if (cpuCoresOpen && !hasOpened.current) {
+		hasOpened.current = true
+	}
+
+	// Get list of CPU cores from the latest stats
+	const cpuCoresData = chartData.systemStats.at(-1)?.stats?.cpuc ?? {}
+	const coreNames = Object.keys(cpuCoresData).sort((a, b) => {
+		const numA = Number.parseInt(a.replace("cpu", ""))
+		const numB = Number.parseInt(b.replace("cpu", ""))
+		return numA - numB
+	})
+
+	if (coreNames.length === 0) {
+		return null
+	}
+
+	return (
+		<Sheet open={cpuCoresOpen} onOpenChange={setCpuCoresOpen}>
+			<DialogTitle className="sr-only">{t`Per-core CPU usage`}</DialogTitle>
+			<SheetTrigger asChild>
+				<Button
+					title={t`View per-core CPU`}
+					variant="outline"
+					size="icon"
+					className="shrink-0 max-sm:absolute max-sm:top-3 max-sm:end-3"
+				>
+					<MoreHorizontalIcon />
+				</Button>
+			</SheetTrigger>
+			{hasOpened.current && (
+				<SheetContent aria-describedby={undefined} className="overflow-auto w-200 !max-w-full p-4 sm:p-6">
+					<ChartTimeSelect className="w-[calc(100%-2em)]" agentVersion={chartData.agentVersion} />
+					{coreNames.map((coreName) => (
+						<ChartCard
+							key={coreName}
+							empty={dataEmpty}
+							grid={grid}
+							title={coreName.toUpperCase()}
+							description={t`CPU usage breakdown for ${coreName}`}
+							legend={true}
+							className="min-h-auto"
+						>
+							<AreaChartDefault
+								chartData={chartData}
+								maxToggled={maxValues}
+								legend={true}
+								dataPoints={[
+									{
+										label: t`Total`,
+										dataKey: ({ stats }: SystemStatsRecord) => {
+											const core = stats?.cpuc?.[coreName]
+											if (!core) return undefined
+											// Sum all metrics: user + system + iowait + steal
+											return core[0] + core[1] + core[2] + core[3]
+										},
+										color: 1,
+										opacity: 0.4,
+									},
+									{
+										label: t`User`,
+										dataKey: ({ stats }: SystemStatsRecord) => stats?.cpuc?.[coreName]?.[0],
+										color: 2,
+										opacity: 0.3,
+									},
+									{
+										label: t`System`,
+										dataKey: ({ stats }: SystemStatsRecord) => stats?.cpuc?.[coreName]?.[1],
+										color: 3,
+										opacity: 0.3,
+									},
+									{
+										label: t`IOWait`,
+										dataKey: ({ stats }: SystemStatsRecord) => stats?.cpuc?.[coreName]?.[2],
+										color: 4,
+										opacity: 0.3,
+									},
+									{
+										label: t`Steal`,
+										dataKey: ({ stats }: SystemStatsRecord) => stats?.cpuc?.[coreName]?.[3],
+										color: 5,
+										opacity: 0.3,
+									},
+								]}
+								tickFormatter={(val) => `${toFixedFloat(val, 2)}%`}
+								contentFormatter={({ value }) => `${decimalString(value)}%`}
+							/>
+						</ChartCard>
+					))}
+				</SheetContent>
+			)}
+		</Sheet>
+	)
+})

internal/site/src/types.d.ts

@@ -84,6 +84,16 @@ export interface SystemStats {
 	cpu: number
 	/** peak cpu */
 	cpum?: number
+	/** cpu user percent */
+	cpuu?: number
+	/** cpu system percent */
+	cpus?: number
+	/** cpu iowait percent */
+	cpui?: number
+	/** cpu steal percent */
+	cpust?: number
+	/** per-core cpu metrics [user, system, iowait, steal] */
+	cpuc?: Record<string, [number, number, number, number]>
 	// TODO: remove these in future release in favor of la
 	/** load average 1 minute */
 	l1?: number