feat(agent): Add docker image update available flag (#2211)

Co-authored-by: henrygd <hank@henrygd.me>

agent/docker.go

@@ -65,10 +65,14 @@ type dockerManager struct {
 	dockerVersionChecked bool                        // Whether a version probe has completed successfully
 	isWindows            bool                        // Whether the Docker Engine API is running on Windows
 	buf                  *bytes.Buffer               // Buffer to store and read response bodies
-	apiStats             *container.ApiStats         // Reusable API stats object
 	excludeContainers    []string                    // Patterns to exclude containers by name
 	usingPodman          bool                        // Whether the Docker Engine API is running on Podman
 
+	registryClient      *http.Client                  // Client for registry requests; nil uses a client with a 10-second timeout
+	imageUpdatesMutex   sync.RWMutex                  // Protects imageUpdates, its entries, and imageUpdatesRunning
+	imageUpdates        map[string]*imageUpdateStatus // Shared update status keyed by normalized image reference
+	imageUpdatesRunning bool                          // Whether a background image-update batch is in progress
+
 	// Cache-time-aware tracking for CPU stats (similar to cpu.go)
 	// Maps cache time intervals to container-specific CPU usage tracking
 	lastCpuContainer map[uint16]map[string]uint64    // cacheTimeMs -> containerId -> last cpu container usage
@@ -161,6 +165,9 @@ func (dm *dockerManager) getDockerStats(cacheTimeMs uint16) ([]*container.Stats,
 		clear(dm.validIds)
 	}
 
+	// Only schedule auxiliary work here; metrics never wait for image discovery.
+	dm.refreshImageUpdates(dm.apiContainerList, time.Now())
+
 	var failedContainers []*container.ApiInfo
 
 	for _, ctr := range dm.apiContainerList {
@@ -506,6 +513,17 @@ func (dm *dockerManager) updateContainerStats(ctr *container.ApiInfo, cacheTimeM
 		}
 	}
 
+	// Read and decode the response before locking shared stats to avoid blocking
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("container stats request failed: %s", resp.Status)
+	}
+	res := &container.ApiStats{}
+	if err := json.NewDecoder(resp.Body).Decode(res); err != nil {
+		return err
+	}
+	updateAvailable := dm.cachedImageUpdate(ctr.Image)
+
 	dm.containerStatsMutex.Lock()
 	defer dm.containerStatsMutex.Unlock()
 
@@ -520,6 +538,9 @@ func (dm *dockerManager) updateContainerStats(ctr *container.ApiInfo, cacheTimeM
 	stats.Status = statusText
 	stats.Health = health
 
+	stats.Image = ctr.Image
+	stats.UpdateAvailable = updateAvailable
+
 	if len(ctr.Ports) > 0 {
 		stats.Ports = convertContainerPortsToString(ctr)
 	}
@@ -532,12 +553,6 @@ func (dm *dockerManager) updateContainerStats(ctr *container.ApiInfo, cacheTimeM
 	stats.NetworkSent = 0
 	stats.NetworkRecv = 0
 
-	res := dm.apiStats
-	res.Networks = nil
-	if err := dm.decode(resp, res); err != nil {
-		return err
-	}
-
 	// Initialize CPU tracking for this cache time interval
 	dm.initializeCpuTracking(cacheTimeMs)
 
@@ -695,7 +710,6 @@ func newDockerManager(agent *Agent) *dockerManager {
 		containerStatsMap: make(map[string]*container.Stats),
 		sem:               make(chan struct{}, 5),
 		apiContainerList:  []*container.ApiInfo{},
-		apiStats:          &container.ApiStats{},
 		excludeContainers: excludeContainers,
 
 		// Initialize cache-time-aware tracking structures

agent/docker_image_updates.go

@@ -0,0 +1,105 @@
+package agent
+
+import (
+	"log/slog"
+	"sync"
+	"time"
+
+	"github.com/distribution/reference"
+	"github.com/henrygd/beszel/internal/entities/container"
+)
+
+const imageUpdateInterval = time.Hour
+
+type imageUpdateStatus struct {
+	available bool
+	checkedAt time.Time
+}
+
+func normalizedImageReference(image string) string {
+	named, err := reference.ParseNormalizedNamed(image)
+	if err != nil {
+		return ""
+	}
+	// Digest-pinned references cannot move to a new version.
+	if _, pinned := named.(reference.Digested); pinned {
+		return ""
+	}
+	return reference.TagNameOnly(named).String()
+}
+
+// refreshImageUpdates starts at most one background batch. Neither its network
+// work nor its completion is part of the container metrics wait group.
+func (dm *dockerManager) refreshImageUpdates(containers []*container.ApiInfo, now time.Time) {
+	dm.imageUpdatesMutex.Lock()
+	defer dm.imageUpdatesMutex.Unlock()
+	if dm.imageUpdatesRunning {
+		return
+	}
+	if dm.imageUpdates == nil {
+		dm.imageUpdates = make(map[string]*imageUpdateStatus)
+	}
+	active := make(map[string]struct{}, len(containers))
+	pending := make(map[string]*imageUpdateStatus)
+	for _, ctr := range containers {
+		if len(ctr.Names) > 0 && dm.shouldExcludeContainer(ctr.Names[0][1:]) {
+			continue
+		}
+		key := normalizedImageReference(ctr.Image)
+		if key == "" {
+			continue
+		}
+		active[key] = struct{}{}
+		entry := dm.imageUpdates[key]
+		if entry == nil {
+			entry = &imageUpdateStatus{}
+			dm.imageUpdates[key] = entry
+		}
+		if entry.checkedAt.IsZero() || now.Sub(entry.checkedAt) >= imageUpdateInterval {
+			pending[key] = entry
+		}
+	}
+	for key := range dm.imageUpdates {
+		if _, ok := active[key]; !ok {
+			delete(dm.imageUpdates, key)
+		}
+	}
+	if len(pending) == 0 {
+		return
+	}
+	dm.imageUpdatesRunning = true
+	go func() {
+		// Limit auxiliary requests even on hosts running many different images.
+		sem := make(chan struct{}, 2)
+		var wg sync.WaitGroup
+		for key, entry := range pending {
+			sem <- struct{}{}
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				defer func() { <-sem }()
+				available, err := dm.checkImageUpdate(key)
+				if err != nil {
+					available = false
+					slog.Debug("Image update check failed", "image", key, "err", err)
+				}
+				dm.imageUpdatesMutex.Lock()
+				entry.available = available
+				entry.checkedAt = time.Now()
+				dm.imageUpdatesMutex.Unlock()
+			}()
+		}
+		wg.Wait()
+		dm.imageUpdatesMutex.Lock()
+		dm.imageUpdatesRunning = false
+		dm.imageUpdatesMutex.Unlock()
+	}()
+}
+
+func (dm *dockerManager) cachedImageUpdate(image string) bool {
+	key := normalizedImageReference(image)
+	dm.imageUpdatesMutex.RLock()
+	defer dm.imageUpdatesMutex.RUnlock()
+	entry := dm.imageUpdates[key]
+	return entry != nil && entry.available
+}

agent/docker_image_updates_test.go

@@ -0,0 +1,225 @@
+//go:build testing
+
+package agent
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/fxamacker/cbor/v2"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/henrygd/beszel/internal/entities/container"
+	"github.com/stretchr/testify/require"
+)
+
+func waitForImageUpdates(t *testing.T, dm *dockerManager) {
+	t.Helper()
+	require.Eventually(t, func() bool {
+		dm.imageUpdatesMutex.RLock()
+		defer dm.imageUpdatesMutex.RUnlock()
+		return !dm.imageUpdatesRunning
+	}, time.Second*3, time.Millisecond)
+}
+
+func TestImageUpdateCacheAndStats(t *testing.T) {
+	local := "sha256:" + strings.Repeat("a", 64)
+	remote := "sha256:" + strings.Repeat("b", 64)
+	var inspections, lookups atomic.Int32
+	var fail atomic.Bool
+	var upToDate atomic.Bool
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch {
+		case strings.HasPrefix(r.URL.Path, "/images/"):
+			inspections.Add(1)
+			fmt.Fprintf(w, `{"RepoDigests":["docker.io/library/nginx@%s"]}`, local)
+		case r.URL.Path == "/containers/json":
+			fmt.Fprint(w, `[{"Id":"aaaaaaaaaaaa","Names":["/one"],"Image":"nginx","Status":"Up 2 hours"},{"Id":"bbbbbbbbbbbb","Names":["/two"],"Image":"docker.io/library/nginx:latest","Status":"Up 2 hours"}]`)
+		case strings.Contains(r.URL.Path, "/stats"):
+			fmt.Fprint(w, `{"memory_stats":{"usage":1048576},"cpu_stats":{},"networks":{}}`)
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer server.Close()
+	dm := newDockerManagerForVersionTest(server)
+	dm.dockerVersionChecked = true
+	dm.registryClient = &http.Client{Timeout: time.Second, Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
+		if fail.Load() {
+			return nil, fmt.Errorf("registry unavailable")
+		}
+		response := &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"token":"test"}`))}
+		if r.Method == http.MethodHead {
+			lookups.Add(1)
+			digest := remote
+			if upToDate.Load() {
+				digest = local
+			}
+			response.Header.Set("Docker-Content-Digest", digest)
+		}
+		return response, nil
+	})}
+	stats, err := dm.getDockerStats(defaultCacheTimeMs)
+	require.NoError(t, err)
+	require.Len(t, stats, 2)
+	waitForImageUpdates(t, dm)
+	require.EqualValues(t, 1, lookups.Load())
+	require.EqualValues(t, 1, inspections.Load())
+	stats, err = dm.getDockerStats(defaultCacheTimeMs)
+	require.NoError(t, err)
+	for _, stat := range stats {
+		require.True(t, stat.UpdateAvailable)
+		if stat.Id == "aaaaaaaaaaaa" {
+			require.Equal(t, "nginx", stat.Image)
+		} else {
+			require.Equal(t, "docker.io/library/nginx:latest", stat.Image)
+		}
+	}
+	require.EqualValues(t, 1, lookups.Load())
+
+	expire := func() {
+		dm.imageUpdatesMutex.Lock()
+		dm.imageUpdates["docker.io/library/nginx:latest"].checkedAt = time.Now().Add(-imageUpdateInterval)
+		dm.imageUpdatesMutex.Unlock()
+	}
+	upToDate.Store(true)
+	expire()
+	_, err = dm.getDockerStats(defaultCacheTimeMs)
+	require.NoError(t, err)
+	waitForImageUpdates(t, dm)
+	require.EqualValues(t, 2, lookups.Load())
+	require.False(t, dm.cachedImageUpdate("nginx:latest"))
+
+	// An expired positive result is cleared on failure, and the failure itself
+	// is cached so realtime stats do not retry a broken registry every second.
+	dm.imageUpdatesMutex.Lock()
+	dm.imageUpdates["docker.io/library/nginx:latest"].available = true
+	dm.imageUpdatesMutex.Unlock()
+	fail.Store(true)
+	expire()
+	_, err = dm.getDockerStats(defaultCacheTimeMs)
+	require.NoError(t, err)
+	waitForImageUpdates(t, dm)
+	failedInspections := inspections.Load()
+	stats, err = dm.getDockerStats(defaultCacheTimeMs)
+	require.NoError(t, err)
+	require.Len(t, stats, 2)
+	require.Equal(t, failedInspections, inspections.Load())
+	for _, stat := range stats {
+		require.False(t, stat.UpdateAvailable)
+		require.Equal(t, 1.0, stat.Mem)
+	}
+}
+
+func TestImageDiscoveryDoesNotBlockStats(t *testing.T) {
+	started := make(chan struct{}, 1)
+	release := make(chan struct{})
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if strings.HasPrefix(r.URL.Path, "/images/") {
+			fmt.Fprintf(w, `{"RepoDigests":["example.com/app@sha256:%s"]}`, strings.Repeat("a", 64))
+		} else {
+			fmt.Fprint(w, `{"memory_stats":{"usage":1048576}}`)
+		}
+	}))
+	defer server.Close()
+	dm := newDockerManagerForVersionTest(server)
+	defer func() { close(release); waitForImageUpdates(t, dm) }()
+	dm.registryClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
+		started <- struct{}{}
+		<-release
+		return nil, fmt.Errorf("timeout")
+	})}
+	ctr := &container.ApiInfo{IdShort: "aaaaaaaaaaaa", Image: "example.com/app", Names: []string{"/one"}}
+	dm.refreshImageUpdates([]*container.ApiInfo{ctr}, time.Now())
+	select {
+	case <-started:
+	case <-time.After(3 * time.Second):
+		t.Fatal("check did not start")
+	}
+	done := make(chan error, 1)
+	go func() { done <- dm.updateContainerStats(ctr, defaultCacheTimeMs) }()
+	select {
+	case err := <-done:
+		require.NoError(t, err)
+	case <-time.After(time.Second):
+		t.Fatal("registry blocked stats")
+	}
+	dm.imageUpdatesMutex.RLock()
+	require.True(t, dm.imageUpdatesRunning)
+	dm.imageUpdatesMutex.RUnlock()
+}
+
+func TestNormalizeImageUpdateReferences(t *testing.T) {
+	require.Equal(t, normalizedImageReference("nginx"), normalizedImageReference("docker.io/library/nginx:latest"))
+	require.Empty(t, normalizedImageReference("bad reference"))
+	require.Empty(t, normalizedImageReference("nginx@sha256:"+strings.Repeat("a", 64)))
+}
+
+// A stats request can return headers promptly and then stall while reading its
+// body. The stats-map mutex must remain available during that read.
+func TestStatsResponseBodyDoesNotHoldStatsLock(t *testing.T) {
+	started := make(chan struct{})
+	release := make(chan struct{})
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		w.(http.Flusher).Flush()
+		close(started)
+		<-release
+		fmt.Fprint(w, `{"memory_stats":{"usage":1048576}}`)
+	}))
+	defer server.Close()
+	dm := newDockerManagerForVersionTest(server)
+	done := make(chan error, 1)
+	go func() {
+		done <- dm.updateContainerStats(&container.ApiInfo{IdShort: "aaaaaaaaaaaa", Names: []string{"/one"}, Image: "nginx"}, defaultCacheTimeMs)
+	}()
+	<-started
+	locked := make(chan struct{})
+	go func() { dm.containerStatsMutex.Lock(); dm.containerStatsMutex.Unlock(); close(locked) }()
+	select {
+	case <-locked:
+	case <-time.After(time.Second):
+		close(release)
+		<-done
+		t.Fatal("Docker response body held the stats mutex")
+	}
+	close(release)
+	require.NoError(t, <-done)
+}
+
+func TestImageUpdateStatsEncoding(t *testing.T) {
+	original := container.Stats{Image: "nginx:latest", UpdateAvailable: true}
+	encoded, err := cbor.Marshal(original)
+	require.NoError(t, err)
+	var fields map[int]any
+	require.NoError(t, cbor.Unmarshal(encoded, &fields))
+	require.Equal(t, true, fields[11])
+	require.Equal(t, "nginx:latest", fields[8])
+	var decoded container.Stats
+	require.NoError(t, cbor.Unmarshal(encoded, &decoded))
+	require.True(t, decoded.UpdateAvailable)
+	require.Equal(t, original.Image, decoded.Image)
+	encoded, err = json.Marshal(original)
+	require.NoError(t, err)
+	require.Contains(t, string(encoded), `"u":true`)
+}
+
+func TestImageUpdateCacheExpiryBoundaryAndPruning(t *testing.T) {
+	now := time.Now()
+	key := normalizedImageReference("nginx")
+	dm := &dockerManager{imageUpdates: map[string]*imageUpdateStatus{
+		key:                           {available: true, checkedAt: now},
+		"unused.example/image:latest": {checkedAt: now},
+	}}
+	dm.refreshImageUpdates([]*container.ApiInfo{{Image: "nginx"}}, now.Add(imageUpdateInterval-time.Nanosecond))
+	require.False(t, dm.imageUpdatesRunning)
+	require.Len(t, dm.imageUpdates, 1)
+	require.True(t, dm.cachedImageUpdate("nginx:latest"))
+	dm.refreshImageUpdates(nil, now)
+	require.Empty(t, dm.imageUpdates)
+}

agent/docker_registry.go

@@ -0,0 +1,222 @@
+package agent
+
+import (
+	_ "crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/distribution/reference"
+	"github.com/opencontainers/go-digest"
+)
+
+const imageRegistryTimeout = 10 * time.Second
+
+const imageManifestAccept = "application/vnd.docker.distribution.manifest.list.v2+json, " +
+	"application/vnd.docker.distribution.manifest.v2+json, " +
+	"application/vnd.oci.image.manifest.v1+json, " +
+	"application/vnd.oci.image.index.v1+json"
+
+// checkImageUpdate compares the digest recorded by Docker for image with the
+// digest currently advertised by its registry. A digest-pinned reference is
+// immutable and therefore never has an update available.
+func (dm *dockerManager) checkImageUpdate(image string) (bool, error) {
+	named, err := reference.ParseNormalizedNamed(image)
+	if err != nil {
+		return false, fmt.Errorf("parse image reference %q: %w", image, err)
+	}
+	if _, pinned := named.(reference.Digested); pinned {
+		return false, nil
+	}
+	named = reference.TagNameOnly(named)
+
+	registry := reference.Domain(named)
+	repository := reference.Path(named)
+	tag := named.(reference.Tagged).Tag()
+
+	localDigest, err := dm.inspectImageDigest(image, registry, repository)
+	if err != nil {
+		return false, err
+	}
+
+	remoteDigest, err := dm.registryImageDigest(registry, repository, tag)
+	if err != nil {
+		return false, err
+	}
+
+	return remoteDigest != localDigest, nil
+}
+
+// inspectImageDigest reads Docker's image metadata without using dm.decode.
+// The checker runs in the image-discovery goroutine, so it must not hold any
+// of the container statistics locks while waiting on the Docker API.
+func (dm *dockerManager) inspectImageDigest(image, registry, repository string) (string, error) {
+	if dm.client == nil {
+		return "", fmt.Errorf("inspect image %q: Docker client is unavailable", image)
+	}
+
+	endpoint := "http://localhost/images/" + url.PathEscape(image) + "/json"
+	resp, err := dm.client.Get(endpoint)
+	if err != nil {
+		return "", fmt.Errorf("inspect image %q: %w", image, err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("inspect image %q failed: %s", image, responseStatus(resp))
+	}
+
+	var inspect struct {
+		RepoDigests []string `json:"RepoDigests"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&inspect); err != nil {
+		return "", fmt.Errorf("decode image inspect %q: %w", image, err)
+	}
+	if len(inspect.RepoDigests) == 0 {
+		return "", fmt.Errorf("inspect image %q returned no repository digests", image)
+	}
+
+	localDigest, ok := matchingRepositoryDigest(inspect.RepoDigests, registry, repository)
+	if !ok {
+		return "", fmt.Errorf("inspect image %q returned no valid digest for %s/%s", image, registry, repository)
+	}
+	return localDigest, nil
+}
+
+// matchingRepositoryDigest returns a valid digest belonging to the requested
+// repository. Docker can return multiple RepoDigests for one local image; an
+// unrelated first entry must never be used for the comparison.
+func matchingRepositoryDigest(repoDigests []string, registry, repository string) (string, bool) {
+	for _, repoDigest := range repoDigests {
+		repoDigest = strings.TrimSpace(repoDigest)
+		at := strings.LastIndexByte(repoDigest, '@')
+		if at <= 0 || at == len(repoDigest)-1 || strings.Contains(repoDigest[:at], "@") {
+			continue
+		}
+
+		repoRef, err := reference.ParseNormalizedNamed(repoDigest[:at])
+		if err != nil || reference.Path(repoRef) != repository || !sameRegistry(reference.Domain(repoRef), registry) {
+			continue
+		}
+		if _, hasTag := repoRef.(reference.Tagged); hasTag {
+			continue
+		}
+
+		d, err := digest.Parse(repoDigest[at+1:])
+		if err != nil {
+			continue
+		}
+		return d.String(), true
+	}
+	return "", false
+}
+
+func sameRegistry(left, right string) bool {
+	left = canonicalRegistry(left)
+	right = canonicalRegistry(right)
+	return left == right ||
+		(left == "ghcr.io" && right == "lscr.io") ||
+		(left == "lscr.io" && right == "ghcr.io")
+}
+
+func canonicalRegistry(registry string) string {
+	if registry == "index.docker.io" {
+		return "docker.io"
+	}
+	return registry
+}
+
+func (dm *dockerManager) registryImageDigest(registry, repository, tag string) (string, error) {
+	client := dm.registryClient
+	if client == nil {
+		client = &http.Client{Timeout: imageRegistryTimeout}
+	}
+
+	token, err := dm.registryToken(client, registry, repository)
+	if err != nil {
+		return "", err
+	}
+
+	host := registry
+	if registry == "docker.io" {
+		host = "registry-1.docker.io"
+	}
+	manifestURL := "https://" + host + "/v2/" + repository + "/manifests/" + url.PathEscape(tag)
+	req, err := http.NewRequest(http.MethodHead, manifestURL, nil)
+	if err != nil {
+		return "", fmt.Errorf("create manifest request: %w", err)
+	}
+	req.Header.Set("Accept", imageManifestAccept)
+	if token != "" {
+		req.Header.Set("Authorization", "Bearer "+token)
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("fetch manifest %s:%s: %w", registry, repository, err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("manifest request for %s:%s failed: %s", repository, tag, responseStatus(resp))
+	}
+
+	remote := strings.TrimSpace(resp.Header.Get("Docker-Content-Digest"))
+	d, err := digest.Parse(remote)
+	if err != nil {
+		return "", fmt.Errorf("manifest request for %s:%s returned invalid digest: %w", repository, tag, err)
+	}
+	return d.String(), nil
+}
+
+func (dm *dockerManager) registryToken(client *http.Client, registry, repository string) (string, error) {
+	var authURL string
+	switch registry {
+	case "docker.io":
+		authURL = "https://auth.docker.io/token?service=registry.docker.io&scope=" + url.QueryEscape("repository:"+repository+":pull")
+	case "ghcr.io", "lscr.io":
+		// lscr.io is the LinuxServer alias for its GHCR-backed images.
+		authURL = "https://ghcr.io/token?service=ghcr.io&scope=" + url.QueryEscape("repository:"+repository+":pull")
+	default:
+		// Anonymous registries remain supported, as they were before the
+		// authenticated Docker Hub and GHCR paths were added.
+		return "", nil
+	}
+
+	req, err := http.NewRequest(http.MethodGet, authURL, nil)
+	if err != nil {
+		return "", fmt.Errorf("create registry auth request: %w", err)
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("fetch registry auth token for %s: %w", repository, err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("registry auth request for %s failed: %s", repository, responseStatus(resp))
+	}
+
+	var tokenResponse struct {
+		Token       string `json:"token"`
+		AccessToken string `json:"access_token"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&tokenResponse); err != nil {
+		return "", fmt.Errorf("decode registry auth response for %s: %w", repository, err)
+	}
+	token := strings.TrimSpace(tokenResponse.Token)
+	if token == "" {
+		token = strings.TrimSpace(tokenResponse.AccessToken)
+	}
+	if token == "" {
+		return "", fmt.Errorf("registry auth response for %s contained no token", repository)
+	}
+	return token, nil
+}
+
+func responseStatus(resp *http.Response) string {
+	if resp.Status != "" {
+		return resp.Status
+	}
+	return http.StatusText(resp.StatusCode)
+}

agent/docker_registry_test.go

@@ -0,0 +1,204 @@
+//go:build testing
+
+package agent
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+type registryTransportFunc func(*http.Request) (*http.Response, error)
+
+func (fn registryTransportFunc) RoundTrip(req *http.Request) (*http.Response, error) {
+	return fn(req)
+}
+
+func registryResponse(status int, body string) *http.Response {
+	return &http.Response{
+		StatusCode: status,
+		Status:     fmt.Sprintf("%d %s", status, http.StatusText(status)),
+		Header:     make(http.Header),
+		Body:       io.NopCloser(strings.NewReader(body)),
+	}
+}
+
+func registryDigest(fill byte) string {
+	return "sha256:" + strings.Repeat(string(fill), 64)
+}
+
+func newRegistryChecker(t *testing.T, inspectBody string, transport http.RoundTripper) *dockerManager {
+	t.Helper()
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if strings.HasPrefix(r.URL.Path, "/images/") {
+			w.Header().Set("Content-Type", "application/json")
+			_, _ = io.WriteString(w, inspectBody)
+			return
+		}
+		http.NotFound(w, r)
+	}))
+	t.Cleanup(server.Close)
+
+	return &dockerManager{
+		client:         newDockerManagerForVersionTest(server).client,
+		registryClient: &http.Client{Transport: transport},
+	}
+}
+
+func TestCheckImageUpdateUsesInspectAndManifestDigests(t *testing.T) {
+	local := registryDigest('a')
+	remote := registryDigest('b')
+	var authCalls, manifestCalls atomic.Int32
+	dm := newRegistryChecker(t, fmt.Sprintf(`{"RepoDigests":["docker.io/library/alpine@%s"]}`, local), registryTransportFunc(func(req *http.Request) (*http.Response, error) {
+		switch {
+		case req.Method == http.MethodGet && req.URL.Host == "auth.docker.io":
+			authCalls.Add(1)
+			require.Equal(t, "/token", req.URL.Path)
+			return registryResponse(http.StatusOK, `{"token":"test-token"}`), nil
+		case req.Method == http.MethodHead && req.URL.Host == "registry-1.docker.io":
+			manifestCalls.Add(1)
+			require.Equal(t, "/v2/library/alpine/manifests/latest", req.URL.Path)
+			require.Equal(t, "Bearer test-token", req.Header.Get("Authorization"))
+			resp := registryResponse(http.StatusOK, "")
+			resp.Header.Set("Docker-Content-Digest", remote)
+			return resp, nil
+		default:
+			return registryResponse(http.StatusNotFound, ""), nil
+		}
+	}))
+
+	available, err := dm.checkImageUpdate("alpine")
+	require.NoError(t, err)
+	require.True(t, available)
+	require.EqualValues(t, 1, authCalls.Load())
+	require.EqualValues(t, 1, manifestCalls.Load())
+}
+
+func TestCheckImageUpdateReportsUnknownInspectState(t *testing.T) {
+	for _, test := range []struct {
+		name string
+		body string
+	}{
+		{name: "missing field", body: `{}`},
+		{name: "empty field", body: `{"RepoDigests":[]}`},
+		{name: "malformed reference", body: `{"RepoDigests":["not-a-repo-digest"]}`},
+		{name: "wrong repository", body: `{"RepoDigests":["docker.io/library/busybox@` + registryDigest('a') + `"]}`},
+		{name: "malformed digest", body: `{"RepoDigests":["docker.io/library/alpine@sha256:not-a-digest"]}`},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			var registryCalls atomic.Int32
+			dm := newRegistryChecker(t, test.body, registryTransportFunc(func(req *http.Request) (*http.Response, error) {
+				registryCalls.Add(1)
+				return registryResponse(http.StatusOK, `{"token":"unexpected"}`), nil
+			}))
+
+			available, err := dm.checkImageUpdate("alpine")
+			require.Error(t, err)
+			require.False(t, available)
+			require.EqualValues(t, 0, registryCalls.Load(), "invalid local state must not query a registry")
+		})
+	}
+}
+
+func TestCheckImageUpdateChecksInspectAuthAndManifestStatuses(t *testing.T) {
+	local := registryDigest('a')
+	validInspect := fmt.Sprintf(`{"RepoDigests":["docker.io/library/alpine@%s"]}`, local)
+
+	tests := []struct {
+		name         string
+		inspectCode  int
+		authCode     int
+		manifestCode int
+		remote       string
+		want         string
+	}{
+		{name: "inspect status", inspectCode: http.StatusNotFound, want: "inspect image"},
+		{name: "auth status", inspectCode: http.StatusOK, authCode: http.StatusUnauthorized, want: "registry auth"},
+		{name: "manifest status", inspectCode: http.StatusOK, authCode: http.StatusOK, manifestCode: http.StatusNotFound, remote: local, want: "manifest request"},
+		{name: "missing digest", inspectCode: http.StatusOK, authCode: http.StatusOK, manifestCode: http.StatusOK, want: "invalid digest"},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				if test.inspectCode != http.StatusOK && strings.HasPrefix(r.URL.Path, "/images/") {
+					w.WriteHeader(test.inspectCode)
+					return
+				}
+				_, _ = io.WriteString(w, validInspect)
+			}))
+			t.Cleanup(server.Close)
+
+			calls := 0
+			dm := &dockerManager{client: newDockerManagerForVersionTest(server).client, registryClient: &http.Client{Transport: registryTransportFunc(func(req *http.Request) (*http.Response, error) {
+				calls++
+				if req.Method == http.MethodGet {
+					return registryResponse(test.authCode, `{"token":"test"}`), nil
+				}
+				response := registryResponse(test.manifestCode, "")
+				response.Header.Set("Docker-Content-Digest", test.remote)
+				return response, nil
+			})}}
+
+			_, err := dm.checkImageUpdate("alpine")
+			require.Error(t, err)
+			require.Contains(t, err.Error(), test.want)
+			if test.inspectCode != http.StatusOK {
+				require.Zero(t, calls)
+			}
+		})
+	}
+}
+
+func TestCheckImageUpdateSupportsAnonymousAndLSCRRegistries(t *testing.T) {
+	t.Run("anonymous registry", func(t *testing.T) {
+		local := registryDigest('a')
+		var calls atomic.Int32
+		dm := newRegistryChecker(t, fmt.Sprintf(`{"RepoDigests":["example.com/app@%s"]}`, local), registryTransportFunc(func(req *http.Request) (*http.Response, error) {
+			calls.Add(1)
+			require.Equal(t, http.MethodHead, req.Method)
+			require.Equal(t, "example.com", req.URL.Host)
+			resp := registryResponse(http.StatusOK, "")
+			resp.Header.Set("Docker-Content-Digest", local)
+			return resp, nil
+		}))
+		available, err := dm.checkImageUpdate("example.com/app")
+		require.NoError(t, err)
+		require.False(t, available)
+		require.EqualValues(t, 1, calls.Load())
+	})
+
+	t.Run("lscr ghcr alias", func(t *testing.T) {
+		local := registryDigest('a')
+		var authCalls, manifestCalls atomic.Int32
+		dm := newRegistryChecker(t, fmt.Sprintf(`{"RepoDigests":["ghcr.io/linuxserver/app@%s"]}`, local), registryTransportFunc(func(req *http.Request) (*http.Response, error) {
+			if req.Method == http.MethodGet {
+				authCalls.Add(1)
+				return registryResponse(http.StatusOK, `{"token":"test"}`), nil
+			}
+			manifestCalls.Add(1)
+			require.Equal(t, "lscr.io", req.URL.Host)
+			resp := registryResponse(http.StatusOK, "")
+			resp.Header.Set("Docker-Content-Digest", local)
+			return resp, nil
+		}))
+		available, err := dm.checkImageUpdate("lscr.io/linuxserver/app")
+		require.NoError(t, err)
+		require.False(t, available)
+		require.EqualValues(t, 1, authCalls.Load())
+		require.EqualValues(t, 1, manifestCalls.Load())
+	})
+}
+
+func TestCheckImageUpdateSkipsPinnedDigest(t *testing.T) {
+	image := "docker.io/library/alpine@" + registryDigest('a')
+	dm := &dockerManager{}
+	available, err := dm.checkImageUpdate(image)
+	require.NoError(t, err)
+	require.False(t, available)
+}

agent/docker_test.go

@@ -1184,7 +1184,6 @@ func TestUpdateContainerStatsPodmanCpuCalculation(t *testing.T) {
 			}
 		})},
 		containerStatsMap: make(map[string]*container.Stats),
-		apiStats:          &container.ApiStats{},
 		usingPodman:       true,
 		lastCpuContainer: map[uint16]map[string]uint64{
 			defaultCacheTimeMs: {"0123456789ab": prevCpuUsage},
@@ -1676,7 +1675,6 @@ func TestUpdateContainerStatsUsesPodmanInspectHealthFallback(t *testing.T) {
 			}
 		})},
 		containerStatsMap:   make(map[string]*container.Stats),
-		apiStats:            &container.ApiStats{},
 		usingPodman:         true,
 		lastCpuContainer:    make(map[uint16]map[string]uint64),
 		lastCpuSystem:       make(map[uint16]map[string]uint64),