feat(agent): Add docker image update available flag (#2211)

Co-authored-by: henrygd <hank@henrygd.me>

agent/docker.go

@@ -65,10 +65,14 @@ type dockerManager struct {
 	dockerVersionChecked bool                        // Whether a version probe has completed successfully
 	isWindows            bool                        // Whether the Docker Engine API is running on Windows
 	buf                  *bytes.Buffer               // Buffer to store and read response bodies
-	apiStats             *container.ApiStats         // Reusable API stats object
 	excludeContainers    []string                    // Patterns to exclude containers by name
 	usingPodman          bool                        // Whether the Docker Engine API is running on Podman
 
+	registryClient      *http.Client                  // Client for registry requests; nil uses a client with a 10-second timeout
+	imageUpdatesMutex   sync.RWMutex                  // Protects imageUpdates, its entries, and imageUpdatesRunning
+	imageUpdates        map[string]*imageUpdateStatus // Shared update status keyed by normalized image reference
+	imageUpdatesRunning bool                          // Whether a background image-update batch is in progress
+
 	// Cache-time-aware tracking for CPU stats (similar to cpu.go)
 	// Maps cache time intervals to container-specific CPU usage tracking
 	lastCpuContainer map[uint16]map[string]uint64    // cacheTimeMs -> containerId -> last cpu container usage
@@ -161,6 +165,9 @@ func (dm *dockerManager) getDockerStats(cacheTimeMs uint16) ([]*container.Stats,
 		clear(dm.validIds)
 	}
 
+	// Only schedule auxiliary work here; metrics never wait for image discovery.
+	dm.refreshImageUpdates(dm.apiContainerList, time.Now())
+
 	var failedContainers []*container.ApiInfo
 
 	for _, ctr := range dm.apiContainerList {
@@ -506,6 +513,17 @@ func (dm *dockerManager) updateContainerStats(ctr *container.ApiInfo, cacheTimeM
 		}
 	}
 
+	// Read and decode the response before locking shared stats to avoid blocking
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("container stats request failed: %s", resp.Status)
+	}
+	res := &container.ApiStats{}
+	if err := json.NewDecoder(resp.Body).Decode(res); err != nil {
+		return err
+	}
+	updateAvailable := dm.cachedImageUpdate(ctr.Image)
+
 	dm.containerStatsMutex.Lock()
 	defer dm.containerStatsMutex.Unlock()
 
@@ -520,6 +538,9 @@ func (dm *dockerManager) updateContainerStats(ctr *container.ApiInfo, cacheTimeM
 	stats.Status = statusText
 	stats.Health = health
 
+	stats.Image = ctr.Image
+	stats.UpdateAvailable = updateAvailable
+
 	if len(ctr.Ports) > 0 {
 		stats.Ports = convertContainerPortsToString(ctr)
 	}
@@ -532,12 +553,6 @@ func (dm *dockerManager) updateContainerStats(ctr *container.ApiInfo, cacheTimeM
 	stats.NetworkSent = 0
 	stats.NetworkRecv = 0
 
-	res := dm.apiStats
-	res.Networks = nil
-	if err := dm.decode(resp, res); err != nil {
-		return err
-	}
-
 	// Initialize CPU tracking for this cache time interval
 	dm.initializeCpuTracking(cacheTimeMs)
 
@@ -695,7 +710,6 @@ func newDockerManager(agent *Agent) *dockerManager {
 		containerStatsMap: make(map[string]*container.Stats),
 		sem:               make(chan struct{}, 5),
 		apiContainerList:  []*container.ApiInfo{},
-		apiStats:          &container.ApiStats{},
 		excludeContainers: excludeContainers,
 
 		// Initialize cache-time-aware tracking structures

agent/docker_image_updates.go

@@ -0,0 +1,105 @@
+package agent
+
+import (
+	"log/slog"
+	"sync"
+	"time"
+
+	"github.com/distribution/reference"
+	"github.com/henrygd/beszel/internal/entities/container"
+)
+
+const imageUpdateInterval = time.Hour
+
+type imageUpdateStatus struct {
+	available bool
+	checkedAt time.Time
+}
+
+func normalizedImageReference(image string) string {
+	named, err := reference.ParseNormalizedNamed(image)
+	if err != nil {
+		return ""
+	}
+	// Digest-pinned references cannot move to a new version.
+	if _, pinned := named.(reference.Digested); pinned {
+		return ""
+	}
+	return reference.TagNameOnly(named).String()
+}
+
+// refreshImageUpdates starts at most one background batch. Neither its network
+// work nor its completion is part of the container metrics wait group.
+func (dm *dockerManager) refreshImageUpdates(containers []*container.ApiInfo, now time.Time) {
+	dm.imageUpdatesMutex.Lock()
+	defer dm.imageUpdatesMutex.Unlock()
+	if dm.imageUpdatesRunning {
+		return
+	}
+	if dm.imageUpdates == nil {
+		dm.imageUpdates = make(map[string]*imageUpdateStatus)
+	}
+	active := make(map[string]struct{}, len(containers))
+	pending := make(map[string]*imageUpdateStatus)
+	for _, ctr := range containers {
+		if len(ctr.Names) > 0 && dm.shouldExcludeContainer(ctr.Names[0][1:]) {
+			continue
+		}
+		key := normalizedImageReference(ctr.Image)
+		if key == "" {
+			continue
+		}
+		active[key] = struct{}{}
+		entry := dm.imageUpdates[key]
+		if entry == nil {
+			entry = &imageUpdateStatus{}
+			dm.imageUpdates[key] = entry
+		}
+		if entry.checkedAt.IsZero() || now.Sub(entry.checkedAt) >= imageUpdateInterval {
+			pending[key] = entry
+		}
+	}
+	for key := range dm.imageUpdates {
+		if _, ok := active[key]; !ok {
+			delete(dm.imageUpdates, key)
+		}
+	}
+	if len(pending) == 0 {
+		return
+	}
+	dm.imageUpdatesRunning = true
+	go func() {
+		// Limit auxiliary requests even on hosts running many different images.
+		sem := make(chan struct{}, 2)
+		var wg sync.WaitGroup
+		for key, entry := range pending {
+			sem <- struct{}{}
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				defer func() { <-sem }()
+				available, err := dm.checkImageUpdate(key)
+				if err != nil {
+					available = false
+					slog.Debug("Image update check failed", "image", key, "err", err)
+				}
+				dm.imageUpdatesMutex.Lock()
+				entry.available = available
+				entry.checkedAt = time.Now()
+				dm.imageUpdatesMutex.Unlock()
+			}()
+		}
+		wg.Wait()
+		dm.imageUpdatesMutex.Lock()
+		dm.imageUpdatesRunning = false
+		dm.imageUpdatesMutex.Unlock()
+	}()
+}
+
+func (dm *dockerManager) cachedImageUpdate(image string) bool {
+	key := normalizedImageReference(image)
+	dm.imageUpdatesMutex.RLock()
+	defer dm.imageUpdatesMutex.RUnlock()
+	entry := dm.imageUpdates[key]
+	return entry != nil && entry.available
+}

agent/docker_image_updates_test.go

@@ -0,0 +1,225 @@
+//go:build testing
+
+package agent
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/fxamacker/cbor/v2"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/henrygd/beszel/internal/entities/container"
+	"github.com/stretchr/testify/require"
+)
+
+func waitForImageUpdates(t *testing.T, dm *dockerManager) {
+	t.Helper()
+	require.Eventually(t, func() bool {
+		dm.imageUpdatesMutex.RLock()
+		defer dm.imageUpdatesMutex.RUnlock()
+		return !dm.imageUpdatesRunning
+	}, time.Second*3, time.Millisecond)
+}
+
+func TestImageUpdateCacheAndStats(t *testing.T) {
+	local := "sha256:" + strings.Repeat("a", 64)
+	remote := "sha256:" + strings.Repeat("b", 64)
+	var inspections, lookups atomic.Int32
+	var fail atomic.Bool
+	var upToDate atomic.Bool
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch {
+		case strings.HasPrefix(r.URL.Path, "/images/"):
+			inspections.Add(1)
+			fmt.Fprintf(w, `{"RepoDigests":["docker.io/library/nginx@%s"]}`, local)
+		case r.URL.Path == "/containers/json":
+			fmt.Fprint(w, `[{"Id":"aaaaaaaaaaaa","Names":["/one"],"Image":"nginx","Status":"Up 2 hours"},{"Id":"bbbbbbbbbbbb","Names":["/two"],"Image":"docker.io/library/nginx:latest","Status":"Up 2 hours"}]`)
+		case strings.Contains(r.URL.Path, "/stats"):
+			fmt.Fprint(w, `{"memory_stats":{"usage":1048576},"cpu_stats":{},"networks":{}}`)
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer server.Close()
+	dm := newDockerManagerForVersionTest(server)
+	dm.dockerVersionChecked = true
+	dm.registryClient = &http.Client{Timeout: time.Second, Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
+		if fail.Load() {
+			return nil, fmt.Errorf("registry unavailable")
+		}
+		response := &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"token":"test"}`))}
+		if r.Method == http.MethodHead {
+			lookups.Add(1)
+			digest := remote
+			if upToDate.Load() {
+				digest = local
+			}
+			response.Header.Set("Docker-Content-Digest", digest)
+		}
+		return response, nil
+	})}
+	stats, err := dm.getDockerStats(defaultCacheTimeMs)
+	require.NoError(t, err)
+	require.Len(t, stats, 2)
+	waitForImageUpdates(t, dm)
+	require.EqualValues(t, 1, lookups.Load())
+	require.EqualValues(t, 1, inspections.Load())
+	stats, err = dm.getDockerStats(defaultCacheTimeMs)
+	require.NoError(t, err)
+	for _, stat := range stats {
+		require.True(t, stat.UpdateAvailable)
+		if stat.Id == "aaaaaaaaaaaa" {
+			require.Equal(t, "nginx", stat.Image)
+		} else {
+			require.Equal(t, "docker.io/library/nginx:latest", stat.Image)
+		}
+	}
+	require.EqualValues(t, 1, lookups.Load())
+
+	expire := func() {
+		dm.imageUpdatesMutex.Lock()
+		dm.imageUpdates["docker.io/library/nginx:latest"].checkedAt = time.Now().Add(-imageUpdateInterval)
+		dm.imageUpdatesMutex.Unlock()
+	}
+	upToDate.Store(true)
+	expire()
+	_, err = dm.getDockerStats(defaultCacheTimeMs)
+	require.NoError(t, err)
+	waitForImageUpdates(t, dm)
+	require.EqualValues(t, 2, lookups.Load())
+	require.False(t, dm.cachedImageUpdate("nginx:latest"))
+
+	// An expired positive result is cleared on failure, and the failure itself
+	// is cached so realtime stats do not retry a broken registry every second.
+	dm.imageUpdatesMutex.Lock()
+	dm.imageUpdates["docker.io/library/nginx:latest"].available = true
+	dm.imageUpdatesMutex.Unlock()
+	fail.Store(true)
+	expire()
+	_, err = dm.getDockerStats(defaultCacheTimeMs)
+	require.NoError(t, err)
+	waitForImageUpdates(t, dm)
+	failedInspections := inspections.Load()
+	stats, err = dm.getDockerStats(defaultCacheTimeMs)
+	require.NoError(t, err)
+	require.Len(t, stats, 2)
+	require.Equal(t, failedInspections, inspections.Load())
+	for _, stat := range stats {
+		require.False(t, stat.UpdateAvailable)
+		require.Equal(t, 1.0, stat.Mem)
+	}
+}
+
+func TestImageDiscoveryDoesNotBlockStats(t *testing.T) {
+	started := make(chan struct{}, 1)
+	release := make(chan struct{})
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if strings.HasPrefix(r.URL.Path, "/images/") {
+			fmt.Fprintf(w, `{"RepoDigests":["example.com/app@sha256:%s"]}`, strings.Repeat("a", 64))
+		} else {
+			fmt.Fprint(w, `{"memory_stats":{"usage":1048576}}`)
+		}
+	}))
+	defer server.Close()
+	dm := newDockerManagerForVersionTest(server)
+	defer func() { close(release); waitForImageUpdates(t, dm) }()
+	dm.registryClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
+		started <- struct{}{}
+		<-release
+		return nil, fmt.Errorf("timeout")
+	})}
+	ctr := &container.ApiInfo{IdShort: "aaaaaaaaaaaa", Image: "example.com/app", Names: []string{"/one"}}
+	dm.refreshImageUpdates([]*container.ApiInfo{ctr}, time.Now())
+	select {
+	case <-started:
+	case <-time.After(3 * time.Second):
+		t.Fatal("check did not start")
+	}
+	done := make(chan error, 1)
+	go func() { done <- dm.updateContainerStats(ctr, defaultCacheTimeMs) }()
+	select {
+	case err := <-done:
+		require.NoError(t, err)
+	case <-time.After(time.Second):
+		t.Fatal("registry blocked stats")
+	}
+	dm.imageUpdatesMutex.RLock()
+	require.True(t, dm.imageUpdatesRunning)
+	dm.imageUpdatesMutex.RUnlock()
+}
+
+func TestNormalizeImageUpdateReferences(t *testing.T) {
+	require.Equal(t, normalizedImageReference("nginx"), normalizedImageReference("docker.io/library/nginx:latest"))
+	require.Empty(t, normalizedImageReference("bad reference"))
+	require.Empty(t, normalizedImageReference("nginx@sha256:"+strings.Repeat("a", 64)))
+}
+
+// A stats request can return headers promptly and then stall while reading its
+// body. The stats-map mutex must remain available during that read.
+func TestStatsResponseBodyDoesNotHoldStatsLock(t *testing.T) {
+	started := make(chan struct{})
+	release := make(chan struct{})
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		w.(http.Flusher).Flush()
+		close(started)
+		<-release
+		fmt.Fprint(w, `{"memory_stats":{"usage":1048576}}`)
+	}))
+	defer server.Close()
+	dm := newDockerManagerForVersionTest(server)
+	done := make(chan error, 1)
+	go func() {
+		done <- dm.updateContainerStats(&container.ApiInfo{IdShort: "aaaaaaaaaaaa", Names: []string{"/one"}, Image: "nginx"}, defaultCacheTimeMs)
+	}()
+	<-started
+	locked := make(chan struct{})
+	go func() { dm.containerStatsMutex.Lock(); dm.containerStatsMutex.Unlock(); close(locked) }()
+	select {
+	case <-locked:
+	case <-time.After(time.Second):
+		close(release)
+		<-done
+		t.Fatal("Docker response body held the stats mutex")
+	}
+	close(release)
+	require.NoError(t, <-done)
+}
+
+func TestImageUpdateStatsEncoding(t *testing.T) {
+	original := container.Stats{Image: "nginx:latest", UpdateAvailable: true}
+	encoded, err := cbor.Marshal(original)
+	require.NoError(t, err)
+	var fields map[int]any
+	require.NoError(t, cbor.Unmarshal(encoded, &fields))
+	require.Equal(t, true, fields[11])
+	require.Equal(t, "nginx:latest", fields[8])
+	var decoded container.Stats
+	require.NoError(t, cbor.Unmarshal(encoded, &decoded))
+	require.True(t, decoded.UpdateAvailable)
+	require.Equal(t, original.Image, decoded.Image)
+	encoded, err = json.Marshal(original)
+	require.NoError(t, err)
+	require.Contains(t, string(encoded), `"u":true`)
+}
+
+func TestImageUpdateCacheExpiryBoundaryAndPruning(t *testing.T) {
+	now := time.Now()
+	key := normalizedImageReference("nginx")
+	dm := &dockerManager{imageUpdates: map[string]*imageUpdateStatus{
+		key:                           {available: true, checkedAt: now},
+		"unused.example/image:latest": {checkedAt: now},
+	}}
+	dm.refreshImageUpdates([]*container.ApiInfo{{Image: "nginx"}}, now.Add(imageUpdateInterval-time.Nanosecond))
+	require.False(t, dm.imageUpdatesRunning)
+	require.Len(t, dm.imageUpdates, 1)
+	require.True(t, dm.cachedImageUpdate("nginx:latest"))
+	dm.refreshImageUpdates(nil, now)
+	require.Empty(t, dm.imageUpdates)
+}

agent/docker_registry.go

@@ -0,0 +1,222 @@
+package agent
+
+import (
+	_ "crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/distribution/reference"
+	"github.com/opencontainers/go-digest"
+)
+
+const imageRegistryTimeout = 10 * time.Second
+
+const imageManifestAccept = "application/vnd.docker.distribution.manifest.list.v2+json, " +
+	"application/vnd.docker.distribution.manifest.v2+json, " +
+	"application/vnd.oci.image.manifest.v1+json, " +
+	"application/vnd.oci.image.index.v1+json"
+
+// checkImageUpdate compares the digest recorded by Docker for image with the
+// digest currently advertised by its registry. A digest-pinned reference is
+// immutable and therefore never has an update available.
+func (dm *dockerManager) checkImageUpdate(image string) (bool, error) {
+	named, err := reference.ParseNormalizedNamed(image)
+	if err != nil {
+		return false, fmt.Errorf("parse image reference %q: %w", image, err)
+	}
+	if _, pinned := named.(reference.Digested); pinned {
+		return false, nil
+	}
+	named = reference.TagNameOnly(named)
+
+	registry := reference.Domain(named)
+	repository := reference.Path(named)
+	tag := named.(reference.Tagged).Tag()
+
+	localDigest, err := dm.inspectImageDigest(image, registry, repository)
+	if err != nil {
+		return false, err
+	}
+
+	remoteDigest, err := dm.registryImageDigest(registry, repository, tag)
+	if err != nil {
+		return false, err
+	}
+
+	return remoteDigest != localDigest, nil
+}
+
+// inspectImageDigest reads Docker's image metadata without using dm.decode.
+// The checker runs in the image-discovery goroutine, so it must not hold any
+// of the container statistics locks while waiting on the Docker API.
+func (dm *dockerManager) inspectImageDigest(image, registry, repository string) (string, error) {
+	if dm.client == nil {
+		return "", fmt.Errorf("inspect image %q: Docker client is unavailable", image)
+	}
+
+	endpoint := "http://localhost/images/" + url.PathEscape(image) + "/json"
+	resp, err := dm.client.Get(endpoint)
+	if err != nil {
+		return "", fmt.Errorf("inspect image %q: %w", image, err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("inspect image %q failed: %s", image, responseStatus(resp))
+	}
+
+	var inspect struct {
+		RepoDigests []string `json:"RepoDigests"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&inspect); err != nil {
+		return "", fmt.Errorf("decode image inspect %q: %w", image, err)
+	}
+	if len(inspect.RepoDigests) == 0 {
+		return "", fmt.Errorf("inspect image %q returned no repository digests", image)
+	}
+
+	localDigest, ok := matchingRepositoryDigest(inspect.RepoDigests, registry, repository)
+	if !ok {
+		return "", fmt.Errorf("inspect image %q returned no valid digest for %s/%s", image, registry, repository)
+	}
+	return localDigest, nil
+}
+
+// matchingRepositoryDigest returns a valid digest belonging to the requested
+// repository. Docker can return multiple RepoDigests for one local image; an
+// unrelated first entry must never be used for the comparison.
+func matchingRepositoryDigest(repoDigests []string, registry, repository string) (string, bool) {
+	for _, repoDigest := range repoDigests {
+		repoDigest = strings.TrimSpace(repoDigest)
+		at := strings.LastIndexByte(repoDigest, '@')
+		if at <= 0 || at == len(repoDigest)-1 || strings.Contains(repoDigest[:at], "@") {
+			continue
+		}
+
+		repoRef, err := reference.ParseNormalizedNamed(repoDigest[:at])
+		if err != nil || reference.Path(repoRef) != repository || !sameRegistry(reference.Domain(repoRef), registry) {
+			continue
+		}
+		if _, hasTag := repoRef.(reference.Tagged); hasTag {
+			continue
+		}
+
+		d, err := digest.Parse(repoDigest[at+1:])
+		if err != nil {
+			continue
+		}
+		return d.String(), true
+	}
+	return "", false
+}
+
+func sameRegistry(left, right string) bool {
+	left = canonicalRegistry(left)
+	right = canonicalRegistry(right)
+	return left == right ||
+		(left == "ghcr.io" && right == "lscr.io") ||
+		(left == "lscr.io" && right == "ghcr.io")
+}
+
+func canonicalRegistry(registry string) string {
+	if registry == "index.docker.io" {
+		return "docker.io"
+	}
+	return registry
+}
+
+func (dm *dockerManager) registryImageDigest(registry, repository, tag string) (string, error) {
+	client := dm.registryClient
+	if client == nil {
+		client = &http.Client{Timeout: imageRegistryTimeout}
+	}
+
+	token, err := dm.registryToken(client, registry, repository)
+	if err != nil {
+		return "", err
+	}
+
+	host := registry
+	if registry == "docker.io" {
+		host = "registry-1.docker.io"
+	}
+	manifestURL := "https://" + host + "/v2/" + repository + "/manifests/" + url.PathEscape(tag)
+	req, err := http.NewRequest(http.MethodHead, manifestURL, nil)
+	if err != nil {
+		return "", fmt.Errorf("create manifest request: %w", err)
+	}
+	req.Header.Set("Accept", imageManifestAccept)
+	if token != "" {
+		req.Header.Set("Authorization", "Bearer "+token)
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("fetch manifest %s:%s: %w", registry, repository, err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("manifest request for %s:%s failed: %s", repository, tag, responseStatus(resp))
+	}
+
+	remote := strings.TrimSpace(resp.Header.Get("Docker-Content-Digest"))
+	d, err := digest.Parse(remote)
+	if err != nil {
+		return "", fmt.Errorf("manifest request for %s:%s returned invalid digest: %w", repository, tag, err)
+	}
+	return d.String(), nil
+}
+
+func (dm *dockerManager) registryToken(client *http.Client, registry, repository string) (string, error) {
+	var authURL string
+	switch registry {
+	case "docker.io":
+		authURL = "https://auth.docker.io/token?service=registry.docker.io&scope=" + url.QueryEscape("repository:"+repository+":pull")
+	case "ghcr.io", "lscr.io":
+		// lscr.io is the LinuxServer alias for its GHCR-backed images.
+		authURL = "https://ghcr.io/token?service=ghcr.io&scope=" + url.QueryEscape("repository:"+repository+":pull")
+	default:
+		// Anonymous registries remain supported, as they were before the
+		// authenticated Docker Hub and GHCR paths were added.
+		return "", nil
+	}
+
+	req, err := http.NewRequest(http.MethodGet, authURL, nil)
+	if err != nil {
+		return "", fmt.Errorf("create registry auth request: %w", err)
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("fetch registry auth token for %s: %w", repository, err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("registry auth request for %s failed: %s", repository, responseStatus(resp))
+	}
+
+	var tokenResponse struct {
+		Token       string `json:"token"`
+		AccessToken string `json:"access_token"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&tokenResponse); err != nil {
+		return "", fmt.Errorf("decode registry auth response for %s: %w", repository, err)
+	}
+	token := strings.TrimSpace(tokenResponse.Token)
+	if token == "" {
+		token = strings.TrimSpace(tokenResponse.AccessToken)
+	}
+	if token == "" {
+		return "", fmt.Errorf("registry auth response for %s contained no token", repository)
+	}
+	return token, nil
+}
+
+func responseStatus(resp *http.Response) string {
+	if resp.Status != "" {
+		return resp.Status
+	}
+	return http.StatusText(resp.StatusCode)
+}

agent/docker_registry_test.go

@@ -0,0 +1,204 @@
+//go:build testing
+
+package agent
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+type registryTransportFunc func(*http.Request) (*http.Response, error)
+
+func (fn registryTransportFunc) RoundTrip(req *http.Request) (*http.Response, error) {
+	return fn(req)
+}
+
+func registryResponse(status int, body string) *http.Response {
+	return &http.Response{
+		StatusCode: status,
+		Status:     fmt.Sprintf("%d %s", status, http.StatusText(status)),
+		Header:     make(http.Header),
+		Body:       io.NopCloser(strings.NewReader(body)),
+	}
+}
+
+func registryDigest(fill byte) string {
+	return "sha256:" + strings.Repeat(string(fill), 64)
+}
+
+func newRegistryChecker(t *testing.T, inspectBody string, transport http.RoundTripper) *dockerManager {
+	t.Helper()
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if strings.HasPrefix(r.URL.Path, "/images/") {
+			w.Header().Set("Content-Type", "application/json")
+			_, _ = io.WriteString(w, inspectBody)
+			return
+		}
+		http.NotFound(w, r)
+	}))
+	t.Cleanup(server.Close)
+
+	return &dockerManager{
+		client:         newDockerManagerForVersionTest(server).client,
+		registryClient: &http.Client{Transport: transport},
+	}
+}
+
+func TestCheckImageUpdateUsesInspectAndManifestDigests(t *testing.T) {
+	local := registryDigest('a')
+	remote := registryDigest('b')
+	var authCalls, manifestCalls atomic.Int32
+	dm := newRegistryChecker(t, fmt.Sprintf(`{"RepoDigests":["docker.io/library/alpine@%s"]}`, local), registryTransportFunc(func(req *http.Request) (*http.Response, error) {
+		switch {
+		case req.Method == http.MethodGet && req.URL.Host == "auth.docker.io":
+			authCalls.Add(1)
+			require.Equal(t, "/token", req.URL.Path)
+			return registryResponse(http.StatusOK, `{"token":"test-token"}`), nil
+		case req.Method == http.MethodHead && req.URL.Host == "registry-1.docker.io":
+			manifestCalls.Add(1)
+			require.Equal(t, "/v2/library/alpine/manifests/latest", req.URL.Path)
+			require.Equal(t, "Bearer test-token", req.Header.Get("Authorization"))
+			resp := registryResponse(http.StatusOK, "")
+			resp.Header.Set("Docker-Content-Digest", remote)
+			return resp, nil
+		default:
+			return registryResponse(http.StatusNotFound, ""), nil
+		}
+	}))
+
+	available, err := dm.checkImageUpdate("alpine")
+	require.NoError(t, err)
+	require.True(t, available)
+	require.EqualValues(t, 1, authCalls.Load())
+	require.EqualValues(t, 1, manifestCalls.Load())
+}
+
+func TestCheckImageUpdateReportsUnknownInspectState(t *testing.T) {
+	for _, test := range []struct {
+		name string
+		body string
+	}{
+		{name: "missing field", body: `{}`},
+		{name: "empty field", body: `{"RepoDigests":[]}`},
+		{name: "malformed reference", body: `{"RepoDigests":["not-a-repo-digest"]}`},
+		{name: "wrong repository", body: `{"RepoDigests":["docker.io/library/busybox@` + registryDigest('a') + `"]}`},
+		{name: "malformed digest", body: `{"RepoDigests":["docker.io/library/alpine@sha256:not-a-digest"]}`},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			var registryCalls atomic.Int32
+			dm := newRegistryChecker(t, test.body, registryTransportFunc(func(req *http.Request) (*http.Response, error) {
+				registryCalls.Add(1)
+				return registryResponse(http.StatusOK, `{"token":"unexpected"}`), nil
+			}))
+
+			available, err := dm.checkImageUpdate("alpine")
+			require.Error(t, err)
+			require.False(t, available)
+			require.EqualValues(t, 0, registryCalls.Load(), "invalid local state must not query a registry")
+		})
+	}
+}
+
+func TestCheckImageUpdateChecksInspectAuthAndManifestStatuses(t *testing.T) {
+	local := registryDigest('a')
+	validInspect := fmt.Sprintf(`{"RepoDigests":["docker.io/library/alpine@%s"]}`, local)
+
+	tests := []struct {
+		name         string
+		inspectCode  int
+		authCode     int
+		manifestCode int
+		remote       string
+		want         string
+	}{
+		{name: "inspect status", inspectCode: http.StatusNotFound, want: "inspect image"},
+		{name: "auth status", inspectCode: http.StatusOK, authCode: http.StatusUnauthorized, want: "registry auth"},
+		{name: "manifest status", inspectCode: http.StatusOK, authCode: http.StatusOK, manifestCode: http.StatusNotFound, remote: local, want: "manifest request"},
+		{name: "missing digest", inspectCode: http.StatusOK, authCode: http.StatusOK, manifestCode: http.StatusOK, want: "invalid digest"},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				if test.inspectCode != http.StatusOK && strings.HasPrefix(r.URL.Path, "/images/") {
+					w.WriteHeader(test.inspectCode)
+					return
+				}
+				_, _ = io.WriteString(w, validInspect)
+			}))
+			t.Cleanup(server.Close)
+
+			calls := 0
+			dm := &dockerManager{client: newDockerManagerForVersionTest(server).client, registryClient: &http.Client{Transport: registryTransportFunc(func(req *http.Request) (*http.Response, error) {
+				calls++
+				if req.Method == http.MethodGet {
+					return registryResponse(test.authCode, `{"token":"test"}`), nil
+				}
+				response := registryResponse(test.manifestCode, "")
+				response.Header.Set("Docker-Content-Digest", test.remote)
+				return response, nil
+			})}}
+
+			_, err := dm.checkImageUpdate("alpine")
+			require.Error(t, err)
+			require.Contains(t, err.Error(), test.want)
+			if test.inspectCode != http.StatusOK {
+				require.Zero(t, calls)
+			}
+		})
+	}
+}
+
+func TestCheckImageUpdateSupportsAnonymousAndLSCRRegistries(t *testing.T) {
+	t.Run("anonymous registry", func(t *testing.T) {
+		local := registryDigest('a')
+		var calls atomic.Int32
+		dm := newRegistryChecker(t, fmt.Sprintf(`{"RepoDigests":["example.com/app@%s"]}`, local), registryTransportFunc(func(req *http.Request) (*http.Response, error) {
+			calls.Add(1)
+			require.Equal(t, http.MethodHead, req.Method)
+			require.Equal(t, "example.com", req.URL.Host)
+			resp := registryResponse(http.StatusOK, "")
+			resp.Header.Set("Docker-Content-Digest", local)
+			return resp, nil
+		}))
+		available, err := dm.checkImageUpdate("example.com/app")
+		require.NoError(t, err)
+		require.False(t, available)
+		require.EqualValues(t, 1, calls.Load())
+	})
+
+	t.Run("lscr ghcr alias", func(t *testing.T) {
+		local := registryDigest('a')
+		var authCalls, manifestCalls atomic.Int32
+		dm := newRegistryChecker(t, fmt.Sprintf(`{"RepoDigests":["ghcr.io/linuxserver/app@%s"]}`, local), registryTransportFunc(func(req *http.Request) (*http.Response, error) {
+			if req.Method == http.MethodGet {
+				authCalls.Add(1)
+				return registryResponse(http.StatusOK, `{"token":"test"}`), nil
+			}
+			manifestCalls.Add(1)
+			require.Equal(t, "lscr.io", req.URL.Host)
+			resp := registryResponse(http.StatusOK, "")
+			resp.Header.Set("Docker-Content-Digest", local)
+			return resp, nil
+		}))
+		available, err := dm.checkImageUpdate("lscr.io/linuxserver/app")
+		require.NoError(t, err)
+		require.False(t, available)
+		require.EqualValues(t, 1, authCalls.Load())
+		require.EqualValues(t, 1, manifestCalls.Load())
+	})
+}
+
+func TestCheckImageUpdateSkipsPinnedDigest(t *testing.T) {
+	image := "docker.io/library/alpine@" + registryDigest('a')
+	dm := &dockerManager{}
+	available, err := dm.checkImageUpdate(image)
+	require.NoError(t, err)
+	require.False(t, available)
+}

agent/docker_test.go

@@ -1184,7 +1184,6 @@ func TestUpdateContainerStatsPodmanCpuCalculation(t *testing.T) {
 			}
 		})},
 		containerStatsMap: make(map[string]*container.Stats),
-		apiStats:          &container.ApiStats{},
 		usingPodman:       true,
 		lastCpuContainer: map[uint16]map[string]uint64{
 			defaultCacheTimeMs: {"0123456789ab": prevCpuUsage},
@@ -1676,7 +1675,6 @@ func TestUpdateContainerStatsUsesPodmanInspectHealthFallback(t *testing.T) {
 			}
 		})},
 		containerStatsMap:   make(map[string]*container.Stats),
-		apiStats:            &container.ApiStats{},
 		usingPodman:         true,
 		lastCpuContainer:    make(map[uint16]map[string]uint64),
 		lastCpuSystem:       make(map[uint16]map[string]uint64),

go.mod

@@ -5,11 +5,13 @@ go 1.27.1
 require (
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/coreos/go-systemd/v22 v22.7.0
+	github.com/distribution/reference v0.6.0
 	github.com/ebitengine/purego v0.11.0
 	github.com/fxamacker/cbor/v2 v2.9.3
 	github.com/gliderlabs/ssh v0.3.8
 	github.com/lxzan/gws v1.10.1
 	github.com/nicholas-fedor/shoutrrr v0.20.0
+	github.com/opencontainers/go-digest v1.0.0
 	github.com/pocketbase/dbx v1.12.0
 	github.com/pocketbase/pocketbase v0.40.2
 	github.com/shirou/gopsutil/v4 v4.26.8

go.sum

@@ -15,6 +15,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6N
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
 github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
+github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
+github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
 github.com/domodwyer/mailyak/v3 v3.6.2 h1:x3tGMsyFhTCaxp6ycgR0FE/bu5QiNp+hetUuCOBXMn8=
 github.com/domodwyer/mailyak/v3 v3.6.2/go.mod h1:lOm/u9CyCVWHeaAmHIdF4RiKVxKUT/H5XX10lIKAL6c=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
@@ -89,6 +91,8 @@ github.com/onsi/ginkgo/v2 v2.32.1 h1:6tlvcDm/3sE8lGJbZ4+d4mO3RLy24/tQWOFzVSQNIfw
 github.com/onsi/ginkgo/v2 v2.32.1/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44=
 github.com/onsi/gomega v1.43.0 h1:VlG/1FxqNxhSO+lq/OHBNaaqwiBK/mO8JbVkX9Y+FeU=
 github.com/onsi/gomega v1.43.0/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg=
+github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
+github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pocketbase/dbx v1.12.0 h1:/oLErM+A0b4xI0PWTGPqSDVjzix48PqI/bng2l0PzoA=
 github.com/pocketbase/dbx v1.12.0/go.mod h1:xXRCIAKTHMgUCyCKZm55pUOdvFziJjQfXaWKhu2vhMs=

internal/entities/container/container.go

@@ -191,6 +191,7 @@ type Stats struct {
 	Id              string       `json:"-" cbor:"7,keyasint"`
 	Image           string       `json:"-" cbor:"8,keyasint"`
 	Ports           string       `json:"-" cbor:"10,keyasint"`
+	UpdateAvailable bool         `json:"u,omitzero" cbor:"11,keyasint,omitzero"`
 	// PrevCpu     [2]uint64    `json:"-"`
 	CpuSystem    uint64       `json:"-"`
 	CpuContainer uint64       `json:"-"`

internal/hub/systems/container_records_test.go

@@ -0,0 +1,46 @@
+//go:build testing
+
+package systems
+
+import (
+	"testing"
+
+	"github.com/henrygd/beszel/internal/entities/container"
+	"github.com/pocketbase/dbx"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCreateContainerRecordsPersistsImageUpdateAvailability(t *testing.T) {
+	_, app := newTestSystemWithHub(t)
+
+	const (
+		systemID    = "system123"
+		containerID = "abcdef123456"
+		image       = "nginx:latest"
+	)
+
+	data := &container.Stats{
+		Id:              containerID,
+		Name:            "web",
+		Image:           image,
+		UpdateAvailable: true,
+	}
+	require.NoError(t, createContainerRecords(app, []*container.Stats{data}, systemID))
+
+	var record struct {
+		Image           string `db:"image"`
+		UpdateAvailable bool   `db:"updatable"`
+	}
+	require.NoError(t, app.DB().Select("image", "updatable").From("containers").
+		Where(dbx.HashExp{"id": containerID}).One(&record))
+	assert.Equal(t, image, record.Image)
+	assert.True(t, record.UpdateAvailable)
+
+	data.UpdateAvailable = false
+	require.NoError(t, createContainerRecords(app, []*container.Stats{data}, systemID))
+	require.NoError(t, app.DB().Select("image", "updatable").From("containers").
+		Where(dbx.HashExp{"id": containerID}).One(&record))
+	assert.Equal(t, image, record.Image)
+	assert.False(t, record.UpdateAvailable)
+}

internal/hub/systems/system.go

@@ -376,7 +376,7 @@ func createContainerRecords(app core.App, data []*container.Stats, systemId stri
 	valueStrings := make([]string, 0, len(data))
 	for i, container := range data {
 		suffix := fmt.Sprintf("%d", i)
-		valueStrings = append(valueStrings, fmt.Sprintf("({:id%[1]s}, {:system}, {:name%[1]s}, {:image%[1]s}, {:ports%[1]s}, {:status%[1]s}, {:health%[1]s}, {:cpu%[1]s}, {:memory%[1]s}, {:net%[1]s}, {:updated})", suffix))
+		valueStrings = append(valueStrings, fmt.Sprintf("({:id%[1]s}, {:system}, {:name%[1]s}, {:image%[1]s}, {:ports%[1]s}, {:status%[1]s}, {:health%[1]s}, {:cpu%[1]s}, {:memory%[1]s}, {:net%[1]s}, {:updateAvailable%[1]s}, {:updated})", suffix))
 		params["id"+suffix] = container.Id
 		params["name"+suffix] = container.Name
 		params["image"+suffix] = container.Image
@@ -390,9 +390,10 @@ func createContainerRecords(app core.App, data []*container.Stats, systemId stri
 			netBytes = uint64((container.NetworkSent + container.NetworkRecv) * 1024 * 1024)
 		}
 		params["net"+suffix] = netBytes
+		params["updateAvailable"+suffix] = container.UpdateAvailable
 	}
 	queryString := fmt.Sprintf(
-		"INSERT INTO containers (id, system, name, image, ports, status, health, cpu, memory, net, updated) VALUES %s ON CONFLICT(id) DO UPDATE SET system = excluded.system, name = excluded.name, image = excluded.image, ports = excluded.ports, status = excluded.status, health = excluded.health, cpu = excluded.cpu, memory = excluded.memory, net = excluded.net, updated = excluded.updated",
+		"INSERT INTO containers (id, system, name, image, ports, status, health, cpu, memory, net, updatable, updated) VALUES %s ON CONFLICT(id) DO UPDATE SET system = excluded.system, name = excluded.name, image = excluded.image, ports = excluded.ports, status = excluded.status, health = excluded.health, cpu = excluded.cpu, memory = excluded.memory, net = excluded.net, updatable = excluded.updatable, updated = excluded.updated",
 		strings.Join(valueStrings, ","),
 	)
 	_, err := app.DB().NewQuery(queryString).Bind(params).Execute()

internal/migrations/1789079746_container_update_available.go

@@ -0,0 +1,24 @@
+package migrations
+
+import (
+	"github.com/pocketbase/pocketbase/core"
+	m "github.com/pocketbase/pocketbase/migrations"
+)
+
+func init() {
+	m.Register(func(app core.App) error {
+		collection, err := app.FindCollectionByNameOrId("containers")
+		if err != nil {
+			return err
+		}
+		collection.Fields.Add(&core.BoolField{Name: "updatable"})
+		return app.Save(collection)
+	}, func(app core.App) error {
+		collection, err := app.FindCollectionByNameOrId("containers")
+		if err != nil {
+			return err
+		}
+		collection.Fields.RemoveByName("updatable")
+		return app.Save(collection)
+	})
+}

internal/site/src/components/containers-table/containers-table-columns.tsx

@@ -4,6 +4,7 @@ import { cn, decimalString, formatBytes, hourWithSeconds } from "@/lib/utils"
 import type { ContainerRecord } from "@/types"
 import { ContainerHealth, ContainerHealthLabels } from "@/lib/enums"
 import {
+	CircleArrowUpIcon,
 	ClockIcon,
 	ContainerIcon,
 	CpuIcon,
@@ -177,11 +178,25 @@ export const containerChartCols: ColumnDef<ContainerRecord>[] = [
 		header: ({ column }) => (
 			<HeaderButton column={column} name={t({ message: "Image", context: "Docker image" })} Icon={LayersIcon} />
 		),
-		cell: ({ getValue }) => {
+		cell: ({ getValue, row }) => {
 			const val = getValue() as string
 			return (
-				<div className="ms-1 xl:w-40 truncate" title={val}>
+				<div className="ms-1 xl:w-40 flex items-center gap-2">
+					<span className="truncate" title={val}>
 						{val}
+					</span>
+					{row.original.updatable && (
+						<Tooltip>
+							<TooltipTrigger
+								className="shrink-0 rounded-sm text-emerald-600 dark:text-emerald-400 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"
+								aria-label={t`Image update available`}
+								onClick={(event) => event.stopPropagation()}
+							>
+								<CircleArrowUpIcon className="size-4" aria-hidden="true" />
+							</TooltipTrigger>
+							<TooltipContent>{t`Image update available`}</TooltipContent>
+						</Tooltip>
+					)}
 				</div>
 			)
 		},

internal/site/src/components/containers-table/containers-table.tsx

@@ -66,7 +66,7 @@ export default function ContainersTable({ systemId }: { systemId?: string }) {
 		function fetchData(systemId?: string) {
 			pb.collection<ContainerRecord>("containers")
 				.getList(0, 2000, {
-					fields: "id,name,image,ports,cpu,memory,net,health,status,system,updated",
+					fields: "id,name,image,updatable,ports,cpu,memory,net,health,status,system,updated",
 					filter: systemId ? pb.filter("system={:system}", { system: systemId }) : undefined,
 				})
 				.then(({ items }) => {

internal/site/src/types.d.ts

@@ -335,6 +335,7 @@ export interface ContainerRecord extends RecordModel {
 	system: string
 	name: string
 	image: string
+	updatable?: boolean
 	ports: string
 	cpu: number
 	memory: number