mirror of
https://github.com/henrygd/beszel.git
synced 2026-08-14 22:37:48 +02:00
c3a432101b
The per-system updater reused an SSH client across ticks and ran the data exchange with no deadline. If a connection went half-open (dead peer that never sends RST/FIN) or an agent accepted the session but never wrote a response, the read in fetchDataViaSSH blocked forever. Because StartUpdater calls update() synchronously on its ticker, a blocked read froze the whole per-system goroutine: the ticker's subsequent ticks were dropped, no error was returned so the system stayed "up", and the agent was never re-dialed until the hub process restarted. Bound each SSH data exchange with sshOperationTimeout via runWithTimeout: on timeout the connection is torn down (unwinding the blocked read) and a retryable error is returned, so the next tick re-dials. Also enable TCP keep-alive on dialed connections as a backstop for genuine network death. Fixes #2041
57 lines
1.9 KiB
Go
57 lines
1.9 KiB
Go
//go:build testing
|
|
|
|
package systems
|
|
|
|
import (
|
|
"errors"
|
|
"testing"
|
|
"testing/synctest"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
)
|
|
|
|
// TestRunWithTimeout covers the guard added for issue #2041: the per-system SSH
|
|
// data exchange must never block the updater indefinitely on a dead connection.
|
|
func TestRunWithTimeout(t *testing.T) {
|
|
t.Run("returns the operation result when it completes before the timeout", func(t *testing.T) {
|
|
synctest.Test(t, func(t *testing.T) {
|
|
wantErr := errors.New("boom")
|
|
onTimeoutCalled := false
|
|
|
|
retry, err := runWithTimeout(10*time.Second, func() (bool, error) {
|
|
return true, wantErr
|
|
}, func() { onTimeoutCalled = true })
|
|
|
|
assert.True(t, retry, "should return the operation's retry value")
|
|
assert.Equal(t, wantErr, err, "should return the operation's error")
|
|
assert.False(t, onTimeoutCalled, "onTimeout must not fire when the op completes")
|
|
})
|
|
})
|
|
|
|
t.Run("times out and tears down the connection when the op blocks", func(t *testing.T) {
|
|
synctest.Test(t, func(t *testing.T) {
|
|
// unblock simulates a half-open connection: the op is stuck reading a
|
|
// response that never arrives until the connection is torn down.
|
|
unblock := make(chan struct{})
|
|
onTimeoutCalled := false
|
|
start := time.Now()
|
|
|
|
retry, err := runWithTimeout(5*time.Second, func() (bool, error) {
|
|
<-unblock
|
|
return false, nil
|
|
}, func() {
|
|
onTimeoutCalled = true
|
|
close(unblock) // tearing down the connection releases the blocked read
|
|
})
|
|
|
|
assert.Equal(t, 5*time.Second, time.Since(start), "should return exactly at the timeout")
|
|
assert.True(t, retry, "a timeout should be retryable so the next tick re-dials")
|
|
assert.Error(t, err, "a timeout must surface an error so the system is set down")
|
|
assert.True(t, onTimeoutCalled, "onTimeout must fire so the dead connection is closed")
|
|
|
|
synctest.Wait() // ensure the released op goroutine exits cleanly
|
|
})
|
|
})
|
|
}
|