mirror of
https://github.com/henrygd/beszel.git
synced 2026-09-25 19:07:47 +02:00
fix: prevent WebSocket reconnect loops on slow agent collections (#2294)
The hub waited only 5s (the request manager default) for stats over WebSocket and closed the connection on any error. On hosts where `zpool list` stalls (seen on Proxmox, up to its 10s timeout), collection exceeded that limit, so the hub sent a close (code 1000) and the agent reconnected. The refresh ran every other cycle, which caused a disconnect roughly every 2 minutes. Hub: - Wait up to 30s for WebSocket stats responses. - Keep the connection open when a stats request times out; only close it (and fall back to SSH) for other errors. Agent: - After the first collection, refresh `zpool list` pool stats and `zfs list` dataset usage in the background and serve cached values meanwhile, so a hung utility cannot delay the stats response.
This commit is contained in:
@@ -697,6 +697,11 @@ func (sys *System) fetchDataFromAgent(options common.DataRequestOptions) (*syste
|
||||
sys.syncPendingNetworkMonitors()
|
||||
return wsData, nil
|
||||
}
|
||||
// A slow collection doesn't mean the connection is broken. Closing it
|
||||
// would force the agent into a reconnect loop, so only report the error.
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
return nil, err
|
||||
}
|
||||
// close the WebSocket connection if error and try SSH
|
||||
sys.closeWebSocketConnection()
|
||||
}
|
||||
@@ -709,12 +714,19 @@ func (sys *System) fetchDataFromAgent(options common.DataRequestOptions) (*syste
|
||||
return sshData, nil
|
||||
}
|
||||
|
||||
// wsDataRequestTimeout bounds how long to wait for stats over WebSocket. Agent
|
||||
// collection can legitimately take several seconds (e.g. a slow `zpool list`),
|
||||
// so this must be well above the request manager's 5s default.
|
||||
var wsDataRequestTimeout = 30 * time.Second
|
||||
|
||||
func (sys *System) fetchDataViaWebSocket(options common.DataRequestOptions) (*system.CombinedData, error) {
|
||||
if sys.WsConn == nil || !sys.WsConn.IsConnected() {
|
||||
return nil, errors.New("no websocket connection")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), wsDataRequestTimeout)
|
||||
defer cancel()
|
||||
wsTransport := transport.NewWebSocketTransport(sys.WsConn)
|
||||
err := wsTransport.Request(context.Background(), common.GetData, options, sys.data)
|
||||
err := wsTransport.Request(ctx, common.GetData, options, sys.data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user