fix: prevent WebSocket reconnect loops on slow agent collections (#2294)

The hub waited only 5s (the request manager default) for stats over
WebSocket and closed the connection on any error. On hosts where
`zpool list` stalls (seen on Proxmox, up to its 10s timeout), collection
exceeded that limit, so the hub sent a close (code 1000) and the agent
reconnected. The refresh ran every other cycle, which caused a
disconnect roughly every 2 minutes.

Hub:
- Wait up to 30s for WebSocket stats responses.
- Keep the connection open when a stats request times out; only close
  it (and fall back to SSH) for other errors.

Agent:
- After the first collection, refresh `zpool list` pool stats and
  `zfs list` dataset usage in the background and serve cached values
  meanwhile, so a hung utility cannot delay the stats response.
This commit is contained in:
henrygd
2026-09-23 11:21:51 -04:00
parent 2d5ea3fa08
commit 8bf6917fe0
4 changed files with 194 additions and 22 deletions

View File

@@ -697,6 +697,11 @@ func (sys *System) fetchDataFromAgent(options common.DataRequestOptions) (*syste
sys.syncPendingNetworkMonitors()
return wsData, nil
}
// A slow collection doesn't mean the connection is broken. Closing it
// would force the agent into a reconnect loop, so only report the error.
if errors.Is(err, context.DeadlineExceeded) {
return nil, err
}
// close the WebSocket connection if error and try SSH
sys.closeWebSocketConnection()
}
@@ -709,12 +714,19 @@ func (sys *System) fetchDataFromAgent(options common.DataRequestOptions) (*syste
return sshData, nil
}
// wsDataRequestTimeout bounds how long to wait for stats over WebSocket. Agent
// collection can legitimately take several seconds (e.g. a slow `zpool list`),
// so this must be well above the request manager's 5s default.
var wsDataRequestTimeout = 30 * time.Second
func (sys *System) fetchDataViaWebSocket(options common.DataRequestOptions) (*system.CombinedData, error) {
if sys.WsConn == nil || !sys.WsConn.IsConnected() {
return nil, errors.New("no websocket connection")
}
ctx, cancel := context.WithTimeout(context.Background(), wsDataRequestTimeout)
defer cancel()
wsTransport := transport.NewWebSocketTransport(sys.WsConn)
err := wsTransport.Request(context.Background(), common.GetData, options, sys.data)
err := wsTransport.Request(ctx, common.GetData, options, sys.data)
if err != nil {
return nil, err
}