From abc15bf650ce723b55f2865f03a2a104352ddafd Mon Sep 17 00:00:00 2001 From: hiroTamada <88675973+hiroTamada@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:56:27 +0000 Subject: [PATCH] Restore an instance when inbound activity races standby A standby guest is only restored when a new inbound connection arrives, so a connection that is already open when the guest is suspended does not stall briefly, it hangs until the client gives up. The standby attempt releases the controller lock for the pause and snapshot, which takes long enough for a request to land mid-transition. The failing path already hands that state back to the reconcile flow, but the successful path dropped it in clearStateLocked and left the guest suspended with a live connection pointed at it. Undo the standby instead, reported as a raced_restored standby attempt. Co-Authored-By: Claude Opus 5 --- cmd/api/api/auto_standby_status_test.go | 2 + lib/autostandby/controller.go | 40 +++++- lib/autostandby/controller_test.go | 115 ++++++++++++++++++ .../auto_standby_integration_linux_test.go | 5 + lib/providers/auto_standby_linux.go | 8 ++ 5 files changed, 168 insertions(+), 2 deletions(-) diff --git a/cmd/api/api/auto_standby_status_test.go b/cmd/api/api/auto_standby_status_test.go index b4aecfd1..a772cc14 100644 --- a/cmd/api/api/auto_standby_status_test.go +++ b/cmd/api/api/auto_standby_status_test.go @@ -38,6 +38,8 @@ func (s *statusStore) ListInstances(context.Context) ([]autostandby.Instance, er func (s *statusStore) StandbyInstance(context.Context, string) error { return nil } +func (s *statusStore) RestoreInstance(context.Context, string) error { return nil } + func (s *statusStore) SetRuntime(_ context.Context, id string, runtime *autostandby.Runtime) error { if s.runtime == nil { s.runtime = make(map[string]*autostandby.Runtime) diff --git a/lib/autostandby/controller.go b/lib/autostandby/controller.go index 6af7197d..ca2104da 100644 --- a/lib/autostandby/controller.go +++ b/lib/autostandby/controller.go @@ -53,6 +53,11 @@ type InstanceEvent struct { type InstanceStore interface { ListInstances(ctx context.Context) ([]Instance, error) StandbyInstance(ctx context.Context, id string) error + // RestoreInstance brings a standby instance back to running. The controller + // calls it when inbound activity turns up on an instance it just suspended; + // wake-on-traffic does not exist, so without this the connection hangs until + // the client gives up. + RestoreInstance(ctx context.Context, id string) error SetRuntime(ctx context.Context, id string, runtime *Runtime) error SubscribeInstanceEvents() (<-chan InstanceEvent, func(), error) } @@ -763,18 +768,49 @@ func (c *Controller) executeStandby(ctx context.Context, id string, instanceName return } - c.recordStandbyAttempt("success") c.log.Info("instance entered standby due to inbound inactivity", "instance_id", id, "instance_name", instanceName, "idle_timeout", idleTimeout) c.mu.Lock() - defer c.mu.Unlock() + // Inbound activity observed while the snapshot was in flight is about to be + // dropped by clearStateLocked, which would leave the guest suspended with a + // live connection pointed at it. The failing path above hands that state to + // the reconcile flow; a successful standby has to undo itself instead. + raced := false if state := c.states[id]; state != nil { + raced = len(state.activeInbound) > 0 c.clearStateLocked(state) if err := c.persistRuntime(ctx, id, nil); err != nil { c.recordControllerError("persist_runtime") c.log.Warn("auto-standby failed to clear runtime after standby", "instance_id", id, "error", err) } } + c.mu.Unlock() + + if raced { + c.recordStandbyAttempt("raced_restored") + c.restoreAfterRacedStandby(ctx, id, instanceName) + return + } + c.recordStandbyAttempt("success") +} + +// restoreAfterRacedStandby wakes an instance that reached standby while inbound +// activity was arriving. Wake-on-traffic does not exist, so the connection that +// raced the snapshot is already pointed at a suspended guest that nothing else +// will wake. +func (c *Controller) restoreAfterRacedStandby(ctx context.Context, id string, instanceName string) { + c.log.Warn("inbound activity raced standby, restoring instance", "instance_id", id, "instance_name", instanceName) + + if err := c.store.RestoreInstance(ctx, id); err != nil { + if errors.Is(err, ErrInstanceNotFound) { + c.mu.Lock() + defer c.mu.Unlock() + c.removeStateLocked(id) + return + } + c.recordControllerError("restore_after_raced_standby") + c.log.Error("failed to restore instance after standby raced inbound activity", "instance_id", id, "instance_name", instanceName, "error", err) + } } func (c *Controller) handleActiveReconcile(ctx context.Context, id string) { diff --git a/lib/autostandby/controller_test.go b/lib/autostandby/controller_test.go index b0cd602e..1489d59b 100644 --- a/lib/autostandby/controller_test.go +++ b/lib/autostandby/controller_test.go @@ -17,9 +17,11 @@ type fakeInstanceStore struct { mu sync.Mutex instances []Instance standbyIDs []string + restoreIDs []string persistedRuntime map[string]*Runtime events chan InstanceEvent standbyErr error + restoreErr error listErr error setRuntimeErr error standbyStarted chan string @@ -66,6 +68,19 @@ func (f *fakeInstanceStore) standbyCalls() []string { return append([]string(nil), f.standbyIDs...) } +func (f *fakeInstanceStore) RestoreInstance(_ context.Context, id string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.restoreIDs = append(f.restoreIDs, id) + return f.restoreErr +} + +func (f *fakeInstanceStore) restoreCalls() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.restoreIDs...) +} + func (f *fakeInstanceStore) SetRuntime(_ context.Context, id string, runtime *Runtime) error { f.mu.Lock() defer f.mu.Unlock() @@ -721,6 +736,106 @@ func TestStandbyCancelledByInboundActivityWhileQueued(t *testing.T) { assert.Equal(t, []string{"inst-first"}, store.standbyCalls()) } +// Standby pauses and snapshots the guest, which takes long enough for a request +// to land mid-transition. Wake-on-traffic does not exist, so the controller has +// to undo the standby or that connection hangs until the client gives up. +func TestStandbyRestoresInstanceWhenInboundActivityRacesSnapshot(t *testing.T) { + t.Parallel() + + idleSince := time.Date(2026, 4, 6, 10, 55, 0, 0, time.UTC) + store := newFakeInstanceStore([]Instance{ + idleTestInstance("inst-raced", "192.168.100.122", idleSince), + }) + store.standbyStarted = make(chan string, 1) + store.standbyRelease = make(chan struct{}) + controller := NewController(store, &fakeConnectionSource{}, ControllerOptions{ + Now: func() time.Time { return idleSince.Add(time.Minute) }, + }) + + require.NoError(t, controller.startupResync(context.Background())) + + controller.handleStandbyTimer(context.Background(), "inst-raced") + require.Equal(t, "inst-raced", <-store.standbyStarted) + + controller.handleConnectionEvent(context.Background(), ConnectionEvent{ + Type: ConnectionEventNew, + Connection: Connection{ + OriginalSourceIP: mustAddr("1.2.3.4"), + OriginalSourcePort: 50011, + OriginalDestinationIP: mustAddr("192.168.100.122"), + OriginalDestinationPort: 8080, + TCPState: TCPStateEstablished, + }, + ObservedAt: idleSince.Add(time.Minute), + }) + + close(store.standbyRelease) + controller.standbyWG.Wait() + + assert.Equal(t, []string{"inst-raced"}, store.standbyCalls()) + assert.Equal(t, []string{"inst-raced"}, store.restoreCalls()) +} + +func TestStandbyRacedRestoreDropsStateWhenInstanceGone(t *testing.T) { + t.Parallel() + + idleSince := time.Date(2026, 4, 6, 10, 55, 0, 0, time.UTC) + store := newFakeInstanceStore([]Instance{ + idleTestInstance("inst-deleted", "192.168.100.123", idleSince), + }) + store.standbyStarted = make(chan string, 1) + store.standbyRelease = make(chan struct{}) + store.restoreErr = fmt.Errorf("%w: deleted mid-transition", ErrInstanceNotFound) + controller := NewController(store, &fakeConnectionSource{}, ControllerOptions{ + Now: func() time.Time { return idleSince.Add(time.Minute) }, + }) + + require.NoError(t, controller.startupResync(context.Background())) + + controller.handleStandbyTimer(context.Background(), "inst-deleted") + require.Equal(t, "inst-deleted", <-store.standbyStarted) + + controller.handleConnectionEvent(context.Background(), ConnectionEvent{ + Type: ConnectionEventNew, + Connection: Connection{ + OriginalSourceIP: mustAddr("1.2.3.4"), + OriginalSourcePort: 50012, + OriginalDestinationIP: mustAddr("192.168.100.123"), + OriginalDestinationPort: 8080, + TCPState: TCPStateEstablished, + }, + ObservedAt: idleSince.Add(time.Minute), + }) + + close(store.standbyRelease) + controller.standbyWG.Wait() + + assert.Equal(t, []string{"inst-deleted"}, store.restoreCalls()) + controller.mu.RLock() + defer controller.mu.RUnlock() + assert.NotContains(t, controller.states, "inst-deleted") +} + +func TestStandbyDoesNotRestoreWhenNoActivityRacedSnapshot(t *testing.T) { + t.Parallel() + + idleSince := time.Date(2026, 4, 6, 10, 55, 0, 0, time.UTC) + store := newFakeInstanceStore([]Instance{ + idleTestInstance("inst-quiet", "192.168.100.124", idleSince), + }) + controller := NewController(store, &fakeConnectionSource{}, ControllerOptions{ + Now: func() time.Time { return idleSince.Add(time.Minute) }, + }) + + require.NoError(t, controller.startupResync(context.Background())) + + controller.handleStandbyTimer(context.Background(), "inst-quiet") + controller.standbyWG.Wait() + + assert.Equal(t, []string{"inst-quiet"}, store.standbyCalls()) + assert.Empty(t, store.restoreCalls()) +} + func TestStandbyDispatchDeduplicatesPendingRequests(t *testing.T) { t.Parallel() diff --git a/lib/instances/auto_standby_integration_linux_test.go b/lib/instances/auto_standby_integration_linux_test.go index 2c542e40..ba537196 100644 --- a/lib/instances/auto_standby_integration_linux_test.go +++ b/lib/instances/auto_standby_integration_linux_test.go @@ -58,6 +58,11 @@ func (s integrationAutoStandbyStore) StandbyInstance(ctx context.Context, id str return err } +func (s integrationAutoStandbyStore) RestoreInstance(ctx context.Context, id string) error { + _, err := s.manager.RestoreInstance(ctx, id) + return err +} + func (s integrationAutoStandbyStore) SetRuntime(ctx context.Context, id string, runtime *autostandby.Runtime) error { return s.manager.SetAutoStandbyRuntime(ctx, id, runtime) } diff --git a/lib/providers/auto_standby_linux.go b/lib/providers/auto_standby_linux.go index d54d7aea..a33c4a12 100644 --- a/lib/providers/auto_standby_linux.go +++ b/lib/providers/auto_standby_linux.go @@ -60,6 +60,14 @@ func (s autoStandbyInstanceStore) StandbyInstance(ctx context.Context, id string return err } +func (s autoStandbyInstanceStore) RestoreInstance(ctx context.Context, id string) error { + _, err := s.manager.RestoreInstance(ctx, id) + if errors.Is(err, instances.ErrNotFound) { + return fmt.Errorf("%w: %v", autostandby.ErrInstanceNotFound, err) + } + return err +} + func (s autoStandbyInstanceStore) SetRuntime(ctx context.Context, id string, runtime *autostandby.Runtime) error { return s.runtimeManager.SetAutoStandbyRuntime(ctx, id, runtime) }