diff --git a/cmd/api/api/auto_standby_status_test.go b/cmd/api/api/auto_standby_status_test.go index b4aecfd1..a772cc14 100644 --- a/cmd/api/api/auto_standby_status_test.go +++ b/cmd/api/api/auto_standby_status_test.go @@ -38,6 +38,8 @@ func (s *statusStore) ListInstances(context.Context) ([]autostandby.Instance, er func (s *statusStore) StandbyInstance(context.Context, string) error { return nil } +func (s *statusStore) RestoreInstance(context.Context, string) error { return nil } + func (s *statusStore) SetRuntime(_ context.Context, id string, runtime *autostandby.Runtime) error { if s.runtime == nil { s.runtime = make(map[string]*autostandby.Runtime) diff --git a/lib/autostandby/controller.go b/lib/autostandby/controller.go index 6af7197d..ca2104da 100644 --- a/lib/autostandby/controller.go +++ b/lib/autostandby/controller.go @@ -53,6 +53,11 @@ type InstanceEvent struct { type InstanceStore interface { ListInstances(ctx context.Context) ([]Instance, error) StandbyInstance(ctx context.Context, id string) error + // RestoreInstance brings a standby instance back to running. The controller + // calls it when inbound activity turns up on an instance it just suspended; + // wake-on-traffic does not exist, so without this the connection hangs until + // the client gives up. + RestoreInstance(ctx context.Context, id string) error SetRuntime(ctx context.Context, id string, runtime *Runtime) error SubscribeInstanceEvents() (<-chan InstanceEvent, func(), error) } @@ -763,18 +768,49 @@ func (c *Controller) executeStandby(ctx context.Context, id string, instanceName return } - c.recordStandbyAttempt("success") c.log.Info("instance entered standby due to inbound inactivity", "instance_id", id, "instance_name", instanceName, "idle_timeout", idleTimeout) c.mu.Lock() - defer c.mu.Unlock() + // Inbound activity observed while the snapshot was in flight is about to be + // dropped by clearStateLocked, which would leave the guest suspended with a + // live connection pointed at it. The failing path above hands that state to + // the reconcile flow; a successful standby has to undo itself instead. + raced := false if state := c.states[id]; state != nil { + raced = len(state.activeInbound) > 0 c.clearStateLocked(state) if err := c.persistRuntime(ctx, id, nil); err != nil { c.recordControllerError("persist_runtime") c.log.Warn("auto-standby failed to clear runtime after standby", "instance_id", id, "error", err) } } + c.mu.Unlock() + + if raced { + c.recordStandbyAttempt("raced_restored") + c.restoreAfterRacedStandby(ctx, id, instanceName) + return + } + c.recordStandbyAttempt("success") +} + +// restoreAfterRacedStandby wakes an instance that reached standby while inbound +// activity was arriving. Wake-on-traffic does not exist, so the connection that +// raced the snapshot is already pointed at a suspended guest that nothing else +// will wake. +func (c *Controller) restoreAfterRacedStandby(ctx context.Context, id string, instanceName string) { + c.log.Warn("inbound activity raced standby, restoring instance", "instance_id", id, "instance_name", instanceName) + + if err := c.store.RestoreInstance(ctx, id); err != nil { + if errors.Is(err, ErrInstanceNotFound) { + c.mu.Lock() + defer c.mu.Unlock() + c.removeStateLocked(id) + return + } + c.recordControllerError("restore_after_raced_standby") + c.log.Error("failed to restore instance after standby raced inbound activity", "instance_id", id, "instance_name", instanceName, "error", err) + } } func (c *Controller) handleActiveReconcile(ctx context.Context, id string) { diff --git a/lib/autostandby/controller_test.go b/lib/autostandby/controller_test.go index b0cd602e..1489d59b 100644 --- a/lib/autostandby/controller_test.go +++ b/lib/autostandby/controller_test.go @@ -17,9 +17,11 @@ type fakeInstanceStore struct { mu sync.Mutex instances []Instance standbyIDs []string + restoreIDs []string persistedRuntime map[string]*Runtime events chan InstanceEvent standbyErr error + restoreErr error listErr error setRuntimeErr error standbyStarted chan string @@ -66,6 +68,19 @@ func (f *fakeInstanceStore) standbyCalls() []string { return append([]string(nil), f.standbyIDs...) } +func (f *fakeInstanceStore) RestoreInstance(_ context.Context, id string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.restoreIDs = append(f.restoreIDs, id) + return f.restoreErr +} + +func (f *fakeInstanceStore) restoreCalls() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.restoreIDs...) +} + func (f *fakeInstanceStore) SetRuntime(_ context.Context, id string, runtime *Runtime) error { f.mu.Lock() defer f.mu.Unlock() @@ -721,6 +736,106 @@ func TestStandbyCancelledByInboundActivityWhileQueued(t *testing.T) { assert.Equal(t, []string{"inst-first"}, store.standbyCalls()) } +// Standby pauses and snapshots the guest, which takes long enough for a request +// to land mid-transition. Wake-on-traffic does not exist, so the controller has +// to undo the standby or that connection hangs until the client gives up. +func TestStandbyRestoresInstanceWhenInboundActivityRacesSnapshot(t *testing.T) { + t.Parallel() + + idleSince := time.Date(2026, 4, 6, 10, 55, 0, 0, time.UTC) + store := newFakeInstanceStore([]Instance{ + idleTestInstance("inst-raced", "192.168.100.122", idleSince), + }) + store.standbyStarted = make(chan string, 1) + store.standbyRelease = make(chan struct{}) + controller := NewController(store, &fakeConnectionSource{}, ControllerOptions{ + Now: func() time.Time { return idleSince.Add(time.Minute) }, + }) + + require.NoError(t, controller.startupResync(context.Background())) + + controller.handleStandbyTimer(context.Background(), "inst-raced") + require.Equal(t, "inst-raced", <-store.standbyStarted) + + controller.handleConnectionEvent(context.Background(), ConnectionEvent{ + Type: ConnectionEventNew, + Connection: Connection{ + OriginalSourceIP: mustAddr("1.2.3.4"), + OriginalSourcePort: 50011, + OriginalDestinationIP: mustAddr("192.168.100.122"), + OriginalDestinationPort: 8080, + TCPState: TCPStateEstablished, + }, + ObservedAt: idleSince.Add(time.Minute), + }) + + close(store.standbyRelease) + controller.standbyWG.Wait() + + assert.Equal(t, []string{"inst-raced"}, store.standbyCalls()) + assert.Equal(t, []string{"inst-raced"}, store.restoreCalls()) +} + +func TestStandbyRacedRestoreDropsStateWhenInstanceGone(t *testing.T) { + t.Parallel() + + idleSince := time.Date(2026, 4, 6, 10, 55, 0, 0, time.UTC) + store := newFakeInstanceStore([]Instance{ + idleTestInstance("inst-deleted", "192.168.100.123", idleSince), + }) + store.standbyStarted = make(chan string, 1) + store.standbyRelease = make(chan struct{}) + store.restoreErr = fmt.Errorf("%w: deleted mid-transition", ErrInstanceNotFound) + controller := NewController(store, &fakeConnectionSource{}, ControllerOptions{ + Now: func() time.Time { return idleSince.Add(time.Minute) }, + }) + + require.NoError(t, controller.startupResync(context.Background())) + + controller.handleStandbyTimer(context.Background(), "inst-deleted") + require.Equal(t, "inst-deleted", <-store.standbyStarted) + + controller.handleConnectionEvent(context.Background(), ConnectionEvent{ + Type: ConnectionEventNew, + Connection: Connection{ + OriginalSourceIP: mustAddr("1.2.3.4"), + OriginalSourcePort: 50012, + OriginalDestinationIP: mustAddr("192.168.100.123"), + OriginalDestinationPort: 8080, + TCPState: TCPStateEstablished, + }, + ObservedAt: idleSince.Add(time.Minute), + }) + + close(store.standbyRelease) + controller.standbyWG.Wait() + + assert.Equal(t, []string{"inst-deleted"}, store.restoreCalls()) + controller.mu.RLock() + defer controller.mu.RUnlock() + assert.NotContains(t, controller.states, "inst-deleted") +} + +func TestStandbyDoesNotRestoreWhenNoActivityRacedSnapshot(t *testing.T) { + t.Parallel() + + idleSince := time.Date(2026, 4, 6, 10, 55, 0, 0, time.UTC) + store := newFakeInstanceStore([]Instance{ + idleTestInstance("inst-quiet", "192.168.100.124", idleSince), + }) + controller := NewController(store, &fakeConnectionSource{}, ControllerOptions{ + Now: func() time.Time { return idleSince.Add(time.Minute) }, + }) + + require.NoError(t, controller.startupResync(context.Background())) + + controller.handleStandbyTimer(context.Background(), "inst-quiet") + controller.standbyWG.Wait() + + assert.Equal(t, []string{"inst-quiet"}, store.standbyCalls()) + assert.Empty(t, store.restoreCalls()) +} + func TestStandbyDispatchDeduplicatesPendingRequests(t *testing.T) { t.Parallel() diff --git a/lib/instances/auto_standby_integration_linux_test.go b/lib/instances/auto_standby_integration_linux_test.go index 2c542e40..ba537196 100644 --- a/lib/instances/auto_standby_integration_linux_test.go +++ b/lib/instances/auto_standby_integration_linux_test.go @@ -58,6 +58,11 @@ func (s integrationAutoStandbyStore) StandbyInstance(ctx context.Context, id str return err } +func (s integrationAutoStandbyStore) RestoreInstance(ctx context.Context, id string) error { + _, err := s.manager.RestoreInstance(ctx, id) + return err +} + func (s integrationAutoStandbyStore) SetRuntime(ctx context.Context, id string, runtime *autostandby.Runtime) error { return s.manager.SetAutoStandbyRuntime(ctx, id, runtime) } diff --git a/lib/providers/auto_standby_linux.go b/lib/providers/auto_standby_linux.go index d54d7aea..a33c4a12 100644 --- a/lib/providers/auto_standby_linux.go +++ b/lib/providers/auto_standby_linux.go @@ -60,6 +60,14 @@ func (s autoStandbyInstanceStore) StandbyInstance(ctx context.Context, id string return err } +func (s autoStandbyInstanceStore) RestoreInstance(ctx context.Context, id string) error { + _, err := s.manager.RestoreInstance(ctx, id) + if errors.Is(err, instances.ErrNotFound) { + return fmt.Errorf("%w: %v", autostandby.ErrInstanceNotFound, err) + } + return err +} + func (s autoStandbyInstanceStore) SetRuntime(ctx context.Context, id string, runtime *autostandby.Runtime) error { return s.runtimeManager.SetAutoStandbyRuntime(ctx, id, runtime) }