Restore an instance when inbound activity races standby

cmd/api/api/auto_standby_status_test.go

@@ -38,6 +38,8 @@ func (s *statusStore) ListInstances(context.Context) ([]autostandby.Instance, er
 
 func (s *statusStore) StandbyInstance(context.Context, string) error { return nil }
 
+func (s *statusStore) RestoreInstance(context.Context, string) error { return nil }
+
 func (s *statusStore) SetRuntime(_ context.Context, id string, runtime *autostandby.Runtime) error {
 	if s.runtime == nil {
 		s.runtime = make(map[string]*autostandby.Runtime)

lib/autostandby/controller.go

@@ -53,6 +53,11 @@ type InstanceEvent struct {
 type InstanceStore interface {
 	ListInstances(ctx context.Context) ([]Instance, error)
 	StandbyInstance(ctx context.Context, id string) error
+	// RestoreInstance brings a standby instance back to running. The controller
+	// calls it when inbound activity turns up on an instance it just suspended;
+	// wake-on-traffic does not exist, so without this the connection hangs until
+	// the client gives up.
+	RestoreInstance(ctx context.Context, id string) error
 	SetRuntime(ctx context.Context, id string, runtime *Runtime) error
 	SubscribeInstanceEvents() (<-chan InstanceEvent, func(), error)
 }
@@ -763,18 +768,49 @@ func (c *Controller) executeStandby(ctx context.Context, id string, instanceName
 		return
 	}
 
-	c.recordStandbyAttempt("success")
 	c.log.Info("instance entered standby due to inbound inactivity", "instance_id", id, "instance_name", instanceName, "idle_timeout", idleTimeout)
 
 	c.mu.Lock()
-	defer c.mu.Unlock()
+	// Inbound activity observed while the snapshot was in flight is about to be
+	// dropped by clearStateLocked, which would leave the guest suspended with a
+	// live connection pointed at it. The failing path above hands that state to
+	// the reconcile flow; a successful standby has to undo itself instead.
+	raced := false
 	if state := c.states[id]; state != nil {
+		raced = len(state.activeInbound) > 0
 		c.clearStateLocked(state)
 		if err := c.persistRuntime(ctx, id, nil); err != nil {
 			c.recordControllerError("persist_runtime")
 			c.log.Warn("auto-standby failed to clear runtime after standby", "instance_id", id, "error", err)
 		}
 	}
+	c.mu.Unlock()
+
+	if raced {
+		c.recordStandbyAttempt("raced_restored")
+		c.restoreAfterRacedStandby(ctx, id, instanceName)
+		return
+	}
+	c.recordStandbyAttempt("success")
+}
+
+// restoreAfterRacedStandby wakes an instance that reached standby while inbound
+// activity was arriving. Wake-on-traffic does not exist, so the connection that
+// raced the snapshot is already pointed at a suspended guest that nothing else
+// will wake.
+func (c *Controller) restoreAfterRacedStandby(ctx context.Context, id string, instanceName string) {
+	c.log.Warn("inbound activity raced standby, restoring instance", "instance_id", id, "instance_name", instanceName)
+
+	if err := c.store.RestoreInstance(ctx, id); err != nil {
+		if errors.Is(err, ErrInstanceNotFound) {
+			c.mu.Lock()
+			defer c.mu.Unlock()
+			c.removeStateLocked(id)
+			return
+		}
+		c.recordControllerError("restore_after_raced_standby")
+		c.log.Error("failed to restore instance after standby raced inbound activity", "instance_id", id, "instance_name", instanceName, "error", err)
+	}
 }
 
 func (c *Controller) handleActiveReconcile(ctx context.Context, id string) {

lib/autostandby/controller_test.go

@@ -17,9 +17,11 @@ type fakeInstanceStore struct {
 	mu               sync.Mutex
 	instances        []Instance
 	standbyIDs       []string
+	restoreIDs       []string
 	persistedRuntime map[string]*Runtime
 	events           chan InstanceEvent
 	standbyErr       error
+	restoreErr       error
 	listErr          error
 	setRuntimeErr    error
 	standbyStarted   chan string
@@ -66,6 +68,19 @@ func (f *fakeInstanceStore) standbyCalls() []string {
 	return append([]string(nil), f.standbyIDs...)
 }
 
+func (f *fakeInstanceStore) RestoreInstance(_ context.Context, id string) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.restoreIDs = append(f.restoreIDs, id)
+	return f.restoreErr
+}
+
+func (f *fakeInstanceStore) restoreCalls() []string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([]string(nil), f.restoreIDs...)
+}
+
 func (f *fakeInstanceStore) SetRuntime(_ context.Context, id string, runtime *Runtime) error {
 	f.mu.Lock()
 	defer f.mu.Unlock()
@@ -721,6 +736,106 @@ func TestStandbyCancelledByInboundActivityWhileQueued(t *testing.T) {
 	assert.Equal(t, []string{"inst-first"}, store.standbyCalls())
 }
 
+// Standby pauses and snapshots the guest, which takes long enough for a request
+// to land mid-transition. Wake-on-traffic does not exist, so the controller has
+// to undo the standby or that connection hangs until the client gives up.
+func TestStandbyRestoresInstanceWhenInboundActivityRacesSnapshot(t *testing.T) {
+	t.Parallel()
+
+	idleSince := time.Date(2026, 4, 6, 10, 55, 0, 0, time.UTC)
+	store := newFakeInstanceStore([]Instance{
+		idleTestInstance("inst-raced", "192.168.100.122", idleSince),
+	})
+	store.standbyStarted = make(chan string, 1)
+	store.standbyRelease = make(chan struct{})
+	controller := NewController(store, &fakeConnectionSource{}, ControllerOptions{
+		Now: func() time.Time { return idleSince.Add(time.Minute) },
+	})
+
+	require.NoError(t, controller.startupResync(context.Background()))
+
+	controller.handleStandbyTimer(context.Background(), "inst-raced")
+	require.Equal(t, "inst-raced", <-store.standbyStarted)
+
+	controller.handleConnectionEvent(context.Background(), ConnectionEvent{
+		Type: ConnectionEventNew,
+		Connection: Connection{
+			OriginalSourceIP:        mustAddr("1.2.3.4"),
+			OriginalSourcePort:      50011,
+			OriginalDestinationIP:   mustAddr("192.168.100.122"),
+			OriginalDestinationPort: 8080,
+			TCPState:                TCPStateEstablished,
+		},
+		ObservedAt: idleSince.Add(time.Minute),
+	})
+
+	close(store.standbyRelease)
+	controller.standbyWG.Wait()
+
+	assert.Equal(t, []string{"inst-raced"}, store.standbyCalls())
+	assert.Equal(t, []string{"inst-raced"}, store.restoreCalls())
+}
+
+func TestStandbyRacedRestoreDropsStateWhenInstanceGone(t *testing.T) {
+	t.Parallel()
+
+	idleSince := time.Date(2026, 4, 6, 10, 55, 0, 0, time.UTC)
+	store := newFakeInstanceStore([]Instance{
+		idleTestInstance("inst-deleted", "192.168.100.123", idleSince),
+	})
+	store.standbyStarted = make(chan string, 1)
+	store.standbyRelease = make(chan struct{})
+	store.restoreErr = fmt.Errorf("%w: deleted mid-transition", ErrInstanceNotFound)
+	controller := NewController(store, &fakeConnectionSource{}, ControllerOptions{
+		Now: func() time.Time { return idleSince.Add(time.Minute) },
+	})
+
+	require.NoError(t, controller.startupResync(context.Background()))
+
+	controller.handleStandbyTimer(context.Background(), "inst-deleted")
+	require.Equal(t, "inst-deleted", <-store.standbyStarted)
+
+	controller.handleConnectionEvent(context.Background(), ConnectionEvent{
+		Type: ConnectionEventNew,
+		Connection: Connection{
+			OriginalSourceIP:        mustAddr("1.2.3.4"),
+			OriginalSourcePort:      50012,
+			OriginalDestinationIP:   mustAddr("192.168.100.123"),
+			OriginalDestinationPort: 8080,
+			TCPState:                TCPStateEstablished,
+		},
+		ObservedAt: idleSince.Add(time.Minute),
+	})
+
+	close(store.standbyRelease)
+	controller.standbyWG.Wait()
+
+	assert.Equal(t, []string{"inst-deleted"}, store.restoreCalls())
+	controller.mu.RLock()
+	defer controller.mu.RUnlock()
+	assert.NotContains(t, controller.states, "inst-deleted")
+}
+
+func TestStandbyDoesNotRestoreWhenNoActivityRacedSnapshot(t *testing.T) {
+	t.Parallel()
+
+	idleSince := time.Date(2026, 4, 6, 10, 55, 0, 0, time.UTC)
+	store := newFakeInstanceStore([]Instance{
+		idleTestInstance("inst-quiet", "192.168.100.124", idleSince),
+	})
+	controller := NewController(store, &fakeConnectionSource{}, ControllerOptions{
+		Now: func() time.Time { return idleSince.Add(time.Minute) },
+	})
+
+	require.NoError(t, controller.startupResync(context.Background()))
+
+	controller.handleStandbyTimer(context.Background(), "inst-quiet")
+	controller.standbyWG.Wait()
+
+	assert.Equal(t, []string{"inst-quiet"}, store.standbyCalls())
+	assert.Empty(t, store.restoreCalls())
+}
+
 func TestStandbyDispatchDeduplicatesPendingRequests(t *testing.T) {
 	t.Parallel()
 

lib/instances/auto_standby_integration_linux_test.go

@@ -58,6 +58,11 @@ func (s integrationAutoStandbyStore) StandbyInstance(ctx context.Context, id str
 	return err
 }
 
+func (s integrationAutoStandbyStore) RestoreInstance(ctx context.Context, id string) error {
+	_, err := s.manager.RestoreInstance(ctx, id)
+	return err
+}
+
 func (s integrationAutoStandbyStore) SetRuntime(ctx context.Context, id string, runtime *autostandby.Runtime) error {
 	return s.manager.SetAutoStandbyRuntime(ctx, id, runtime)
 }

lib/providers/auto_standby_linux.go

@@ -60,6 +60,14 @@ func (s autoStandbyInstanceStore) StandbyInstance(ctx context.Context, id string
 	return err
 }
 
+func (s autoStandbyInstanceStore) RestoreInstance(ctx context.Context, id string) error {
+	_, err := s.manager.RestoreInstance(ctx, id)
+	if errors.Is(err, instances.ErrNotFound) {
+		return fmt.Errorf("%w: %v", autostandby.ErrInstanceNotFound, err)
+	}
+	return err
+}
+
 func (s autoStandbyInstanceStore) SetRuntime(ctx context.Context, id string, runtime *autostandby.Runtime) error {
 	return s.runtimeManager.SetAutoStandbyRuntime(ctx, id, runtime)
 }