fix(cloud): eliminate periodic runtime CPU spikes

tests/unit_tests/cloud/test_directory_projection.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import datetime
 import logging
 from types import SimpleNamespace
+from unittest.mock import Mock
 
 import pytest
 import sqlalchemy
@@ -181,6 +182,12 @@ def _delta(
 
 async def test_initial_snapshot_projects_core_owned_rows(projection_context):
     application, session_factory = projection_context
+    reconcile_execution_projection = Mock()
+    application.tool_mgr = SimpleNamespace(
+        mcp_tool_loader=SimpleNamespace(
+            reconcile_execution_projection=reconcile_execution_projection,
+        )
+    )
     service = DirectoryProjectionService(
         application,
         _Provider([_snapshot(1)]),
@@ -212,6 +219,11 @@ async def test_initial_snapshot_projects_core_owned_rows(projection_context):
     assert state is not None
     assert state.cursor == 1
     assert state.snapshot_coverage_cursor == 1
+    reconcile_execution_projection.assert_called_once_with(
+        INSTANCE_UUID,
+        {WORKSPACE_UUID: 1},
+        affected_workspace_uuids=None,
+    )
 
 
 async def test_same_cursor_equivocation_and_rollback_fail_closed(projection_context):
@@ -297,6 +309,12 @@ async def test_archived_and_absent_workspaces_are_execution_fenced(projection_co
 
 async def test_event_poll_fetches_workspace_delta_and_records_receipt(projection_context):
     application, session_factory = projection_context
+    reconcile_execution_projection = Mock()
+    application.tool_mgr = SimpleNamespace(
+        mcp_tool_loader=SimpleNamespace(
+            reconcile_execution_projection=reconcile_execution_projection,
+        )
+    )
     event = DirectoryEvent(
         cursor=2,
         uuid='40000000-0000-0000-0000-000000000001',
@@ -320,6 +338,7 @@ async def test_event_poll_fetches_workspace_delta_and_records_receipt(projection
     )
     service = DirectoryProjectionService(application, provider, INSTANCE_UUID)
     await service.initialize()
+    reconcile_execution_projection.reset_mock()
 
     await service.sync_once()
 
@@ -335,6 +354,11 @@ async def test_event_poll_fetches_workspace_delta_and_records_receipt(projection
     assert inbox.applied_at is not None
     assert provider.snapshot_calls == 1
     assert provider.delta_calls == 1
+    reconcile_execution_projection.assert_called_once_with(
+        INSTANCE_UUID,
+        {WORKSPACE_UUID: 1},
+        affected_workspace_uuids={WORKSPACE_UUID},
+    )
 
 
 async def test_directory_delta_does_not_skip_unfetched_event_cursors(projection_context):

tests/unit_tests/pipeline/test_aggregator.py

@@ -875,6 +875,57 @@ class TestMessageAggregatorWorkspaceIsolation:
         assert {key[1] for key in agg.buffers} == {'workspace-a', 'workspace-b'}
         await agg.flush_all()
 
+    @pytest.mark.asyncio
+    async def test_new_buffer_uses_scope_counter_without_global_scan(self):
+        class NoGlobalIterationDict(dict):
+            def __iter__(self):
+                raise AssertionError('aggregation admission scanned all buffers')
+
+            def items(self):
+                raise AssertionError('aggregation admission scanned all buffers')
+
+            def values(self):
+                raise AssertionError('aggregation admission scanned all buffers')
+
+        app = make_aggregator_app()
+        enable_aggregation(app)
+        agg = get_aggregator_module().MessageAggregator(app)
+        agg.max_buffers = 2_000
+        agg.max_buffers_per_workspace = 2_000
+        existing = {
+            (
+                'instance-test',
+                f'workspace-{index}',
+                1,
+                'bot',
+                'pipeline',
+                'person',
+                index,
+            ): object()
+            for index in range(1_000)
+        }
+        agg.buffers = NoGlobalIterationDict(existing)
+        agg._buffer_counts_by_scope = {key[:3]: 1 for key in existing}
+        context = execution_context(
+            'workspace-target',
+            pipeline_uuid='test-pipeline',
+        )
+
+        await agg.add_message(**scoped_message_kwargs(context))
+
+        key = aggregation_key(
+            context,
+            pipeline_uuid='test-pipeline',
+        )
+        assert key in agg.buffers
+        assert agg._buffer_counts_by_scope[key[:3]] == 1
+        timer_task = agg.buffers[key].timer_task
+        assert timer_task is not None
+        timer_task.cancel()
+        await asyncio.gather(timer_task, return_exceptions=True)
+        await agg._flush_buffer(key, context)
+        assert key[:3] not in agg._buffer_counts_by_scope
+
     @pytest.mark.asyncio
     async def test_same_launcher_in_two_bots_uses_separate_buffers(self):
         app = make_aggregator_app()

tests/unit_tests/provider/test_mcp_resources.py

@@ -558,27 +558,72 @@ async def test_mcp_resource_cache_is_not_served_to_stale_generation():
 
 
 @pytest.mark.asyncio
-async def test_mcp_idle_lifecycle_stops_without_retry_after_generation_bump():
-    session = _connected_session()
-    session.server_config.update({'mode': 'remote', 'url': 'https://example.com/mcp'})
-    session._FENCE_POLL_INTERVAL = 0
-    session._init_remote_server = AsyncMock()
-    session.refresh = AsyncMock()
-    session._assert_execution_active = AsyncMock(
-        side_effect=[
-            None,
-            None,
-            None,
-            WorkspaceGenerationMismatchError('generation changed while idle'),
-        ]
+async def test_directory_projection_retires_idle_mcp_scope_without_db_poll():
+    loader = MCPLoader(_app())
+    sessions = []
+    for index in range(100):
+        context = ExecutionContext(
+            instance_uuid='instance-a',
+            workspace_uuid=f'workspace-{index}',
+            placement_generation=1,
+        )
+        session = RuntimeMCPSession(
+            f'server-{index}',
+            {'uuid': f'srv-{index}', 'mode': 'remote'},
+            True,
+            loader.ap,
+            context,
+        )
+        session.shutdown = AsyncMock()
+        loader._register_session(context, session.server_name, session)
+        sessions.append(session)
+
+    loader.reconcile_execution_projection('instance-a', {})
+    reconcile_task = loader._projection_reconcile_task
+    assert reconcile_task is not None
+    assert len(loader._pending_projection_retirements) == 100
+
+    # A second projection coalesces into the same worker instead of creating
+    # one timer or task per Workspace.
+    loader.reconcile_execution_projection('instance-a', {})
+    assert loader._projection_reconcile_task is reconcile_task
+
+    await asyncio.wait_for(reconcile_task, timeout=1)
+
+    assert loader.sessions == {}
+    assert loader._scope_generations == {}
+    assert loader._pending_projection_retirements == set()
+    assert sum(session.shutdown.await_count for session in sessions) == 100
+    loader.ap.workspace_service.get_execution_binding.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_directory_projection_keeps_matching_and_unaffected_mcp_scopes():
+    loader = MCPLoader(_app())
+    matching = _connected_session()
+    other_context = ExecutionContext(
+        instance_uuid='instance-a',
+        workspace_uuid='workspace-b',
+        placement_generation=1,
     )
+    unaffected = _connected_session(
+        name='other',
+        uuid='srv-2',
+        execution_context=other_context,
+    )
+    _register_session(loader, matching)
+    _register_session(loader, unaffected)
 
-    await session._lifecycle_loop_with_retry()
+    loader.reconcile_execution_projection(
+        'instance-a',
+        {'workspace-a': 1},
+        affected_workspace_uuids={'workspace-a'},
+    )
+    await asyncio.sleep(0)
 
-    session._init_remote_server.assert_awaited_once_with()
-    assert session.status == MCPSessionStatus.ERROR
-    assert session.error_message == 'Workspace execution binding is stale'
-    assert session._shutdown_event.is_set()
+    assert loader.get_session(TEST_EXECUTION_CONTEXT, 'docs') is matching
+    assert loader.get_session(other_context, 'other') is unaffected
+    assert loader._projection_reconcile_task is None
 
 
 @pytest.mark.asyncio

tests/unit_tests/utils/test_cloud_runtime_soak.py

@@ -397,6 +397,7 @@ def test_evaluate_gate_detects_restart_circuit_and_stuck_launch() -> None:
                 0,
                 **{
                     f'{prefix}.active_launches': 1,
+                    f'{prefix}.gate_waiters': 2,
                     f'{prefix}.half_open_probe_inflight': 1,
                     f'{prefix}.open_remaining_seconds': 60,
                     f'{prefix}.circuit_open_total': 0,
@@ -406,6 +407,7 @@ def test_evaluate_gate_detects_restart_circuit_and_stuck_launch() -> None:
                 60,
                 **{
                     f'{prefix}.active_launches': 1,
+                    f'{prefix}.gate_waiters': 2,
                     f'{prefix}.half_open_probe_inflight': 1,
                     f'{prefix}.open_remaining_seconds': 1,
                     f'{prefix}.circuit_open_total': 1,
@@ -422,10 +424,43 @@ def test_evaluate_gate_detects_restart_circuit_and_stuck_launch() -> None:
 
     assert any('circuit_open_total by 1' in failure for failure in result.failures)
     assert any('active_launches above zero' in failure for failure in result.failures)
+    assert any('gate_waiters above zero' in failure for failure in result.failures)
     assert any('half_open_probe_inflight above zero' in failure for failure in result.failures)
     assert any('open_remaining_seconds above zero' in failure for failure in result.failures)
 
 
+def test_evaluate_gate_detects_stuck_mcp_projection_cleanup() -> None:
+    prefix = 'body.resources.runtimes'
+    state = _state(
+        'endpoint',
+        [
+            _sample(
+                0,
+                **{
+                    f'{prefix}.mcp_projection_retirements': 3,
+                    f'{prefix}.mcp_projection_reconcile_active': 1,
+                },
+            ),
+            _sample(
+                60,
+                **{
+                    f'{prefix}.mcp_projection_retirements': 1,
+                    f'{prefix}.mcp_projection_reconcile_active': 1,
+                },
+            ),
+        ],
+    )
+
+    result = soak.evaluate_gate(
+        [state],
+        analysis_start_seconds=0,
+        thresholds=_thresholds(),
+    )
+
+    assert any('mcp_projection_retirements above zero' in failure for failure in result.failures)
+    assert any('mcp_projection_reconcile_active above zero' in failure for failure in result.failures)
+
+
 def test_evaluate_gate_detects_event_loop_stall_and_sustained_lag() -> None:
     prefix = 'body.resources.event_loop'
     samples = [