fix(cloud): bound runtime restart storms

tests/unit_tests/core/test_load_config.py

@@ -298,6 +298,10 @@ class TestApplyEnvOverridesToConfig:
                     'max_pids': 128,
                     'max_open_files': 256,
                     'max_file_size_mb': 512,
+                    'max_concurrent_restarts': 1,
+                    'restart_failure_threshold': 8,
+                    'restart_failure_window_seconds': 30.0,
+                    'restart_circuit_open_seconds': 60.0,
                 }
             },
             'mcp': {'stdio': {'enabled': True}},
@@ -308,6 +312,10 @@ class TestApplyEnvOverridesToConfig:
             'PLUGIN__WORKER__MAX_PIDS': '64',
             'PLUGIN__WORKER__MAX_OPEN_FILES': '128',
             'PLUGIN__WORKER__MAX_FILE_SIZE_MB': '256',
+            'PLUGIN__WORKER__MAX_CONCURRENT_RESTARTS': '2',
+            'PLUGIN__WORKER__RESTART_FAILURE_THRESHOLD': '12',
+            'PLUGIN__WORKER__RESTART_FAILURE_WINDOW_SECONDS': '45.5',
+            'PLUGIN__WORKER__RESTART_CIRCUIT_OPEN_SECONDS': '90.0',
             'MCP__STDIO__ENABLED': 'false',
         }
 
@@ -320,6 +328,10 @@ class TestApplyEnvOverridesToConfig:
             'max_pids': 64,
             'max_open_files': 128,
             'max_file_size_mb': 256,
+            'max_concurrent_restarts': 2,
+            'restart_failure_threshold': 12,
+            'restart_failure_window_seconds': 45.5,
+            'restart_circuit_open_seconds': 90.0,
         }
         assert result['mcp']['stdio']['enabled'] is False
 

tests/unit_tests/platform/test_http_bot_tenancy.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import asyncio
+import time
 from types import SimpleNamespace
 
 import pytest
@@ -92,3 +93,58 @@ async def test_http_bot_bounds_inbound_listener_tasks(monkeypatch):
     await first
     await asyncio.sleep(0)
     assert adapter.inbound_tasks == set()
+
+
+def test_http_bot_outbound_state_has_a_hard_capacity(monkeypatch):
+    monkeypatch.setattr(http_bot_module, '_OUTBOUND_STATE_MAX', 2)
+    monkeypatch.setattr(http_bot_module, '_OUTBOUND_PRUNE_SCAN_MAX', 2)
+    adapter = _adapter(SimpleNamespace(), None)
+    first = adapter._outbound_state('first')
+    second = adapter._outbound_state('second')
+    first.queue.put_nowait({})
+    second.queue.put_nowait({})
+
+    with pytest.raises(RuntimeError, match='outbound session capacity reached'):
+        adapter._next_sequence('third', is_final=True)
+
+    assert len(adapter.outbound_states) == 2
+    assert adapter._next_sequence('first', is_final=True) == 1
+
+
+def test_http_bot_outbound_state_pruning_is_bounded_and_reclaims_stale(monkeypatch):
+    monkeypatch.setattr(http_bot_module, '_OUTBOUND_STATE_MAX', 2)
+    monkeypatch.setattr(http_bot_module, '_OUTBOUND_PRUNE_SCAN_MAX', 1)
+    monkeypatch.setattr(http_bot_module, '_OUTBOUND_IDLE_SECONDS', 10)
+    adapter = _adapter(SimpleNamespace(), None)
+    stale = adapter._outbound_state('stale')
+    stale.last_active = time.monotonic() - 11
+    adapter._outbound_state('active')
+
+    assert adapter._next_sequence('replacement', is_final=True) == 1
+    assert set(adapter.outbound_states) == {'active', 'replacement'}
+
+
+def test_http_bot_idempotency_cache_has_a_hard_capacity(monkeypatch):
+    monkeypatch.setattr(http_bot_module, '_IDEMPOTENCY_MAX', 2)
+    monkeypatch.setattr(http_bot_module, '_IDEMPOTENCY_PRUNE_SCAN_MAX', 1)
+    adapter = _adapter(SimpleNamespace(), None)
+
+    assert adapter._reserve_idempotency_key('first') == 'accepted'
+    assert adapter._reserve_idempotency_key('second') == 'accepted'
+    assert adapter._reserve_idempotency_key('third') == 'overloaded'
+    assert len(adapter.idempotency_cache) == 2
+    assert adapter._reserve_idempotency_key('first') == 'duplicate'
+
+
+def test_http_bot_idempotency_cache_reclaims_expired_oldest(monkeypatch):
+    monkeypatch.setattr(http_bot_module, '_IDEMPOTENCY_MAX', 2)
+    monkeypatch.setattr(http_bot_module, '_IDEMPOTENCY_PRUNE_SCAN_MAX', 1)
+    monkeypatch.setattr(http_bot_module, '_IDEMPOTENCY_TTL', 10)
+    adapter = _adapter(SimpleNamespace(), None)
+    adapter.idempotency_cache = {
+        'expired': time.monotonic() - 11,
+        'active': time.monotonic(),
+    }
+
+    assert adapter._reserve_idempotency_key('replacement') == 'accepted'
+    assert set(adapter.idempotency_cache) == {'active', 'replacement'}

tests/unit_tests/plugin/test_connector_ping.py

@@ -337,6 +337,10 @@ def test_worker_policy_is_loaded_only_from_instance_configuration():
                         'max_pids': 64,
                         'max_open_files': 128,
                         'max_file_size_mb': 32,
+                        'max_concurrent_restarts': 2,
+                        'restart_failure_threshold': 12,
+                        'restart_failure_window_seconds': 45,
+                        'restart_circuit_open_seconds': 90,
                         'require_hard_limits': True,
                     },
                     # A plugin-controlled value at any other path is ignored.
@@ -354,6 +358,10 @@ def test_worker_policy_is_loaded_only_from_instance_configuration():
     assert policy.max_pids == 64
     assert policy.max_open_files == 128
     assert policy.max_file_size_mb == 32
+    assert policy.max_concurrent_restarts == 2
+    assert policy.restart_failure_threshold == 12
+    assert policy.restart_failure_window_seconds == 45
+    assert policy.restart_circuit_open_seconds == 90
     assert policy.require_hard_limits is True
 
 

tests/unit_tests/utils/test_cloud_runtime_soak.py

@@ -212,7 +212,11 @@ def test_read_endpoint_snapshot_flattens_resource_metrics() -> None:
                     'blocking_executor': {
                         'pending': 0,
                         'global_rejected_total': 2,
-                    }
+                    },
+                    'restart_coordinator': {
+                        'active_launches': 1,
+                        'circuit_open_total': 3,
+                    },
                 },
             }
         )
@@ -226,6 +230,8 @@ def test_read_endpoint_snapshot_flattens_resource_metrics() -> None:
     assert metrics['http.ok'] == 1
     assert metrics['body.resources.blocking_executor.pending'] == 0
     assert metrics['body.resources.blocking_executor.global_rejected_total'] == 2
+    assert metrics['body.resources.restart_coordinator.active_launches'] == 1
+    assert metrics['body.resources.restart_coordinator.circuit_open_total'] == 3
 
 
 def test_read_endpoint_snapshot_fails_closed_on_not_ready() -> None:
@@ -382,6 +388,44 @@ def test_evaluate_gate_detects_executor_rejection_and_stuck_pending() -> None:
     assert any('pending above zero' in failure for failure in result.failures)
 
 
+def test_evaluate_gate_detects_restart_circuit_and_stuck_launch() -> None:
+    prefix = 'body.resources.restart_coordinator'
+    state = _state(
+        'endpoint',
+        [
+            _sample(
+                0,
+                **{
+                    f'{prefix}.active_launches': 1,
+                    f'{prefix}.half_open_probe_inflight': 1,
+                    f'{prefix}.open_remaining_seconds': 60,
+                    f'{prefix}.circuit_open_total': 0,
+                },
+            ),
+            _sample(
+                60,
+                **{
+                    f'{prefix}.active_launches': 1,
+                    f'{prefix}.half_open_probe_inflight': 1,
+                    f'{prefix}.open_remaining_seconds': 1,
+                    f'{prefix}.circuit_open_total': 1,
+                },
+            ),
+        ],
+    )
+
+    result = soak.evaluate_gate(
+        [state],
+        analysis_start_seconds=0,
+        thresholds=_thresholds(),
+    )
+
+    assert any('circuit_open_total by 1' in failure for failure in result.failures)
+    assert any('active_launches above zero' in failure for failure in result.failures)
+    assert any('half_open_probe_inflight above zero' in failure for failure in result.failures)
+    assert any('open_remaining_seconds above zero' in failure for failure in result.failures)
+
+
 def test_evaluate_gate_detects_event_loop_stall_and_sustained_lag() -> None:
     prefix = 'body.resources.event_loop'
     samples = [