fix(cloud): tolerate slow plugin runtime reconciliation (#2405)

Co-authored-by: Chan <dadachann@users.noreply.github.com>

src/langbot/pkg/core/stages/load_config.py

@@ -41,6 +41,7 @@ _RUNTIME_POLICY_DEFAULTS = {
         }
     },
     'plugin': {
+        'connect_timeout_seconds': 180.0,
         'worker': {
             'max_cpus': 1.0,
             'max_memory_mb': 512,
@@ -56,7 +57,7 @@ _RUNTIME_POLICY_DEFAULTS = {
             'restart_failure_window_seconds': 30.0,
             'restart_circuit_open_seconds': 60.0,
             'require_hard_limits': False,
-        }
+        },
     },
     'mcp': {'stdio': {'enabled': True}},
     'monitoring': {

src/langbot/pkg/plugin/connector.py

@@ -6,6 +6,7 @@ import contextlib
 import contextvars
 import hashlib
 import json
+import math
 import time
 import uuid
 from typing import Any
@@ -76,7 +77,7 @@ _GITHUB_ASSET_HOSTS = frozenset(
     }
 )
 _HTTP_REDIRECT_STATUSES = frozenset({301, 302, 303, 307, 308})
-_CONNECT_TIMEOUT_SEC = 30.0
+_DEFAULT_CONNECT_TIMEOUT_SECONDS = 180.0
 _HEARTBEAT_INTERVAL_SEC = 20.0
 _HEARTBEAT_FAILURE_THRESHOLD = 3
 _RECONNECT_MAX_DELAY_SEC = 60.0
@@ -206,6 +207,17 @@ class PluginRuntimeConnector(ManagedRuntimeConnector):
 
         return f'{constants.instance_id}:plugin-runtime'
 
+    @staticmethod
+    def _runtime_connect_timeout(plugin_config: dict[str, Any]) -> float:
+        value = plugin_config.get('connect_timeout_seconds', _DEFAULT_CONNECT_TIMEOUT_SECONDS)
+        if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value <= 0:
+            raise ValueError('plugin.connect_timeout_seconds must be a positive number')
+        return float(value)
+
+    @staticmethod
+    def _runtime_connect_timeout_error(timeout_seconds: float) -> str:
+        return f'Plugin runtime did not become ready within {timeout_seconds:g} seconds'
+
     def _runtime_handler(self) -> handler.RuntimeConnectionHandler:
         runtime_handler = getattr(self, 'handler', None)
         if runtime_handler is None:
@@ -701,10 +713,13 @@ class PluginRuntimeConnector(ManagedRuntimeConnector):
         """
 
         runtime_handler = self._runtime_handler()
+        started_at = time.monotonic()
         async with self._state_lock:
             all_states: dict[str, PluginInstallationDesiredState] = {}
             workspace_installations: dict[str, set[str]] = {}
+            workspace_count = 0
             for context in contexts:
+                workspace_count += 1
                 execution_context = await self._validate_execution_context(context)
                 states = await self._load_workspace_desired_states(execution_context)
                 installation_ids = {state.binding.installation_uuid for state in states}
@@ -724,6 +739,13 @@ class PluginRuntimeConnector(ManagedRuntimeConnector):
                     runtime_handler.unregister_installation_binding(previous.binding)
             self._known_desired_states = all_states
             self._workspace_installations = workspace_installations
+            self.ap.logger.info(
+                'Shared plugin runtime reconcile completed: workspaces=%d desired_installations=%d '
+                'elapsed_seconds=%.3f',
+                workspace_count,
+                len(all_states),
+                time.monotonic() - started_at,
+            )
             return result
 
     async def _validate_execution_context(self, context: TenantContext) -> ExecutionContext:
@@ -959,11 +981,14 @@ class PluginRuntimeConnector(ManagedRuntimeConnector):
                 task_coro = self.ctrl.run(new_connection_callback)
 
             self._transport_task = asyncio.create_task(task_coro)
+            connect_timeout_seconds = self._runtime_connect_timeout(self.ap.instance_config.data.get('plugin', {}))
             try:
-                await asyncio.wait_for(self._connected.wait(), timeout=_CONNECT_TIMEOUT_SEC)
+                await asyncio.wait_for(self._connected.wait(), timeout=connect_timeout_seconds)
             except asyncio.TimeoutError as exc:
                 await self._stop_transport()
-                raise PluginRuntimeNotConnectedError('Plugin runtime did not become ready within 30 seconds') from exc
+                raise PluginRuntimeNotConnectedError(
+                    self._runtime_connect_timeout_error(connect_timeout_seconds)
+                ) from exc
             if connect_errors:
                 await self._stop_transport()
                 raise PluginRuntimeNotConnectedError(f'Plugin runtime connection failed: {connect_errors[-1]}')