fix(runtime): complete reconnect recovery paths

This commit is contained in:
Junyan Qin
2026-07-23 17:28:31 +08:00
parent 9cbbaf617b
commit 06e03af994
14 changed files with 223 additions and 34 deletions
+17 -5
View File
@@ -307,10 +307,10 @@ class RuntimeMCPSession:
await self._box_stdio_runtime.initialize()
return
# Box is configured (ap.box_service exists) but currently unavailable
# (disabled by config or connection failed). Refuse stdio MCP rather
# than silently falling through to host-stdio — the operator asked
# for the sandbox and the failure mode should be visible.
# Box is configured but explicitly disabled. Refuse stdio MCP rather
# than silently falling through to host-stdio — the operator asked for
# the sandbox and the failure mode should be visible. An enabled Box
# that is reconnecting is handled above and waits for availability.
#
# Set ``error_phase = BOX_UNAVAILABLE`` BEFORE raising so the retry
# wrapper can short-circuit (retrying is pointless when Box is
@@ -1823,11 +1823,23 @@ class MCPLoader(loader.ToolLoader):
async def shutdown(self):
"""关闭所有工具"""
self.ap.logger.info('Shutting down all MCP sessions...')
for server_name, session in list(self.sessions.items()):
hosted_tasks = [task for task in self._hosted_mcp_tasks if not task.done()]
for task in hosted_tasks:
task.cancel()
if hosted_tasks:
await asyncio.gather(*hosted_tasks, return_exceptions=True)
self._hosted_mcp_tasks.clear()
async def shutdown_session(server_name: str, session: RuntimeMCPSession) -> None:
try:
await session.shutdown()
self.ap.logger.debug(f'Shutdown MCP session: {server_name}')
except Exception as e:
self.ap.logger.error(f'Error shutting down MCP session {server_name}: {e}\n{traceback.format_exc()}')
await asyncio.gather(
*(shutdown_session(server_name, session) for server_name, session in list(self.sessions.items()))
)
self.sessions.clear()
self.ap.logger.info('All MCP sessions shutdown complete')
@@ -185,11 +185,10 @@ class BoxStdioSessionRuntime:
box_service = getattr(self.ap, 'box_service', None)
if box_service is None:
return False
# When Box is configured but currently unavailable (disabled or
# connection failed), do NOT silently fall through to host-stdio —
# that would bypass the sandbox the operator asked for. The caller
# is expected to refuse the stdio MCP server with a clear error.
return bool(getattr(box_service, 'available', False))
# An enabled Box service remains the required transport while it is
# reconnecting. initialize() waits for availability instead of
# permanently failing the MCP server or falling through to host stdio.
return bool(getattr(box_service, 'enabled', True))
async def initialize(self) -> None:
await self._wait_for_box_runtime()
@@ -488,7 +487,7 @@ class BoxStdioSessionRuntime:
)
warned = True
if asyncio.get_running_loop().time() >= deadline:
self.owner.error_phase = MCPSessionErrorPhase.SESSION_CREATE
self.owner.error_phase = MCPSessionErrorPhase.BOX_UNAVAILABLE
raise Exception(f'Box runtime is not available after {int(timeout_sec)} seconds')
await asyncio.sleep(1)
@@ -57,7 +57,7 @@ class NativeToolLoader(loader.ToolLoader):
return await is_box_backend_available(self.ap)
async def get_tools(self, bound_plugins: list[str] | None = None) -> list[resource_tool.LLMTool]:
if not self._is_sandbox_available():
if not await self._is_sandbox_available():
return []
if self._tools is None:
self._tools = [
@@ -71,7 +71,7 @@ class NativeToolLoader(loader.ToolLoader):
return list(self._tools)
async def has_tool(self, name: str) -> bool:
return name in _ALL_TOOL_NAMES and self._is_sandbox_available()
return name in _ALL_TOOL_NAMES and await self._is_sandbox_available()
async def invoke_tool(self, name: str, parameters: dict, query: pipeline_query.Query):
if name == EXEC_TOOL_NAME:
@@ -652,12 +652,9 @@ else:
if callable(refresh_skill):
refresh_skill(selected_skill.get('name', ''))
def _is_sandbox_available(self) -> bool:
"""Check if sandbox backend is available.
This checks the cached backend availability from initialization,
not just whether the box_service process is running.
"""
async def _is_sandbox_available(self) -> bool:
"""Refresh backend availability so Box reconnects restore tool exposure."""
self._backend_available = await self._check_backend_available()
return bool(self._backend_available)
def _build_exec_tool(self) -> resource_tool.LLMTool:
@@ -49,19 +49,27 @@ class SkillToolLoader(loader.ToolLoader):
return await is_box_backend_available(self.ap)
async def get_tools(self, bound_plugins: list[str] | None = None) -> list[resource_tool.LLMTool]:
if not self._is_available():
if not await self._is_available():
return []
if not self._tools:
self._tools = [
self._build_activate_skill_tool(),
self._build_register_skill_tool(),
]
return list(self._tools)
async def has_tool(self, name: str) -> bool:
return self._is_available() and name in SKILL_TOOL_NAMES
return await self._is_available() and name in SKILL_TOOL_NAMES
def _is_available(self) -> bool:
async def _is_available(self) -> bool:
"""Check if skill tools should be available.
Skill tools require both a skill manager and a sandbox backend.
"""
return self._has_skill_manager() and self._sandbox_available
if not self._has_skill_manager():
return False
self._sandbox_available = await self._check_sandbox_available()
return self._sandbox_available
async def invoke_tool(self, name: str, parameters: dict, query) -> typing.Any:
if name == ACTIVATE_SKILL_TOOL_NAME: