fix(agent-debug): stream execution traces with platform mocks and coverage

src/langbot/pkg/agent/runner/orchestrator.py

@@ -95,9 +95,7 @@ class AgentRunOrchestrator:
         if event_workspace_id and event_workspace_id != execution_context.workspace_uuid:
             raise ValueError('Agent event Workspace does not match its trusted ExecutionContext')
         if not event_workspace_id:
-            event = event.model_copy(
-                update={'workspace_id': execution_context.workspace_uuid}
-            )
+            event = event.model_copy(update={'workspace_id': execution_context.workspace_uuid})
         descriptor = await self.registry.get(
             execution_context,
             runner_id,
@@ -106,6 +104,9 @@ class AgentRunOrchestrator:
 
         if execution_query is None:
             execution_query = build_execution_query(event, [])
+            # Synthetic events must expose the same trusted scope as pipeline queries.
+            for field_name in ('instance_uuid', 'workspace_uuid', 'placement_generation', 'query_uuid'):
+                object.__setattr__(execution_query, field_name, getattr(execution_context, field_name))
             project_mcp_resource_config(execution_query, binding.runner_config)
         object.__setattr__(execution_query, '_execution_context', execution_context)
 
@@ -268,6 +269,11 @@ class AgentRunOrchestrator:
                     sequence=sequence_int,
                 )
 
+                # Trusted Host observers receive validated events before message-only normalization.
+                result_observer = (adapter_context or {}).get('_result_observer')
+                if result_observer is not None:
+                    await result_observer(result_dict)
+
                 if result_type == 'state.updated':
                     await self.journal.handle_state_updated_event(
                         result_dict,

src/langbot/pkg/agent/runner/platform_tools.py

@@ -8,6 +8,8 @@ import typing
 from dataclasses import dataclass
 
 import langbot_plugin.api.entities.builtin.platform.message as platform_message
+from langbot_plugin.api.entities.builtin.platform import entities as platform_entities
+from langbot_plugin.api.entities.builtin.platform import events as platform_events
 
 from .host_models import AgentEventEnvelope
 
@@ -388,6 +390,25 @@ def platform_tool_catalog() -> list[dict[str, typing.Any]]:
     ]
 
 
+def validate_debug_mock_options(value: typing.Any) -> dict[str, typing.Any]:
+    if not isinstance(value, dict) or set(value) - {'errors', 'results', 'unsupported_apis'}:
+        raise ValueError('Mock options must contain only errors, results and unsupported_apis')
+    for field in ('errors', 'results'):
+        entries = value.get(field, {})
+        if not isinstance(entries, dict) or set(entries) - set(PLATFORM_TOOLS_BY_NAME):
+            raise ValueError(f'Mock {field} must map platform tool names to outcomes')
+    if any(not isinstance(error, str) or not error.strip() for error in value.get('errors', {}).values()):
+        raise ValueError('Mock errors must be non-empty strings')
+    if set(value.get('errors', {})) & set(value.get('results', {})):
+        raise ValueError('A mock tool cannot have both an error and a result')
+    unsupported = value.get('unsupported_apis', [])
+    if not isinstance(unsupported, list) or any(
+        not isinstance(api, str) or api not in {tool.api for tool in PLATFORM_TOOL_DEFINITIONS} for api in unsupported
+    ):
+        raise ValueError('Mock unsupported_apis must be an array of platform API names')
+    return copy.deepcopy(value)
+
+
 def _event_matches(event_type: str, patterns: tuple[str, ...]) -> bool:
     return any(fnmatch.fnmatchcase(event_type, pattern) for pattern in patterns)
 
@@ -587,6 +608,14 @@ async def execute_platform_tool(
     if definition is None:
         raise ValueError(f'Unknown platform tool: {tool_name}')
     authorization = session.get('authorization', {})
+    context = authorization.get('platform_context') or {}
+    delivery = context.get('delivery') or {}
+    normalized = _normalize_platform_params(definition, parameters)
+    if definition.scope == 'event':
+        normalized = _event_params(definition, context, normalized)
+    # This flag is frozen by the Host from the synthetic debug envelope, not tool arguments.
+    if delivery.get('surface') == 'webui' and (delivery.get('platform_capabilities') or {}).get('debug_mock') is True:
+        return _execute_mock_platform_tool(definition, context, normalized)
     bot_id = authorization.get('bot_id')
     if not bot_id:
         raise ValueError('This run is not associated with a platform bot')
@@ -598,9 +627,6 @@ async def execute_platform_tool(
     api_func = getattr(bot.adapter, definition.api, None)
     if not callable(api_func):
         raise ValueError(f'Platform API {definition.api} is declared but not implemented')
-    normalized = _normalize_platform_params(definition, parameters)
-    if definition.scope == 'event':
-        normalized = _event_params(definition, authorization.get('platform_context') or {}, normalized)
     if definition.api == 'send_message':
         normalized = {
             'target_type': _require_string(normalized, 'target_type'),
@@ -612,6 +638,66 @@ async def execute_platform_tool(
     return await api_func(**normalized)
 
 
+def _execute_mock_platform_tool(
+    definition: PlatformToolDefinition, context: dict[str, typing.Any], parameters: dict[str, typing.Any]
+) -> dict[str, typing.Any]:
+    """Simulate the adapter boundary while preserving real model calls and validated targets."""
+    data = context.get('data') or {}
+    actor = context.get('actor') or {}
+    options = ((context.get('delivery') or {}).get('platform_capabilities') or {}).get('mock_options') or {}
+    result: typing.Any = None
+    user_id = parameters.get('user_id') or actor.get('actor_id') or 'debug-user'
+    user = platform_entities.User(
+        id=user_id,
+        nickname=(actor.get('actor_name') or 'Debug User')
+        if user_id == actor.get('actor_id')
+        else f'Mock user {user_id}',
+    )
+    group_id = parameters.get('group_id') or data.get('group_id') or 'debug-group'
+    group = platform_entities.UserGroup(id=group_id, name=data.get('group_name') or 'Mock group')
+    member = platform_entities.UserGroupMember(user=user, group_id=group_id)
+    if definition.api == 'send_message':
+        for field in ('target_type', 'target_id', 'text'):
+            _require_string(parameters, field)
+        result = {'message_id': 'mock-message'}
+    elif definition.api == 'get_user_info':
+        result = user.model_dump(mode='json')
+    elif definition.api == 'get_group_member_info':
+        result = member.model_dump(mode='json')
+    elif definition.api == 'get_group_info':
+        result = group.model_dump(mode='json')
+    elif definition.api == 'get_group_list':
+        result = [group.model_dump(mode='json')]
+    elif definition.api == 'get_group_member_list':
+        result = [member.model_dump(mode='json')]
+    elif definition.api == 'get_friend_list':
+        result = [user.model_dump(mode='json')]
+    elif definition.api == 'get_message':
+        result = platform_events.MessageReceivedEvent(
+            message_id=parameters['message_id'],
+            chat_id=parameters['chat_id'],
+            chat_type='group' if parameters['chat_type'] == 'group' else 'private',
+            sender=user,
+            message_chain=platform_message.MessageChain(
+                [platform_message.Plain(text=str(data.get('text') or 'Mock message'))]
+            ),
+        ).model_dump(mode='json')
+    error = options.get('errors', {}).get(definition.name)
+    if definition.name in options.get('results', {}):
+        result = copy.deepcopy(options['results'][definition.name])
+    return {
+        'ok': error is None,
+        'mock': True,
+        'delivery': 'simulated',
+        'tool': definition.name,
+        'api': definition.api,
+        'parameters': parameters,
+        'result': None if error else result,
+        **({'error': error} if error else {}),
+        'notice': 'Simulated platform operation. No real platform API was called.',
+    }
+
+
 __all__ = [
     'PLATFORM_TOOL_DEFINITIONS',
     'build_platform_tool_resources',

src/langbot/pkg/api/http/controller/groups/agent_debug_stream.py

@@ -0,0 +1,68 @@
+"""Bounded, cancellable NDJSON transport for Agent debug execution."""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import json
+
+import quart
+
+from .....agent.runner.errors import (
+    AgentRunnerError,
+    RunnerExecutionError,
+    RunnerNotAuthorizedError,
+    RunnerNotFoundError,
+    RunnerProtocolError,
+)
+
+
+def debug_stream_response(service, context, agent_uuid: str, payload: dict) -> quart.Response:
+    async def stream():
+        queue: asyncio.Queue[dict] = asyncio.Queue(maxsize=32)
+
+        async def on_result(result: dict) -> None:
+            await queue.put({'kind': 'result', 'data': result})
+
+        async def execute() -> None:
+            try:
+                result = await service.debug_agent(context, agent_uuid, payload, on_result=on_result)
+                await queue.put({'kind': 'completed', 'data': result})
+            except Exception as exc:
+                if isinstance(exc, RunnerExecutionError):
+                    code, message = exc.error_code or 'runner_execution_failed', exc.message
+                elif isinstance(exc, RunnerNotFoundError):
+                    code, message = 'runner_not_found', 'The configured Agent runner is unavailable'
+                elif isinstance(exc, RunnerNotAuthorizedError):
+                    code, message = 'runner_not_authorized', 'The configured Agent runner is not authorized'
+                elif isinstance(exc, RunnerProtocolError):
+                    code, message = 'runner_protocol_error', 'The Agent runner returned an invalid response'
+                elif isinstance(exc, ValueError):
+                    code, message = 'invalid_request', str(exc)
+                elif isinstance(exc, AgentRunnerError):
+                    code, message = 'runner_error', 'The Agent runner could not complete this test'
+                else:
+                    code, message = 'runner_error', 'The Agent debug execution failed'
+                await queue.put({'kind': 'error', 'code': code, 'msg': message})
+
+        task = asyncio.create_task(execute())
+        try:
+            while True:
+                try:
+                    frame = await asyncio.wait_for(queue.get(), timeout=15)
+                except TimeoutError:
+                    yield '\n'
+                    continue
+                yield json.dumps(frame, ensure_ascii=False) + '\n'
+                if frame['kind'] in {'completed', 'error'}:
+                    break
+        finally:
+            task.cancel()
+            with contextlib.suppress(asyncio.CancelledError):
+                await task
+
+    response = quart.Response(stream(), content_type='application/x-ndjson; charset=utf-8')
+    response.timeout = None
+    response.headers['Cache-Control'] = 'no-store'
+    response.headers['X-Accel-Buffering'] = 'no'
+    return response

src/langbot/pkg/api/http/controller/groups/agents.py

@@ -12,11 +12,24 @@ from .....agent.runner.errors import (
 from ...authz import Permission, require_permission
 from ...context import RequestContext
 from .. import group
+from .agent_debug_stream import debug_stream_response
 
 
 @group.group_class('agents', '/api/v1/agents')
 class AgentsRouterGroup(group.RouterGroup):
     async def initialize(self) -> None:
+        @self.route(
+            '/<agent_uuid>/debug/stream',
+            methods=['POST'],
+            auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
+            permission=Permission.RUNTIME_OPERATE,
+        )
+        async def stream_debug(agent_uuid: str, request_context: RequestContext):
+            payload = await quart.request.get_json()
+            if not isinstance(payload, dict):
+                return self.http_status(400, -1, 'Debug payload must be an object')
+            return debug_stream_response(self.ap.agent_service, request_context, agent_uuid, payload)
+
         @self.route(
             '',
             methods=['GET', 'POST'],

src/langbot/pkg/api/http/service/agent.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import datetime
+import copy
 import fnmatch
 import time
 import uuid
@@ -26,8 +27,10 @@ from ....agent.runner.host_models import (
 )
 from ....agent.runner.resource_policy import ResourcePolicyProjector
 from ....agent.runner.platform_tools import (
+    PLATFORM_TOOL_DEFINITIONS,
     platform_tool_catalog,
     resolve_agent_platform_tool_names,
+    validate_debug_mock_options,
 )
 from ....entity.persistence import agent as persistence_agent
 from ....workspace.errors import WorkspaceNotFoundError
@@ -118,6 +121,8 @@ class AgentService:
         context: RequestContext,
         agent_uuid: str,
         payload: dict[str, typing.Any],
+        *,
+        on_result: typing.Callable[[dict[str, typing.Any]], typing.Awaitable[None]] | None = None,
     ) -> dict[str, typing.Any]:
         """Execute one synthetic event against a configured Agent.
 
@@ -129,7 +134,7 @@ class AgentService:
         if agent is None or agent.get('kind') != AGENT_KIND_AGENT:
             raise ValueError('Agent not found')
 
-        event_type = str(payload.get('event_type') or 'message.received').strip()
+        event_type = str(payload.get('event_type', 'message.received')).strip()
         if not event_type or len(event_type) > 128:
             raise ValueError('Invalid event_type')
         if not self._supports_event_type(
@@ -141,9 +146,10 @@ class AgentService:
         text = str(payload.get('text') or '').strip()
         if len(text) > 20_000:
             raise ValueError('Debug input is too long')
-        event_data = payload.get('data') or {}
+        event_data = payload.get('data', {})
         if not isinstance(event_data, dict):
             raise ValueError('Debug event data must be an object')
+        mock_options = validate_debug_mock_options(payload.get('mock', {}))
 
         config = agent.get('config')
         if not isinstance(config, dict):
@@ -156,20 +162,39 @@ class AgentService:
         if not conversation_id or len(conversation_id) > 256:
             raise ValueError('Invalid debug conversation_id')
 
-        actor_payload = payload.get('actor') or {
-            'actor_type': 'user',
-            'actor_id': 'debug-user',
-            'actor_name': 'Debug User',
-        }
-        subject_payload = payload.get('subject') or {
-            'subject_type': 'message' if event_type.startswith('message.') else event_type.split('.', 1)[0],
-            'subject_id': 'debug-subject',
-            'data': event_data,
-        }
+        actor_payload = payload.get(
+            'actor',
+            {
+                'actor_type': 'user',
+                'actor_id': str(
+                    event_data.get('member_id')
+                    or event_data.get('requester_id')
+                    or event_data.get('user_id')
+                    or 'debug-user'
+                ),
+                'actor_name': str(
+                    event_data.get('member_name')
+                    or event_data.get('requester_name')
+                    or event_data.get('user_name')
+                    or 'Debug User'
+                ),
+            },
+        )
+        subject_payload = payload.get(
+            'subject',
+            {
+                'subject_type': 'message' if event_type.startswith('message.') else event_type.split('.', 1)[0],
+                'subject_id': 'debug-subject',
+                'data': event_data,
+            },
+        )
         if not isinstance(actor_payload, dict) or not isinstance(subject_payload, dict):
             raise ValueError('Debug actor and subject must be objects')
 
         event_id = f'debug:{agent_uuid}:{uuid.uuid4()}'
+        is_group = event_type.startswith(('group.', 'bot.')) or bool(event_data.get('group_id'))
+        actor = ActorContext.model_validate(actor_payload)
+        target_id = str(event_data.get('group_id') or 'debug-group') if is_group else actor.actor_id
         event = AgentEventEnvelope(
             event_id=event_id,
             event_type=event_type,
@@ -178,7 +203,7 @@ class AgentService:
             source_event_type=event_type,
             workspace_id=context.workspace_uuid,
             conversation_id=conversation_id,
-            actor=ActorContext.model_validate(actor_payload),
+            actor=actor,
             subject=SubjectContext.model_validate(subject_payload),
             input=AgentInput.model_validate(
                 {
@@ -191,13 +216,27 @@ class AgentService:
             ),
             delivery=DeliveryContext(
                 surface='webui',
-                reply_target=None,
-                supports_streaming=False,
+                reply_target={
+                    'target_type': 'group' if is_group else 'person',
+                    'target_id': target_id,
+                    **({'group_id': target_id} if is_group else {}),
+                    **(
+                        {'message_id': str(event_data.get('message_id') or 'debug-message')}
+                        if event_type.startswith('message.')
+                        else {}
+                    ),
+                },
+                supports_streaming=on_result is not None,
                 supports_edit=False,
                 supports_reaction=False,
                 platform_capabilities={
                     'event_type': event_type,
                     'debug': True,
+                    'debug_mock': True,
+                    'supported_apis': sorted(
+                        {tool.api for tool in PLATFORM_TOOL_DEFINITIONS} - set(mock_options.get('unsupported_apis', []))
+                    ),
+                    'mock_options': mock_options,
                 },
             ),
             raw_ref=RawEventRef(ref_id=event_id, storage_key=None),
@@ -219,7 +258,7 @@ class AgentService:
                 state_scopes=['conversation', 'actor', 'subject', 'runner'],
             ),
             delivery_policy=DeliveryPolicy(
-                enable_streaming=False,
+                enable_streaming=on_result is not None,
                 enable_reply=False,
                 enable_interactions=False,
             ),
@@ -233,11 +272,35 @@ class AgentService:
         )
 
         output_items: list[dict[str, typing.Any]] = []
+        execution_events: list[dict[str, typing.Any]] = []
+
+        async def observe_result(result: dict[str, typing.Any]) -> None:
+            if result.get('type') not in {
+                'message.delta',
+                'message.completed',
+                'tool.call.started',
+                'tool.call.completed',
+                'run.completed',
+                'run.failed',
+            }:
+                return
+            visible_result = copy.deepcopy(
+                {
+                    key: result[key]
+                    for key in ('type', 'data', 'sequence', 'timestamp', 'run_id', 'usage')
+                    if key in result
+                }
+            )
+            if on_result is not None:
+                await on_result(visible_result)
+            elif len(execution_events) < 1000:
+                execution_events.append(visible_result)
+
         final_text = ''
         async for output in self.ap.agent_run_orchestrator.run(
             event,
             binding,
-            adapter_context={'_execution_context': execution_context},
+            adapter_context={'_execution_context': execution_context, '_result_observer': observe_result},
         ):
             output_text = self._provider_output_to_text(output)
             if output_text:
@@ -256,6 +319,7 @@ class AgentService:
             'conversation_id': conversation_id,
             'final_text': final_text,
             'outputs': output_items,
+            'execution_events': execution_events,
         }
 
     @staticmethod

src/langbot/pkg/api/mcp/server.py

@@ -208,6 +208,19 @@ class LangBotMCPServer:
             return _dump({'ok': True})
 
         # ----- Models -------------------------------------------------- #
+        @mcp.tool(
+            description=(
+                'Run a synthetic event against an Agent processor without platform delivery. '
+                'Returns final text and execution_events containing reported messages/thinking and tool calls. '
+                'Platform tools use mock adapters; other tools execute normally. '
+                'Requires runtime.operate; payload accepts event_type, text, data, conversation_id, actor, subject and '
+                'mock (errors/results keyed by platform tool name; unsupported_apis lists unavailable platform APIs).'
+            )
+        )
+        async def debug_agent(processor_uuid: str, payload: dict) -> str:
+            context = _authorized(Permission.RUNTIME_OPERATE)
+            return _dump(await ap.agent_service.debug_agent(context, processor_uuid, payload))
+
         @mcp.tool(description='List all configured LLM models. Secrets are redacted.')
         async def list_llm_models() -> str:
             context = _authorized(Permission.RESOURCE_VIEW)

src/langbot/pkg/provider/tools/loaders/native.py

@@ -369,7 +369,7 @@ class NativeToolLoader(loader.ToolLoader):
         *,
         include_visible: bool,
         include_activated: bool,
-    ) -> _HostLocation:
+    ) -> _HostLocation | None:
         selected_skill, rewritten_path = skill_loader.resolve_virtual_skill_path(
             self.ap,
             query,
@@ -378,6 +378,10 @@ class NativeToolLoader(loader.ToolLoader):
             include_activated=include_activated,
         )
 
+        # Validate the virtual path before choosing remote or local file operations.
+        relative_parts = _relative_workspace_parts(rewritten_path)
+        if self._should_use_box_workspace_files(selected_skill):
+            return None
         box_service = self.ap.box_service
         if selected_skill is not None:
             if not self._can_interpret_skill_host_paths():
@@ -395,7 +399,7 @@ class NativeToolLoader(loader.ToolLoader):
 
         return _HostLocation(
             root=str(host_root),
-            relative_parts=_relative_workspace_parts(rewritten_path),
+            relative_parts=relative_parts,
             selected_skill=selected_skill,
             workspace_anchor=str(workspace_anchor) if workspace_anchor else None,
         )
@@ -440,7 +444,11 @@ class NativeToolLoader(loader.ToolLoader):
             # host-path fallback.
             return True
         default_workspace = getattr(box_service, 'default_workspace', None)
-        return bool(default_workspace and not os.path.isdir(os.path.realpath(default_workspace)))
+        return (
+            not default_workspace
+            or getattr(box_service, 'shares_filesystem_with_box', True) is False
+            or not os.path.isdir(os.path.realpath(default_workspace))
+        )
 
     def _read_host_location(self, location: _HostLocation, parameters: dict) -> dict:
         with _open_host_root(location, create=False) as root_fd:
@@ -1004,7 +1012,7 @@ import json, os, re, signal, time
 from pathlib import Path
 path = {json.dumps(path)}
 pattern = {json.dumps(pattern)}
-include = {json.dumps(include)}
+include = {include!r}
 skip_dirs = {json.dumps(sorted(_SKIP_DIRS))}
 def regex_timeout(_signum, _frame):
     raise TimeoutError
@@ -1160,7 +1168,7 @@ else:
             include_visible=True,
             include_activated=True,
         )
-        if self._should_use_box_workspace_files(host_location.selected_skill):
+        if host_location is None:
             return await self._read_workspace_via_box(path, parameters, query)
         try:
             return await asyncio.to_thread(self._read_host_location, host_location, parameters)
@@ -1193,7 +1201,7 @@ else:
             include_visible=False,
             include_activated=True,
         )
-        if self._should_use_box_workspace_files(host_location.selected_skill):
+        if host_location is None:
             return await self._write_workspace_via_box(path, content, parameters, query)
         try:
             await run_blocking_atomic(self._write_host_location, host_location, content, parameters)
@@ -1253,7 +1261,7 @@ else:
             include_visible=False,
             include_activated=True,
         )
-        if self._should_use_box_workspace_files(host_location.selected_skill):
+        if host_location is None:
             return await self._edit_workspace_via_box(path, old_string, new_string, query)
         try:
             changed, error = await run_blocking_atomic(
@@ -1548,7 +1556,7 @@ else:
             include_visible=True,
             include_activated=True,
         )
-        if self._should_use_box_workspace_files(host_location.selected_skill):
+        if host_location is None:
             return await self._glob_workspace_via_box(path, pattern, query)
         try:
             return await asyncio.to_thread(self._glob_host_location, host_location, pattern, path)
@@ -1570,7 +1578,7 @@ else:
             include_visible=True,
             include_activated=True,
         )
-        if self._should_use_box_workspace_files(host_location.selected_skill):
+        if host_location is None:
             return await self._grep_workspace_via_box(path, pattern, include, query)
         try:
             return await asyncio.to_thread(