fix(agent-debug): stream execution traces with platform mocks and coverage

tests/unit_tests/agent/test_orchestrator_integration.py

@@ -1445,3 +1445,30 @@ class TestQueryEntryAdapterHostCapabilities:
         assert user_item['attachment_refs'][0]['content'] is None
         assert 'aGVsbG8=' not in str(user_item)
         assert 'Pinned documentation' not in str(user_item)
+
+
+@pytest.mark.asyncio
+async def test_synthetic_event_query_exposes_trusted_workspace_to_tools(clean_agent_state):
+    from langbot.pkg.provider.tools.loaders.skill import get_visible_skills
+    from langbot.pkg.pipeline.pool import get_query_execution_context
+
+    plugin = FakePluginConnector(results=[{'type': 'run.completed', 'data': {'finish_reason': 'stop'}}])
+    app = FakeApplication(plugin, clean_agent_state)
+    orchestrator = AgentRunOrchestrator(app, FakeRegistry(make_descriptor()))
+    query = make_query()
+    plan = orchestrator.query_bridge.build_plan(query)
+    context = get_query_execution_context(query)
+    outputs = [
+        item
+        async for item in orchestrator.run(plan.event, plan.binding, adapter_context={'_execution_context': context})
+    ]
+    assert outputs == []
+    synthetic = plugin.sessions_during_run[0]['execution_query']
+    assert synthetic.workspace_uuid == context.workspace_uuid
+    assert synthetic.instance_uuid == context.instance_uuid
+    assert synthetic.placement_generation == context.placement_generation
+    assert synthetic.query_uuid == context.query_uuid
+    received = []
+    app.skill_mgr.get_skills = lambda scope: received.append(scope) or {}
+    get_visible_skills(app, synthetic)
+    assert received[0].workspace_uuid == context.workspace_uuid

tests/unit_tests/agent/test_platform_tools.py

@@ -10,16 +10,107 @@ from langbot_plugin.api.entities.builtin.agent_runner import (
     SubjectContext,
 )
 from langbot_plugin.api.entities.builtin.platform import message as platform_message
+from langbot_plugin.api.entities.builtin.platform import entities as platform_entities
+from langbot_plugin.api.entities.builtin.platform import events as platform_events
 
 from langbot.pkg.agent.runner.host_models import AgentEventEnvelope
 from langbot.pkg.agent.runner.platform_tools import (
+    PLATFORM_TOOL_DEFINITIONS,
     build_platform_tool_resources,
     execute_platform_tool,
     freeze_platform_context,
     resolve_agent_platform_tool_names,
+    validate_debug_mock_options,
 )
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize('definition', PLATFORM_TOOL_DEFINITIONS, ids=lambda tool: tool.name)
+@pytest.mark.parametrize('failure', [False, True])
+async def test_every_platform_tool_mock_preserves_validation_and_never_calls_bot(definition, failure):
+    pattern = definition.event_patterns[0]
+    event_type = 'group.member_joined' if pattern == '*' else pattern.replace('*', 'received')
+    event = _event(event_type)
+    event.delivery.surface = 'webui'
+    event.delivery.platform_capabilities = {
+        'debug_mock': True,
+        'supported_apis': [definition.api],
+        'mock_options': {'errors': {definition.name: 'E2E denied'}} if failure else {},
+    }
+    resources, capabilities = build_platform_tool_resources(event, [definition.name], ['call'])
+    assert capabilities['unavailable_tools'] == []
+    assert len(resources) == 1
+    parameters = {}
+    for name in definition.parameters.get('required', []):
+        schema = definition.parameters['properties'][name]
+        parameters[name] = schema.get(
+            'enum', [False if schema['type'] == 'boolean' else 30 if schema['type'] == 'integer' else 'fixture-' + name]
+        )[0]
+    ap = SimpleNamespace(
+        platform_mgr=SimpleNamespace(get_bot_by_uuid=AsyncMock(side_effect=AssertionError('real adapter accessed')))
+    )
+    session = {'authorization': {'resources': {'tools': resources}, 'platform_context': freeze_platform_context(event)}}
+    result = await execute_platform_tool(ap, object(), session, definition.name, parameters)
+    assert result['mock'] is True
+    assert result['ok'] is not failure
+    assert result['api'] == definition.api
+    if failure:
+        assert result['error'] == 'E2E denied'
+    else:
+        schemas = {
+            'get_user_info': platform_entities.User,
+            'get_group_info': platform_entities.UserGroup,
+            'get_group_member_info': platform_entities.UserGroupMember,
+            'get_message': platform_events.MessageReceivedEvent,
+            'get_friend_list': platform_entities.User,
+            'get_group_list': platform_entities.UserGroup,
+            'get_group_member_list': platform_entities.UserGroupMember,
+        }
+        if definition.api in schemas:
+            items = result['result'] if isinstance(result['result'], list) else [result['result']]
+            for item in items:
+                schemas[definition.api].model_validate(item)
+    ap.platform_mgr.get_bot_by_uuid.assert_not_awaited()
+    with pytest.raises(ValueError):
+        await execute_platform_tool(ap, object(), session, definition.name, {**parameters, 'forged_target': 'other'})
+
+
+@pytest.mark.parametrize(
+    'options',
+    [
+        None,
+        [],
+        {'unexpected': True},
+        {'errors': []},
+        {'errors': {'exec': 'oops'}},
+        {'errors': {'event_reply': ''}},
+        {'results': {'missing': {}}},
+        {'errors': {'event_reply': 'oops'}, 'results': {'event_reply': {}}},
+        {'unsupported_apis': 'send_message'},
+        {'unsupported_apis': ['unknown']},
+    ],
+)
+def test_invalid_mock_options_are_rejected(options):
+    with pytest.raises(ValueError):
+        validate_debug_mock_options(options)
+
+
+@pytest.mark.asyncio
+async def test_mock_fixture_does_not_mutate_options():
+    event = _event()
+    fixture = {'name': 'Fixture User', 'nested': {'value': 42}}
+    event.delivery.surface = 'webui'
+    event.delivery.platform_capabilities = {
+        'debug_mock': True,
+        'mock_options': {'results': {'event_get_actor': fixture}},
+    }
+    session = {'authorization': {'platform_context': freeze_platform_context(event)}}
+    result = await execute_platform_tool(SimpleNamespace(), object(), session, 'event_get_actor', {})
+    assert result['result'] == fixture
+    result['result']['nested']['value'] = 100
+    assert fixture['nested']['value'] == 42
+
+
 def _event(event_type: str = 'friend.request_received') -> AgentEventEnvelope:
     return AgentEventEnvelope(
         event_id='event-1',
@@ -88,6 +179,40 @@ def test_platform_resources_require_runner_call_permission() -> None:
     assert capabilities['unavailable_tools'] == [{'name': 'event_reply', 'reason': 'runner_call_permission_missing'}]
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ('name', 'params', 'api'),
+    [
+        ('event_reply', {'text': 'Hello'}, 'send_message'),
+        ('event_get_actor', {}, 'get_user_info'),
+        ('event_get_group', {}, 'get_group_info'),
+        ('event_respond_friend_request', {'approve': False}, 'approve_friend_request'),
+        (
+            'platform_send_message',
+            {'target_type': 'group', 'target_id': 'explicit-group', 'text': 'Hi'},
+            'send_message',
+        ),
+    ],
+)
+async def test_debug_mock_executes_without_accessing_a_real_adapter(name, params, api):
+    event = _event()
+    event.delivery.surface = 'webui'
+    event.delivery.platform_capabilities['debug_mock'] = True
+    ap = SimpleNamespace(platform_mgr=SimpleNamespace(get_bot_by_uuid=AsyncMock()))
+    session = {'authorization': {'platform_context': freeze_platform_context(event)}}
+    result = await execute_platform_tool(ap, object(), session, name, params)
+    assert result['mock'] is True and result['ok'] is True
+    assert result['delivery'] == 'simulated'
+    assert result['api'] == api
+    if name == 'event_reply':
+        assert result['parameters'] == {'target_type': 'group', 'target_id': 'group-1', 'text': 'Hello'}
+    if name == 'platform_send_message':
+        assert result['parameters']['target_id'] == 'explicit-group'
+    ap.platform_mgr.get_bot_by_uuid.assert_not_awaited()
+    with pytest.raises(ValueError):
+        await execute_platform_tool(ap, object(), session, name, {**params, 'unexpected': True})
+
+
 def test_agent_platform_tools_are_resolved_for_the_current_event() -> None:
     selected = resolve_agent_platform_tool_names(
         {

tests/unit_tests/api/service/test_agent_service.py

@@ -149,7 +149,8 @@ class TestAgentServiceMetadata:
 
 
 class TestAgentServiceDebug:
-    async def test_debug_agent_runs_configured_runner_with_synthetic_event(self):
+    @pytest.mark.parametrize('streaming', [False, True])
+    async def test_debug_agent_runs_configured_runner_with_synthetic_event(self, streaming):
         app = _make_app()
         agent_config = _agent_row().config
         agent_config['allowed_platform_tools'] = ['platform_get_user_info']
@@ -159,7 +160,16 @@ class TestAgentServiceDebug:
         }
         agent_config['allowed_tools'] = ['exec', 'weather']
 
+        visible_event = {
+            'type': 'tool.call.started',
+            'data': {'tool_name': 'exec', 'parameters': {'command': 'echo hi'}},
+        }
+        observer = AsyncMock() if streaming else None
+
         async def run_agent(event, binding, adapter_context):
+            assert binding.delivery_policy.enable_streaming is streaming
+            await adapter_context['_result_observer']({**visible_event, 'private_context': 'must not leak'})
+            await adapter_context['_result_observer']({'type': 'state.updated', 'data': {'private': True}})
             yield SimpleNamespace(
                 role='assistant',
                 content='debug result',
@@ -193,8 +203,14 @@ class TestAgentServiceDebug:
                 'data': {'member_id': 'user-1'},
                 'conversation_id': 'debug-session',
             },
+            on_result=observer,
         )
 
+        if streaming:
+            observer.assert_awaited_once_with(visible_event)
+            assert result['execution_events'] == []
+        else:
+            assert result['execution_events'] == [visible_event]
         assert result['event_type'] == 'group.member.joined'
         assert result['conversation_id'] == 'debug-session'
         assert result['final_text'] == 'debug result'
@@ -207,6 +223,10 @@ class TestAgentServiceDebug:
         ]
         event, binding = app.agent_run_orchestrator.run.call_args.args
         assert event.workspace_id == WORKSPACE_UUID
+        assert event.delivery.platform_capabilities['debug_mock'] is True
+        assert 'send_message' in event.delivery.platform_capabilities['supported_apis']
+        assert event.delivery.reply_target['target_id'] == 'debug-group'
+        assert binding.delivery_policy.enable_reply is False
         assert event.data == {'member_id': 'user-1'}
         assert binding.agent_id == 'agent-1'
         assert binding.runner_id == 'plugin:test/runner/default'
@@ -245,6 +265,100 @@ class TestAgentServiceDebug:
             )
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    'event_type',
+    [
+        'message.received',
+        'message.edited',
+        'message.deleted',
+        'message.reaction',
+        'group.member_joined',
+        'group.member_left',
+        'group.member_banned',
+        'friend.request_received',
+        'friend.added',
+        'feedback.received',
+        'bot.invited_to_group',
+        'bot.muted',
+        'bot.unmuted',
+        'bot.removed_from_group',
+        'platform.specific',
+        'custom.probe',
+    ],
+)
+async def test_debug_event_matrix_preserves_scope_targets_and_mock_options(event_type):
+    app = _make_app()
+    captured = []
+
+    async def run(event, binding, adapter_context):
+        captured.append((event, binding))
+        if False:
+            yield
+
+    app.agent_run_orchestrator = SimpleNamespace(run=run)
+    service = AgentService(app)
+    service.get_agent = AsyncMock(
+        return_value={'kind': AGENT_KIND_AGENT, 'supported_event_patterns': ['*'], 'config': _agent_row().config}
+    )
+    context = SimpleNamespace(
+        instance_uuid='instance-test',
+        workspace_uuid=WORKSPACE_UUID,
+        placement_generation=1,
+        principal=SimpleNamespace(account_uuid='account-test'),
+        entitlement_revision=0,
+    )
+    data = {
+        'group_id': '群组-42',
+        'member_id': 'user-42',
+        'member_name': '测试用户🙂',
+        'request_id': 'request-42',
+        'nested': {'value': [1, 2]},
+    }
+    await service.debug_agent(
+        context,
+        'agent-1',
+        {
+            'event_type': event_type,
+            'text': 'probe',
+            'data': data,
+            'mock': {'errors': {'event_reply': 'denied'}, 'unsupported_apis': ['delete_message']},
+        },
+    )
+    event, binding = captured[0]
+    assert event.data == data
+    assert event.actor.actor_name == '测试用户🙂'
+    assert event.delivery.reply_target['target_id'] == '群组-42'
+    assert 'delete_message' not in event.delivery.platform_capabilities['supported_apis']
+    assert event.delivery.platform_capabilities['mock_options']['errors'] == {'event_reply': 'denied'}
+    assert event.bot_id is None
+    assert binding.delivery_policy.enable_reply is False
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    'payload',
+    [
+        {'event_type': ''},
+        {'data': []},
+        {'data': None},
+        {'data': False},
+        {'mock': []},
+        {'mock': {'errors': {'event_reply': None}}},
+        {'actor': []},
+    ],
+)
+async def test_debug_rejects_invalid_envelope_before_execution(payload):
+    app = _make_app()
+    app.agent_run_orchestrator = SimpleNamespace(run=Mock(side_effect=AssertionError('invalid request executed')))
+    service = AgentService(app)
+    service.get_agent = AsyncMock(
+        return_value={'kind': AGENT_KIND_AGENT, 'supported_event_patterns': ['*'], 'config': _agent_row().config}
+    )
+    with pytest.raises(ValueError):
+        await service.debug_agent(SimpleNamespace(workspace_uuid=WORKSPACE_UUID), 'agent-1', payload)
+
+
 class TestAgentServiceListAndLookup:
     async def test_get_agents_merges_agents_and_pipelines_without_leaking_config(self):
         app = _make_app()

tests/unit_tests/api/test_agents_controller.py

@@ -169,3 +169,44 @@ async def test_debug_agent_returns_actionable_runner_error():
         'code': 'dify.config_invalid',
         'msg': 'api-key is required',
     }
+
+
+async def test_debug_stream_preserves_events_before_error():
+    import json
+
+    async def debug_agent(context, agent_uuid, payload, *, on_result):
+        await on_result({'type': 'tool.call.started', 'data': {'tool_name': 'exec'}})
+        raise RunnerExecutionError('test/runner', 'partial failure', error_code='runner.timeout')
+
+    client = await _create_test_client(SimpleNamespace(debug_agent=debug_agent))
+    response = await client.post(
+        '/api/v1/agents/agent-1/debug/stream',
+        headers={'Authorization': 'Bearer test-token'},
+        json={'event_type': 'message.received', 'text': 'hello'},
+    )
+    assert response.status_code == 200
+    frames = [json.loads(line) for line in (await response.get_data()).splitlines() if line]
+    assert frames[0]['kind'] == 'result'
+    assert frames[1]['kind'] == 'error'
+    assert frames[1]['code'] == 'runner.timeout'
+
+
+async def test_debug_stream_cancels_execution_when_closed():
+    import asyncio
+    from langbot.pkg.api.http.controller.groups.agent_debug_stream import debug_stream_response
+
+    cancelled = asyncio.Event()
+
+    async def debug_agent(context, agent_uuid, payload, *, on_result):
+        try:
+            await on_result({'type': 'message.delta', 'data': {}})
+            await asyncio.Event().wait()
+        finally:
+            cancelled.set()
+
+    response = debug_stream_response(SimpleNamespace(debug_agent=debug_agent), None, 'agent-1', {})
+    iterator = response.response.__aiter__()
+    first = await anext(iterator)
+    assert 'message.delta' in first
+    await iterator.aclose()
+    await asyncio.wait_for(cancelled.wait(), timeout=1)

tests/unit_tests/provider/test_skill_tools.py

@@ -540,6 +540,16 @@ class TestNativeToolLoaderSkillPaths:
             ap.skill_mgr = _make_skill_manager({'demo': _make_skill_data(name='demo', package_root=tmpdir)})
             loader = NativeToolLoader(ap)
 
+            if not loader._can_interpret_skill_host_paths():
+                # Windows lacks the secure descriptor-relative host file operations.
+                with pytest.raises(ValueError, match='owned by the Box Runtime'):
+                    await loader.invoke_tool(
+                        'read',
+                        {'path': '/workspace/.skills/demo/SKILL.md'},
+                        _make_query(query_id='q1', variables={PIPELINE_BOUND_SKILLS_KEY: ['demo']}),
+                    )
+                return
+
             result = await loader.invoke_tool(
                 'read',
                 {'path': '/workspace/.skills/demo/SKILL.md'},

tests/unit_tests/provider/test_tool_manager_native.py

@@ -246,6 +246,8 @@ async def test_native_tool_loader_rechecks_admission_at_the_final_invoke_boundar
 
 
 def _make_loader_with_workspace(tmpdir: str) -> tuple[NativeToolLoader, Mock]:
+    if not native_loader._SECURE_HOST_FILE_OPS_AVAILABLE:
+        pytest.skip('Host file operations require POSIX descriptor-relative APIs; remote Box is tested separately')
     logger = Mock()
     box_service = SimpleNamespace(
         available=True,
@@ -508,6 +510,7 @@ async def test_path_escape_blocked():
     ],
 )
 @pytest.mark.asyncio
+@pytest.mark.skipif(not native_loader._SECURE_HOST_FILE_OPS_AVAILABLE, reason='Requires POSIX descriptor-relative host APIs')
 async def test_host_workspace_operations_do_not_follow_a_swapped_ancestor(
     monkeypatch,
     tool_name: str,
@@ -794,3 +797,52 @@ async def test_grep_interrupts_catastrophic_regex(monkeypatch):
         )
 
         assert result == {'ok': False, 'error': 'Regex search timed out'}
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    'tool,parameters',
+    [
+        ('read', {'path': '/workspace/probe.txt'}),
+        ('write', {'path': '/workspace/probe.txt', 'content': 'ok'}),
+        ('edit', {'path': '/workspace/probe.txt', 'old_string': 'ok', 'new_string': 'done'}),
+        ('glob', {'path': '/workspace', 'pattern': '*.txt'}),
+        ('grep', {'path': '/workspace', 'pattern': 'ok'}),
+    ],
+)
+@pytest.mark.parametrize('secure_host_apis', [False, True])
+async def test_standalone_box_files_do_not_require_core_host_workspace(monkeypatch, tool, parameters, secure_host_apis):
+    monkeypatch.setattr(native_loader, '_SECURE_HOST_FILE_OPS_AVAILABLE', secure_host_apis)
+    box = SimpleNamespace(
+        available=True,
+        default_workspace=None,
+        shares_filesystem_with_box=False,
+        execute_tool=AsyncMock(),
+        _tenant_workspace=Mock(side_effect=AssertionError('must not resolve Core path')),
+    )
+    loader = NativeToolLoader(SimpleNamespace(box_service=box, logger=Mock()))
+    remote = AsyncMock(return_value={'ok': True, 'sentinel': tool})
+    monkeypatch.setattr(loader, f'_{tool}_workspace_via_box', remote)
+    assert await loader.invoke_tool(tool, parameters, _make_query()) == {'ok': True, 'sentinel': tool}
+    remote.assert_awaited_once()
+    with pytest.raises(ValueError, match='workspace boundary'):
+        await loader.invoke_tool(tool, {**parameters, 'path': '/workspace/../outside'}, _make_query())
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize('include', [None, '*.txt', 'quote"\n*.md'])
+async def test_box_grep_script_serializes_optional_include_as_python(monkeypatch, include):
+    import ast
+
+    loader = NativeToolLoader(SimpleNamespace(logger=Mock()))
+    captured = AsyncMock(return_value={'ok': True})
+    monkeypatch.setattr(loader, '_run_workspace_file_script', captured)
+    await loader._grep_workspace_via_box('/workspace', 'probe', include, _make_query())
+    script = captured.call_args.args[0]
+    tree = ast.parse(script)
+    values = {}
+    for statement in tree.body:
+        if isinstance(statement, ast.Assign) and isinstance(statement.targets[0], ast.Name):
+            values[statement.targets[0].id] = ast.literal_eval(statement.value)
+    assert values['include'] == include
+    assert values['path'] == '/workspace'