mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-09-16 14:57:15 +00:00
feat(runner): unify plugin execution across agents and event processors
This commit is contained in:
+34
-39
@@ -28,7 +28,7 @@ from tests.e2e.utils.process_manager import find_project_root
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
|
||||
LOCAL_AGENT_RUNNER_ID = 'plugin:langbot-team/LocalAgent/default'
|
||||
LOCAL_RUNNER_ID = 'plugin:langbot-team/LocalAgent/default'
|
||||
FAKE_PROVIDER_UUID = 'e2e-fake-provider'
|
||||
FAKE_MODEL_UUID = 'e2e-fake-local-agent-model'
|
||||
E2E_TOOL_NAME = 'e2e_lookup'
|
||||
@@ -112,11 +112,11 @@ def _event(
|
||||
text: str,
|
||||
thread_id: str = 'e2e-local-agent-thread',
|
||||
):
|
||||
"""Build an AgentRunner event envelope for Local Agent E2E probes."""
|
||||
"""Build an Runner event envelope for Local Agent E2E probes."""
|
||||
from langbot.pkg.agent.runner.host_models import AgentEventEnvelope
|
||||
from langbot_plugin.api.entities.builtin.agent_runner.delivery import DeliveryContext
|
||||
from langbot_plugin.api.entities.builtin.agent_runner.event import ActorContext, SubjectContext
|
||||
from langbot_plugin.api.entities.builtin.agent_runner.input import AgentInput
|
||||
from langbot_plugin.api.entities.builtin.runner.delivery import DeliveryContext
|
||||
from langbot_plugin.api.entities.builtin.runner.event import ActorContext, SubjectContext
|
||||
from langbot_plugin.api.entities.builtin.runner.input import AgentInput
|
||||
|
||||
return AgentEventEnvelope(
|
||||
event_id=event_id,
|
||||
@@ -163,7 +163,7 @@ def _binding(
|
||||
return AgentBinding(
|
||||
binding_id=binding_id,
|
||||
scope=BindingScope(scope_type='global'),
|
||||
runner_id=LOCAL_AGENT_RUNNER_ID,
|
||||
runner_id=LOCAL_RUNNER_ID,
|
||||
runner_config=config,
|
||||
resource_policy=ResourcePolicy(
|
||||
allowed_model_uuids=[FAKE_MODEL_UUID],
|
||||
@@ -205,7 +205,7 @@ class _FakeToolManager:
|
||||
del query
|
||||
self.calls.append({'name': name, 'parameters': dict(parameters)})
|
||||
return {
|
||||
'value': f"tool-result:{parameters.get('query')}",
|
||||
'value': f'tool-result:{parameters.get("query")}',
|
||||
'source': 'fake-tool-manager',
|
||||
}
|
||||
|
||||
@@ -402,7 +402,7 @@ async def _inject_fake_llm_model(ap) -> Any:
|
||||
return fake_requester
|
||||
|
||||
|
||||
async def _run_agent(ap, event, binding) -> list[Any]:
|
||||
async def _run_runner(ap, event, binding) -> list[Any]:
|
||||
"""Execute through the trusted Workspace context used by the real Host."""
|
||||
execution_context = await ap.plugin_connector._current_execution_context()
|
||||
return [
|
||||
@@ -463,20 +463,20 @@ async def _boot_local_agent_app(tmpdir: Path):
|
||||
)
|
||||
|
||||
execution_context = await ap.plugin_connector._current_execution_context()
|
||||
runners = await ap.agent_runner_registry.list_runners(execution_context, use_cache=False)
|
||||
if not any(runner.id == LOCAL_AGENT_RUNNER_ID for runner in runners):
|
||||
runners = await ap.runner_registry.list_runners(execution_context, use_cache=False)
|
||||
if not any(runner.id == LOCAL_RUNNER_ID for runner in runners):
|
||||
await ap.plugin_connector.install_plugin(
|
||||
PluginInstallSource.LOCAL,
|
||||
{'plugin_file': (tmpdir / 'langbot-local-agent.zip').read_bytes()},
|
||||
)
|
||||
|
||||
for _ in range(60):
|
||||
runners = await ap.agent_runner_registry.list_runners(execution_context, use_cache=False)
|
||||
if any(runner.id == LOCAL_AGENT_RUNNER_ID for runner in runners):
|
||||
runners = await ap.runner_registry.list_runners(execution_context, use_cache=False)
|
||||
if any(runner.id == LOCAL_RUNNER_ID for runner in runners):
|
||||
break
|
||||
await asyncio.sleep(1)
|
||||
else:
|
||||
raise AssertionError(f'{LOCAL_AGENT_RUNNER_ID} was not discovered after installation')
|
||||
raise AssertionError(f'{LOCAL_RUNNER_ID} was not discovered after installation')
|
||||
|
||||
return ap, run_task
|
||||
|
||||
@@ -521,7 +521,7 @@ def _run_local_agent_probe(tmpdir: Path, probe):
|
||||
return asyncio.run(_run())
|
||||
|
||||
|
||||
def test_local_agent_runner_uses_host_fake_provider_and_persists_ledger(
|
||||
def test_local_runner_uses_host_fake_provider_and_persists_ledger(
|
||||
local_agent_e2e_tmpdir,
|
||||
local_agent_e2e_config_path,
|
||||
local_agent_runtime_process,
|
||||
@@ -536,7 +536,7 @@ def test_local_agent_runner_uses_host_fake_provider_and_persists_ledger(
|
||||
conversation_id='e2e-local-agent-conversation',
|
||||
text='Say pong through the fake provider.',
|
||||
)
|
||||
messages = await _run_agent(ap, event, _binding())
|
||||
messages = await _run_runner(ap, event, _binding())
|
||||
return messages, list(fake_requester._count_tokens_payloads)
|
||||
|
||||
messages, token_payloads = _run_local_agent_probe(local_agent_e2e_tmpdir, _run_probe)
|
||||
@@ -553,13 +553,13 @@ def test_local_agent_runner_uses_host_fake_provider_and_persists_ledger(
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
try:
|
||||
run_row = conn.execute(
|
||||
"SELECT run_id, status, runner_id, status_reason FROM agent_run WHERE event_id = ?",
|
||||
'SELECT run_id, status, runner_id, status_reason FROM agent_run WHERE event_id = ?',
|
||||
('e2e-local-agent-event-001',),
|
||||
).fetchone()
|
||||
assert run_row is not None
|
||||
run_id, status, runner_id, status_reason = run_row
|
||||
assert status == 'completed'
|
||||
assert runner_id == LOCAL_AGENT_RUNNER_ID
|
||||
assert runner_id == LOCAL_RUNNER_ID
|
||||
assert status_reason == 'stop'
|
||||
|
||||
event_rows = conn.execute(
|
||||
@@ -578,12 +578,12 @@ def test_local_agent_runner_uses_host_fake_provider_and_persists_ledger(
|
||||
assert transcript_rows[0][1] == 'Say pong through the fake provider.'
|
||||
assert transcript_rows[1][1] == 'Fake LLM response'
|
||||
assert transcript_rows[1][2] == run_id
|
||||
assert transcript_rows[1][3] == LOCAL_AGENT_RUNNER_ID
|
||||
assert transcript_rows[1][3] == LOCAL_RUNNER_ID
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_local_agent_runner_executes_authorized_tool_loop_through_host_action(
|
||||
def test_local_runner_executes_authorized_tool_loop_through_host_action(
|
||||
local_agent_e2e_tmpdir,
|
||||
local_agent_e2e_config_path,
|
||||
local_agent_runtime_process,
|
||||
@@ -613,7 +613,7 @@ def test_local_agent_runner_executes_authorized_tool_loop_through_host_action(
|
||||
'tool-execution-mode': 'serial',
|
||||
},
|
||||
)
|
||||
messages = await _run_agent(ap, event, binding)
|
||||
messages = await _run_runner(ap, event, binding)
|
||||
return messages, tool_mgr.calls, _invoke_payload_texts(fake_requester)
|
||||
|
||||
messages, tool_calls, invoke_payload_texts = _run_local_agent_probe(local_agent_e2e_tmpdir, _run_probe)
|
||||
@@ -627,7 +627,7 @@ def test_local_agent_runner_executes_authorized_tool_loop_through_host_action(
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
try:
|
||||
run_row = conn.execute(
|
||||
"SELECT run_id, status, status_reason FROM agent_run WHERE event_id = ?",
|
||||
'SELECT run_id, status, status_reason FROM agent_run WHERE event_id = ?',
|
||||
('e2e-local-agent-tool-event-001',),
|
||||
).fetchone()
|
||||
assert run_row is not None
|
||||
@@ -652,7 +652,7 @@ def test_local_agent_runner_executes_authorized_tool_loop_through_host_action(
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_local_agent_runner_retrieves_authorized_rag_context_through_host_action(
|
||||
def test_local_runner_retrieves_authorized_rag_context_through_host_action(
|
||||
local_agent_e2e_tmpdir,
|
||||
local_agent_e2e_config_path,
|
||||
local_agent_runtime_process,
|
||||
@@ -679,7 +679,7 @@ def test_local_agent_runner_retrieves_authorized_rag_context_through_host_action
|
||||
'retrieval-top-k': 1,
|
||||
},
|
||||
)
|
||||
messages = await _run_agent(ap, event, binding)
|
||||
messages = await _run_runner(ap, event, binding)
|
||||
return messages, fake_kb.retrieve_calls, _invoke_payload_texts(fake_requester)
|
||||
|
||||
messages, retrieve_calls, invoke_payload_texts = _run_local_agent_probe(local_agent_e2e_tmpdir, _run_probe)
|
||||
@@ -702,7 +702,7 @@ def test_local_agent_runner_retrieves_authorized_rag_context_through_host_action
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
try:
|
||||
run_row = conn.execute(
|
||||
"SELECT run_id, status FROM agent_run WHERE event_id = ?",
|
||||
'SELECT run_id, status FROM agent_run WHERE event_id = ?',
|
||||
('e2e-local-agent-rag-event-001',),
|
||||
).fetchone()
|
||||
assert run_row is not None
|
||||
@@ -720,7 +720,7 @@ def test_local_agent_runner_retrieves_authorized_rag_context_through_host_action
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_local_agent_runner_compacts_history_and_persists_checkpoint(
|
||||
def test_local_runner_compacts_history_and_persists_checkpoint(
|
||||
local_agent_e2e_tmpdir,
|
||||
local_agent_e2e_config_path,
|
||||
local_agent_runtime_process,
|
||||
@@ -745,8 +745,7 @@ def test_local_agent_runner_compacts_history_and_persists_checkpoint(
|
||||
conversation_id='e2e-local-agent-compaction-conversation',
|
||||
role='user' if index % 2 == 0 else 'assistant',
|
||||
content=(
|
||||
f'HIST_SENTINEL-{index} '
|
||||
'This is intentionally long deterministic history for compaction. ' * 10
|
||||
f'HIST_SENTINEL-{index} This is intentionally long deterministic history for compaction. ' * 10
|
||||
),
|
||||
thread_id='e2e-local-agent-thread',
|
||||
item_type='message',
|
||||
@@ -767,7 +766,7 @@ def test_local_agent_runner_compacts_history_and_persists_checkpoint(
|
||||
'context-history-fetch-limit': 20,
|
||||
},
|
||||
)
|
||||
messages = await _run_agent(ap, event, binding)
|
||||
messages = await _run_runner(ap, event, binding)
|
||||
return messages, _invoke_payload_texts(fake_requester), fake_requester._invoke_count
|
||||
|
||||
messages, invoke_payload_texts, invoke_count = _run_local_agent_probe(local_agent_e2e_tmpdir, _run_probe)
|
||||
@@ -782,7 +781,7 @@ def test_local_agent_runner_compacts_history_and_persists_checkpoint(
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
try:
|
||||
run_row = conn.execute(
|
||||
"SELECT run_id, status FROM agent_run WHERE event_id = ?",
|
||||
'SELECT run_id, status FROM agent_run WHERE event_id = ?',
|
||||
('e2e-local-agent-compaction-event-001',),
|
||||
).fetchone()
|
||||
assert run_row is not None
|
||||
@@ -799,7 +798,7 @@ def test_local_agent_runner_compacts_history_and_persists_checkpoint(
|
||||
assert event_types == ['message.completed', 'run.completed']
|
||||
|
||||
state_row = conn.execute(
|
||||
"SELECT value_json FROM agent_runner_state WHERE state_key = 'runner.compaction.checkpoint'"
|
||||
"SELECT value_json FROM runner_state WHERE state_key = 'runner.compaction.checkpoint'"
|
||||
).fetchone()
|
||||
assert state_row is not None
|
||||
checkpoint = json.loads(state_row[0])
|
||||
@@ -812,7 +811,7 @@ def test_local_agent_runner_compacts_history_and_persists_checkpoint(
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_local_agent_runner_combines_rag_compaction_and_multi_turn_tool_loop(
|
||||
def test_local_runner_combines_rag_compaction_and_multi_turn_tool_loop(
|
||||
local_agent_e2e_tmpdir,
|
||||
local_agent_e2e_config_path,
|
||||
local_agent_runtime_process,
|
||||
@@ -883,7 +882,7 @@ def test_local_agent_runner_combines_rag_compaction_and_multi_turn_tool_loop(
|
||||
'context-history-fetch-limit': 25,
|
||||
},
|
||||
)
|
||||
messages = await _run_agent(ap, event, binding)
|
||||
messages = await _run_runner(ap, event, binding)
|
||||
return (
|
||||
messages,
|
||||
tool_mgr.calls,
|
||||
@@ -926,7 +925,7 @@ def test_local_agent_runner_combines_rag_compaction_and_multi_turn_tool_loop(
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
try:
|
||||
run_row = conn.execute(
|
||||
"SELECT run_id, status, status_reason FROM agent_run WHERE event_id = ?",
|
||||
'SELECT run_id, status, status_reason FROM agent_run WHERE event_id = ?',
|
||||
('e2e-local-agent-combo-event-001',),
|
||||
).fetchone()
|
||||
assert run_row is not None
|
||||
@@ -952,15 +951,11 @@ def test_local_agent_runner_combines_rag_compaction_and_multi_turn_tool_loop(
|
||||
assert 'COMBO_FINAL' in event_rows[4][1]
|
||||
|
||||
state_rows = conn.execute(
|
||||
"SELECT value_json FROM agent_runner_state WHERE state_key = 'runner.compaction.checkpoint'"
|
||||
"SELECT value_json FROM runner_state WHERE state_key = 'runner.compaction.checkpoint'"
|
||||
).fetchall()
|
||||
checkpoints = [json.loads(row[0]) for row in state_rows]
|
||||
checkpoint = next(
|
||||
(
|
||||
item
|
||||
for item in checkpoints
|
||||
if item.get('conversation_id') == 'e2e-local-agent-combo-conversation'
|
||||
),
|
||||
(item for item in checkpoints if item.get('conversation_id') == 'e2e-local-agent-combo-conversation'),
|
||||
None,
|
||||
)
|
||||
assert checkpoint is not None
|
||||
+95
-104
@@ -1,7 +1,7 @@
|
||||
"""E2E tests for pluginized AgentRunner execution.
|
||||
"""E2E tests for pluginized Runner execution.
|
||||
|
||||
This module starts the real LangBot backend with the plugin system enabled and
|
||||
loads a deterministic AgentRunner plugin through the real SDK Plugin Runtime.
|
||||
loads a deterministic Runner plugin through the real SDK Plugin Runtime.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -29,22 +29,22 @@ QA_RUNNER_ID = 'plugin:e2e/agent-runner-qa/default'
|
||||
|
||||
|
||||
@pytest.fixture(scope='session')
|
||||
def agent_runner_e2e_port():
|
||||
"""Port for the AgentRunner plugin-runtime E2E process."""
|
||||
def runner_e2e_port():
|
||||
"""Port for the Runner plugin-runtime E2E process."""
|
||||
return 15310
|
||||
|
||||
|
||||
@pytest.fixture(scope='session')
|
||||
def agent_runner_e2e_tmpdir():
|
||||
"""Create temporary directory for AgentRunner E2E testing."""
|
||||
tmpdir = Path(tempfile.mkdtemp(prefix='langbot_agent_runner_e2e_'))
|
||||
def runner_e2e_tmpdir():
|
||||
"""Create temporary directory for Runner E2E testing."""
|
||||
tmpdir = Path(tempfile.mkdtemp(prefix='langbot_runner_e2e_'))
|
||||
yield tmpdir
|
||||
shutil.rmtree(tmpdir, ignore_errors=True)
|
||||
|
||||
|
||||
def _write_qa_agent_runner_plugin(plugin_root: Path) -> None:
|
||||
"""Write a deterministic AgentRunner plugin used by this E2E."""
|
||||
runner_dir = plugin_root / 'components' / 'agent_runner'
|
||||
def _write_qa_runner_plugin(plugin_root: Path) -> None:
|
||||
"""Write a deterministic Runner plugin used by this E2E."""
|
||||
runner_dir = plugin_root / 'components' / 'runner'
|
||||
runner_dir.mkdir(parents=True, exist_ok=True)
|
||||
(plugin_root / 'assets').mkdir(parents=True, exist_ok=True)
|
||||
(plugin_root / 'assets' / 'icon.svg').write_text(
|
||||
@@ -61,27 +61,24 @@ def _write_qa_agent_runner_plugin(plugin_root: Path) -> None:
|
||||
name: agent-runner-qa
|
||||
version: 0.1.0
|
||||
label:
|
||||
en_US: AgentRunner QA
|
||||
zh_Hans: AgentRunner QA
|
||||
en_US: Runner QA
|
||||
zh_Hans: Runner QA
|
||||
description:
|
||||
en_US: Deterministic AgentRunner E2E probe.
|
||||
zh_Hans: 确定性的 AgentRunner E2E 探针。
|
||||
en_US: Deterministic Runner E2E probe.
|
||||
zh_Hans: 确定性的 Runner E2E 探针。
|
||||
icon: assets/icon.svg
|
||||
spec:
|
||||
version: 0.1.0
|
||||
config: []
|
||||
components:
|
||||
AgentRunner:
|
||||
Runner:
|
||||
fromDirs:
|
||||
- path: components/agent_runner/
|
||||
EventProcessor:
|
||||
fromDirs:
|
||||
- path: components/event_processor/
|
||||
- path: components/runner/
|
||||
pages: []
|
||||
execution:
|
||||
python:
|
||||
path: main.py
|
||||
attr: AgentRunnerQAPlugin
|
||||
attr: RunnerQAPlugin
|
||||
"""
|
||||
).strip()
|
||||
+ '\n',
|
||||
@@ -95,7 +92,7 @@ def _write_qa_agent_runner_plugin(plugin_root: Path) -> None:
|
||||
from langbot_plugin.api.definition.plugin import BasePlugin
|
||||
|
||||
|
||||
class AgentRunnerQAPlugin(BasePlugin):
|
||||
class RunnerQAPlugin(BasePlugin):
|
||||
async def initialize(self) -> None:
|
||||
pass
|
||||
"""
|
||||
@@ -107,7 +104,7 @@ def _write_qa_agent_runner_plugin(plugin_root: Path) -> None:
|
||||
textwrap.dedent(
|
||||
"""
|
||||
apiVersion: langbot/v1
|
||||
kind: AgentRunner
|
||||
kind: Runner
|
||||
metadata:
|
||||
name: default
|
||||
label:
|
||||
@@ -124,7 +121,7 @@ def _write_qa_agent_runner_plugin(plugin_root: Path) -> None:
|
||||
execution:
|
||||
python:
|
||||
path: default.py
|
||||
attr: DefaultAgentRunner
|
||||
attr: DefaultRunner
|
||||
"""
|
||||
).strip()
|
||||
+ '\n',
|
||||
@@ -137,42 +134,42 @@ def _write_qa_agent_runner_plugin(plugin_root: Path) -> None:
|
||||
|
||||
from typing import AsyncGenerator
|
||||
|
||||
from langbot_plugin.api.definition.components.agent_runner.runner import AgentRunner
|
||||
from langbot_plugin.api.entities.builtin.agent_runner.context import AgentRunContext
|
||||
from langbot_plugin.api.entities.builtin.agent_runner.result import AgentRunResult
|
||||
from langbot_plugin.api.definition.components.runner.runner import Runner
|
||||
from langbot_plugin.api.entities.builtin.runner.context import RunnerContext
|
||||
from langbot_plugin.api.entities.builtin.runner.result import RunnerResult
|
||||
from langbot_plugin.api.entities.builtin.provider.message import Message
|
||||
|
||||
|
||||
class DefaultAgentRunner(AgentRunner):
|
||||
async def run(self, ctx: AgentRunContext) -> AsyncGenerator[AgentRunResult, None]:
|
||||
class DefaultRunner(Runner):
|
||||
async def run(self, ctx: RunnerContext) -> AsyncGenerator[RunnerResult, None]:
|
||||
text = ctx.input.to_text()
|
||||
yield AgentRunResult.message_completed(
|
||||
yield RunnerResult.message_completed(
|
||||
ctx.run_id,
|
||||
Message(role='assistant', content=f'e2e echo: {text}'),
|
||||
)
|
||||
yield AgentRunResult.state_updated(
|
||||
yield RunnerResult.state_updated(
|
||||
ctx.run_id,
|
||||
'e2e.echo_count',
|
||||
{'count': 1},
|
||||
scope='conversation',
|
||||
)
|
||||
yield AgentRunResult.run_completed(ctx.run_id, finish_reason='stop')
|
||||
yield RunnerResult.run_completed(ctx.run_id, finish_reason='stop')
|
||||
"""
|
||||
).strip()
|
||||
+ '\n',
|
||||
encoding='utf-8',
|
||||
)
|
||||
|
||||
processor_dir = plugin_root / 'components' / 'event_processor'
|
||||
processor_dir.mkdir(parents=True)
|
||||
(processor_dir / 'default.yaml').write_text(
|
||||
processor_dir = runner_dir
|
||||
(processor_dir / 'welcome.yaml').write_text(
|
||||
textwrap.dedent("""
|
||||
apiVersion: langbot/v1
|
||||
kind: EventProcessor
|
||||
kind: Runner
|
||||
metadata:
|
||||
name: default
|
||||
name: welcome
|
||||
label: {en_US: Welcome processor, zh_Hans: Welcome processor}
|
||||
spec:
|
||||
usages: [event]
|
||||
events: [group.member_joined]
|
||||
config:
|
||||
- name: greeting
|
||||
@@ -184,20 +181,20 @@ def _write_qa_agent_runner_plugin(plugin_root: Path) -> None:
|
||||
permissions:
|
||||
tools: [detail, call]
|
||||
execution:
|
||||
python: {path: default.py, attr: WelcomeProcessor}
|
||||
python: {path: welcome.py, attr: WelcomeProcessor}
|
||||
""")
|
||||
)
|
||||
(processor_dir / 'default.py').write_text(
|
||||
(processor_dir / 'welcome.py').write_text(
|
||||
textwrap.dedent("""
|
||||
from langbot_plugin.api.definition.components.event_processor import EventProcessor, EventProcessorContext
|
||||
from langbot_plugin.api.definition.components.runner import Runner, RunnerContext
|
||||
from langbot_plugin.api.entities.builtin.platform.events import MemberJoinedEvent
|
||||
|
||||
class WelcomeProcessor(EventProcessor):
|
||||
class WelcomeProcessor(Runner):
|
||||
async def initialize(self):
|
||||
@self.handler(MemberJoinedEvent)
|
||||
async def handle(ctx: EventProcessorContext):
|
||||
await ctx.log('Handling ' + str(ctx.event.member.id))
|
||||
result = await ctx.reply(ctx.config['greeting'] + ', ' + (ctx.event.member.nickname or str(ctx.event.member.id)))
|
||||
async def handle(ctx: RunnerContext):
|
||||
await ctx.log('Handling ' + str(ctx.platform_event.member.id))
|
||||
result = await ctx.reply(ctx.config['greeting'] + ', ' + (ctx.platform_event.member.nickname or str(ctx.platform_event.member.id)))
|
||||
await ctx.log('Reply simulated: ' + str(result.get('mock')))
|
||||
""")
|
||||
)
|
||||
@@ -211,7 +208,7 @@ def _free_port() -> int:
|
||||
|
||||
|
||||
@pytest.fixture(scope='session')
|
||||
def agent_runner_runtime_ports():
|
||||
def runner_runtime_ports():
|
||||
"""Control/debug ports for the standalone plugin runtime."""
|
||||
control_port = _free_port()
|
||||
debug_port = _free_port()
|
||||
@@ -221,17 +218,17 @@ def agent_runner_runtime_ports():
|
||||
|
||||
|
||||
@pytest.fixture(scope='session')
|
||||
def agent_runner_e2e_config_path(agent_runner_e2e_tmpdir, agent_runner_e2e_port, agent_runner_runtime_ports):
|
||||
"""Create a plugin-enabled config and deterministic AgentRunner fixture."""
|
||||
config_path = create_minimal_config(agent_runner_e2e_tmpdir, port=agent_runner_e2e_port)
|
||||
create_test_directories(agent_runner_e2e_tmpdir)
|
||||
def runner_e2e_config_path(runner_e2e_tmpdir, runner_e2e_port, runner_runtime_ports):
|
||||
"""Create a plugin-enabled config and deterministic Runner fixture."""
|
||||
config_path = create_minimal_config(runner_e2e_tmpdir, port=runner_e2e_port)
|
||||
create_test_directories(runner_e2e_tmpdir)
|
||||
|
||||
import yaml
|
||||
|
||||
with open(config_path, encoding='utf-8') as f:
|
||||
config = yaml.safe_load(f)
|
||||
config['api']['global_api_key'] = 'e2e-agent-runner-key'
|
||||
runtime_control_port, _runtime_debug_port = agent_runner_runtime_ports
|
||||
runtime_control_port, _runtime_debug_port = runner_runtime_ports
|
||||
config['plugin']['enable'] = True
|
||||
config['plugin']['runtime_ws_url'] = f'ws://127.0.0.1:{runtime_control_port}/control/ws'
|
||||
config['plugin']['enable_marketplace'] = False
|
||||
@@ -240,10 +237,10 @@ def agent_runner_e2e_config_path(agent_runner_e2e_tmpdir, agent_runner_e2e_port,
|
||||
with open(config_path, 'w', encoding='utf-8') as f:
|
||||
yaml.safe_dump(config, f, default_flow_style=False)
|
||||
|
||||
plugin_source = agent_runner_e2e_tmpdir / 'agent-runner-qa-package'
|
||||
_write_qa_agent_runner_plugin(plugin_source)
|
||||
plugin_source = runner_e2e_tmpdir / 'agent-runner-qa-package'
|
||||
_write_qa_runner_plugin(plugin_source)
|
||||
shutil.make_archive(
|
||||
str(agent_runner_e2e_tmpdir / 'agent-runner-qa'),
|
||||
str(runner_e2e_tmpdir / 'agent-runner-qa'),
|
||||
'zip',
|
||||
root_dir=plugin_source,
|
||||
)
|
||||
@@ -251,11 +248,11 @@ def agent_runner_e2e_config_path(agent_runner_e2e_tmpdir, agent_runner_e2e_port,
|
||||
|
||||
|
||||
@pytest.fixture(scope='session')
|
||||
def agent_runner_runtime_process(agent_runner_e2e_tmpdir, agent_runner_runtime_ports):
|
||||
def runner_runtime_process(runner_e2e_tmpdir, runner_runtime_ports):
|
||||
"""Start the real SDK plugin runtime over WebSocket."""
|
||||
control_port, debug_port = agent_runner_runtime_ports
|
||||
stdout_path = agent_runner_e2e_tmpdir / 'plugin-runtime.stdout.log'
|
||||
stderr_path = agent_runner_e2e_tmpdir / 'plugin-runtime.stderr.log'
|
||||
control_port, debug_port = runner_runtime_ports
|
||||
stdout_path = runner_e2e_tmpdir / 'plugin-runtime.stdout.log'
|
||||
stderr_path = runner_e2e_tmpdir / 'plugin-runtime.stderr.log'
|
||||
stdout_file = open(stdout_path, 'wb')
|
||||
stderr_file = open(stderr_path, 'wb')
|
||||
proc = subprocess.Popen(
|
||||
@@ -269,7 +266,7 @@ def agent_runner_runtime_process(agent_runner_e2e_tmpdir, agent_runner_runtime_p
|
||||
'--ws-debug-port',
|
||||
str(debug_port),
|
||||
],
|
||||
cwd=agent_runner_e2e_tmpdir,
|
||||
cwd=runner_e2e_tmpdir,
|
||||
stdout=stdout_file,
|
||||
stderr=stderr_file,
|
||||
start_new_session=True,
|
||||
@@ -286,18 +283,18 @@ def agent_runner_runtime_process(agent_runner_e2e_tmpdir, agent_runner_runtime_p
|
||||
|
||||
|
||||
@pytest.fixture(scope='session')
|
||||
def agent_runner_langbot_process(
|
||||
agent_runner_e2e_config_path,
|
||||
agent_runner_e2e_port,
|
||||
agent_runner_e2e_tmpdir,
|
||||
agent_runner_runtime_process,
|
||||
def runner_langbot_process(
|
||||
runner_e2e_config_path,
|
||||
runner_e2e_port,
|
||||
runner_e2e_tmpdir,
|
||||
runner_runtime_process,
|
||||
):
|
||||
"""Start real LangBot with plugin runtime enabled."""
|
||||
project_root = find_project_root()
|
||||
proc = LangBotProcess(
|
||||
project_root=project_root,
|
||||
work_dir=agent_runner_e2e_tmpdir,
|
||||
port=agent_runner_e2e_port,
|
||||
work_dir=runner_e2e_tmpdir,
|
||||
port=runner_e2e_port,
|
||||
timeout=180,
|
||||
debug=True,
|
||||
cli_args=['--standalone-runtime'],
|
||||
@@ -306,7 +303,7 @@ def agent_runner_langbot_process(
|
||||
success = proc.start()
|
||||
if not success:
|
||||
stdout, stderr = proc.get_logs()
|
||||
pytest.fail(f'LangBot failed to start with AgentRunner plugin runtime:\nstdout: {stdout}\nstderr: {stderr}')
|
||||
pytest.fail(f'LangBot failed to start with Runner plugin runtime:\nstdout: {stdout}\nstderr: {stderr}')
|
||||
|
||||
yield proc
|
||||
|
||||
@@ -314,10 +311,10 @@ def agent_runner_langbot_process(
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def agent_runner_client(agent_runner_e2e_port, agent_runner_langbot_process):
|
||||
"""HTTP client for the AgentRunner E2E backend."""
|
||||
def runner_client(runner_e2e_port, runner_langbot_process):
|
||||
"""HTTP client for the Runner E2E backend."""
|
||||
with httpx.Client(
|
||||
base_url=f'http://127.0.0.1:{agent_runner_e2e_port}',
|
||||
base_url=f'http://127.0.0.1:{runner_e2e_port}',
|
||||
timeout=90.0,
|
||||
trust_env=False,
|
||||
) as client:
|
||||
@@ -396,29 +393,25 @@ def _ensure_qa_plugin(client: httpx.Client, token: str, package_path: Path) -> N
|
||||
_install_qa_plugin(client, token, package_path)
|
||||
|
||||
|
||||
def test_plugin_runtime_discovers_agent_runner(
|
||||
agent_runner_client,
|
||||
agent_runner_langbot_process,
|
||||
agent_runner_e2e_tmpdir,
|
||||
def test_plugin_runtime_discovers_runner(
|
||||
runner_client,
|
||||
runner_langbot_process,
|
||||
runner_e2e_tmpdir,
|
||||
):
|
||||
"""Pipeline metadata should include the real runtime-discovered QA runner."""
|
||||
token = _init_and_auth(agent_runner_client)
|
||||
token = _init_and_auth(runner_client)
|
||||
_ensure_qa_plugin(
|
||||
agent_runner_client,
|
||||
runner_client,
|
||||
token,
|
||||
agent_runner_e2e_tmpdir / 'agent-runner-qa.zip',
|
||||
runner_e2e_tmpdir / 'agent-runner-qa.zip',
|
||||
)
|
||||
option_names = _wait_for_qa_runner(agent_runner_client, token)
|
||||
option_names = _wait_for_qa_runner(runner_client, token)
|
||||
if QA_RUNNER_ID in option_names:
|
||||
return
|
||||
|
||||
host_stdout, host_stderr = agent_runner_langbot_process.get_logs()
|
||||
runtime_stdout = (agent_runner_e2e_tmpdir / 'plugin-runtime.stdout.log').read_text(
|
||||
encoding='utf-8', errors='replace'
|
||||
)
|
||||
runtime_stderr = (agent_runner_e2e_tmpdir / 'plugin-runtime.stderr.log').read_text(
|
||||
encoding='utf-8', errors='replace'
|
||||
)
|
||||
host_stdout, host_stderr = runner_langbot_process.get_logs()
|
||||
runtime_stdout = (runner_e2e_tmpdir / 'plugin-runtime.stdout.log').read_text(encoding='utf-8', errors='replace')
|
||||
runtime_stderr = (runner_e2e_tmpdir / 'plugin-runtime.stderr.log').read_text(encoding='utf-8', errors='replace')
|
||||
assert QA_RUNNER_ID in option_names, (
|
||||
f'{QA_RUNNER_ID} was not discovered\n'
|
||||
f'Host stdout (tail):\n{host_stdout[-20_000:]}\nHost stderr (tail):\n{host_stderr[-20_000:]}\n'
|
||||
@@ -427,26 +420,26 @@ def test_plugin_runtime_discovers_agent_runner(
|
||||
)
|
||||
|
||||
|
||||
def test_host_orchestrator_runs_agent_runner_and_records_ledger(
|
||||
agent_runner_client,
|
||||
agent_runner_langbot_process,
|
||||
agent_runner_e2e_tmpdir,
|
||||
def test_host_orchestrator_runs_runner_and_records_ledger(
|
||||
runner_client,
|
||||
runner_langbot_process,
|
||||
runner_e2e_tmpdir,
|
||||
):
|
||||
"""Create/configure/debug an Agent through HTTP and persist Runner side effects."""
|
||||
del agent_runner_langbot_process
|
||||
token = _init_and_auth(agent_runner_client)
|
||||
del runner_langbot_process
|
||||
token = _init_and_auth(runner_client)
|
||||
_ensure_qa_plugin(
|
||||
agent_runner_client,
|
||||
runner_client,
|
||||
token,
|
||||
agent_runner_e2e_tmpdir / 'agent-runner-qa.zip',
|
||||
runner_e2e_tmpdir / 'agent-runner-qa.zip',
|
||||
)
|
||||
headers = {'Authorization': f'Bearer {token}'}
|
||||
create_response = agent_runner_client.post(
|
||||
create_response = runner_client.post(
|
||||
'/api/v1/agents',
|
||||
headers=headers,
|
||||
json={
|
||||
'kind': 'agent',
|
||||
'name': 'AgentRunner E2E Agent',
|
||||
'name': 'Runner E2E Agent',
|
||||
'description': 'Exercises the installed QA Runner.',
|
||||
'emoji': 'QA',
|
||||
'supported_event_patterns': ['message.*'],
|
||||
@@ -462,7 +455,7 @@ def test_host_orchestrator_runs_agent_runner_and_records_ledger(
|
||||
assert create_payload['code'] == 0, create_payload
|
||||
agent_uuid = create_payload['data']['uuid']
|
||||
|
||||
get_response = agent_runner_client.get(f'/api/v1/agents/{agent_uuid}', headers=headers)
|
||||
get_response = runner_client.get(f'/api/v1/agents/{agent_uuid}', headers=headers)
|
||||
assert get_response.status_code == 200, get_response.text
|
||||
stored_agent = get_response.json()['data']['agent']
|
||||
assert stored_agent['config']['allowed_platform_tools'] == [
|
||||
@@ -470,7 +463,7 @@ def test_host_orchestrator_runs_agent_runner_and_records_ledger(
|
||||
'platform_get_user_info',
|
||||
]
|
||||
|
||||
debug_response = agent_runner_client.post(
|
||||
debug_response = runner_client.post(
|
||||
f'/api/v1/agents/{agent_uuid}/debug',
|
||||
headers=headers,
|
||||
json={
|
||||
@@ -486,7 +479,7 @@ def test_host_orchestrator_runs_agent_runner_and_records_ledger(
|
||||
assert result['final_text'] == 'e2e echo: hello from orchestrator e2e'
|
||||
assert result['outputs'][0]['role'] == 'assistant'
|
||||
|
||||
db_path = agent_runner_e2e_tmpdir / 'data' / 'langbot.db'
|
||||
db_path = runner_e2e_tmpdir / 'data' / 'langbot.db'
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
try:
|
||||
run_row = conn.execute(
|
||||
@@ -504,9 +497,7 @@ def test_host_orchestrator_runs_agent_runner_and_records_ledger(
|
||||
}
|
||||
assert {'state.updated', 'message.completed', 'run.completed'}.issubset(event_types)
|
||||
|
||||
state_row = conn.execute(
|
||||
"SELECT value_json FROM agent_runner_state WHERE state_key = 'e2e.echo_count'"
|
||||
).fetchone()
|
||||
state_row = conn.execute("SELECT value_json FROM runner_state WHERE state_key = 'e2e.echo_count'").fetchone()
|
||||
assert state_row is not None
|
||||
assert '"count": 1' in state_row[0]
|
||||
finally:
|
||||
@@ -514,17 +505,17 @@ def test_host_orchestrator_runs_agent_runner_and_records_ledger(
|
||||
|
||||
|
||||
def test_event_processor_real_runtime_logs_actions_and_instance_isolation(
|
||||
agent_runner_client,
|
||||
agent_runner_e2e_tmpdir,
|
||||
runner_client,
|
||||
runner_e2e_tmpdir,
|
||||
):
|
||||
client = agent_runner_client
|
||||
client = runner_client
|
||||
token = _init_and_auth(client)
|
||||
_ensure_qa_plugin(client, token, agent_runner_e2e_tmpdir / 'agent-runner-qa.zip')
|
||||
_ensure_qa_plugin(client, token, runner_e2e_tmpdir / 'agent-runner-qa.zip')
|
||||
headers = {'Authorization': f'Bearer {token}'}
|
||||
metadata_response = client.get('/api/v1/agents/_/metadata', headers=headers).json()
|
||||
assert metadata_response['code'] == 0, metadata_response
|
||||
metadata = metadata_response['data']
|
||||
ref = 'event_processor:e2e/agent-runner-qa/default'
|
||||
ref = 'plugin:e2e/agent-runner-qa/welcome'
|
||||
assert any(item['id'] == ref for item in metadata['event_processors']), metadata
|
||||
assert ref not in _wait_for_qa_runner(client, token)
|
||||
created = []
|
||||
Reference in New Issue
Block a user