feat(runner): unify plugin execution across agents and event processors

This commit is contained in:
RockChinQ
2026-09-10 18:04:38 +08:00
parent 8903a40c41
commit f24a7c9bb2
223 changed files with 4091 additions and 3068 deletions
+7 -7
View File
@@ -37,10 +37,10 @@ LANGBOT_NO_PROXY=localhost,127.0.0.1,::1
# LANGBOT_PIPELINE_NAME=Generic QA Pipeline
# LANGBOT_LOCAL_AGENT_PIPELINE_URL=http://127.0.0.1:3000/home/pipelines?id=<local-agent-pipeline-uuid>
# LANGBOT_LOCAL_AGENT_PIPELINE_NAME=Local Agent QA Pipeline
# LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL=http://127.0.0.1:3000/home/pipelines?id=<acp-agent-runner-pipeline-uuid>
# LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME=ACP AgentRunner QA Pipeline
# LANGBOT_ACP_AGENT_RUNNER_SSH_TARGET=yhh@101.34.71.12
# LANGBOT_ACP_AGENT_RUNNER_SSH_PORT=22
# LANGBOT_ACP_AGENT_RUNNER_SSH_IDENTITY_FILE=
# LANGBOT_ACP_AGENT_RUNNER_SSH_EXTRA_OPTIONS=
# LANGBOT_ACP_AGENT_RUNNER_REMOTE_WORKSPACE=/home/yhh/langbot-e2e/acp-workspace
# LANGBOT_ACP_RUNNER_PIPELINE_URL=http://127.0.0.1:3000/home/pipelines?id=<acp-agent-runner-pipeline-uuid>
# LANGBOT_ACP_RUNNER_PIPELINE_NAME=ACP Runner QA Pipeline
# LANGBOT_ACP_RUNNER_SSH_TARGET=yhh@101.34.71.12
# LANGBOT_ACP_RUNNER_SSH_PORT=22
# LANGBOT_ACP_RUNNER_SSH_IDENTITY_FILE=
# LANGBOT_ACP_RUNNER_SSH_EXTRA_OPTIONS=
# LANGBOT_ACP_RUNNER_REMOTE_WORKSPACE=/home/yhh/langbot-e2e/acp-workspace
+1 -1
View File
@@ -66,7 +66,7 @@ The tools wrap the LangBot service layer. Current tools (v1):
| `list_bots` / `get_bot` / `create_bot` / `update_bot` / `delete_bot` | Manage messaging-platform bots (secrets redacted on read) |
| `list_bot_event_route_statuses` | Inspect bot event-route runtime status |
| `list_processors` / `get_processor` / `create_processor` / `update_processor` / `delete_processor` | Manage the peer Agent, Pipeline and Event processor types |
| `get_processor_metadata` | Discover installed EventProcessor components, schemas and supported event patterns. |
| `get_processor_metadata` | Discover installed event-capable Runner components, schemas and supported event patterns. |
| `list_processor_runs` / `get_processor_run_events` | Read one Event processor instance run history and logs; paginate with `before_id` / `after_sequence`. |
| `debug_agent` | Execute a synthetic Agent event (`processor_uuid`, `payload`); requires `runtime.operate`. Returns final text and up to 1000 execution events (thinking, text, tool arguments/results). Platform tools use Mock; other configured tools execute normally. Optional `payload.mock`: `errors`/`results` keyed by platform tool name, `unsupported_apis` lists unavailable platform APIs. |
| `list_pipelines` / `get_pipeline` / `create_pipeline` / `update_pipeline` / `delete_pipeline` | Manage pipelines |
+5 -5
View File
@@ -11,13 +11,13 @@ Use this skill when an agent needs to verify LangBot behavior through the WebUI
- **General WebUI testing**: read `references/web-ui-testing.md`.
- **Pipeline Debug Chat**: read `references/pipeline-debug-chat.md`.
- **Dify AgentRunner**: read `references/dify-agent-runner.md`.
- **Dify Runner**: read `references/dify-agent-runner.md`.
- **Model provider setup or test button**: read `references/model-provider-testing.md`.
- **Plugin install/runtime/tool/page smoke**: read `references/plugin-e2e-smoke.md`.
- **Local Agent Runner**: read `references/local-agent-runner.md`.
- **Local Agent Runner path coverage**: read `references/local-agent-runner-coverage.md`.
- **Diff-aware AgentRunner QA after code changes**: read `references/agent-runner-qa-workflow.md`.
- **Agent Runner release gate**: read `references/agent-runner-release-gate.md`.
- **Local Runner**: read `references/local-agent-runner.md`.
- **Local Runner path coverage**: read `references/local-agent-runner-coverage.md`.
- **Diff-aware Runner QA after code changes**: read `references/agent-runner-qa-workflow.md`.
- **Runner release gate**: read `references/agent-runner-release-gate.md`.
- **Sandbox-backed skill authoring**: read `references/sandbox-skill-authoring.md`.
- **LangRAG knowledge bases**: read `references/langrag-knowledge-base.md`.
- **MCP stdio tool testing**: read `references/mcp-stdio-testing.md`.
@@ -1,5 +1,5 @@
id: acp-agent-runner-debug-chat
title: "ACP AgentRunner can answer through Debug Chat using real remote Claude"
title: "ACP Runner can answer through Debug Chat using real remote Claude"
mode: agent-browser
area: pipeline
type: regression
@@ -19,42 +19,42 @@ env:
- LANGBOT_FRONTEND_URL
- LANGBOT_BACKEND_URL
env_any:
- LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL|LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME
- LANGBOT_ACP_RUNNER_PIPELINE_URL|LANGBOT_ACP_RUNNER_PIPELINE_NAME
automation: scripts/e2e/pipeline-debug-chat.mjs
automation_env:
- LANGBOT_FRONTEND_URL
- LANGBOT_BACKEND_URL
- LANGBOT_BROWSER_PROFILE
- LANGBOT_CHROMIUM_EXECUTABLE
- LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL
- LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME
- LANGBOT_ACP_RUNNER_PIPELINE_URL
- LANGBOT_ACP_RUNNER_PIPELINE_NAME
- LANGBOT_E2E_PROMPT
- LANGBOT_E2E_EXPECTED_TEXT
- LANGBOT_E2E_EXPECTED_RUNNER_ID
- LANGBOT_E2E_RESPONSE_TIMEOUT_MS
automation_pipeline_url_env: LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL
automation_pipeline_name_env: LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME
automation_expected_runner_id: "plugin:langbot-team/ACPAgentRunner/default"
automation_prompt: "Do not launch any background agent, subagent, task, or worker. In this current ACP session, directly call the MCP tool named langbot_get_current_event exactly once and wait for its result. After it returns, reply exactly ACP_AGENT_RUNNER_E2E_OK with no other text."
automation_expected_text: "ACP_AGENT_RUNNER_E2E_OK"
automation_pipeline_url_env: LANGBOT_ACP_RUNNER_PIPELINE_URL
automation_pipeline_name_env: LANGBOT_ACP_RUNNER_PIPELINE_NAME
automation_expected_runner_id: "plugin:langbot-team/ACPRunner/default"
automation_prompt: "Do not launch any background agent, subagent, task, or worker. In this current ACP session, directly call the MCP tool named langbot_get_current_event exactly once and wait for its result. After it returns, reply exactly ACP_RUNNER_E2E_OK with no other text."
automation_expected_text: "ACP_RUNNER_E2E_OK"
automation_response_timeout_ms: "300000"
setup_automation:
- "node:scripts/e2e/ensure-acp-agent-runner-pipeline.mjs --write-env"
setup_provides_env:
- LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL
- LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME
- LANGBOT_ACP_RUNNER_PIPELINE_URL
- LANGBOT_ACP_RUNNER_PIPELINE_NAME
preconditions:
- "The remote machine has a working Claude Code login and can run npx -y @agentclientprotocol/claude-agent-acp."
- "LangBot can non-interactively SSH to the remote machine; the runner opens the MCP reverse tunnel automatically."
steps:
- "Open LANGBOT_FRONTEND_URL."
- "Open the ACP AgentRunner QA pipeline."
- "Confirm the pipeline AI runner is plugin:langbot-team/ACPAgentRunner/default."
- "Open the ACP Runner QA pipeline."
- "Confirm the pipeline AI runner is plugin:langbot-team/ACPRunner/default."
- "Open Debug Chat."
- "Ask the real remote Claude ACP agent to call langbot_get_current_event and return ACP_AGENT_RUNNER_E2E_OK exactly."
- "Ask the real remote Claude ACP agent to call langbot_get_current_event and return ACP_RUNNER_E2E_OK exactly."
checks:
- "UI: Debug Chat shows the user prompt."
- "UI: Debug Chat shows a Bot response containing ACP_AGENT_RUNNER_E2E_OK."
- "UI: Debug Chat shows a Bot response containing ACP_RUNNER_E2E_OK."
- "Logs: Backend logs include Processing request from person_websocket and Streaming completed for this run."
- "Logs: No acp runner request error appears for this run."
- "Console: No unexpected frontend errors appear during Debug Chat."
@@ -66,13 +66,13 @@ diagnostics:
- "Use scripts/e2e/ensure-acp-agent-runner-pipeline.mjs --write-env to create/update the pipeline."
- "For remote Claude on 101, verify ssh yhh@101.34.71.12 can run without password prompts; no separate ssh -R process is required."
success_patterns:
- "ACP_AGENT_RUNNER_E2E_OK"
- "ACP_RUNNER_E2E_OK"
- "Processing request from person_websocket"
- "Streaming completed"
failure_patterns:
- "acp.command_not_found"
- "acp.process_exited"
- "Agent runner plugin:langbot-team/ACPAgentRunner/default execution failed"
- "Agent runner plugin:langbot-team/ACPRunner/default execution failed"
troubleshooting:
- backend-not-listening
- plugin-runtime-timeout
@@ -1,5 +1,5 @@
id: agent-run-ledger-audit
title: "Persisted AgentRunner run ledger passes end-to-end invariants"
title: "Persisted Runner run ledger passes end-to-end invariants"
mode: probe
area: agent
type: regression
@@ -15,7 +15,7 @@ skills:
- langbot-testing
automation: scripts/e2e/agent-run-ledger-audit.mjs
steps:
- "Set LANGBOT_AGENT_RUN_ID to audit a specific run, or leave it unset to audit the latest persisted AgentRunner run."
- "Set LANGBOT_AGENT_RUN_ID to audit a specific run, or leave it unset to audit the latest persisted Runner run."
- "For an external runner's own CLI tools, set LANGBOT_AGENT_TOOL_AUTHORIZATION_MODE=runner-native; keep the default strict mode for Host tool calls."
- "Read the active LangBot database configuration and inspect the selected run and its ordered events."
- "Verify completed terminal state, run.completed, paired tool.call.started/completed events, stable tool names, and monotonic ordering."
@@ -1,5 +1,5 @@
id: agent-runner-async-db-readiness
title: "AgentRunner async DB readiness probe"
title: "Runner async DB readiness probe"
mode: probe
area: release
type: smoke
@@ -1,5 +1,5 @@
id: agent-runner-behavior-matrix
title: "AgentRunner deterministic behavior matrix probe"
title: "Runner deterministic behavior matrix probe"
mode: probe
area: release
type: regression
@@ -1,5 +1,5 @@
id: agent-runner-fixture-contract
title: "QA AgentRunner fixture contract probe"
title: "QA Runner fixture contract probe"
mode: probe
area: release
type: regression
@@ -17,19 +17,19 @@ env:
automation: skills/langbot-testing/probes/agent-runner-fixture-contract.mjs
steps:
- "Run `rtk bin/lbs test run agent-runner-fixture-contract --dry-run` first; remove `--dry-run` after checking the planned evidence directory."
- "Automation imports the QA AgentRunner fixture source and executes normal, streaming, and controlled-failure paths with SDK entities."
- "Automation imports the QA Runner fixture source and executes normal, streaming, and controlled-failure paths with SDK entities."
checks:
- "automation-result.json status is pass."
- "probe-stdout.log contains QA_AGENT_RUNNER_FIXTURE_CONTRACT_OK."
- "Normal input returns QA_AGENT_RUNNER_OK:<input>."
- "probe-stdout.log contains QA_RUNNER_FIXTURE_CONTRACT_OK."
- "Normal input returns QA_RUNNER_OK:<input>."
- "Streaming input emits message.delta chunks and completes."
- "Failure input returns QA_AGENT_RUNNER_CONTROLLED_FAILURE."
- "Failure input returns QA_RUNNER_CONTROLLED_FAILURE."
evidence_required:
- filesystem
diagnostics:
- "This validates the deterministic fixture source contract. It does not prove the plugin package is installed in a live LangBot instance."
success_patterns:
- "QA_AGENT_RUNNER_FIXTURE_CONTRACT_OK"
- "QA_RUNNER_FIXTURE_CONTRACT_OK"
failure_patterns:
- "AssertionError"
- "fixture contract exited"
@@ -25,7 +25,7 @@ automation_env:
- LANGBOT_BROWSER_PROFILE
- LANGBOT_CHROMIUM_EXECUTABLE
preconditions:
- "The plugin runtime is enabled and connected, with at least one AgentRunner registered."
- "The plugin runtime is enabled and connected, with at least one Runner registered."
- "The target is a local test instance where a temporary Agent may be created and deleted."
steps:
- "Read the live plugin runtime status and select a registered runner from Agent metadata."
@@ -1,5 +1,5 @@
id: agent-runner-ledger-concurrency
title: "AgentRunner run ledger concurrency and auth pytest probe"
title: "Runner run ledger concurrency and auth pytest probe"
mode: probe
area: release
type: regression
@@ -1,5 +1,5 @@
id: agent-runner-ledger-contention
title: "AgentRunner ledger SQLite contention probe"
title: "Runner ledger SQLite contention probe"
mode: probe
area: release
type: regression
@@ -1,5 +1,5 @@
id: agent-runner-ledger-invariants
title: "AgentRunner ledger schema and status invariants probe"
title: "Runner ledger schema and status invariants probe"
mode: probe
area: release
type: regression
@@ -1,5 +1,5 @@
id: agent-runner-ledger-stress
title: "AgentRunner ledger lightweight stress probe"
title: "Runner ledger lightweight stress probe"
mode: probe
area: release
type: regression
@@ -1,5 +1,5 @@
id: agent-runner-live-install
title: "QA AgentRunner package installs and registers in LangBot"
title: "QA Runner package installs and registers in LangBot"
mode: probe
area: release
type: regression
@@ -25,7 +25,7 @@ automation_expected_tool: ""
automation_expected_runner_id: "plugin:qa/agent-runner/default"
steps:
- "Run `rtk bin/lbs test run agent-runner-live-install --dry-run` first; remove `--dry-run` only after readiness points at a local test LangBot instance."
- "Automation authenticates the local test user, uploads the QA AgentRunner .lbpkg package, waits for the install task, and reads pipeline metadata."
- "Automation authenticates the local test user, uploads the QA Runner .lbpkg package, waits for the install task, and reads pipeline metadata."
checks:
- "automation-result.json status is pass."
- "/api/v1/plugins lists qa/agent-runner after install."
@@ -1,5 +1,5 @@
id: agent-runner-qa-debug-chat
title: "QA AgentRunner returns deterministic output through Debug Chat"
title: "QA Runner returns deterministic output through Debug Chat"
mode: agent-browser
area: pipeline
type: regression
@@ -17,21 +17,21 @@ skills:
env:
- LANGBOT_FRONTEND_URL
- LANGBOT_BACKEND_URL
- LANGBOT_QA_AGENT_RUNNER_PIPELINE_URL
- LANGBOT_QA_AGENT_RUNNER_PIPELINE_NAME
- LANGBOT_QA_RUNNER_PIPELINE_URL
- LANGBOT_QA_RUNNER_PIPELINE_NAME
automation: scripts/e2e/pipeline-debug-chat.mjs
automation_env:
- LANGBOT_FRONTEND_URL
- LANGBOT_BACKEND_URL
- LANGBOT_BROWSER_PROFILE
- LANGBOT_CHROMIUM_EXECUTABLE
- LANGBOT_QA_AGENT_RUNNER_PIPELINE_URL
- LANGBOT_QA_AGENT_RUNNER_PIPELINE_NAME
automation_pipeline_url_env: LANGBOT_QA_AGENT_RUNNER_PIPELINE_URL
automation_pipeline_name_env: LANGBOT_QA_AGENT_RUNNER_PIPELINE_NAME
- LANGBOT_QA_RUNNER_PIPELINE_URL
- LANGBOT_QA_RUNNER_PIPELINE_NAME
automation_pipeline_url_env: LANGBOT_QA_RUNNER_PIPELINE_URL
automation_pipeline_name_env: LANGBOT_QA_RUNNER_PIPELINE_NAME
automation_expected_runner_id: "plugin:qa/agent-runner/default"
automation_prompt: "hello-live"
automation_expected_text: "QA_AGENT_RUNNER_OK:hello-live"
automation_expected_text: "QA_RUNNER_OK:hello-live"
automation_response_timeout_ms: "120000"
automation_debug_chat_response_p95_ms: "120000"
automation_reset_debug_chat: "1"
@@ -39,17 +39,17 @@ setup_automation:
- "case:agent-runner-live-install"
- "node:scripts/e2e/ensure-qa-agent-runner-pipeline.mjs --write-env"
setup_provides_env:
- LANGBOT_QA_AGENT_RUNNER_PIPELINE_URL
- LANGBOT_QA_AGENT_RUNNER_PIPELINE_NAME
- LANGBOT_QA_RUNNER_PIPELINE_URL
- LANGBOT_QA_RUNNER_PIPELINE_NAME
steps:
- "Open LANGBOT_FRONTEND_URL."
- "Open the pipeline from LANGBOT_QA_AGENT_RUNNER_PIPELINE_URL or LANGBOT_QA_AGENT_RUNNER_PIPELINE_NAME."
- "Open the pipeline from LANGBOT_QA_RUNNER_PIPELINE_URL or LANGBOT_QA_RUNNER_PIPELINE_NAME."
- "Confirm the pipeline AI runner is plugin:qa/agent-runner/default."
- "Open Debug Chat."
- "Send: hello-live."
checks:
- "UI: The user message appears in Debug Chat."
- "UI: A Bot message appears and contains QA_AGENT_RUNNER_OK:hello-live."
- "UI: A Bot message appears and contains QA_RUNNER_OK:hello-live."
- "API diagnostic: pipeline config uses plugin:qa/agent-runner/default."
- "Console: No unexpected frontend runtime errors appear during the send/receive path."
evidence_required:
@@ -62,7 +62,7 @@ diagnostics:
- "This is the deterministic live execution proof that sits after fixture contract and live install."
- "If the runner id mismatch is reported, rerun ensure-qa-agent-runner-pipeline.mjs --write-env."
success_patterns:
- "QA_AGENT_RUNNER_OK:hello-live"
- "QA_RUNNER_OK:hello-live"
failure_patterns:
- "plugin:qa/agent-runner/default execution failed"
- "Action invoke_llm_stream call timed out"
@@ -19,7 +19,7 @@ env:
- LANGBOT_BACKEND_URL
env_any:
- LANGBOT_LOCAL_AGENT_PIPELINE_URL|LANGBOT_LOCAL_AGENT_PIPELINE_NAME
- LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL|LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME
- LANGBOT_ACP_RUNNER_PIPELINE_URL|LANGBOT_ACP_RUNNER_PIPELINE_NAME
automation: scripts/e2e/agent-runner-release-preflight.mjs
automation_env:
- LANGBOT_FRONTEND_URL
@@ -28,24 +28,24 @@ automation_env:
- LANGBOT_CHROMIUM_EXECUTABLE
automation_env_any:
- LANGBOT_LOCAL_AGENT_PIPELINE_URL|LANGBOT_LOCAL_AGENT_PIPELINE_NAME
- LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL|LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME
- LANGBOT_ACP_RUNNER_PIPELINE_URL|LANGBOT_ACP_RUNNER_PIPELINE_NAME
preconditions:
- "LANGBOT_LOCAL_AGENT_PIPELINE_URL or LANGBOT_LOCAL_AGENT_PIPELINE_NAME points to the local-agent release pipeline."
- "LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL or LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME points to the ACP AgentRunner release pipeline."
- "LANGBOT_ACP_RUNNER_PIPELINE_URL or LANGBOT_ACP_RUNNER_PIPELINE_NAME points to the ACP Runner release pipeline."
- "The active browser profile is authenticated for the same LangBot backend."
- "By default the preflight performs a cheap model test for the local-agent primary model; set LANGBOT_PREFLIGHT_TEST_MODELS=0 only when deliberately classifying model credentials outside this run."
steps:
- "Open LANGBOT_FRONTEND_URL with the configured browser profile."
- "Use the browser token to call LangBot backend readiness APIs without printing token values."
- "Check plugin runtime status, Box status, required runner plugins, qa-plugin-smoke, and qa_plugin_echo."
- "Resolve the local-agent and ACP AgentRunner QA pipelines from their case-specific env vars."
- "Resolve the local-agent and ACP Runner QA pipelines from their case-specific env vars."
- "Assert each pipeline uses the expected runner id."
- "Assert the external runner pipeline uses the expected runner id."
- "Assert the local-agent primary model advertises func_call and vision for the full release gate."
- "Run the local-agent primary model test endpoint unless LANGBOT_PREFLIGHT_TEST_MODELS=0."
checks:
- "API diagnostic: api-diagnostic.json has no blockers and no env_issues."
- "API diagnostic: required pipelines resolve to plugin:langbot-team/LocalAgent/default and plugin:langbot-team/ACPAgentRunner/default."
- "API diagnostic: required pipelines resolve to plugin:langbot-team/LocalAgent/default and plugin:langbot-team/ACPRunner/default."
- "API diagnostic: qa_plugin_echo is exposed by /api/v1/tools."
- "API diagnostic: local-agent model check catches invalid credentials or missing func_call/vision before release E2E starts."
- "Secret safety: token values, api keys, and provider secrets are not printed."
@@ -1,5 +1,5 @@
id: agent-runner-runtime-chaos
title: "AgentRunner SDK runtime chaos pytest probe"
title: "Runner SDK runtime chaos pytest probe"
mode: probe
area: release
type: regression
@@ -19,10 +19,10 @@ automation: skills/langbot-testing/probes/agent-runner-runtime-chaos.mjs
steps:
- "Run `rtk bin/lbs test run agent-runner-runtime-chaos --dry-run` first; remove `--dry-run` after checking the SDK repo target and evidence directory."
- "Automation resolves LANGBOT_PLUGIN_SDK_REPO, defaulting to ../../langbot-plugin-sdk when the env var is unset."
- "Automation runs the existing SDK pytest files tests/runtime/plugin/test_mgr_agent_runner.py and tests/runtime/test_pull_api_handlers.py."
- "Automation runs the existing SDK pytest files tests/runtime/plugin/test_mgr_runner.py and tests/runtime/test_pull_api_handlers.py."
checks:
- "automation-result.json status is pass."
- "pytest exit status is 0 for the existing AgentRunner runtime and pull API handler tests."
- "pytest exit status is 0 for the existing Runner runtime and pull API handler tests."
- "pytest-stdout.log and pytest-stderr.log are written under LBS_EVIDENCE_DIR."
evidence_required:
- filesystem
@@ -45,7 +45,7 @@ steps:
- "Select exactly one Box child whose parent is main.py running from LANGBOT_REPO; abort on zero or multiple matches."
- "Send SIGTERM to that Box child and poll process, Box status, MCP runtime info, and global tools for up to 30 seconds."
- "Without running MCP setup or registration, reset Debug Chat and call qa_mcp_echo with a unique per-run value through the browser."
- "Audit the matching AgentRunner ledger run and require the exact qa_mcp_echo arguments plus the complete tool result text."
- "Audit the matching Runner ledger run and require the exact qa_mcp_echo arguments plus the complete tool result text."
checks:
- "The old Box PID exits and a new Box PID appears under the same LangBot parent."
- "Box returns available=true with at least one active session and managed process."
@@ -1,5 +1,5 @@
id: dify-agent-debug-chat
title: "Dify AgentRunner returns a response through Pipeline Debug Chat"
title: "Dify Runner returns a response through Pipeline Debug Chat"
mode: agent-browser
area: pipeline
type: provider
@@ -18,8 +18,8 @@ skills:
env:
- LANGBOT_FRONTEND_URL
- LANGBOT_BACKEND_URL
- LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL
- LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME
- LANGBOT_ACP_RUNNER_PIPELINE_URL
- LANGBOT_ACP_RUNNER_PIPELINE_NAME
preconditions:
- "An external-harness runner pipeline (e.g. ACP remote claude-code) is configured with langbot-assets-enabled=true so the LangBot MCP gateway is exposed to the harness."
- "The remote harness (claude-code) is reachable and responsive (claude -p returns within the runner timeout)."
@@ -29,10 +29,10 @@ automation_env:
- LANGBOT_FRONTEND_URL
- LANGBOT_BROWSER_PROFILE
- LANGBOT_CHROMIUM_EXECUTABLE
- LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL
- LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME
automation_pipeline_url_env: LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL
automation_pipeline_name_env: LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME
- LANGBOT_ACP_RUNNER_PIPELINE_URL
- LANGBOT_ACP_RUNNER_PIPELINE_NAME
automation_pipeline_url_env: LANGBOT_ACP_RUNNER_PIPELINE_URL
automation_pipeline_name_env: LANGBOT_ACP_RUNNER_PIPELINE_NAME
automation_prompt: "You have LangBot tools available via an MCP server (tools prefixed langbot_). Call langbot_list_assets with asset_types = [\"skills\",\"tools\"]. Then reply with one single line: the literal token PROBEDONE, a space, the number of skills you found, a space, and the number of tools you found."
automation_expected_text: "PROBEDONE"
automation_response_timeout_ms: "540000"
@@ -1,5 +1,5 @@
id: wizard-runner-marketplace-catalog
title: "Quick Start installs a published AgentRunner on a clean instance"
title: "Quick Start installs a published Runner on a clean instance"
mode: agent-browser
area: wizard
type: feature
@@ -27,19 +27,19 @@ preconditions:
steps:
- "Start an isolated first-run instance and confirm zero installed plugins and zero registered runners."
- "Resume Quick Start at the AI Engine step with a temporary disabled Bot."
- "Confirm the browser requests Marketplace plugins with component_filter=AgentRunner."
- "Confirm the browser requests Marketplace plugins with component_filter=Runner."
- "Confirm langbot-team/LocalAgent is published with an installable version and the Runner Extensions link is correct."
- "Install LocalAgent and wait for plugin initialization and AgentRunner registration."
- "Install LocalAgent and wait for plugin initialization and Runner registration."
- "Confirm Create & Deploy is disabled before installation and enabled after LocalAgent is selected."
- "Verify the layout at desktop and mobile widths."
checks:
- "API: The instance has zero installed plugins and zero registered runners."
- "API: The instance wizard status is none."
- "Network: Marketplace search uses component_filter=AgentRunner and type_filter=plugin."
- "Network: Marketplace search uses component_filter=Runner and type_filter=plugin."
- "UI: The AI Engine step displays the published langbot-team/LocalAgent card."
- "Marketplace: LocalAgent includes latest_version so installation can proceed."
- "Runtime: LocalAgent installs and registers plugin:langbot-team/LocalAgent/default."
- "UI: Browse Runner Extensions links to the AgentRunner-filtered market."
- "UI: Browse Runner Extensions links to the Runner-filtered market."
- "UI: Create & Deploy transitions from disabled to enabled only after Runner selection."
- "Console: No unexpected frontend errors appear during the flow."
- "Cleanup: Wizard progress and the temporary Bot are removed."
@@ -18,7 +18,7 @@ steps:
- "Discover the active LangBot checkout and sibling workspace repositories, with LANGBOT_WORKSPACE_ROOT and repository-specific env overrides available for non-default layouts."
- "Verify every checkout is on main or dev/4.11.x and record dirty worktrees as warnings only."
- "Validate plugin manifest identities for LocalAgent, Control Plane, LongTermMemory, GeneralParsers, LangRAG, and Skill Authoring."
- "Use the LangBot virtualenv to import langbot_plugin and AgentRunner Protocol entities from the workspace SDK source tree."
- "Use the LangBot virtualenv to import langbot_plugin and Runner Protocol entities from the workspace SDK source tree."
checks:
- "workspace-preflight.json contains no failed checks."
- "The langbot_plugin import path is inside the discovered langbot-plugin-sdk/src directory."
@@ -16,7 +16,7 @@ skills:
automation: scripts/e2e/workspace-repository-contracts.mjs
steps:
- "Resolve the active LangBot virtualenv and workspace SDK source path."
- "Run tests independently for AgentRunner, Control Plane, LongTermMemory, Parser, RAG, Skill Authoring, LocalAgent, LangBot Agent/Provider, the skills CLI, and SDK runtime contracts."
- "Run tests independently for Runner, Control Plane, LongTermMemory, Parser, RAG, Skill Authoring, LocalAgent, LangBot Agent/Provider, the skills CLI, and SDK runtime contracts."
- "Run SDK packaging blackbox separately so an isolated build dependency network failure is classified as env_issue without masking product test failures."
- "Write per-repository stdout, stderr, status, and duration under repository-contracts/."
checks:
@@ -1,7 +1,7 @@
[
{
"id": "qa-agent-runner-behaviors",
"title": "Deterministic AgentRunner behavior matrix",
"title": "Deterministic Runner behavior matrix",
"kind": "json",
"path": "fixtures/agent-runner/qa-runner-behaviors.json",
"related_cases": [
@@ -13,7 +13,7 @@
},
{
"id": "qa-agent-runner-source",
"title": "QA deterministic AgentRunner fixture source",
"title": "QA deterministic Runner fixture source",
"kind": "plugin_source",
"path": "fixtures/plugins/qa-agent-runner/manifest.yaml",
"related_cases": [
@@ -22,11 +22,11 @@
"agent-runner-live-install",
"agent-runner-qa-debug-chat"
],
"checks": ["exists", "qa_agent_runner_source"]
"checks": ["exists", "qa_runner_source"]
},
{
"id": "qa-agent-runner-package",
"title": "QA deterministic AgentRunner prebuilt package",
"title": "QA deterministic Runner prebuilt package",
"kind": "plugin_package",
"path": "fixtures/plugins/qa-agent-runner/dist/qa-agent-runner-0.1.0.lbpkg",
"related_cases": [
@@ -1,6 +1,6 @@
# QA AgentRunner Fixture
# QA Runner Fixture
Deterministic AgentRunner plugin source used by `langbot-skills` probes and future browser release-gate cases.
Deterministic Runner plugin source used by `langbot-skills` probes and future browser release-gate cases.
Runner id after installation should be:
@@ -10,6 +10,6 @@ plugin:qa/agent-runner/default
Expected behavior:
- normal input returns `QA_AGENT_RUNNER_OK:<input>`
- normal input returns `QA_RUNNER_OK:<input>`
- input containing `stream` emits streaming chunks then completes
- input containing `fail` returns `QA_AGENT_RUNNER_CONTROLLED_FAILURE`
- input containing `fail` returns `QA_RUNNER_CONTROLLED_FAILURE`
@@ -1,4 +1,4 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64" role="img" aria-label="QA AgentRunner icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64" role="img" aria-label="QA Runner icon">
<rect width="64" height="64" rx="12" fill="#111827"/>
<path d="M16 20h32v22H35l-7 8v-8H16z" fill="#22c55e"/>
<path d="M24 30h16" stroke="#111827" stroke-width="4" stroke-linecap="round"/>

Before

Width:  |  Height:  |  Size: 306 B

After

Width:  |  Height:  |  Size: 301 B

@@ -1,39 +0,0 @@
from __future__ import annotations
import typing
from langbot_plugin.api.definition.components.agent_runner.runner import AgentRunner
from langbot_plugin.api.entities.builtin.agent_runner import AgentRunContext, AgentRunResult
from langbot_plugin.api.entities.builtin.provider.message import Message, MessageChunk
class DefaultAgentRunner(AgentRunner):
async def run(
self,
ctx: AgentRunContext,
) -> typing.AsyncGenerator[AgentRunResult, None]:
text = (ctx.input.to_text() or "").strip()
if "fail" in text.lower():
yield AgentRunResult.run_failed(
ctx.run_id,
error="QA_AGENT_RUNNER_CONTROLLED_FAILURE",
code="qa.controlled_failure",
retryable=False,
)
return
content = f"QA_AGENT_RUNNER_OK:{text or 'empty'}"
if "stream" in text.lower():
for chunk in ("QA_", "AGENT_", f"RUNNER_OK:{text}"):
yield AgentRunResult.message_delta(
ctx.run_id,
MessageChunk(role="assistant", content=chunk),
)
yield AgentRunResult.run_completed(ctx.run_id, finish_reason="stop")
return
yield AgentRunResult.run_completed(
ctx.run_id,
Message(role="assistant", content=content),
finish_reason="stop",
)
@@ -0,0 +1,39 @@
from __future__ import annotations
import typing
from langbot_plugin.api.definition.components.runner.runner import Runner
from langbot_plugin.api.entities.builtin.runner import RunnerContext, RunnerResult
from langbot_plugin.api.entities.builtin.provider.message import Message, MessageChunk
class DefaultRunner(Runner):
async def run(
self,
ctx: RunnerContext,
) -> typing.AsyncGenerator[RunnerResult, None]:
text = (ctx.input.to_text() or '').strip()
if 'fail' in text.lower():
yield RunnerResult.run_failed(
ctx.run_id,
error='QA_RUNNER_CONTROLLED_FAILURE',
code='qa.controlled_failure',
retryable=False,
)
return
content = f'QA_RUNNER_OK:{text or "empty"}'
if 'stream' in text.lower():
for chunk in ('QA_', 'AGENT_', f'RUNNER_OK:{text}'):
yield RunnerResult.message_delta(
ctx.run_id,
MessageChunk(role='assistant', content=chunk),
)
yield RunnerResult.run_completed(ctx.run_id, finish_reason='stop')
return
yield RunnerResult.run_completed(
ctx.run_id,
Message(role='assistant', content=content),
finish_reason='stop',
)
@@ -1,5 +1,5 @@
apiVersion: langbot/v1
kind: AgentRunner
kind: Runner
metadata:
name: default
label:
@@ -27,4 +27,4 @@ spec:
execution:
python:
path: default.py
attr: DefaultAgentRunner
attr: DefaultRunner
@@ -3,6 +3,6 @@ from __future__ import annotations
from langbot_plugin.api.definition.plugin import BasePlugin
class QAAgentRunnerPlugin(BasePlugin):
class QARunnerPlugin(BasePlugin):
async def initialize(self) -> None:
self.ready_marker = "qa-agent-runner-ready"
self.ready_marker = 'qa-agent-runner-ready'
@@ -6,20 +6,20 @@ metadata:
repository: https://example.invalid/langbot/qa-agent-runner
version: 0.1.0
description:
en_US: Deterministic AgentRunner fixture for LangBot QA.
zh_Hans: LangBot QA 使用的确定性 AgentRunner 夹具。
en_US: Deterministic Runner fixture for LangBot QA.
zh_Hans: LangBot QA 使用的确定性 Runner 夹具。
label:
en_US: QA AgentRunner
zh_Hans: QA AgentRunner
en_US: QA Runner
zh_Hans: QA Runner
icon: assets/icon.svg
spec:
config: []
components:
AgentRunner:
Runner:
fromDirs:
- path: components/agent_runner/
- path: components/runner/
maxDepth: 1
execution:
python:
path: main.py
attr: QAAgentRunnerPlugin
attr: QARunnerPlugin
@@ -7,7 +7,11 @@ import { delimiter, join, resolve } from "node:path";
import { env } from "node:process";
function timestampSlug(date = new Date()) {
return date.toISOString().replace(/\.\d{3}Z$/, "Z").replace(/[^0-9A-Za-z]+/g, "-").replace(/^-|-$/g, "");
return date
.toISOString()
.replace(/\.\d{3}Z$/, "Z")
.replace(/[^0-9A-Za-z]+/g, "-")
.replace(/^-|-$/g, "");
}
function localIsoWithOffset(date = new Date()) {
@@ -51,7 +55,14 @@ function run(command, timeoutMs, childEnv) {
});
child.on("error", (error) => {
clearTimeout(timeout);
resolveDone({ stdout, stderr, error, timedOut, status: null, signal: null });
resolveDone({
stdout,
stderr,
error,
timedOut,
status: null,
signal: null,
});
});
child.on("close", (status, signal) => {
clearTimeout(timeout);
@@ -66,7 +77,7 @@ import json
import sys
from pathlib import Path
from langbot.pkg.agent.runner.descriptor import AgentRunnerDescriptor
from langbot.pkg.agent.runner.descriptor import RunnerDescriptor
from langbot.pkg.agent.runner.errors import RunnerExecutionError, RunnerProtocolError
from langbot.pkg.agent.runner.result_normalizer import AgentResultNormalizer
@@ -80,10 +91,10 @@ class App:
logger = Logger()
def descriptor():
return AgentRunnerDescriptor(
return RunnerDescriptor(
id='plugin:qa/agent-runner/default',
source='plugin',
label={'en_US': 'QA AgentRunner'},
label={'en_US': 'QA Runner'},
plugin_author='qa',
plugin_name='agent-runner',
runner_name='default',
@@ -139,18 +150,26 @@ async function main() {
const root = resolve(env.LBS_ROOT || process.cwd());
const caseId = "agent-runner-behavior-matrix";
const runId = env.LBS_RUN_ID || `${timestampSlug()}-${caseId}`;
const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId));
const evidenceDir = resolve(
env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId),
);
await mkdir(evidenceDir, { recursive: true });
const startedAt = new Date();
const langbotRepo = resolve(root, env.LANGBOT_REPO || "..");
const sdkRepo = resolve(root, env.LANGBOT_PLUGIN_SDK_REPO || "../../langbot-plugin-sdk");
const sdkRepo = resolve(
root,
env.LANGBOT_PLUGIN_SDK_REPO || "../../langbot-plugin-sdk",
);
const sdkSrc = resolve(sdkRepo, "src");
const fixturePath = resolve(root, "skills/langbot-testing/fixtures/agent-runner/qa-runner-behaviors.json");
const fixturePath = resolve(
root,
"skills/langbot-testing/fixtures/agent-runner/qa-runner-behaviors.json",
);
const stdoutLog = join(evidenceDir, "probe-stdout.log");
const stderrLog = join(evidenceDir, "probe-stderr.log");
const automationResultJson = join(evidenceDir, "automation-result.json");
const resultJson = join(evidenceDir, "result.json");
const timeoutMs = Number(env.LANGBOT_AGENT_RUNNER_PROBE_TIMEOUT_MS || "30000");
const timeoutMs = Number(env.LANGBOT_RUNNER_PROBE_TIMEOUT_MS || "30000");
const command = {
executable: "rtk",
args: [resolve(langbotRepo, ".venv/bin/python"), "-c", script, fixturePath],
@@ -175,7 +194,12 @@ async function main() {
timeout_ms: timeoutMs,
exit_status: null,
signal: null,
evidence: { stdout_log: stdoutLog, stderr_log: stderrLog, automation_result_json: automationResultJson, result_json: resultJson },
evidence: {
stdout_log: stdoutLog,
stderr_log: stderrLog,
automation_result_json: automationResultJson,
result_json: resultJson,
},
evidence_collected: ["filesystem"],
};
try {
@@ -185,7 +209,9 @@ async function main() {
} else {
const proc = await run(command, timeoutMs, {
...process.env,
PYTHONPATH: [sdkSrc, process.env.PYTHONPATH].filter(Boolean).join(delimiter),
PYTHONPATH: [sdkSrc, process.env.PYTHONPATH]
.filter(Boolean)
.join(delimiter),
UV_CACHE_DIR: env.UV_CACHE_DIR || join(evidenceDir, ".uv-cache"),
});
await writeFile(stdoutLog, proc.stdout, "utf8");
@@ -198,7 +224,10 @@ async function main() {
} else if (proc.timedOut) {
result.status = "fail";
result.reason = `behavior matrix timed out after ${timeoutMs}ms`;
} else if (proc.status === 0 && proc.stdout.includes("QA_RUNNER_BEHAVIOR_MATRIX_OK")) {
} else if (
proc.status === 0 &&
proc.stdout.includes("QA_RUNNER_BEHAVIOR_MATRIX_OK")
) {
result.status = "pass";
result.reason = "behavior matrix passed";
} else {
@@ -219,7 +248,9 @@ async function main() {
await writeFile(resultJson, resultText, "utf8");
console.log(JSON.stringify(result, null, 2));
}
process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1);
process.exit(
result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1,
);
}
await main();
@@ -7,7 +7,11 @@ import { delimiter, join, resolve } from "node:path";
import { env } from "node:process";
function timestampSlug(date = new Date()) {
return date.toISOString().replace(/\.\d{3}Z$/, "Z").replace(/[^0-9A-Za-z]+/g, "-").replace(/^-|-$/g, "");
return date
.toISOString()
.replace(/\.\d{3}Z$/, "Z")
.replace(/[^0-9A-Za-z]+/g, "-")
.replace(/^-|-$/g, "");
}
function localIsoWithOffset(date = new Date()) {
@@ -51,7 +55,14 @@ function run(command, timeoutMs, childEnv) {
});
child.on("error", (error) => {
clearTimeout(timeout);
resolveDone({ stdout, stderr, error, timedOut, status: null, signal: null });
resolveDone({
stdout,
stderr,
error,
timedOut,
status: null,
signal: null,
});
});
child.on("close", (status, signal) => {
clearTimeout(timeout);
@@ -66,27 +77,27 @@ import importlib.util
import sys
from pathlib import Path
from langbot_plugin.api.entities.builtin.agent_runner.context import AgentRunContext
from langbot_plugin.api.entities.builtin.agent_runner.delivery import DeliveryContext
from langbot_plugin.api.entities.builtin.agent_runner.event import AgentEventContext
from langbot_plugin.api.entities.builtin.agent_runner.input import AgentInput
from langbot_plugin.api.entities.builtin.agent_runner.resources import AgentResources
from langbot_plugin.api.entities.builtin.agent_runner.runtime import AgentRuntimeContext
from langbot_plugin.api.entities.builtin.agent_runner.trigger import AgentTrigger
from langbot_plugin.api.entities.builtin.runner.context import RunnerContext
from langbot_plugin.api.entities.builtin.runner.delivery import DeliveryContext
from langbot_plugin.api.entities.builtin.runner.event import AgentEventContext
from langbot_plugin.api.entities.builtin.runner.input import AgentInput
from langbot_plugin.api.entities.builtin.runner.resources import AgentResources
from langbot_plugin.api.entities.builtin.runner.runtime import AgentRuntimeContext
from langbot_plugin.api.entities.builtin.runner.trigger import AgentTrigger
fixture = Path(sys.argv[1])
runner_py = fixture / "components" / "agent_runner" / "default.py"
runner_py = fixture / "components" / "runner" / "default.py"
manifest = fixture / "manifest.yaml"
runner_yaml = fixture / "components" / "agent_runner" / "default.yaml"
runner_yaml = fixture / "components" / "runner" / "default.yaml"
assert manifest.exists(), manifest
assert runner_yaml.exists(), runner_yaml
spec = importlib.util.spec_from_file_location("qa_agent_runner_fixture", runner_py)
spec = importlib.util.spec_from_file_location("qa_runner_fixture", runner_py)
module = importlib.util.module_from_spec(spec)
assert spec and spec.loader
spec.loader.exec_module(module)
def context(run_id, text):
return AgentRunContext(
return RunnerContext(
run_id=run_id,
trigger=AgentTrigger(type="message.received", source="webui"),
event=AgentEventContext(event_id=f"evt-{run_id}", event_type="message.received", source="webui"),
@@ -97,7 +108,7 @@ def context(run_id, text):
)
async def collect(text):
runner = module.DefaultAgentRunner()
runner = module.DefaultRunner()
results = []
async for result in runner.run(context(f"run-{len(text)}", text)):
results.append(result)
@@ -107,17 +118,17 @@ async def main():
normal = await collect("hello")
assert len(normal) == 1, normal
assert normal[0].type.value == "run.completed"
assert normal[0].data["message"]["content"] == "QA_AGENT_RUNNER_OK:hello"
assert normal[0].data["message"]["content"] == "QA_RUNNER_OK:hello"
stream = await collect("stream hello")
assert [item.type.value for item in stream] == ["message.delta", "message.delta", "message.delta", "run.completed"]
assert "".join(item.data["chunk"]["content"] for item in stream[:3]) == "QA_AGENT_RUNNER_OK:stream hello"
assert "".join(item.data["chunk"]["content"] for item in stream[:3]) == "QA_RUNNER_OK:stream hello"
failed = await collect("please fail")
assert len(failed) == 1
assert failed[0].type.value == "run.failed"
assert failed[0].data["error"] == "QA_AGENT_RUNNER_CONTROLLED_FAILURE"
print("QA_AGENT_RUNNER_FIXTURE_CONTRACT_OK")
assert failed[0].data["error"] == "QA_RUNNER_CONTROLLED_FAILURE"
print("QA_RUNNER_FIXTURE_CONTRACT_OK")
asyncio.run(main())
`;
@@ -126,18 +137,30 @@ async function main() {
const root = resolve(env.LBS_ROOT || process.cwd());
const caseId = "agent-runner-fixture-contract";
const runId = env.LBS_RUN_ID || `${timestampSlug()}-${caseId}`;
const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId));
const evidenceDir = resolve(
env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId),
);
await mkdir(evidenceDir, { recursive: true });
const startedAt = new Date();
const sdkRepo = resolve(root, env.LANGBOT_PLUGIN_SDK_REPO || "../../langbot-plugin-sdk");
const sdkRepo = resolve(
root,
env.LANGBOT_PLUGIN_SDK_REPO || "../../langbot-plugin-sdk",
);
const sdkSrc = resolve(sdkRepo, "src");
const fixturePath = resolve(root, "skills/langbot-testing/fixtures/plugins/qa-agent-runner");
const fixturePath = resolve(
root,
"skills/langbot-testing/fixtures/plugins/qa-agent-runner",
);
const stdoutLog = join(evidenceDir, "probe-stdout.log");
const stderrLog = join(evidenceDir, "probe-stderr.log");
const automationResultJson = join(evidenceDir, "automation-result.json");
const resultJson = join(evidenceDir, "result.json");
const timeoutMs = Number(env.LANGBOT_AGENT_RUNNER_PROBE_TIMEOUT_MS || "30000");
const command = { executable: "rtk", args: ["uv", "run", "--no-sync", "python", "-c", script, fixturePath], cwd: sdkRepo };
const timeoutMs = Number(env.LANGBOT_RUNNER_PROBE_TIMEOUT_MS || "30000");
const command = {
executable: "rtk",
args: ["uv", "run", "--no-sync", "python", "-c", script, fixturePath],
cwd: sdkRepo,
};
const result = {
source: "automation",
probe: "agent-runner-fixture-contract",
@@ -156,7 +179,12 @@ async function main() {
timeout_ms: timeoutMs,
exit_status: null,
signal: null,
evidence: { stdout_log: stdoutLog, stderr_log: stderrLog, automation_result_json: automationResultJson, result_json: resultJson },
evidence: {
stdout_log: stdoutLog,
stderr_log: stderrLog,
automation_result_json: automationResultJson,
result_json: resultJson,
},
evidence_collected: ["filesystem"],
};
try {
@@ -166,7 +194,9 @@ async function main() {
} else {
const proc = await run(command, timeoutMs, {
...process.env,
PYTHONPATH: [sdkSrc, process.env.PYTHONPATH].filter(Boolean).join(delimiter),
PYTHONPATH: [sdkSrc, process.env.PYTHONPATH]
.filter(Boolean)
.join(delimiter),
UV_CACHE_DIR: env.UV_CACHE_DIR || join(evidenceDir, ".uv-cache"),
});
await writeFile(stdoutLog, proc.stdout, "utf8");
@@ -179,9 +209,12 @@ async function main() {
} else if (proc.timedOut) {
result.status = "fail";
result.reason = `fixture contract probe timed out after ${timeoutMs}ms`;
} else if (proc.status === 0 && proc.stdout.includes("QA_AGENT_RUNNER_FIXTURE_CONTRACT_OK")) {
} else if (
proc.status === 0 &&
proc.stdout.includes("QA_RUNNER_FIXTURE_CONTRACT_OK")
) {
result.status = "pass";
result.reason = "QA AgentRunner fixture contract passed";
result.reason = "QA Runner fixture contract passed";
} else {
result.status = "fail";
result.reason = `fixture contract exited with status ${proc.status}`;
@@ -200,7 +233,9 @@ async function main() {
await writeFile(resultJson, resultText, "utf8");
console.log(JSON.stringify(result, null, 2));
}
process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1);
process.exit(
result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1,
);
}
await main();
@@ -8,7 +8,8 @@ await runPytestProbe({
defaultRepo: "..",
pythonPathEnvKeys: ["LANGBOT_PLUGIN_SDK_REPO"],
defaultPythonPaths: ["../../langbot-plugin-sdk/src"],
description: "LangBot AgentRunner run ledger claim, lease, authorization, and runtime-admin pytest probe.",
description:
"LangBot Runner run ledger claim, lease, authorization, and runtime-admin pytest probe.",
testTargets: [
"tests/unit_tests/agent/test_run_ledger_store.py::test_create_queued_run_claim_renew_release",
"tests/unit_tests/agent/test_run_ledger_store.py::test_expired_claim_can_be_reclaimed",
@@ -7,7 +7,11 @@ import { delimiter, join, resolve } from "node:path";
import { env } from "node:process";
function timestampSlug(date = new Date()) {
return date.toISOString().replace(/\.\d{3}Z$/, "Z").replace(/[^0-9A-Za-z]+/g, "-").replace(/^-|-$/g, "");
return date
.toISOString()
.replace(/\.\d{3}Z$/, "Z")
.replace(/[^0-9A-Za-z]+/g, "-")
.replace(/^-|-$/g, "");
}
function localIsoWithOffset(date = new Date()) {
@@ -51,7 +55,14 @@ function run(command, timeoutMs, childEnv) {
});
child.on("error", (error) => {
clearTimeout(timeout);
resolveDone({ stdout, stderr, error, timedOut, status: null, signal: null });
resolveDone({
stdout,
stderr,
error,
timedOut,
status: null,
signal: null,
});
});
child.on("close", (status, signal) => {
clearTimeout(timeout);
@@ -149,18 +160,23 @@ async function main() {
const root = resolve(env.LBS_ROOT || process.cwd());
const caseId = "agent-runner-ledger-contention";
const runId = env.LBS_RUN_ID || `${timestampSlug()}-${caseId}`;
const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId));
const evidenceDir = resolve(
env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId),
);
await mkdir(evidenceDir, { recursive: true });
const startedAt = new Date();
const langbotRepo = resolve(root, env.LANGBOT_REPO || "..");
const sdkRepo = resolve(root, env.LANGBOT_PLUGIN_SDK_REPO || "../../langbot-plugin-sdk");
const sdkRepo = resolve(
root,
env.LANGBOT_PLUGIN_SDK_REPO || "../../langbot-plugin-sdk",
);
const sdkSrc = resolve(sdkRepo, "src");
const dbPath = join(evidenceDir, "ledger-contention.sqlite3");
const stdoutLog = join(evidenceDir, "probe-stdout.log");
const stderrLog = join(evidenceDir, "probe-stderr.log");
const automationResultJson = join(evidenceDir, "automation-result.json");
const resultJson = join(evidenceDir, "result.json");
const timeoutMs = Number(env.LANGBOT_AGENT_RUNNER_PROBE_TIMEOUT_MS || "30000");
const timeoutMs = Number(env.LANGBOT_RUNNER_PROBE_TIMEOUT_MS || "30000");
const command = {
executable: "rtk",
args: [resolve(langbotRepo, ".venv/bin/python"), "-c", script, dbPath],
@@ -185,7 +201,13 @@ async function main() {
timeout_ms: timeoutMs,
exit_status: null,
signal: null,
evidence: { stdout_log: stdoutLog, stderr_log: stderrLog, database: dbPath, automation_result_json: automationResultJson, result_json: resultJson },
evidence: {
stdout_log: stdoutLog,
stderr_log: stderrLog,
database: dbPath,
automation_result_json: automationResultJson,
result_json: resultJson,
},
evidence_collected: ["filesystem"],
};
try {
@@ -195,7 +217,9 @@ async function main() {
} else {
const proc = await run(command, timeoutMs, {
...process.env,
PYTHONPATH: [sdkSrc, process.env.PYTHONPATH].filter(Boolean).join(delimiter),
PYTHONPATH: [sdkSrc, process.env.PYTHONPATH]
.filter(Boolean)
.join(delimiter),
UV_CACHE_DIR: env.UV_CACHE_DIR || join(evidenceDir, ".uv-cache"),
});
await writeFile(stdoutLog, proc.stdout, "utf8");
@@ -208,7 +232,10 @@ async function main() {
} else if (proc.timedOut) {
result.status = "fail";
result.reason = `ledger contention timed out after ${timeoutMs}ms`;
} else if (proc.status === 0 && proc.stdout.includes("LEDGER_CONTENTION_OK")) {
} else if (
proc.status === 0 &&
proc.stdout.includes("LEDGER_CONTENTION_OK")
) {
result.status = "pass";
result.reason = "ledger contention probe passed";
} else {
@@ -229,7 +256,9 @@ async function main() {
await writeFile(resultJson, resultText, "utf8");
console.log(JSON.stringify(result, null, 2));
}
process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1);
process.exit(
result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1,
);
}
await main();
@@ -7,7 +7,11 @@ import { delimiter, join, resolve } from "node:path";
import { env } from "node:process";
function timestampSlug(date = new Date()) {
return date.toISOString().replace(/\.\d{3}Z$/, "Z").replace(/[^0-9A-Za-z]+/g, "-").replace(/^-|-$/g, "");
return date
.toISOString()
.replace(/\.\d{3}Z$/, "Z")
.replace(/[^0-9A-Za-z]+/g, "-")
.replace(/^-|-$/g, "");
}
function localIsoWithOffset(date = new Date()) {
@@ -55,7 +59,14 @@ function runProcess(command, timeoutMs, childEnv) {
});
child.on("error", (error) => {
clearTimeout(timeout);
resolveDone({ stdout, stderr, error, timedOut, status: null, signal: null });
resolveDone({
stdout,
stderr,
error,
timedOut,
status: null,
signal: null,
});
});
child.on("close", (status, signal) => {
clearTimeout(timeout);
@@ -124,11 +135,16 @@ async function main() {
const root = resolve(env.LBS_ROOT || process.cwd());
const caseId = "agent-runner-ledger-invariants";
const runId = env.LBS_RUN_ID || `${timestampSlug()}-${caseId}`;
const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId));
const evidenceDir = resolve(
env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId),
);
await mkdir(evidenceDir, { recursive: true });
const startedAt = new Date();
const langbotRepo = resolveFromRoot(root, env.LANGBOT_REPO || "..");
const sdkRepo = resolveFromRoot(root, env.LANGBOT_PLUGIN_SDK_REPO || "../../langbot-plugin-sdk");
const sdkRepo = resolveFromRoot(
root,
env.LANGBOT_PLUGIN_SDK_REPO || "../../langbot-plugin-sdk",
);
const sdkSrc = resolve(sdkRepo, "src");
const stdoutLog = join(evidenceDir, "probe-stdout.log");
const stderrLog = join(evidenceDir, "probe-stderr.log");
@@ -139,7 +155,7 @@ async function main() {
args: [resolve(langbotRepo, ".venv/bin/python"), "-c", probeScript],
cwd: langbotRepo,
};
const timeoutMs = Number(env.LANGBOT_AGENT_RUNNER_PROBE_TIMEOUT_MS || "30000");
const timeoutMs = Number(env.LANGBOT_RUNNER_PROBE_TIMEOUT_MS || "30000");
const result = {
source: "automation",
probe: "python-sync",
@@ -174,7 +190,9 @@ async function main() {
} else {
const childEnv = {
...process.env,
PYTHONPATH: [sdkSrc, process.env.PYTHONPATH].filter(Boolean).join(delimiter),
PYTHONPATH: [sdkSrc, process.env.PYTHONPATH]
.filter(Boolean)
.join(delimiter),
UV_CACHE_DIR: env.UV_CACHE_DIR || join(evidenceDir, ".uv-cache"),
};
await mkdir(childEnv.UV_CACHE_DIR, { recursive: true });
@@ -210,7 +228,9 @@ async function main() {
await writeFile(resultJson, resultText, "utf8");
console.log(JSON.stringify(result, null, 2));
}
process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1);
process.exit(
result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1,
);
}
await main();
@@ -7,7 +7,11 @@ import { delimiter, join, resolve } from "node:path";
import { env } from "node:process";
function timestampSlug(date = new Date()) {
return date.toISOString().replace(/\.\d{3}Z$/, "Z").replace(/[^0-9A-Za-z]+/g, "-").replace(/^-|-$/g, "");
return date
.toISOString()
.replace(/\.\d{3}Z$/, "Z")
.replace(/[^0-9A-Za-z]+/g, "-")
.replace(/^-|-$/g, "");
}
function localIsoWithOffset(date = new Date()) {
@@ -51,7 +55,14 @@ function run(command, timeoutMs, childEnv) {
});
child.on("error", (error) => {
clearTimeout(timeout);
resolveDone({ stdout, stderr, error, timedOut, status: null, signal: null });
resolveDone({
stdout,
stderr,
error,
timedOut,
status: null,
signal: null,
});
});
child.on("close", (status, signal) => {
clearTimeout(timeout);
@@ -116,17 +127,22 @@ async function main() {
const root = resolve(env.LBS_ROOT || process.cwd());
const caseId = "agent-runner-ledger-stress";
const runId = env.LBS_RUN_ID || `${timestampSlug()}-${caseId}`;
const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId));
const evidenceDir = resolve(
env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId),
);
await mkdir(evidenceDir, { recursive: true });
const startedAt = new Date();
const langbotRepo = resolve(root, env.LANGBOT_REPO || "..");
const sdkRepo = resolve(root, env.LANGBOT_PLUGIN_SDK_REPO || "../../langbot-plugin-sdk");
const sdkRepo = resolve(
root,
env.LANGBOT_PLUGIN_SDK_REPO || "../../langbot-plugin-sdk",
);
const sdkSrc = resolve(sdkRepo, "src");
const stdoutLog = join(evidenceDir, "probe-stdout.log");
const stderrLog = join(evidenceDir, "probe-stderr.log");
const automationResultJson = join(evidenceDir, "automation-result.json");
const resultJson = join(evidenceDir, "result.json");
const timeoutMs = Number(env.LANGBOT_AGENT_RUNNER_PROBE_TIMEOUT_MS || "30000");
const timeoutMs = Number(env.LANGBOT_RUNNER_PROBE_TIMEOUT_MS || "30000");
const command = {
executable: "rtk",
args: [resolve(langbotRepo, ".venv/bin/python"), "-c", script],
@@ -150,7 +166,12 @@ async function main() {
timeout_ms: timeoutMs,
exit_status: null,
signal: null,
evidence: { stdout_log: stdoutLog, stderr_log: stderrLog, automation_result_json: automationResultJson, result_json: resultJson },
evidence: {
stdout_log: stdoutLog,
stderr_log: stderrLog,
automation_result_json: automationResultJson,
result_json: resultJson,
},
evidence_collected: ["filesystem"],
};
try {
@@ -160,7 +181,9 @@ async function main() {
} else {
const proc = await run(command, timeoutMs, {
...process.env,
PYTHONPATH: [sdkSrc, process.env.PYTHONPATH].filter(Boolean).join(delimiter),
PYTHONPATH: [sdkSrc, process.env.PYTHONPATH]
.filter(Boolean)
.join(delimiter),
UV_CACHE_DIR: env.UV_CACHE_DIR || join(evidenceDir, ".uv-cache"),
});
await writeFile(stdoutLog, proc.stdout, "utf8");
@@ -173,7 +196,10 @@ async function main() {
} else if (proc.timedOut) {
result.status = "fail";
result.reason = `ledger stress timed out after ${timeoutMs}ms`;
} else if (proc.status === 0 && proc.stdout.includes("LEDGER_STRESS_OK")) {
} else if (
proc.status === 0 &&
proc.stdout.includes("LEDGER_STRESS_OK")
) {
result.status = "pass";
result.reason = "ledger stress probe passed";
} else {
@@ -194,7 +220,9 @@ async function main() {
await writeFile(resultJson, resultText, "utf8");
console.log(JSON.stringify(result, null, 2));
}
process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1);
process.exit(
result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1,
);
}
await main();
@@ -6,9 +6,10 @@ await runPytestProbe({
caseId: "agent-runner-runtime-chaos",
repoEnvKey: "LANGBOT_PLUGIN_SDK_REPO",
defaultRepo: "../../langbot-plugin-sdk",
description: "LangBot plugin SDK AgentRunner runtime failure, timeout, forwarding, and pull API pytest probe.",
description:
"LangBot plugin SDK Runner runtime failure, timeout, forwarding, and pull API pytest probe.",
testTargets: [
"tests/runtime/plugin/test_mgr_agent_runner.py",
"tests/runtime/plugin/test_mgr_runner.py",
"tests/runtime/test_pull_api_handlers.py",
],
});
@@ -5,7 +5,10 @@ import { basename, delimiter, join, resolve } from "node:path";
import { env } from "node:process";
function loadEnvDefaults(root) {
for (const path of [join(root, "skills/.env"), join(root, "skills/.env.local")]) {
for (const path of [
join(root, "skills/.env"),
join(root, "skills/.env.local"),
]) {
if (!existsSync(path)) continue;
for (const rawLine of readFileSync(path, "utf8").split(/\r?\n/)) {
const line = rawLine.trim();
@@ -14,13 +17,20 @@ function loadEnvDefaults(root) {
if (sep === -1) continue;
const key = line.slice(0, sep).trim();
if (env[key]) continue;
env[key] = line.slice(sep + 1).trim().replace(/^["']|["']$/g, "");
env[key] = line
.slice(sep + 1)
.trim()
.replace(/^["']|["']$/g, "");
}
}
}
function timestampSlug(date = new Date()) {
return date.toISOString().replace(/\.\d{3}Z$/, "Z").replace(/[^0-9A-Za-z]+/g, "-").replace(/^-|-$/g, "");
return date
.toISOString()
.replace(/\.\d{3}Z$/, "Z")
.replace(/[^0-9A-Za-z]+/g, "-")
.replace(/^-|-$/g, "");
}
function localIsoWithOffset(date = new Date()) {
@@ -88,7 +98,14 @@ async function runProcess(command, timeoutMs, childEnv) {
});
child.on("error", (error) => {
clearTimeout(timeout);
resolveDone({ stdout, stderr, error, timedOut, status: null, signal: null });
resolveDone({
stdout,
stderr,
error,
timedOut,
status: null,
signal: null,
});
});
child.on("close", (status, signal) => {
clearTimeout(timeout);
@@ -109,10 +126,14 @@ export async function runPytestProbe({
}) {
const root = resolve(env.LBS_ROOT || process.cwd());
loadEnvDefaults(root);
const resolvedTimeoutMs = Number(timeoutMs || env.LANGBOT_AGENT_RUNNER_PROBE_TIMEOUT_MS || "180000");
const resolvedTimeoutMs = Number(
timeoutMs || env.LANGBOT_RUNNER_PROBE_TIMEOUT_MS || "180000",
);
const runId = env.LBS_RUN_ID || `${timestampSlug()}-${caseId}`;
const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId));
const evidenceDir = resolve(
env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId),
);
await mkdir(evidenceDir, { recursive: true });
const uvCacheDir = env.UV_CACHE_DIR || join(evidenceDir, ".uv-cache");
await mkdir(uvCacheDir, { recursive: true });
@@ -171,14 +192,21 @@ export async function runPytestProbe({
result.status = "env_issue";
result.reason = `${repoEnvKey || "repo"} did not resolve to an existing directory: ${repoPath}`;
} else {
const missingTargets = testTargets.filter((target) => !existsSync(join(repoPath, target.split("::")[0])));
const missingTargets = testTargets.filter(
(target) => !existsSync(join(repoPath, target.split("::")[0])),
);
if (missingTargets.length > 0) {
result.status = "env_issue";
result.reason = `pytest target file(s) not found in ${basename(repoPath)}: ${missingTargets.join(", ")}`;
} else {
const childEnv = { ...process.env, UV_CACHE_DIR: uvCacheDir };
if (pythonPaths.length > 0) {
childEnv.PYTHONPATH = [pythonPaths.join(delimiter), childEnv.PYTHONPATH].filter(Boolean).join(delimiter);
childEnv.PYTHONPATH = [
pythonPaths.join(delimiter),
childEnv.PYTHONPATH,
]
.filter(Boolean)
.join(delimiter);
}
const proc = await runProcess(command, resolvedTimeoutMs, childEnv);
result.exit_status = proc.status;
@@ -195,7 +223,11 @@ export async function runPytestProbe({
} else if (proc.status === 0) {
result.status = "pass";
result.reason = `pytest passed for ${testTargets.join(", ")}.`;
} else if (/command not found|no such file or directory|executable file not found/i.test(`${proc.stdout}\n${proc.stderr}`)) {
} else if (
/command not found|no such file or directory|executable file not found/i.test(
`${proc.stdout}\n${proc.stderr}`,
)
) {
result.status = "env_issue";
result.reason = `pytest command could not run in ${repoPath}. See ${stdoutLog} and ${stderrLog}.`;
} else {
@@ -1,6 +1,6 @@
# AgentRunner QA Workflow
# Runner QA Workflow
Use this workflow when an agent finishes AgentRunner-related code and enters a
Use this workflow when an agent finishes Runner-related code and enters a
test phase.
## Order
@@ -25,7 +25,7 @@ test phase.
backend is available and installing the QA fixture is acceptable.
- `rtk bin/lbs test run agent-runner-qa-debug-chat --dry-run` when WebUI live
execution needs deterministic coverage without a model provider. This
case runs its setup automation first: install the QA AgentRunner fixture,
case runs its setup automation first: install the QA Runner fixture,
create/update the QA pipeline, write the case-specific pipeline env, then
execute Debug Chat.
- `rtk bin/lbs test run agent-runner-ledger-invariants --dry-run`
@@ -51,8 +51,8 @@ only to review or adjust the generated list.
| --- | --- | --- |
| `LangBot/src/langbot/pkg/agent/runner/*`, `tests/unit_tests/agent/test_result_normalizer.py`, protocol/result/context/resource builders | `rtk bin/lbs test run agent-runner-fixture-contract --dry-run`; `rtk bin/lbs test run agent-runner-behavior-matrix --dry-run`; targeted LangBot unit tests for touched files | Result shape, user-visible runner output, or Debug Chat delivery changed: add `pipeline-debug-chat` or `local-agent-basic-debug-chat`. |
| `LangBot/src/langbot/pkg/entity/persistence/agent_run.py`, `run_journal.py`, run ledger store/API/auth tests, claim/lease/status code | `rtk bin/lbs test run agent-runner-ledger-invariants --dry-run`; `rtk bin/lbs test run agent-runner-ledger-stress --dry-run`; `rtk bin/lbs test run agent-runner-ledger-contention --dry-run`; `rtk bin/lbs test run agent-runner-async-db-readiness --dry-run` before `rtk bin/lbs test run agent-runner-ledger-concurrency --dry-run` | Debug Chat run lifecycle, resume, or visible completion changed: add `local-agent-basic-debug-chat`. |
| `langbot-plugin-sdk/src/langbot_plugin/api/entities/builtin/agent_runner/*`, `api/proxies/agent_run_api.py`, runtime pull handlers, plugin manager/runtime IO | `rtk bin/lbs test run agent-runner-runtime-chaos --dry-run`; `rtk bin/lbs test run agent-runner-behavior-matrix --dry-run`; targeted SDK pytest | Runtime delivery or tool-call surface changed: add `agent-runner-release-preflight`, then `local-agent-basic-debug-chat`. |
| `langbot-agent-runner/*/components/agent_runner/*`, external runner daemon/client code, ACP/Codex/Claude runner command wrappers | Repo-local targeted tests; `rtk bin/lbs test run agent-runner-runtime-chaos --dry-run`; `rtk bin/lbs test run agent-runner-release-preflight --dry-run` | ACP or external coding runner behavior changed: add `acp-agent-runner-debug-chat`. |
| `langbot-plugin-sdk/src/langbot_plugin/api/entities/builtin/runner/*`, `api/proxies/agent_run_api.py`, runtime pull handlers, plugin manager/runtime IO | `rtk bin/lbs test run agent-runner-runtime-chaos --dry-run`; `rtk bin/lbs test run agent-runner-behavior-matrix --dry-run`; targeted SDK pytest | Runtime delivery or tool-call surface changed: add `agent-runner-release-preflight`, then `local-agent-basic-debug-chat`. |
| `langbot-agent-runner/*/components/runner/*`, external runner daemon/client code, ACP/Codex/Claude runner command wrappers | Repo-local targeted tests; `rtk bin/lbs test run agent-runner-runtime-chaos --dry-run`; `rtk bin/lbs test run agent-runner-release-preflight --dry-run` | ACP or external coding runner behavior changed: add `acp-agent-runner-debug-chat`. |
| Prompt preprocessing, effective prompt, pipeline AI config, runner binding/default runner migration | `rtk bin/lbs test run agent-runner-behavior-matrix --dry-run`; targeted LangBot pipeline/agent tests | The runner reads host-provided prompt or saved runner config: add `local-agent-effective-prompt-debug-chat`. |
| Context window, transcript, history/event state, compaction, checkpoint/steering | `rtk bin/lbs test run agent-runner-behavior-matrix --dry-run`; targeted LangBot agent state/context tests | Multi-turn memory, compaction, or steering behavior changed: add `local-agent-context-compaction-debug-chat` and, for steering-specific changes, `local-agent-steering-debug-chat`. |
| Plugin tool authorization, host tool listing, MCP tool bridge, function-call conversion | `rtk bin/lbs test run agent-runner-behavior-matrix --dry-run`; targeted plugin/MCP/tool tests | Tool execution is user-visible: add `local-agent-plugin-tool-call-debug-chat`; for MCP-specific changes add `mcp-stdio-register` then `mcp-stdio-tool-call`. |
@@ -1,4 +1,4 @@
# Agent Runner Release Gate
# Runner Release Gate
Use this reference when judging whether runner externalization is release-ready. The goal is not to enumerate every possible prompt. The gate covers product abilities and trust boundaries with deterministic normal-path cases, then leaves rare negative branches to unit and contract tests.
@@ -35,7 +35,7 @@ For a quick early blocker check, run:
rtk bin/lbs test run agent-runner-release-preflight --dry-run
```
For the code-level AgentRunner probes, run:
For the code-level Runner probes, run:
```bash
rtk bin/lbs test run agent-runner-behavior-matrix --dry-run
@@ -70,7 +70,7 @@ API integration gate, not a Debug Chat execution proof.
`agent-runner-qa-debug-chat` is the deterministic live execution proof. It uses
a pipeline created by `scripts/e2e/ensure-qa-agent-runner-pipeline.mjs` and
expects Debug Chat to return `QA_AGENT_RUNNER_OK:<input>` through
expects Debug Chat to return `QA_RUNNER_OK:<input>` through
`plugin:qa/agent-runner/default`.
`agent-runner-ledger-invariants` is the fast Host ledger probe. It uses
@@ -97,7 +97,7 @@ If it times out before any test result and a direct `aiosqlite.connect()` script
also hangs, classify the run with troubleshooting id
`aiosqlite-connect-hangs` instead of treating it as a browser E2E failure.
`agent-runner-runtime-chaos` runs SDK AgentRunner runtime and pull API handler
`agent-runner-runtime-chaos` runs SDK Runner runtime and pull API handler
tests from `LANGBOT_PLUGIN_SDK_REPO` or `../langbot-plugin-sdk`.
Each probe writes `automation-result.json` and probe logs under
`LBS_EVIDENCE_DIR`.
@@ -108,9 +108,9 @@ Each probe writes `automation-result.json` and probe logs under
| --- | --- | --- |
| Authenticated WebUI session | `webui-login-state`, `agent-runner-release-preflight` | The browser profile can operate the same backend that later cases use. |
| Generic Pipeline Debug Chat | `pipeline-debug-chat` | The WebUI Debug Chat path itself works before runner-specific failures are diagnosed. |
| Deterministic QA runner install | `agent-runner-live-install` | A local `.lbpkg` AgentRunner package can install and register a runner. |
| Deterministic QA runner install | `agent-runner-live-install` | A local `.lbpkg` Runner package can install and register a runner. |
| Deterministic QA runner Debug Chat | `agent-runner-qa-debug-chat` | The installed QA runner executes through WebUI Debug Chat without a model provider. |
| Required runner plugins | `agent-runner-release-preflight` | `langbot-team/LocalAgent` and `langbot-team/ACPAgentRunner` are visible to the host. |
| Required runner plugins | `agent-runner-release-preflight` | `langbot-team/LocalAgent` and `langbot-team/ACPRunner` are visible to the host. |
| Required QA plugin tools | `plugin-e2e-smoke`, `agent-runner-release-preflight`, `qa-plugin-smoke-live-install` | The deterministic `qa_plugin_echo` and `qa_plugin_fail` tools are exposed before tool-loop and tool-error cases start. |
| Knowledge base fixture | `langrag-kb-retrieve`, `local-agent-rag-debug-chat` | LangRAG data is queryable and the runner inserts retrieved context. |
| Effective prompt bridge | `local-agent-effective-prompt-debug-chat` | Host prompt preprocessing reaches the runner. |
@@ -149,7 +149,7 @@ rtk uv run pytest -q
# langbot-plugin-sdk
rtk uv run pytest -q
# langbot-skills saved AgentRunner probes
# langbot-skills saved Runner probes
rtk bin/lbs test run agent-runner-behavior-matrix --dry-run
rtk bin/lbs test run agent-runner-ledger-invariants --dry-run
rtk bin/lbs test run agent-runner-ledger-stress --dry-run
@@ -1,4 +1,4 @@
# Dify AgentRunner
# Dify Runner
Use this reference when validating `langbot-team/DifyAgent` through LangBot WebUI.
@@ -1,4 +1,4 @@
# Local Agent Runner Coverage
# Local Runner Coverage
Use this matrix when judging whether the external `langbot-team/LocalAgent` plugin still behaves like the old built-in local-agent runner.
@@ -10,7 +10,7 @@ The QA target is end-to-end behavior. UI cases prove the host, SDK, plugin runti
- `LangBot/src/langbot/pkg/agent/runner/pipeline_adapter.py` adapts Pipeline-only fields into `ctx.adapter.extra.prompt`, `ctx.adapter.extra.params`, and optional `ctx.bootstrap.messages`.
- `LangBot/src/langbot/pkg/agent/runner/resource_builder.py` authorizes models, fallback models, rerank models, tools, and knowledge bases for the current run.
- `LangBot/src/langbot/pkg/plugin/handler.py` validates run-scoped model/tool/rerank access and calls the host model provider or tool manager with the current query.
- `langbot-local-agent/components/agent_runner/default.py` selects streaming or non-streaming execution, retrieves RAG context, builds messages, invokes models with fallback, and runs tool loops.
- `langbot-local-agent/components/runner/default.py` selects streaming or non-streaming execution, retrieves RAG context, builds messages, invokes models with fallback, and runs tool loops.
- `langbot-local-agent/pkg/messages.py` prefers the host effective prompt from `ctx.adapter.extra.prompt`, uses `ctx.bootstrap.messages` only as a small bootstrap window, and preserves structured/multimodal input while inserting RAG context.
TODO: Treat `ctx.adapter.extra.prompt` as a temporary Pipeline bridge for old
@@ -1,10 +1,10 @@
# Local Agent Runner
# Local Runner
Use this reference when validating the pluginized `langbot-team/LocalAgent` runner through the WebUI.
The goal is behavior parity with the old built-in local-agent runner. The code does not need to be identical, but the visible behavior should match: effective prompt, current input, history, model selection and fallback, tool calling, knowledge retrieval, multimodal input, streaming and non-streaming output all have to reach the runner through the host and SDK.
For path-by-path coverage, read [Local Agent Runner Coverage](local-agent-runner-coverage.md).
For path-by-path coverage, read [Local Runner Coverage](local-agent-runner-coverage.md).
## Main Surface
@@ -35,7 +35,7 @@ Measure user experience and internal composition separately:
- WebUI load and interaction latency.
- Debug Chat send-to-first-visible-token and send-to-completion latency.
- Pipeline, RAG, plugin runtime, MCP, AgentRunner, and persistence segment
- Pipeline, RAG, plugin runtime, MCP, Runner, and persistence segment
latency.
- Queue wait time, concurrency, throughput, timeout rate, and p95/p99 latency.
- Startup, plugin install, knowledge-base ingestion, migration, and recovery
@@ -33,7 +33,7 @@ Both external runners receive the same host-generated gateway `AgentMCPServerCon
This is a **runner-plugin transport detail, not a host all-tool-branch issue** — proven by **both** runners discovering skills end-to-end with the unmodified branch (see cases below).
> **Correction (2026-06-22).** An earlier revision of this doc claimed acp was "blocked" on remote-ssh and *required* `langbot-assets-gateway-public-url`, based on a run that returned `PROBEDONE 0 0` / timeout. That was an **environment artifact, not an acp defect**: a duplicate backend instance (a second checkout `LangBot-master/` whose box runtime contended for the same `--ws-control-port 5410`) plus a wedged plugin runtime (host `emit_event` / `list_agent_runners` action calls timing out with `ActionCallTimeoutError`). Re-run on a clean single-instance runtime, **acp passes via the reverse tunnel with no `public-url`** (`PROBEDONE 1 17`, 824s).
> **Correction (2026-06-22).** An earlier revision of this doc claimed acp was "blocked" on remote-ssh and *required* `langbot-assets-gateway-public-url`, based on a run that returned `PROBEDONE 0 0` / timeout. That was an **environment artifact, not an acp defect**: a duplicate backend instance (a second checkout `LangBot-master/` whose box runtime contended for the same `--ws-control-port 5410`) plus a wedged plugin runtime (host `emit_event` / `list_runners` action calls timing out with `ActionCallTimeoutError`). Re-run on a clean single-instance runtime, **acp passes via the reverse tunnel with no `public-url`** (`PROBEDONE 1 17`, 824s).
- **Lifecycle**: discover → activate → operate (native exec under the activated mount path) → register.
- **Backend**: docker · nsjail · e2b.
@@ -12,7 +12,7 @@ Date: 2026-05-16
### Symptom
The WebUI can send a Debug Chat message, but the bot response is missing or says `Agent runner temporarily unavailable`. Backend logs may include `Action list_plugins call timed out`, `Action list_agent_runners call timed out`, or `Action invoke_llm_stream call timed out`.
The WebUI can send a Debug Chat message, but the bot response is missing or says `Agent runner temporarily unavailable`. Backend logs may include `Action list_plugins call timed out`, `Action list_runners call timed out`, or `Action invoke_llm_stream call timed out`.
### Likely Cause
@@ -78,7 +78,7 @@ Structured entry: `../troubleshooting/marketplace-network-flaky.yaml`
Marketplace icon/tag/recommendation requests can fail while plugin cards are already visible. Retry first, and use backend component endpoints only to confirm installation results.
## agent-runner-actor-context-fields: AgentRunner reads old actor fields
## agent-runner-actor-context-fields: Runner reads old actor fields
Structured entry: `../troubleshooting/agent-runner-actor-context-fields.yaml`
@@ -1,6 +1,6 @@
# Workspace Release Testing
Use the workspace gates when changes span LangBot core, the plugin SDK, AgentRunner, or multiple first-party plugins.
Use the workspace gates when changes span LangBot core, the plugin SDK, Runner, or multiple first-party plugins.
## Cost Ladder
@@ -1,6 +1,6 @@
id: langbot-workspace-release-gate
title: "LangBot workspace top-down release gate"
description: "Broad release gate combining deterministic repository contracts with representative browser workflows, plugin pages, RAG/parser, EBA, external AgentRunner, and one complex LocalAgent task."
description: "Broad release gate combining deterministic repository contracts with representative browser workflows, plugin pages, RAG/parser, EBA, external Runner, and one complex LocalAgent task."
type: release_gate
priority: p0
tags:
@@ -1,9 +1,9 @@
id: agent-runner-actor-context-fields
title: "AgentRunner reads old actor.type and actor.id fields"
title: "Runner reads old actor.type and actor.id fields"
date: 2026-05-17
symptoms:
- "Pipeline Debug Chat shows Agent runner execution failed."
- "Backend logs show an AttributeError from an AgentRunner plugin."
- "Backend logs show an AttributeError from an Runner plugin."
patterns:
- "AttributeError: 'ActorContext' object has no attribute 'type'"
- "AttributeError: 'ActorContext' object has no attribute 'id'"
@@ -15,8 +15,8 @@ fix_steps:
- "Update runner code to read actor.actor_type and actor.actor_id."
- "Keep getattr fallback to type/id only if compatibility with older host data is required."
- "Restart LangBot or the plugin runtime so the updated plugin code is loaded."
- "Add a regression test that builds AgentRunContext with ActorContext(actor_type=..., actor_id=...)."
verification: "Run dify-agent-debug-chat or another AgentRunner Debug Chat and confirm the assistant/bot message contains the expected sentinel while backend logs show Streaming completed."
- "Add a regression test that builds RunnerContext with ActorContext(actor_type=..., actor_id=...)."
verification: "Run dify-agent-debug-chat or another Runner Debug Chat and confirm the assistant/bot message contains the expected sentinel while backend logs show Streaming completed."
related_cases:
- dify-agent-debug-chat
- pipeline-debug-chat
@@ -2,7 +2,7 @@ id: aiosqlite-connect-hangs
title: "aiosqlite connect hangs before ledger pytest starts"
category: env_issue
symptoms:
- "AgentRunner ledger pytest probe times out after collecting tests but before reporting a test result."
- "Runner ledger pytest probe times out after collecting tests but before reporting a test result."
- "pytest stdout stops at a line like tests/unit_tests/agent/test_run_ledger_store.py."
- "A direct aiosqlite.connect(':memory:') script prints its first line and then hangs."
patterns:
@@ -1,5 +1,5 @@
id: ambiguous-runner-default-label
title: "AgentRunner selector shows multiple Default or 默认 options"
title: "Runner selector shows multiple Default or 默认 options"
date: 2026-05-17
symptoms:
- "The Pipeline AI runner selector shows multiple options named Default or 默认."
@@ -9,7 +9,7 @@ patterns:
- "label.zh_Hans: 默认"
- "label.en_US: Default"
likely_causes:
- "AgentRunner component ids are commonly named default, but the user-facing metadata.label was also left generic."
- "Runner component ids are commonly named default, but the user-facing metadata.label was also left generic."
- "The frontend displays metadata.label as the primary option label."
fix_steps:
- "Keep metadata.name as default if the plugin component id is intended to remain stable."
@@ -8,7 +8,7 @@ symptoms:
- "Knowledge sidebar or plugin sidebar loading may hang or time out."
patterns:
- "Action list_plugins call timed out"
- "Action list_agent_runners call timed out"
- "Action list_runners call timed out"
- "Action invoke_llm_stream call timed out"
- "All models failed during streaming setup"
- "Failed to fetch plugins for sidebar"