mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-09-17 15:27:15 +00:00
feat(runner): unify plugin execution across agents and event processors
This commit is contained in:
+7
-7
@@ -37,10 +37,10 @@ LANGBOT_NO_PROXY=localhost,127.0.0.1,::1
|
||||
# LANGBOT_PIPELINE_NAME=Generic QA Pipeline
|
||||
# LANGBOT_LOCAL_AGENT_PIPELINE_URL=http://127.0.0.1:3000/home/pipelines?id=<local-agent-pipeline-uuid>
|
||||
# LANGBOT_LOCAL_AGENT_PIPELINE_NAME=Local Agent QA Pipeline
|
||||
# LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL=http://127.0.0.1:3000/home/pipelines?id=<acp-agent-runner-pipeline-uuid>
|
||||
# LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME=ACP AgentRunner QA Pipeline
|
||||
# LANGBOT_ACP_AGENT_RUNNER_SSH_TARGET=yhh@101.34.71.12
|
||||
# LANGBOT_ACP_AGENT_RUNNER_SSH_PORT=22
|
||||
# LANGBOT_ACP_AGENT_RUNNER_SSH_IDENTITY_FILE=
|
||||
# LANGBOT_ACP_AGENT_RUNNER_SSH_EXTRA_OPTIONS=
|
||||
# LANGBOT_ACP_AGENT_RUNNER_REMOTE_WORKSPACE=/home/yhh/langbot-e2e/acp-workspace
|
||||
# LANGBOT_ACP_RUNNER_PIPELINE_URL=http://127.0.0.1:3000/home/pipelines?id=<acp-agent-runner-pipeline-uuid>
|
||||
# LANGBOT_ACP_RUNNER_PIPELINE_NAME=ACP Runner QA Pipeline
|
||||
# LANGBOT_ACP_RUNNER_SSH_TARGET=yhh@101.34.71.12
|
||||
# LANGBOT_ACP_RUNNER_SSH_PORT=22
|
||||
# LANGBOT_ACP_RUNNER_SSH_IDENTITY_FILE=
|
||||
# LANGBOT_ACP_RUNNER_SSH_EXTRA_OPTIONS=
|
||||
# LANGBOT_ACP_RUNNER_REMOTE_WORKSPACE=/home/yhh/langbot-e2e/acp-workspace
|
||||
|
||||
@@ -66,7 +66,7 @@ The tools wrap the LangBot service layer. Current tools (v1):
|
||||
| `list_bots` / `get_bot` / `create_bot` / `update_bot` / `delete_bot` | Manage messaging-platform bots (secrets redacted on read) |
|
||||
| `list_bot_event_route_statuses` | Inspect bot event-route runtime status |
|
||||
| `list_processors` / `get_processor` / `create_processor` / `update_processor` / `delete_processor` | Manage the peer Agent, Pipeline and Event processor types |
|
||||
| `get_processor_metadata` | Discover installed EventProcessor components, schemas and supported event patterns. |
|
||||
| `get_processor_metadata` | Discover installed event-capable Runner components, schemas and supported event patterns. |
|
||||
| `list_processor_runs` / `get_processor_run_events` | Read one Event processor instance run history and logs; paginate with `before_id` / `after_sequence`. |
|
||||
| `debug_agent` | Execute a synthetic Agent event (`processor_uuid`, `payload`); requires `runtime.operate`. Returns final text and up to 1000 execution events (thinking, text, tool arguments/results). Platform tools use Mock; other configured tools execute normally. Optional `payload.mock`: `errors`/`results` keyed by platform tool name, `unsupported_apis` lists unavailable platform APIs. |
|
||||
| `list_pipelines` / `get_pipeline` / `create_pipeline` / `update_pipeline` / `delete_pipeline` | Manage pipelines |
|
||||
|
||||
@@ -11,13 +11,13 @@ Use this skill when an agent needs to verify LangBot behavior through the WebUI
|
||||
|
||||
- **General WebUI testing**: read `references/web-ui-testing.md`.
|
||||
- **Pipeline Debug Chat**: read `references/pipeline-debug-chat.md`.
|
||||
- **Dify AgentRunner**: read `references/dify-agent-runner.md`.
|
||||
- **Dify Runner**: read `references/dify-agent-runner.md`.
|
||||
- **Model provider setup or test button**: read `references/model-provider-testing.md`.
|
||||
- **Plugin install/runtime/tool/page smoke**: read `references/plugin-e2e-smoke.md`.
|
||||
- **Local Agent Runner**: read `references/local-agent-runner.md`.
|
||||
- **Local Agent Runner path coverage**: read `references/local-agent-runner-coverage.md`.
|
||||
- **Diff-aware AgentRunner QA after code changes**: read `references/agent-runner-qa-workflow.md`.
|
||||
- **Agent Runner release gate**: read `references/agent-runner-release-gate.md`.
|
||||
- **Local Runner**: read `references/local-agent-runner.md`.
|
||||
- **Local Runner path coverage**: read `references/local-agent-runner-coverage.md`.
|
||||
- **Diff-aware Runner QA after code changes**: read `references/agent-runner-qa-workflow.md`.
|
||||
- **Runner release gate**: read `references/agent-runner-release-gate.md`.
|
||||
- **Sandbox-backed skill authoring**: read `references/sandbox-skill-authoring.md`.
|
||||
- **LangRAG knowledge bases**: read `references/langrag-knowledge-base.md`.
|
||||
- **MCP stdio tool testing**: read `references/mcp-stdio-testing.md`.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
id: acp-agent-runner-debug-chat
|
||||
title: "ACP AgentRunner can answer through Debug Chat using real remote Claude"
|
||||
title: "ACP Runner can answer through Debug Chat using real remote Claude"
|
||||
mode: agent-browser
|
||||
area: pipeline
|
||||
type: regression
|
||||
@@ -19,42 +19,42 @@ env:
|
||||
- LANGBOT_FRONTEND_URL
|
||||
- LANGBOT_BACKEND_URL
|
||||
env_any:
|
||||
- LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL|LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME
|
||||
- LANGBOT_ACP_RUNNER_PIPELINE_URL|LANGBOT_ACP_RUNNER_PIPELINE_NAME
|
||||
automation: scripts/e2e/pipeline-debug-chat.mjs
|
||||
automation_env:
|
||||
- LANGBOT_FRONTEND_URL
|
||||
- LANGBOT_BACKEND_URL
|
||||
- LANGBOT_BROWSER_PROFILE
|
||||
- LANGBOT_CHROMIUM_EXECUTABLE
|
||||
- LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL
|
||||
- LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME
|
||||
- LANGBOT_ACP_RUNNER_PIPELINE_URL
|
||||
- LANGBOT_ACP_RUNNER_PIPELINE_NAME
|
||||
- LANGBOT_E2E_PROMPT
|
||||
- LANGBOT_E2E_EXPECTED_TEXT
|
||||
- LANGBOT_E2E_EXPECTED_RUNNER_ID
|
||||
- LANGBOT_E2E_RESPONSE_TIMEOUT_MS
|
||||
automation_pipeline_url_env: LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL
|
||||
automation_pipeline_name_env: LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME
|
||||
automation_expected_runner_id: "plugin:langbot-team/ACPAgentRunner/default"
|
||||
automation_prompt: "Do not launch any background agent, subagent, task, or worker. In this current ACP session, directly call the MCP tool named langbot_get_current_event exactly once and wait for its result. After it returns, reply exactly ACP_AGENT_RUNNER_E2E_OK with no other text."
|
||||
automation_expected_text: "ACP_AGENT_RUNNER_E2E_OK"
|
||||
automation_pipeline_url_env: LANGBOT_ACP_RUNNER_PIPELINE_URL
|
||||
automation_pipeline_name_env: LANGBOT_ACP_RUNNER_PIPELINE_NAME
|
||||
automation_expected_runner_id: "plugin:langbot-team/ACPRunner/default"
|
||||
automation_prompt: "Do not launch any background agent, subagent, task, or worker. In this current ACP session, directly call the MCP tool named langbot_get_current_event exactly once and wait for its result. After it returns, reply exactly ACP_RUNNER_E2E_OK with no other text."
|
||||
automation_expected_text: "ACP_RUNNER_E2E_OK"
|
||||
automation_response_timeout_ms: "300000"
|
||||
setup_automation:
|
||||
- "node:scripts/e2e/ensure-acp-agent-runner-pipeline.mjs --write-env"
|
||||
setup_provides_env:
|
||||
- LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL
|
||||
- LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME
|
||||
- LANGBOT_ACP_RUNNER_PIPELINE_URL
|
||||
- LANGBOT_ACP_RUNNER_PIPELINE_NAME
|
||||
preconditions:
|
||||
- "The remote machine has a working Claude Code login and can run npx -y @agentclientprotocol/claude-agent-acp."
|
||||
- "LangBot can non-interactively SSH to the remote machine; the runner opens the MCP reverse tunnel automatically."
|
||||
steps:
|
||||
- "Open LANGBOT_FRONTEND_URL."
|
||||
- "Open the ACP AgentRunner QA pipeline."
|
||||
- "Confirm the pipeline AI runner is plugin:langbot-team/ACPAgentRunner/default."
|
||||
- "Open the ACP Runner QA pipeline."
|
||||
- "Confirm the pipeline AI runner is plugin:langbot-team/ACPRunner/default."
|
||||
- "Open Debug Chat."
|
||||
- "Ask the real remote Claude ACP agent to call langbot_get_current_event and return ACP_AGENT_RUNNER_E2E_OK exactly."
|
||||
- "Ask the real remote Claude ACP agent to call langbot_get_current_event and return ACP_RUNNER_E2E_OK exactly."
|
||||
checks:
|
||||
- "UI: Debug Chat shows the user prompt."
|
||||
- "UI: Debug Chat shows a Bot response containing ACP_AGENT_RUNNER_E2E_OK."
|
||||
- "UI: Debug Chat shows a Bot response containing ACP_RUNNER_E2E_OK."
|
||||
- "Logs: Backend logs include Processing request from person_websocket and Streaming completed for this run."
|
||||
- "Logs: No acp runner request error appears for this run."
|
||||
- "Console: No unexpected frontend errors appear during Debug Chat."
|
||||
@@ -66,13 +66,13 @@ diagnostics:
|
||||
- "Use scripts/e2e/ensure-acp-agent-runner-pipeline.mjs --write-env to create/update the pipeline."
|
||||
- "For remote Claude on 101, verify ssh yhh@101.34.71.12 can run without password prompts; no separate ssh -R process is required."
|
||||
success_patterns:
|
||||
- "ACP_AGENT_RUNNER_E2E_OK"
|
||||
- "ACP_RUNNER_E2E_OK"
|
||||
- "Processing request from person_websocket"
|
||||
- "Streaming completed"
|
||||
failure_patterns:
|
||||
- "acp.command_not_found"
|
||||
- "acp.process_exited"
|
||||
- "Agent runner plugin:langbot-team/ACPAgentRunner/default execution failed"
|
||||
- "Agent runner plugin:langbot-team/ACPRunner/default execution failed"
|
||||
troubleshooting:
|
||||
- backend-not-listening
|
||||
- plugin-runtime-timeout
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
id: agent-run-ledger-audit
|
||||
title: "Persisted AgentRunner run ledger passes end-to-end invariants"
|
||||
title: "Persisted Runner run ledger passes end-to-end invariants"
|
||||
mode: probe
|
||||
area: agent
|
||||
type: regression
|
||||
@@ -15,7 +15,7 @@ skills:
|
||||
- langbot-testing
|
||||
automation: scripts/e2e/agent-run-ledger-audit.mjs
|
||||
steps:
|
||||
- "Set LANGBOT_AGENT_RUN_ID to audit a specific run, or leave it unset to audit the latest persisted AgentRunner run."
|
||||
- "Set LANGBOT_AGENT_RUN_ID to audit a specific run, or leave it unset to audit the latest persisted Runner run."
|
||||
- "For an external runner's own CLI tools, set LANGBOT_AGENT_TOOL_AUTHORIZATION_MODE=runner-native; keep the default strict mode for Host tool calls."
|
||||
- "Read the active LangBot database configuration and inspect the selected run and its ordered events."
|
||||
- "Verify completed terminal state, run.completed, paired tool.call.started/completed events, stable tool names, and monotonic ordering."
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
id: agent-runner-async-db-readiness
|
||||
title: "AgentRunner async DB readiness probe"
|
||||
title: "Runner async DB readiness probe"
|
||||
mode: probe
|
||||
area: release
|
||||
type: smoke
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
id: agent-runner-behavior-matrix
|
||||
title: "AgentRunner deterministic behavior matrix probe"
|
||||
title: "Runner deterministic behavior matrix probe"
|
||||
mode: probe
|
||||
area: release
|
||||
type: regression
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
id: agent-runner-fixture-contract
|
||||
title: "QA AgentRunner fixture contract probe"
|
||||
title: "QA Runner fixture contract probe"
|
||||
mode: probe
|
||||
area: release
|
||||
type: regression
|
||||
@@ -17,19 +17,19 @@ env:
|
||||
automation: skills/langbot-testing/probes/agent-runner-fixture-contract.mjs
|
||||
steps:
|
||||
- "Run `rtk bin/lbs test run agent-runner-fixture-contract --dry-run` first; remove `--dry-run` after checking the planned evidence directory."
|
||||
- "Automation imports the QA AgentRunner fixture source and executes normal, streaming, and controlled-failure paths with SDK entities."
|
||||
- "Automation imports the QA Runner fixture source and executes normal, streaming, and controlled-failure paths with SDK entities."
|
||||
checks:
|
||||
- "automation-result.json status is pass."
|
||||
- "probe-stdout.log contains QA_AGENT_RUNNER_FIXTURE_CONTRACT_OK."
|
||||
- "Normal input returns QA_AGENT_RUNNER_OK:<input>."
|
||||
- "probe-stdout.log contains QA_RUNNER_FIXTURE_CONTRACT_OK."
|
||||
- "Normal input returns QA_RUNNER_OK:<input>."
|
||||
- "Streaming input emits message.delta chunks and completes."
|
||||
- "Failure input returns QA_AGENT_RUNNER_CONTROLLED_FAILURE."
|
||||
- "Failure input returns QA_RUNNER_CONTROLLED_FAILURE."
|
||||
evidence_required:
|
||||
- filesystem
|
||||
diagnostics:
|
||||
- "This validates the deterministic fixture source contract. It does not prove the plugin package is installed in a live LangBot instance."
|
||||
success_patterns:
|
||||
- "QA_AGENT_RUNNER_FIXTURE_CONTRACT_OK"
|
||||
- "QA_RUNNER_FIXTURE_CONTRACT_OK"
|
||||
failure_patterns:
|
||||
- "AssertionError"
|
||||
- "fixture contract exited"
|
||||
|
||||
@@ -25,7 +25,7 @@ automation_env:
|
||||
- LANGBOT_BROWSER_PROFILE
|
||||
- LANGBOT_CHROMIUM_EXECUTABLE
|
||||
preconditions:
|
||||
- "The plugin runtime is enabled and connected, with at least one AgentRunner registered."
|
||||
- "The plugin runtime is enabled and connected, with at least one Runner registered."
|
||||
- "The target is a local test instance where a temporary Agent may be created and deleted."
|
||||
steps:
|
||||
- "Read the live plugin runtime status and select a registered runner from Agent metadata."
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
id: agent-runner-ledger-concurrency
|
||||
title: "AgentRunner run ledger concurrency and auth pytest probe"
|
||||
title: "Runner run ledger concurrency and auth pytest probe"
|
||||
mode: probe
|
||||
area: release
|
||||
type: regression
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
id: agent-runner-ledger-contention
|
||||
title: "AgentRunner ledger SQLite contention probe"
|
||||
title: "Runner ledger SQLite contention probe"
|
||||
mode: probe
|
||||
area: release
|
||||
type: regression
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
id: agent-runner-ledger-invariants
|
||||
title: "AgentRunner ledger schema and status invariants probe"
|
||||
title: "Runner ledger schema and status invariants probe"
|
||||
mode: probe
|
||||
area: release
|
||||
type: regression
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
id: agent-runner-ledger-stress
|
||||
title: "AgentRunner ledger lightweight stress probe"
|
||||
title: "Runner ledger lightweight stress probe"
|
||||
mode: probe
|
||||
area: release
|
||||
type: regression
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
id: agent-runner-live-install
|
||||
title: "QA AgentRunner package installs and registers in LangBot"
|
||||
title: "QA Runner package installs and registers in LangBot"
|
||||
mode: probe
|
||||
area: release
|
||||
type: regression
|
||||
@@ -25,7 +25,7 @@ automation_expected_tool: ""
|
||||
automation_expected_runner_id: "plugin:qa/agent-runner/default"
|
||||
steps:
|
||||
- "Run `rtk bin/lbs test run agent-runner-live-install --dry-run` first; remove `--dry-run` only after readiness points at a local test LangBot instance."
|
||||
- "Automation authenticates the local test user, uploads the QA AgentRunner .lbpkg package, waits for the install task, and reads pipeline metadata."
|
||||
- "Automation authenticates the local test user, uploads the QA Runner .lbpkg package, waits for the install task, and reads pipeline metadata."
|
||||
checks:
|
||||
- "automation-result.json status is pass."
|
||||
- "/api/v1/plugins lists qa/agent-runner after install."
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
id: agent-runner-qa-debug-chat
|
||||
title: "QA AgentRunner returns deterministic output through Debug Chat"
|
||||
title: "QA Runner returns deterministic output through Debug Chat"
|
||||
mode: agent-browser
|
||||
area: pipeline
|
||||
type: regression
|
||||
@@ -17,21 +17,21 @@ skills:
|
||||
env:
|
||||
- LANGBOT_FRONTEND_URL
|
||||
- LANGBOT_BACKEND_URL
|
||||
- LANGBOT_QA_AGENT_RUNNER_PIPELINE_URL
|
||||
- LANGBOT_QA_AGENT_RUNNER_PIPELINE_NAME
|
||||
- LANGBOT_QA_RUNNER_PIPELINE_URL
|
||||
- LANGBOT_QA_RUNNER_PIPELINE_NAME
|
||||
automation: scripts/e2e/pipeline-debug-chat.mjs
|
||||
automation_env:
|
||||
- LANGBOT_FRONTEND_URL
|
||||
- LANGBOT_BACKEND_URL
|
||||
- LANGBOT_BROWSER_PROFILE
|
||||
- LANGBOT_CHROMIUM_EXECUTABLE
|
||||
- LANGBOT_QA_AGENT_RUNNER_PIPELINE_URL
|
||||
- LANGBOT_QA_AGENT_RUNNER_PIPELINE_NAME
|
||||
automation_pipeline_url_env: LANGBOT_QA_AGENT_RUNNER_PIPELINE_URL
|
||||
automation_pipeline_name_env: LANGBOT_QA_AGENT_RUNNER_PIPELINE_NAME
|
||||
- LANGBOT_QA_RUNNER_PIPELINE_URL
|
||||
- LANGBOT_QA_RUNNER_PIPELINE_NAME
|
||||
automation_pipeline_url_env: LANGBOT_QA_RUNNER_PIPELINE_URL
|
||||
automation_pipeline_name_env: LANGBOT_QA_RUNNER_PIPELINE_NAME
|
||||
automation_expected_runner_id: "plugin:qa/agent-runner/default"
|
||||
automation_prompt: "hello-live"
|
||||
automation_expected_text: "QA_AGENT_RUNNER_OK:hello-live"
|
||||
automation_expected_text: "QA_RUNNER_OK:hello-live"
|
||||
automation_response_timeout_ms: "120000"
|
||||
automation_debug_chat_response_p95_ms: "120000"
|
||||
automation_reset_debug_chat: "1"
|
||||
@@ -39,17 +39,17 @@ setup_automation:
|
||||
- "case:agent-runner-live-install"
|
||||
- "node:scripts/e2e/ensure-qa-agent-runner-pipeline.mjs --write-env"
|
||||
setup_provides_env:
|
||||
- LANGBOT_QA_AGENT_RUNNER_PIPELINE_URL
|
||||
- LANGBOT_QA_AGENT_RUNNER_PIPELINE_NAME
|
||||
- LANGBOT_QA_RUNNER_PIPELINE_URL
|
||||
- LANGBOT_QA_RUNNER_PIPELINE_NAME
|
||||
steps:
|
||||
- "Open LANGBOT_FRONTEND_URL."
|
||||
- "Open the pipeline from LANGBOT_QA_AGENT_RUNNER_PIPELINE_URL or LANGBOT_QA_AGENT_RUNNER_PIPELINE_NAME."
|
||||
- "Open the pipeline from LANGBOT_QA_RUNNER_PIPELINE_URL or LANGBOT_QA_RUNNER_PIPELINE_NAME."
|
||||
- "Confirm the pipeline AI runner is plugin:qa/agent-runner/default."
|
||||
- "Open Debug Chat."
|
||||
- "Send: hello-live."
|
||||
checks:
|
||||
- "UI: The user message appears in Debug Chat."
|
||||
- "UI: A Bot message appears and contains QA_AGENT_RUNNER_OK:hello-live."
|
||||
- "UI: A Bot message appears and contains QA_RUNNER_OK:hello-live."
|
||||
- "API diagnostic: pipeline config uses plugin:qa/agent-runner/default."
|
||||
- "Console: No unexpected frontend runtime errors appear during the send/receive path."
|
||||
evidence_required:
|
||||
@@ -62,7 +62,7 @@ diagnostics:
|
||||
- "This is the deterministic live execution proof that sits after fixture contract and live install."
|
||||
- "If the runner id mismatch is reported, rerun ensure-qa-agent-runner-pipeline.mjs --write-env."
|
||||
success_patterns:
|
||||
- "QA_AGENT_RUNNER_OK:hello-live"
|
||||
- "QA_RUNNER_OK:hello-live"
|
||||
failure_patterns:
|
||||
- "plugin:qa/agent-runner/default execution failed"
|
||||
- "Action invoke_llm_stream call timed out"
|
||||
|
||||
@@ -19,7 +19,7 @@ env:
|
||||
- LANGBOT_BACKEND_URL
|
||||
env_any:
|
||||
- LANGBOT_LOCAL_AGENT_PIPELINE_URL|LANGBOT_LOCAL_AGENT_PIPELINE_NAME
|
||||
- LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL|LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME
|
||||
- LANGBOT_ACP_RUNNER_PIPELINE_URL|LANGBOT_ACP_RUNNER_PIPELINE_NAME
|
||||
automation: scripts/e2e/agent-runner-release-preflight.mjs
|
||||
automation_env:
|
||||
- LANGBOT_FRONTEND_URL
|
||||
@@ -28,24 +28,24 @@ automation_env:
|
||||
- LANGBOT_CHROMIUM_EXECUTABLE
|
||||
automation_env_any:
|
||||
- LANGBOT_LOCAL_AGENT_PIPELINE_URL|LANGBOT_LOCAL_AGENT_PIPELINE_NAME
|
||||
- LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL|LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME
|
||||
- LANGBOT_ACP_RUNNER_PIPELINE_URL|LANGBOT_ACP_RUNNER_PIPELINE_NAME
|
||||
preconditions:
|
||||
- "LANGBOT_LOCAL_AGENT_PIPELINE_URL or LANGBOT_LOCAL_AGENT_PIPELINE_NAME points to the local-agent release pipeline."
|
||||
- "LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL or LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME points to the ACP AgentRunner release pipeline."
|
||||
- "LANGBOT_ACP_RUNNER_PIPELINE_URL or LANGBOT_ACP_RUNNER_PIPELINE_NAME points to the ACP Runner release pipeline."
|
||||
- "The active browser profile is authenticated for the same LangBot backend."
|
||||
- "By default the preflight performs a cheap model test for the local-agent primary model; set LANGBOT_PREFLIGHT_TEST_MODELS=0 only when deliberately classifying model credentials outside this run."
|
||||
steps:
|
||||
- "Open LANGBOT_FRONTEND_URL with the configured browser profile."
|
||||
- "Use the browser token to call LangBot backend readiness APIs without printing token values."
|
||||
- "Check plugin runtime status, Box status, required runner plugins, qa-plugin-smoke, and qa_plugin_echo."
|
||||
- "Resolve the local-agent and ACP AgentRunner QA pipelines from their case-specific env vars."
|
||||
- "Resolve the local-agent and ACP Runner QA pipelines from their case-specific env vars."
|
||||
- "Assert each pipeline uses the expected runner id."
|
||||
- "Assert the external runner pipeline uses the expected runner id."
|
||||
- "Assert the local-agent primary model advertises func_call and vision for the full release gate."
|
||||
- "Run the local-agent primary model test endpoint unless LANGBOT_PREFLIGHT_TEST_MODELS=0."
|
||||
checks:
|
||||
- "API diagnostic: api-diagnostic.json has no blockers and no env_issues."
|
||||
- "API diagnostic: required pipelines resolve to plugin:langbot-team/LocalAgent/default and plugin:langbot-team/ACPAgentRunner/default."
|
||||
- "API diagnostic: required pipelines resolve to plugin:langbot-team/LocalAgent/default and plugin:langbot-team/ACPRunner/default."
|
||||
- "API diagnostic: qa_plugin_echo is exposed by /api/v1/tools."
|
||||
- "API diagnostic: local-agent model check catches invalid credentials or missing func_call/vision before release E2E starts."
|
||||
- "Secret safety: token values, api keys, and provider secrets are not printed."
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
id: agent-runner-runtime-chaos
|
||||
title: "AgentRunner SDK runtime chaos pytest probe"
|
||||
title: "Runner SDK runtime chaos pytest probe"
|
||||
mode: probe
|
||||
area: release
|
||||
type: regression
|
||||
@@ -19,10 +19,10 @@ automation: skills/langbot-testing/probes/agent-runner-runtime-chaos.mjs
|
||||
steps:
|
||||
- "Run `rtk bin/lbs test run agent-runner-runtime-chaos --dry-run` first; remove `--dry-run` after checking the SDK repo target and evidence directory."
|
||||
- "Automation resolves LANGBOT_PLUGIN_SDK_REPO, defaulting to ../../langbot-plugin-sdk when the env var is unset."
|
||||
- "Automation runs the existing SDK pytest files tests/runtime/plugin/test_mgr_agent_runner.py and tests/runtime/test_pull_api_handlers.py."
|
||||
- "Automation runs the existing SDK pytest files tests/runtime/plugin/test_mgr_runner.py and tests/runtime/test_pull_api_handlers.py."
|
||||
checks:
|
||||
- "automation-result.json status is pass."
|
||||
- "pytest exit status is 0 for the existing AgentRunner runtime and pull API handler tests."
|
||||
- "pytest exit status is 0 for the existing Runner runtime and pull API handler tests."
|
||||
- "pytest-stdout.log and pytest-stderr.log are written under LBS_EVIDENCE_DIR."
|
||||
evidence_required:
|
||||
- filesystem
|
||||
|
||||
@@ -45,7 +45,7 @@ steps:
|
||||
- "Select exactly one Box child whose parent is main.py running from LANGBOT_REPO; abort on zero or multiple matches."
|
||||
- "Send SIGTERM to that Box child and poll process, Box status, MCP runtime info, and global tools for up to 30 seconds."
|
||||
- "Without running MCP setup or registration, reset Debug Chat and call qa_mcp_echo with a unique per-run value through the browser."
|
||||
- "Audit the matching AgentRunner ledger run and require the exact qa_mcp_echo arguments plus the complete tool result text."
|
||||
- "Audit the matching Runner ledger run and require the exact qa_mcp_echo arguments plus the complete tool result text."
|
||||
checks:
|
||||
- "The old Box PID exits and a new Box PID appears under the same LangBot parent."
|
||||
- "Box returns available=true with at least one active session and managed process."
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
id: dify-agent-debug-chat
|
||||
title: "Dify AgentRunner returns a response through Pipeline Debug Chat"
|
||||
title: "Dify Runner returns a response through Pipeline Debug Chat"
|
||||
mode: agent-browser
|
||||
area: pipeline
|
||||
type: provider
|
||||
|
||||
@@ -18,8 +18,8 @@ skills:
|
||||
env:
|
||||
- LANGBOT_FRONTEND_URL
|
||||
- LANGBOT_BACKEND_URL
|
||||
- LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL
|
||||
- LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME
|
||||
- LANGBOT_ACP_RUNNER_PIPELINE_URL
|
||||
- LANGBOT_ACP_RUNNER_PIPELINE_NAME
|
||||
preconditions:
|
||||
- "An external-harness runner pipeline (e.g. ACP remote claude-code) is configured with langbot-assets-enabled=true so the LangBot MCP gateway is exposed to the harness."
|
||||
- "The remote harness (claude-code) is reachable and responsive (claude -p returns within the runner timeout)."
|
||||
@@ -29,10 +29,10 @@ automation_env:
|
||||
- LANGBOT_FRONTEND_URL
|
||||
- LANGBOT_BROWSER_PROFILE
|
||||
- LANGBOT_CHROMIUM_EXECUTABLE
|
||||
- LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL
|
||||
- LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME
|
||||
automation_pipeline_url_env: LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL
|
||||
automation_pipeline_name_env: LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME
|
||||
- LANGBOT_ACP_RUNNER_PIPELINE_URL
|
||||
- LANGBOT_ACP_RUNNER_PIPELINE_NAME
|
||||
automation_pipeline_url_env: LANGBOT_ACP_RUNNER_PIPELINE_URL
|
||||
automation_pipeline_name_env: LANGBOT_ACP_RUNNER_PIPELINE_NAME
|
||||
automation_prompt: "You have LangBot tools available via an MCP server (tools prefixed langbot_). Call langbot_list_assets with asset_types = [\"skills\",\"tools\"]. Then reply with one single line: the literal token PROBEDONE, a space, the number of skills you found, a space, and the number of tools you found."
|
||||
automation_expected_text: "PROBEDONE"
|
||||
automation_response_timeout_ms: "540000"
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
id: wizard-runner-marketplace-catalog
|
||||
title: "Quick Start installs a published AgentRunner on a clean instance"
|
||||
title: "Quick Start installs a published Runner on a clean instance"
|
||||
mode: agent-browser
|
||||
area: wizard
|
||||
type: feature
|
||||
@@ -27,19 +27,19 @@ preconditions:
|
||||
steps:
|
||||
- "Start an isolated first-run instance and confirm zero installed plugins and zero registered runners."
|
||||
- "Resume Quick Start at the AI Engine step with a temporary disabled Bot."
|
||||
- "Confirm the browser requests Marketplace plugins with component_filter=AgentRunner."
|
||||
- "Confirm the browser requests Marketplace plugins with component_filter=Runner."
|
||||
- "Confirm langbot-team/LocalAgent is published with an installable version and the Runner Extensions link is correct."
|
||||
- "Install LocalAgent and wait for plugin initialization and AgentRunner registration."
|
||||
- "Install LocalAgent and wait for plugin initialization and Runner registration."
|
||||
- "Confirm Create & Deploy is disabled before installation and enabled after LocalAgent is selected."
|
||||
- "Verify the layout at desktop and mobile widths."
|
||||
checks:
|
||||
- "API: The instance has zero installed plugins and zero registered runners."
|
||||
- "API: The instance wizard status is none."
|
||||
- "Network: Marketplace search uses component_filter=AgentRunner and type_filter=plugin."
|
||||
- "Network: Marketplace search uses component_filter=Runner and type_filter=plugin."
|
||||
- "UI: The AI Engine step displays the published langbot-team/LocalAgent card."
|
||||
- "Marketplace: LocalAgent includes latest_version so installation can proceed."
|
||||
- "Runtime: LocalAgent installs and registers plugin:langbot-team/LocalAgent/default."
|
||||
- "UI: Browse Runner Extensions links to the AgentRunner-filtered market."
|
||||
- "UI: Browse Runner Extensions links to the Runner-filtered market."
|
||||
- "UI: Create & Deploy transitions from disabled to enabled only after Runner selection."
|
||||
- "Console: No unexpected frontend errors appear during the flow."
|
||||
- "Cleanup: Wizard progress and the temporary Bot are removed."
|
||||
|
||||
@@ -18,7 +18,7 @@ steps:
|
||||
- "Discover the active LangBot checkout and sibling workspace repositories, with LANGBOT_WORKSPACE_ROOT and repository-specific env overrides available for non-default layouts."
|
||||
- "Verify every checkout is on main or dev/4.11.x and record dirty worktrees as warnings only."
|
||||
- "Validate plugin manifest identities for LocalAgent, Control Plane, LongTermMemory, GeneralParsers, LangRAG, and Skill Authoring."
|
||||
- "Use the LangBot virtualenv to import langbot_plugin and AgentRunner Protocol entities from the workspace SDK source tree."
|
||||
- "Use the LangBot virtualenv to import langbot_plugin and Runner Protocol entities from the workspace SDK source tree."
|
||||
checks:
|
||||
- "workspace-preflight.json contains no failed checks."
|
||||
- "The langbot_plugin import path is inside the discovered langbot-plugin-sdk/src directory."
|
||||
|
||||
@@ -16,7 +16,7 @@ skills:
|
||||
automation: scripts/e2e/workspace-repository-contracts.mjs
|
||||
steps:
|
||||
- "Resolve the active LangBot virtualenv and workspace SDK source path."
|
||||
- "Run tests independently for AgentRunner, Control Plane, LongTermMemory, Parser, RAG, Skill Authoring, LocalAgent, LangBot Agent/Provider, the skills CLI, and SDK runtime contracts."
|
||||
- "Run tests independently for Runner, Control Plane, LongTermMemory, Parser, RAG, Skill Authoring, LocalAgent, LangBot Agent/Provider, the skills CLI, and SDK runtime contracts."
|
||||
- "Run SDK packaging blackbox separately so an isolated build dependency network failure is classified as env_issue without masking product test failures."
|
||||
- "Write per-repository stdout, stderr, status, and duration under repository-contracts/."
|
||||
checks:
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
[
|
||||
{
|
||||
"id": "qa-agent-runner-behaviors",
|
||||
"title": "Deterministic AgentRunner behavior matrix",
|
||||
"title": "Deterministic Runner behavior matrix",
|
||||
"kind": "json",
|
||||
"path": "fixtures/agent-runner/qa-runner-behaviors.json",
|
||||
"related_cases": [
|
||||
@@ -13,7 +13,7 @@
|
||||
},
|
||||
{
|
||||
"id": "qa-agent-runner-source",
|
||||
"title": "QA deterministic AgentRunner fixture source",
|
||||
"title": "QA deterministic Runner fixture source",
|
||||
"kind": "plugin_source",
|
||||
"path": "fixtures/plugins/qa-agent-runner/manifest.yaml",
|
||||
"related_cases": [
|
||||
@@ -22,11 +22,11 @@
|
||||
"agent-runner-live-install",
|
||||
"agent-runner-qa-debug-chat"
|
||||
],
|
||||
"checks": ["exists", "qa_agent_runner_source"]
|
||||
"checks": ["exists", "qa_runner_source"]
|
||||
},
|
||||
{
|
||||
"id": "qa-agent-runner-package",
|
||||
"title": "QA deterministic AgentRunner prebuilt package",
|
||||
"title": "QA deterministic Runner prebuilt package",
|
||||
"kind": "plugin_package",
|
||||
"path": "fixtures/plugins/qa-agent-runner/dist/qa-agent-runner-0.1.0.lbpkg",
|
||||
"related_cases": [
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# QA AgentRunner Fixture
|
||||
# QA Runner Fixture
|
||||
|
||||
Deterministic AgentRunner plugin source used by `langbot-skills` probes and future browser release-gate cases.
|
||||
Deterministic Runner plugin source used by `langbot-skills` probes and future browser release-gate cases.
|
||||
|
||||
Runner id after installation should be:
|
||||
|
||||
@@ -10,6 +10,6 @@ plugin:qa/agent-runner/default
|
||||
|
||||
Expected behavior:
|
||||
|
||||
- normal input returns `QA_AGENT_RUNNER_OK:<input>`
|
||||
- normal input returns `QA_RUNNER_OK:<input>`
|
||||
- input containing `stream` emits streaming chunks then completes
|
||||
- input containing `fail` returns `QA_AGENT_RUNNER_CONTROLLED_FAILURE`
|
||||
- input containing `fail` returns `QA_RUNNER_CONTROLLED_FAILURE`
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64" role="img" aria-label="QA AgentRunner icon">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64" role="img" aria-label="QA Runner icon">
|
||||
<rect width="64" height="64" rx="12" fill="#111827"/>
|
||||
<path d="M16 20h32v22H35l-7 8v-8H16z" fill="#22c55e"/>
|
||||
<path d="M24 30h16" stroke="#111827" stroke-width="4" stroke-linecap="round"/>
|
||||
|
||||
|
Before Width: | Height: | Size: 306 B After Width: | Height: | Size: 301 B |
-39
@@ -1,39 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import typing
|
||||
|
||||
from langbot_plugin.api.definition.components.agent_runner.runner import AgentRunner
|
||||
from langbot_plugin.api.entities.builtin.agent_runner import AgentRunContext, AgentRunResult
|
||||
from langbot_plugin.api.entities.builtin.provider.message import Message, MessageChunk
|
||||
|
||||
|
||||
class DefaultAgentRunner(AgentRunner):
|
||||
async def run(
|
||||
self,
|
||||
ctx: AgentRunContext,
|
||||
) -> typing.AsyncGenerator[AgentRunResult, None]:
|
||||
text = (ctx.input.to_text() or "").strip()
|
||||
if "fail" in text.lower():
|
||||
yield AgentRunResult.run_failed(
|
||||
ctx.run_id,
|
||||
error="QA_AGENT_RUNNER_CONTROLLED_FAILURE",
|
||||
code="qa.controlled_failure",
|
||||
retryable=False,
|
||||
)
|
||||
return
|
||||
|
||||
content = f"QA_AGENT_RUNNER_OK:{text or 'empty'}"
|
||||
if "stream" in text.lower():
|
||||
for chunk in ("QA_", "AGENT_", f"RUNNER_OK:{text}"):
|
||||
yield AgentRunResult.message_delta(
|
||||
ctx.run_id,
|
||||
MessageChunk(role="assistant", content=chunk),
|
||||
)
|
||||
yield AgentRunResult.run_completed(ctx.run_id, finish_reason="stop")
|
||||
return
|
||||
|
||||
yield AgentRunResult.run_completed(
|
||||
ctx.run_id,
|
||||
Message(role="assistant", content=content),
|
||||
finish_reason="stop",
|
||||
)
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import typing
|
||||
|
||||
from langbot_plugin.api.definition.components.runner.runner import Runner
|
||||
from langbot_plugin.api.entities.builtin.runner import RunnerContext, RunnerResult
|
||||
from langbot_plugin.api.entities.builtin.provider.message import Message, MessageChunk
|
||||
|
||||
|
||||
class DefaultRunner(Runner):
|
||||
async def run(
|
||||
self,
|
||||
ctx: RunnerContext,
|
||||
) -> typing.AsyncGenerator[RunnerResult, None]:
|
||||
text = (ctx.input.to_text() or '').strip()
|
||||
if 'fail' in text.lower():
|
||||
yield RunnerResult.run_failed(
|
||||
ctx.run_id,
|
||||
error='QA_RUNNER_CONTROLLED_FAILURE',
|
||||
code='qa.controlled_failure',
|
||||
retryable=False,
|
||||
)
|
||||
return
|
||||
|
||||
content = f'QA_RUNNER_OK:{text or "empty"}'
|
||||
if 'stream' in text.lower():
|
||||
for chunk in ('QA_', 'AGENT_', f'RUNNER_OK:{text}'):
|
||||
yield RunnerResult.message_delta(
|
||||
ctx.run_id,
|
||||
MessageChunk(role='assistant', content=chunk),
|
||||
)
|
||||
yield RunnerResult.run_completed(ctx.run_id, finish_reason='stop')
|
||||
return
|
||||
|
||||
yield RunnerResult.run_completed(
|
||||
ctx.run_id,
|
||||
Message(role='assistant', content=content),
|
||||
finish_reason='stop',
|
||||
)
|
||||
+2
-2
@@ -1,5 +1,5 @@
|
||||
apiVersion: langbot/v1
|
||||
kind: AgentRunner
|
||||
kind: Runner
|
||||
metadata:
|
||||
name: default
|
||||
label:
|
||||
@@ -27,4 +27,4 @@ spec:
|
||||
execution:
|
||||
python:
|
||||
path: default.py
|
||||
attr: DefaultAgentRunner
|
||||
attr: DefaultRunner
|
||||
@@ -3,6 +3,6 @@ from __future__ import annotations
|
||||
from langbot_plugin.api.definition.plugin import BasePlugin
|
||||
|
||||
|
||||
class QAAgentRunnerPlugin(BasePlugin):
|
||||
class QARunnerPlugin(BasePlugin):
|
||||
async def initialize(self) -> None:
|
||||
self.ready_marker = "qa-agent-runner-ready"
|
||||
self.ready_marker = 'qa-agent-runner-ready'
|
||||
|
||||
@@ -6,20 +6,20 @@ metadata:
|
||||
repository: https://example.invalid/langbot/qa-agent-runner
|
||||
version: 0.1.0
|
||||
description:
|
||||
en_US: Deterministic AgentRunner fixture for LangBot QA.
|
||||
zh_Hans: LangBot QA 使用的确定性 AgentRunner 夹具。
|
||||
en_US: Deterministic Runner fixture for LangBot QA.
|
||||
zh_Hans: LangBot QA 使用的确定性 Runner 夹具。
|
||||
label:
|
||||
en_US: QA AgentRunner
|
||||
zh_Hans: QA AgentRunner
|
||||
en_US: QA Runner
|
||||
zh_Hans: QA Runner
|
||||
icon: assets/icon.svg
|
||||
spec:
|
||||
config: []
|
||||
components:
|
||||
AgentRunner:
|
||||
Runner:
|
||||
fromDirs:
|
||||
- path: components/agent_runner/
|
||||
- path: components/runner/
|
||||
maxDepth: 1
|
||||
execution:
|
||||
python:
|
||||
path: main.py
|
||||
attr: QAAgentRunnerPlugin
|
||||
attr: QARunnerPlugin
|
||||
|
||||
@@ -7,7 +7,11 @@ import { delimiter, join, resolve } from "node:path";
|
||||
import { env } from "node:process";
|
||||
|
||||
function timestampSlug(date = new Date()) {
|
||||
return date.toISOString().replace(/\.\d{3}Z$/, "Z").replace(/[^0-9A-Za-z]+/g, "-").replace(/^-|-$/g, "");
|
||||
return date
|
||||
.toISOString()
|
||||
.replace(/\.\d{3}Z$/, "Z")
|
||||
.replace(/[^0-9A-Za-z]+/g, "-")
|
||||
.replace(/^-|-$/g, "");
|
||||
}
|
||||
|
||||
function localIsoWithOffset(date = new Date()) {
|
||||
@@ -51,7 +55,14 @@ function run(command, timeoutMs, childEnv) {
|
||||
});
|
||||
child.on("error", (error) => {
|
||||
clearTimeout(timeout);
|
||||
resolveDone({ stdout, stderr, error, timedOut, status: null, signal: null });
|
||||
resolveDone({
|
||||
stdout,
|
||||
stderr,
|
||||
error,
|
||||
timedOut,
|
||||
status: null,
|
||||
signal: null,
|
||||
});
|
||||
});
|
||||
child.on("close", (status, signal) => {
|
||||
clearTimeout(timeout);
|
||||
@@ -66,7 +77,7 @@ import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from langbot.pkg.agent.runner.descriptor import AgentRunnerDescriptor
|
||||
from langbot.pkg.agent.runner.descriptor import RunnerDescriptor
|
||||
from langbot.pkg.agent.runner.errors import RunnerExecutionError, RunnerProtocolError
|
||||
from langbot.pkg.agent.runner.result_normalizer import AgentResultNormalizer
|
||||
|
||||
@@ -80,10 +91,10 @@ class App:
|
||||
logger = Logger()
|
||||
|
||||
def descriptor():
|
||||
return AgentRunnerDescriptor(
|
||||
return RunnerDescriptor(
|
||||
id='plugin:qa/agent-runner/default',
|
||||
source='plugin',
|
||||
label={'en_US': 'QA AgentRunner'},
|
||||
label={'en_US': 'QA Runner'},
|
||||
plugin_author='qa',
|
||||
plugin_name='agent-runner',
|
||||
runner_name='default',
|
||||
@@ -139,18 +150,26 @@ async function main() {
|
||||
const root = resolve(env.LBS_ROOT || process.cwd());
|
||||
const caseId = "agent-runner-behavior-matrix";
|
||||
const runId = env.LBS_RUN_ID || `${timestampSlug()}-${caseId}`;
|
||||
const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId));
|
||||
const evidenceDir = resolve(
|
||||
env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId),
|
||||
);
|
||||
await mkdir(evidenceDir, { recursive: true });
|
||||
const startedAt = new Date();
|
||||
const langbotRepo = resolve(root, env.LANGBOT_REPO || "..");
|
||||
const sdkRepo = resolve(root, env.LANGBOT_PLUGIN_SDK_REPO || "../../langbot-plugin-sdk");
|
||||
const sdkRepo = resolve(
|
||||
root,
|
||||
env.LANGBOT_PLUGIN_SDK_REPO || "../../langbot-plugin-sdk",
|
||||
);
|
||||
const sdkSrc = resolve(sdkRepo, "src");
|
||||
const fixturePath = resolve(root, "skills/langbot-testing/fixtures/agent-runner/qa-runner-behaviors.json");
|
||||
const fixturePath = resolve(
|
||||
root,
|
||||
"skills/langbot-testing/fixtures/agent-runner/qa-runner-behaviors.json",
|
||||
);
|
||||
const stdoutLog = join(evidenceDir, "probe-stdout.log");
|
||||
const stderrLog = join(evidenceDir, "probe-stderr.log");
|
||||
const automationResultJson = join(evidenceDir, "automation-result.json");
|
||||
const resultJson = join(evidenceDir, "result.json");
|
||||
const timeoutMs = Number(env.LANGBOT_AGENT_RUNNER_PROBE_TIMEOUT_MS || "30000");
|
||||
const timeoutMs = Number(env.LANGBOT_RUNNER_PROBE_TIMEOUT_MS || "30000");
|
||||
const command = {
|
||||
executable: "rtk",
|
||||
args: [resolve(langbotRepo, ".venv/bin/python"), "-c", script, fixturePath],
|
||||
@@ -175,7 +194,12 @@ async function main() {
|
||||
timeout_ms: timeoutMs,
|
||||
exit_status: null,
|
||||
signal: null,
|
||||
evidence: { stdout_log: stdoutLog, stderr_log: stderrLog, automation_result_json: automationResultJson, result_json: resultJson },
|
||||
evidence: {
|
||||
stdout_log: stdoutLog,
|
||||
stderr_log: stderrLog,
|
||||
automation_result_json: automationResultJson,
|
||||
result_json: resultJson,
|
||||
},
|
||||
evidence_collected: ["filesystem"],
|
||||
};
|
||||
try {
|
||||
@@ -185,7 +209,9 @@ async function main() {
|
||||
} else {
|
||||
const proc = await run(command, timeoutMs, {
|
||||
...process.env,
|
||||
PYTHONPATH: [sdkSrc, process.env.PYTHONPATH].filter(Boolean).join(delimiter),
|
||||
PYTHONPATH: [sdkSrc, process.env.PYTHONPATH]
|
||||
.filter(Boolean)
|
||||
.join(delimiter),
|
||||
UV_CACHE_DIR: env.UV_CACHE_DIR || join(evidenceDir, ".uv-cache"),
|
||||
});
|
||||
await writeFile(stdoutLog, proc.stdout, "utf8");
|
||||
@@ -198,7 +224,10 @@ async function main() {
|
||||
} else if (proc.timedOut) {
|
||||
result.status = "fail";
|
||||
result.reason = `behavior matrix timed out after ${timeoutMs}ms`;
|
||||
} else if (proc.status === 0 && proc.stdout.includes("QA_RUNNER_BEHAVIOR_MATRIX_OK")) {
|
||||
} else if (
|
||||
proc.status === 0 &&
|
||||
proc.stdout.includes("QA_RUNNER_BEHAVIOR_MATRIX_OK")
|
||||
) {
|
||||
result.status = "pass";
|
||||
result.reason = "behavior matrix passed";
|
||||
} else {
|
||||
@@ -219,7 +248,9 @@ async function main() {
|
||||
await writeFile(resultJson, resultText, "utf8");
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
}
|
||||
process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1);
|
||||
process.exit(
|
||||
result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1,
|
||||
);
|
||||
}
|
||||
|
||||
await main();
|
||||
|
||||
@@ -7,7 +7,11 @@ import { delimiter, join, resolve } from "node:path";
|
||||
import { env } from "node:process";
|
||||
|
||||
function timestampSlug(date = new Date()) {
|
||||
return date.toISOString().replace(/\.\d{3}Z$/, "Z").replace(/[^0-9A-Za-z]+/g, "-").replace(/^-|-$/g, "");
|
||||
return date
|
||||
.toISOString()
|
||||
.replace(/\.\d{3}Z$/, "Z")
|
||||
.replace(/[^0-9A-Za-z]+/g, "-")
|
||||
.replace(/^-|-$/g, "");
|
||||
}
|
||||
|
||||
function localIsoWithOffset(date = new Date()) {
|
||||
@@ -51,7 +55,14 @@ function run(command, timeoutMs, childEnv) {
|
||||
});
|
||||
child.on("error", (error) => {
|
||||
clearTimeout(timeout);
|
||||
resolveDone({ stdout, stderr, error, timedOut, status: null, signal: null });
|
||||
resolveDone({
|
||||
stdout,
|
||||
stderr,
|
||||
error,
|
||||
timedOut,
|
||||
status: null,
|
||||
signal: null,
|
||||
});
|
||||
});
|
||||
child.on("close", (status, signal) => {
|
||||
clearTimeout(timeout);
|
||||
@@ -66,27 +77,27 @@ import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from langbot_plugin.api.entities.builtin.agent_runner.context import AgentRunContext
|
||||
from langbot_plugin.api.entities.builtin.agent_runner.delivery import DeliveryContext
|
||||
from langbot_plugin.api.entities.builtin.agent_runner.event import AgentEventContext
|
||||
from langbot_plugin.api.entities.builtin.agent_runner.input import AgentInput
|
||||
from langbot_plugin.api.entities.builtin.agent_runner.resources import AgentResources
|
||||
from langbot_plugin.api.entities.builtin.agent_runner.runtime import AgentRuntimeContext
|
||||
from langbot_plugin.api.entities.builtin.agent_runner.trigger import AgentTrigger
|
||||
from langbot_plugin.api.entities.builtin.runner.context import RunnerContext
|
||||
from langbot_plugin.api.entities.builtin.runner.delivery import DeliveryContext
|
||||
from langbot_plugin.api.entities.builtin.runner.event import AgentEventContext
|
||||
from langbot_plugin.api.entities.builtin.runner.input import AgentInput
|
||||
from langbot_plugin.api.entities.builtin.runner.resources import AgentResources
|
||||
from langbot_plugin.api.entities.builtin.runner.runtime import AgentRuntimeContext
|
||||
from langbot_plugin.api.entities.builtin.runner.trigger import AgentTrigger
|
||||
|
||||
fixture = Path(sys.argv[1])
|
||||
runner_py = fixture / "components" / "agent_runner" / "default.py"
|
||||
runner_py = fixture / "components" / "runner" / "default.py"
|
||||
manifest = fixture / "manifest.yaml"
|
||||
runner_yaml = fixture / "components" / "agent_runner" / "default.yaml"
|
||||
runner_yaml = fixture / "components" / "runner" / "default.yaml"
|
||||
assert manifest.exists(), manifest
|
||||
assert runner_yaml.exists(), runner_yaml
|
||||
spec = importlib.util.spec_from_file_location("qa_agent_runner_fixture", runner_py)
|
||||
spec = importlib.util.spec_from_file_location("qa_runner_fixture", runner_py)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
assert spec and spec.loader
|
||||
spec.loader.exec_module(module)
|
||||
|
||||
def context(run_id, text):
|
||||
return AgentRunContext(
|
||||
return RunnerContext(
|
||||
run_id=run_id,
|
||||
trigger=AgentTrigger(type="message.received", source="webui"),
|
||||
event=AgentEventContext(event_id=f"evt-{run_id}", event_type="message.received", source="webui"),
|
||||
@@ -97,7 +108,7 @@ def context(run_id, text):
|
||||
)
|
||||
|
||||
async def collect(text):
|
||||
runner = module.DefaultAgentRunner()
|
||||
runner = module.DefaultRunner()
|
||||
results = []
|
||||
async for result in runner.run(context(f"run-{len(text)}", text)):
|
||||
results.append(result)
|
||||
@@ -107,17 +118,17 @@ async def main():
|
||||
normal = await collect("hello")
|
||||
assert len(normal) == 1, normal
|
||||
assert normal[0].type.value == "run.completed"
|
||||
assert normal[0].data["message"]["content"] == "QA_AGENT_RUNNER_OK:hello"
|
||||
assert normal[0].data["message"]["content"] == "QA_RUNNER_OK:hello"
|
||||
|
||||
stream = await collect("stream hello")
|
||||
assert [item.type.value for item in stream] == ["message.delta", "message.delta", "message.delta", "run.completed"]
|
||||
assert "".join(item.data["chunk"]["content"] for item in stream[:3]) == "QA_AGENT_RUNNER_OK:stream hello"
|
||||
assert "".join(item.data["chunk"]["content"] for item in stream[:3]) == "QA_RUNNER_OK:stream hello"
|
||||
|
||||
failed = await collect("please fail")
|
||||
assert len(failed) == 1
|
||||
assert failed[0].type.value == "run.failed"
|
||||
assert failed[0].data["error"] == "QA_AGENT_RUNNER_CONTROLLED_FAILURE"
|
||||
print("QA_AGENT_RUNNER_FIXTURE_CONTRACT_OK")
|
||||
assert failed[0].data["error"] == "QA_RUNNER_CONTROLLED_FAILURE"
|
||||
print("QA_RUNNER_FIXTURE_CONTRACT_OK")
|
||||
|
||||
asyncio.run(main())
|
||||
`;
|
||||
@@ -126,18 +137,30 @@ async function main() {
|
||||
const root = resolve(env.LBS_ROOT || process.cwd());
|
||||
const caseId = "agent-runner-fixture-contract";
|
||||
const runId = env.LBS_RUN_ID || `${timestampSlug()}-${caseId}`;
|
||||
const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId));
|
||||
const evidenceDir = resolve(
|
||||
env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId),
|
||||
);
|
||||
await mkdir(evidenceDir, { recursive: true });
|
||||
const startedAt = new Date();
|
||||
const sdkRepo = resolve(root, env.LANGBOT_PLUGIN_SDK_REPO || "../../langbot-plugin-sdk");
|
||||
const sdkRepo = resolve(
|
||||
root,
|
||||
env.LANGBOT_PLUGIN_SDK_REPO || "../../langbot-plugin-sdk",
|
||||
);
|
||||
const sdkSrc = resolve(sdkRepo, "src");
|
||||
const fixturePath = resolve(root, "skills/langbot-testing/fixtures/plugins/qa-agent-runner");
|
||||
const fixturePath = resolve(
|
||||
root,
|
||||
"skills/langbot-testing/fixtures/plugins/qa-agent-runner",
|
||||
);
|
||||
const stdoutLog = join(evidenceDir, "probe-stdout.log");
|
||||
const stderrLog = join(evidenceDir, "probe-stderr.log");
|
||||
const automationResultJson = join(evidenceDir, "automation-result.json");
|
||||
const resultJson = join(evidenceDir, "result.json");
|
||||
const timeoutMs = Number(env.LANGBOT_AGENT_RUNNER_PROBE_TIMEOUT_MS || "30000");
|
||||
const command = { executable: "rtk", args: ["uv", "run", "--no-sync", "python", "-c", script, fixturePath], cwd: sdkRepo };
|
||||
const timeoutMs = Number(env.LANGBOT_RUNNER_PROBE_TIMEOUT_MS || "30000");
|
||||
const command = {
|
||||
executable: "rtk",
|
||||
args: ["uv", "run", "--no-sync", "python", "-c", script, fixturePath],
|
||||
cwd: sdkRepo,
|
||||
};
|
||||
const result = {
|
||||
source: "automation",
|
||||
probe: "agent-runner-fixture-contract",
|
||||
@@ -156,7 +179,12 @@ async function main() {
|
||||
timeout_ms: timeoutMs,
|
||||
exit_status: null,
|
||||
signal: null,
|
||||
evidence: { stdout_log: stdoutLog, stderr_log: stderrLog, automation_result_json: automationResultJson, result_json: resultJson },
|
||||
evidence: {
|
||||
stdout_log: stdoutLog,
|
||||
stderr_log: stderrLog,
|
||||
automation_result_json: automationResultJson,
|
||||
result_json: resultJson,
|
||||
},
|
||||
evidence_collected: ["filesystem"],
|
||||
};
|
||||
try {
|
||||
@@ -166,7 +194,9 @@ async function main() {
|
||||
} else {
|
||||
const proc = await run(command, timeoutMs, {
|
||||
...process.env,
|
||||
PYTHONPATH: [sdkSrc, process.env.PYTHONPATH].filter(Boolean).join(delimiter),
|
||||
PYTHONPATH: [sdkSrc, process.env.PYTHONPATH]
|
||||
.filter(Boolean)
|
||||
.join(delimiter),
|
||||
UV_CACHE_DIR: env.UV_CACHE_DIR || join(evidenceDir, ".uv-cache"),
|
||||
});
|
||||
await writeFile(stdoutLog, proc.stdout, "utf8");
|
||||
@@ -179,9 +209,12 @@ async function main() {
|
||||
} else if (proc.timedOut) {
|
||||
result.status = "fail";
|
||||
result.reason = `fixture contract probe timed out after ${timeoutMs}ms`;
|
||||
} else if (proc.status === 0 && proc.stdout.includes("QA_AGENT_RUNNER_FIXTURE_CONTRACT_OK")) {
|
||||
} else if (
|
||||
proc.status === 0 &&
|
||||
proc.stdout.includes("QA_RUNNER_FIXTURE_CONTRACT_OK")
|
||||
) {
|
||||
result.status = "pass";
|
||||
result.reason = "QA AgentRunner fixture contract passed";
|
||||
result.reason = "QA Runner fixture contract passed";
|
||||
} else {
|
||||
result.status = "fail";
|
||||
result.reason = `fixture contract exited with status ${proc.status}`;
|
||||
@@ -200,7 +233,9 @@ async function main() {
|
||||
await writeFile(resultJson, resultText, "utf8");
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
}
|
||||
process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1);
|
||||
process.exit(
|
||||
result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1,
|
||||
);
|
||||
}
|
||||
|
||||
await main();
|
||||
|
||||
@@ -8,7 +8,8 @@ await runPytestProbe({
|
||||
defaultRepo: "..",
|
||||
pythonPathEnvKeys: ["LANGBOT_PLUGIN_SDK_REPO"],
|
||||
defaultPythonPaths: ["../../langbot-plugin-sdk/src"],
|
||||
description: "LangBot AgentRunner run ledger claim, lease, authorization, and runtime-admin pytest probe.",
|
||||
description:
|
||||
"LangBot Runner run ledger claim, lease, authorization, and runtime-admin pytest probe.",
|
||||
testTargets: [
|
||||
"tests/unit_tests/agent/test_run_ledger_store.py::test_create_queued_run_claim_renew_release",
|
||||
"tests/unit_tests/agent/test_run_ledger_store.py::test_expired_claim_can_be_reclaimed",
|
||||
|
||||
@@ -7,7 +7,11 @@ import { delimiter, join, resolve } from "node:path";
|
||||
import { env } from "node:process";
|
||||
|
||||
function timestampSlug(date = new Date()) {
|
||||
return date.toISOString().replace(/\.\d{3}Z$/, "Z").replace(/[^0-9A-Za-z]+/g, "-").replace(/^-|-$/g, "");
|
||||
return date
|
||||
.toISOString()
|
||||
.replace(/\.\d{3}Z$/, "Z")
|
||||
.replace(/[^0-9A-Za-z]+/g, "-")
|
||||
.replace(/^-|-$/g, "");
|
||||
}
|
||||
|
||||
function localIsoWithOffset(date = new Date()) {
|
||||
@@ -51,7 +55,14 @@ function run(command, timeoutMs, childEnv) {
|
||||
});
|
||||
child.on("error", (error) => {
|
||||
clearTimeout(timeout);
|
||||
resolveDone({ stdout, stderr, error, timedOut, status: null, signal: null });
|
||||
resolveDone({
|
||||
stdout,
|
||||
stderr,
|
||||
error,
|
||||
timedOut,
|
||||
status: null,
|
||||
signal: null,
|
||||
});
|
||||
});
|
||||
child.on("close", (status, signal) => {
|
||||
clearTimeout(timeout);
|
||||
@@ -149,18 +160,23 @@ async function main() {
|
||||
const root = resolve(env.LBS_ROOT || process.cwd());
|
||||
const caseId = "agent-runner-ledger-contention";
|
||||
const runId = env.LBS_RUN_ID || `${timestampSlug()}-${caseId}`;
|
||||
const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId));
|
||||
const evidenceDir = resolve(
|
||||
env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId),
|
||||
);
|
||||
await mkdir(evidenceDir, { recursive: true });
|
||||
const startedAt = new Date();
|
||||
const langbotRepo = resolve(root, env.LANGBOT_REPO || "..");
|
||||
const sdkRepo = resolve(root, env.LANGBOT_PLUGIN_SDK_REPO || "../../langbot-plugin-sdk");
|
||||
const sdkRepo = resolve(
|
||||
root,
|
||||
env.LANGBOT_PLUGIN_SDK_REPO || "../../langbot-plugin-sdk",
|
||||
);
|
||||
const sdkSrc = resolve(sdkRepo, "src");
|
||||
const dbPath = join(evidenceDir, "ledger-contention.sqlite3");
|
||||
const stdoutLog = join(evidenceDir, "probe-stdout.log");
|
||||
const stderrLog = join(evidenceDir, "probe-stderr.log");
|
||||
const automationResultJson = join(evidenceDir, "automation-result.json");
|
||||
const resultJson = join(evidenceDir, "result.json");
|
||||
const timeoutMs = Number(env.LANGBOT_AGENT_RUNNER_PROBE_TIMEOUT_MS || "30000");
|
||||
const timeoutMs = Number(env.LANGBOT_RUNNER_PROBE_TIMEOUT_MS || "30000");
|
||||
const command = {
|
||||
executable: "rtk",
|
||||
args: [resolve(langbotRepo, ".venv/bin/python"), "-c", script, dbPath],
|
||||
@@ -185,7 +201,13 @@ async function main() {
|
||||
timeout_ms: timeoutMs,
|
||||
exit_status: null,
|
||||
signal: null,
|
||||
evidence: { stdout_log: stdoutLog, stderr_log: stderrLog, database: dbPath, automation_result_json: automationResultJson, result_json: resultJson },
|
||||
evidence: {
|
||||
stdout_log: stdoutLog,
|
||||
stderr_log: stderrLog,
|
||||
database: dbPath,
|
||||
automation_result_json: automationResultJson,
|
||||
result_json: resultJson,
|
||||
},
|
||||
evidence_collected: ["filesystem"],
|
||||
};
|
||||
try {
|
||||
@@ -195,7 +217,9 @@ async function main() {
|
||||
} else {
|
||||
const proc = await run(command, timeoutMs, {
|
||||
...process.env,
|
||||
PYTHONPATH: [sdkSrc, process.env.PYTHONPATH].filter(Boolean).join(delimiter),
|
||||
PYTHONPATH: [sdkSrc, process.env.PYTHONPATH]
|
||||
.filter(Boolean)
|
||||
.join(delimiter),
|
||||
UV_CACHE_DIR: env.UV_CACHE_DIR || join(evidenceDir, ".uv-cache"),
|
||||
});
|
||||
await writeFile(stdoutLog, proc.stdout, "utf8");
|
||||
@@ -208,7 +232,10 @@ async function main() {
|
||||
} else if (proc.timedOut) {
|
||||
result.status = "fail";
|
||||
result.reason = `ledger contention timed out after ${timeoutMs}ms`;
|
||||
} else if (proc.status === 0 && proc.stdout.includes("LEDGER_CONTENTION_OK")) {
|
||||
} else if (
|
||||
proc.status === 0 &&
|
||||
proc.stdout.includes("LEDGER_CONTENTION_OK")
|
||||
) {
|
||||
result.status = "pass";
|
||||
result.reason = "ledger contention probe passed";
|
||||
} else {
|
||||
@@ -229,7 +256,9 @@ async function main() {
|
||||
await writeFile(resultJson, resultText, "utf8");
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
}
|
||||
process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1);
|
||||
process.exit(
|
||||
result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1,
|
||||
);
|
||||
}
|
||||
|
||||
await main();
|
||||
|
||||
@@ -7,7 +7,11 @@ import { delimiter, join, resolve } from "node:path";
|
||||
import { env } from "node:process";
|
||||
|
||||
function timestampSlug(date = new Date()) {
|
||||
return date.toISOString().replace(/\.\d{3}Z$/, "Z").replace(/[^0-9A-Za-z]+/g, "-").replace(/^-|-$/g, "");
|
||||
return date
|
||||
.toISOString()
|
||||
.replace(/\.\d{3}Z$/, "Z")
|
||||
.replace(/[^0-9A-Za-z]+/g, "-")
|
||||
.replace(/^-|-$/g, "");
|
||||
}
|
||||
|
||||
function localIsoWithOffset(date = new Date()) {
|
||||
@@ -55,7 +59,14 @@ function runProcess(command, timeoutMs, childEnv) {
|
||||
});
|
||||
child.on("error", (error) => {
|
||||
clearTimeout(timeout);
|
||||
resolveDone({ stdout, stderr, error, timedOut, status: null, signal: null });
|
||||
resolveDone({
|
||||
stdout,
|
||||
stderr,
|
||||
error,
|
||||
timedOut,
|
||||
status: null,
|
||||
signal: null,
|
||||
});
|
||||
});
|
||||
child.on("close", (status, signal) => {
|
||||
clearTimeout(timeout);
|
||||
@@ -124,11 +135,16 @@ async function main() {
|
||||
const root = resolve(env.LBS_ROOT || process.cwd());
|
||||
const caseId = "agent-runner-ledger-invariants";
|
||||
const runId = env.LBS_RUN_ID || `${timestampSlug()}-${caseId}`;
|
||||
const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId));
|
||||
const evidenceDir = resolve(
|
||||
env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId),
|
||||
);
|
||||
await mkdir(evidenceDir, { recursive: true });
|
||||
const startedAt = new Date();
|
||||
const langbotRepo = resolveFromRoot(root, env.LANGBOT_REPO || "..");
|
||||
const sdkRepo = resolveFromRoot(root, env.LANGBOT_PLUGIN_SDK_REPO || "../../langbot-plugin-sdk");
|
||||
const sdkRepo = resolveFromRoot(
|
||||
root,
|
||||
env.LANGBOT_PLUGIN_SDK_REPO || "../../langbot-plugin-sdk",
|
||||
);
|
||||
const sdkSrc = resolve(sdkRepo, "src");
|
||||
const stdoutLog = join(evidenceDir, "probe-stdout.log");
|
||||
const stderrLog = join(evidenceDir, "probe-stderr.log");
|
||||
@@ -139,7 +155,7 @@ async function main() {
|
||||
args: [resolve(langbotRepo, ".venv/bin/python"), "-c", probeScript],
|
||||
cwd: langbotRepo,
|
||||
};
|
||||
const timeoutMs = Number(env.LANGBOT_AGENT_RUNNER_PROBE_TIMEOUT_MS || "30000");
|
||||
const timeoutMs = Number(env.LANGBOT_RUNNER_PROBE_TIMEOUT_MS || "30000");
|
||||
const result = {
|
||||
source: "automation",
|
||||
probe: "python-sync",
|
||||
@@ -174,7 +190,9 @@ async function main() {
|
||||
} else {
|
||||
const childEnv = {
|
||||
...process.env,
|
||||
PYTHONPATH: [sdkSrc, process.env.PYTHONPATH].filter(Boolean).join(delimiter),
|
||||
PYTHONPATH: [sdkSrc, process.env.PYTHONPATH]
|
||||
.filter(Boolean)
|
||||
.join(delimiter),
|
||||
UV_CACHE_DIR: env.UV_CACHE_DIR || join(evidenceDir, ".uv-cache"),
|
||||
};
|
||||
await mkdir(childEnv.UV_CACHE_DIR, { recursive: true });
|
||||
@@ -210,7 +228,9 @@ async function main() {
|
||||
await writeFile(resultJson, resultText, "utf8");
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
}
|
||||
process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1);
|
||||
process.exit(
|
||||
result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1,
|
||||
);
|
||||
}
|
||||
|
||||
await main();
|
||||
|
||||
@@ -7,7 +7,11 @@ import { delimiter, join, resolve } from "node:path";
|
||||
import { env } from "node:process";
|
||||
|
||||
function timestampSlug(date = new Date()) {
|
||||
return date.toISOString().replace(/\.\d{3}Z$/, "Z").replace(/[^0-9A-Za-z]+/g, "-").replace(/^-|-$/g, "");
|
||||
return date
|
||||
.toISOString()
|
||||
.replace(/\.\d{3}Z$/, "Z")
|
||||
.replace(/[^0-9A-Za-z]+/g, "-")
|
||||
.replace(/^-|-$/g, "");
|
||||
}
|
||||
|
||||
function localIsoWithOffset(date = new Date()) {
|
||||
@@ -51,7 +55,14 @@ function run(command, timeoutMs, childEnv) {
|
||||
});
|
||||
child.on("error", (error) => {
|
||||
clearTimeout(timeout);
|
||||
resolveDone({ stdout, stderr, error, timedOut, status: null, signal: null });
|
||||
resolveDone({
|
||||
stdout,
|
||||
stderr,
|
||||
error,
|
||||
timedOut,
|
||||
status: null,
|
||||
signal: null,
|
||||
});
|
||||
});
|
||||
child.on("close", (status, signal) => {
|
||||
clearTimeout(timeout);
|
||||
@@ -116,17 +127,22 @@ async function main() {
|
||||
const root = resolve(env.LBS_ROOT || process.cwd());
|
||||
const caseId = "agent-runner-ledger-stress";
|
||||
const runId = env.LBS_RUN_ID || `${timestampSlug()}-${caseId}`;
|
||||
const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId));
|
||||
const evidenceDir = resolve(
|
||||
env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId),
|
||||
);
|
||||
await mkdir(evidenceDir, { recursive: true });
|
||||
const startedAt = new Date();
|
||||
const langbotRepo = resolve(root, env.LANGBOT_REPO || "..");
|
||||
const sdkRepo = resolve(root, env.LANGBOT_PLUGIN_SDK_REPO || "../../langbot-plugin-sdk");
|
||||
const sdkRepo = resolve(
|
||||
root,
|
||||
env.LANGBOT_PLUGIN_SDK_REPO || "../../langbot-plugin-sdk",
|
||||
);
|
||||
const sdkSrc = resolve(sdkRepo, "src");
|
||||
const stdoutLog = join(evidenceDir, "probe-stdout.log");
|
||||
const stderrLog = join(evidenceDir, "probe-stderr.log");
|
||||
const automationResultJson = join(evidenceDir, "automation-result.json");
|
||||
const resultJson = join(evidenceDir, "result.json");
|
||||
const timeoutMs = Number(env.LANGBOT_AGENT_RUNNER_PROBE_TIMEOUT_MS || "30000");
|
||||
const timeoutMs = Number(env.LANGBOT_RUNNER_PROBE_TIMEOUT_MS || "30000");
|
||||
const command = {
|
||||
executable: "rtk",
|
||||
args: [resolve(langbotRepo, ".venv/bin/python"), "-c", script],
|
||||
@@ -150,7 +166,12 @@ async function main() {
|
||||
timeout_ms: timeoutMs,
|
||||
exit_status: null,
|
||||
signal: null,
|
||||
evidence: { stdout_log: stdoutLog, stderr_log: stderrLog, automation_result_json: automationResultJson, result_json: resultJson },
|
||||
evidence: {
|
||||
stdout_log: stdoutLog,
|
||||
stderr_log: stderrLog,
|
||||
automation_result_json: automationResultJson,
|
||||
result_json: resultJson,
|
||||
},
|
||||
evidence_collected: ["filesystem"],
|
||||
};
|
||||
try {
|
||||
@@ -160,7 +181,9 @@ async function main() {
|
||||
} else {
|
||||
const proc = await run(command, timeoutMs, {
|
||||
...process.env,
|
||||
PYTHONPATH: [sdkSrc, process.env.PYTHONPATH].filter(Boolean).join(delimiter),
|
||||
PYTHONPATH: [sdkSrc, process.env.PYTHONPATH]
|
||||
.filter(Boolean)
|
||||
.join(delimiter),
|
||||
UV_CACHE_DIR: env.UV_CACHE_DIR || join(evidenceDir, ".uv-cache"),
|
||||
});
|
||||
await writeFile(stdoutLog, proc.stdout, "utf8");
|
||||
@@ -173,7 +196,10 @@ async function main() {
|
||||
} else if (proc.timedOut) {
|
||||
result.status = "fail";
|
||||
result.reason = `ledger stress timed out after ${timeoutMs}ms`;
|
||||
} else if (proc.status === 0 && proc.stdout.includes("LEDGER_STRESS_OK")) {
|
||||
} else if (
|
||||
proc.status === 0 &&
|
||||
proc.stdout.includes("LEDGER_STRESS_OK")
|
||||
) {
|
||||
result.status = "pass";
|
||||
result.reason = "ledger stress probe passed";
|
||||
} else {
|
||||
@@ -194,7 +220,9 @@ async function main() {
|
||||
await writeFile(resultJson, resultText, "utf8");
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
}
|
||||
process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1);
|
||||
process.exit(
|
||||
result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1,
|
||||
);
|
||||
}
|
||||
|
||||
await main();
|
||||
|
||||
@@ -6,9 +6,10 @@ await runPytestProbe({
|
||||
caseId: "agent-runner-runtime-chaos",
|
||||
repoEnvKey: "LANGBOT_PLUGIN_SDK_REPO",
|
||||
defaultRepo: "../../langbot-plugin-sdk",
|
||||
description: "LangBot plugin SDK AgentRunner runtime failure, timeout, forwarding, and pull API pytest probe.",
|
||||
description:
|
||||
"LangBot plugin SDK Runner runtime failure, timeout, forwarding, and pull API pytest probe.",
|
||||
testTargets: [
|
||||
"tests/runtime/plugin/test_mgr_agent_runner.py",
|
||||
"tests/runtime/plugin/test_mgr_runner.py",
|
||||
"tests/runtime/test_pull_api_handlers.py",
|
||||
],
|
||||
});
|
||||
|
||||
@@ -5,7 +5,10 @@ import { basename, delimiter, join, resolve } from "node:path";
|
||||
import { env } from "node:process";
|
||||
|
||||
function loadEnvDefaults(root) {
|
||||
for (const path of [join(root, "skills/.env"), join(root, "skills/.env.local")]) {
|
||||
for (const path of [
|
||||
join(root, "skills/.env"),
|
||||
join(root, "skills/.env.local"),
|
||||
]) {
|
||||
if (!existsSync(path)) continue;
|
||||
for (const rawLine of readFileSync(path, "utf8").split(/\r?\n/)) {
|
||||
const line = rawLine.trim();
|
||||
@@ -14,13 +17,20 @@ function loadEnvDefaults(root) {
|
||||
if (sep === -1) continue;
|
||||
const key = line.slice(0, sep).trim();
|
||||
if (env[key]) continue;
|
||||
env[key] = line.slice(sep + 1).trim().replace(/^["']|["']$/g, "");
|
||||
env[key] = line
|
||||
.slice(sep + 1)
|
||||
.trim()
|
||||
.replace(/^["']|["']$/g, "");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function timestampSlug(date = new Date()) {
|
||||
return date.toISOString().replace(/\.\d{3}Z$/, "Z").replace(/[^0-9A-Za-z]+/g, "-").replace(/^-|-$/g, "");
|
||||
return date
|
||||
.toISOString()
|
||||
.replace(/\.\d{3}Z$/, "Z")
|
||||
.replace(/[^0-9A-Za-z]+/g, "-")
|
||||
.replace(/^-|-$/g, "");
|
||||
}
|
||||
|
||||
function localIsoWithOffset(date = new Date()) {
|
||||
@@ -88,7 +98,14 @@ async function runProcess(command, timeoutMs, childEnv) {
|
||||
});
|
||||
child.on("error", (error) => {
|
||||
clearTimeout(timeout);
|
||||
resolveDone({ stdout, stderr, error, timedOut, status: null, signal: null });
|
||||
resolveDone({
|
||||
stdout,
|
||||
stderr,
|
||||
error,
|
||||
timedOut,
|
||||
status: null,
|
||||
signal: null,
|
||||
});
|
||||
});
|
||||
child.on("close", (status, signal) => {
|
||||
clearTimeout(timeout);
|
||||
@@ -109,10 +126,14 @@ export async function runPytestProbe({
|
||||
}) {
|
||||
const root = resolve(env.LBS_ROOT || process.cwd());
|
||||
loadEnvDefaults(root);
|
||||
const resolvedTimeoutMs = Number(timeoutMs || env.LANGBOT_AGENT_RUNNER_PROBE_TIMEOUT_MS || "180000");
|
||||
const resolvedTimeoutMs = Number(
|
||||
timeoutMs || env.LANGBOT_RUNNER_PROBE_TIMEOUT_MS || "180000",
|
||||
);
|
||||
|
||||
const runId = env.LBS_RUN_ID || `${timestampSlug()}-${caseId}`;
|
||||
const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId));
|
||||
const evidenceDir = resolve(
|
||||
env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId),
|
||||
);
|
||||
await mkdir(evidenceDir, { recursive: true });
|
||||
const uvCacheDir = env.UV_CACHE_DIR || join(evidenceDir, ".uv-cache");
|
||||
await mkdir(uvCacheDir, { recursive: true });
|
||||
@@ -171,14 +192,21 @@ export async function runPytestProbe({
|
||||
result.status = "env_issue";
|
||||
result.reason = `${repoEnvKey || "repo"} did not resolve to an existing directory: ${repoPath}`;
|
||||
} else {
|
||||
const missingTargets = testTargets.filter((target) => !existsSync(join(repoPath, target.split("::")[0])));
|
||||
const missingTargets = testTargets.filter(
|
||||
(target) => !existsSync(join(repoPath, target.split("::")[0])),
|
||||
);
|
||||
if (missingTargets.length > 0) {
|
||||
result.status = "env_issue";
|
||||
result.reason = `pytest target file(s) not found in ${basename(repoPath)}: ${missingTargets.join(", ")}`;
|
||||
} else {
|
||||
const childEnv = { ...process.env, UV_CACHE_DIR: uvCacheDir };
|
||||
if (pythonPaths.length > 0) {
|
||||
childEnv.PYTHONPATH = [pythonPaths.join(delimiter), childEnv.PYTHONPATH].filter(Boolean).join(delimiter);
|
||||
childEnv.PYTHONPATH = [
|
||||
pythonPaths.join(delimiter),
|
||||
childEnv.PYTHONPATH,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(delimiter);
|
||||
}
|
||||
const proc = await runProcess(command, resolvedTimeoutMs, childEnv);
|
||||
result.exit_status = proc.status;
|
||||
@@ -195,7 +223,11 @@ export async function runPytestProbe({
|
||||
} else if (proc.status === 0) {
|
||||
result.status = "pass";
|
||||
result.reason = `pytest passed for ${testTargets.join(", ")}.`;
|
||||
} else if (/command not found|no such file or directory|executable file not found/i.test(`${proc.stdout}\n${proc.stderr}`)) {
|
||||
} else if (
|
||||
/command not found|no such file or directory|executable file not found/i.test(
|
||||
`${proc.stdout}\n${proc.stderr}`,
|
||||
)
|
||||
) {
|
||||
result.status = "env_issue";
|
||||
result.reason = `pytest command could not run in ${repoPath}. See ${stdoutLog} and ${stderrLog}.`;
|
||||
} else {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# AgentRunner QA Workflow
|
||||
# Runner QA Workflow
|
||||
|
||||
Use this workflow when an agent finishes AgentRunner-related code and enters a
|
||||
Use this workflow when an agent finishes Runner-related code and enters a
|
||||
test phase.
|
||||
|
||||
## Order
|
||||
@@ -25,7 +25,7 @@ test phase.
|
||||
backend is available and installing the QA fixture is acceptable.
|
||||
- `rtk bin/lbs test run agent-runner-qa-debug-chat --dry-run` when WebUI live
|
||||
execution needs deterministic coverage without a model provider. This
|
||||
case runs its setup automation first: install the QA AgentRunner fixture,
|
||||
case runs its setup automation first: install the QA Runner fixture,
|
||||
create/update the QA pipeline, write the case-specific pipeline env, then
|
||||
execute Debug Chat.
|
||||
- `rtk bin/lbs test run agent-runner-ledger-invariants --dry-run`
|
||||
@@ -51,8 +51,8 @@ only to review or adjust the generated list.
|
||||
| --- | --- | --- |
|
||||
| `LangBot/src/langbot/pkg/agent/runner/*`, `tests/unit_tests/agent/test_result_normalizer.py`, protocol/result/context/resource builders | `rtk bin/lbs test run agent-runner-fixture-contract --dry-run`; `rtk bin/lbs test run agent-runner-behavior-matrix --dry-run`; targeted LangBot unit tests for touched files | Result shape, user-visible runner output, or Debug Chat delivery changed: add `pipeline-debug-chat` or `local-agent-basic-debug-chat`. |
|
||||
| `LangBot/src/langbot/pkg/entity/persistence/agent_run.py`, `run_journal.py`, run ledger store/API/auth tests, claim/lease/status code | `rtk bin/lbs test run agent-runner-ledger-invariants --dry-run`; `rtk bin/lbs test run agent-runner-ledger-stress --dry-run`; `rtk bin/lbs test run agent-runner-ledger-contention --dry-run`; `rtk bin/lbs test run agent-runner-async-db-readiness --dry-run` before `rtk bin/lbs test run agent-runner-ledger-concurrency --dry-run` | Debug Chat run lifecycle, resume, or visible completion changed: add `local-agent-basic-debug-chat`. |
|
||||
| `langbot-plugin-sdk/src/langbot_plugin/api/entities/builtin/agent_runner/*`, `api/proxies/agent_run_api.py`, runtime pull handlers, plugin manager/runtime IO | `rtk bin/lbs test run agent-runner-runtime-chaos --dry-run`; `rtk bin/lbs test run agent-runner-behavior-matrix --dry-run`; targeted SDK pytest | Runtime delivery or tool-call surface changed: add `agent-runner-release-preflight`, then `local-agent-basic-debug-chat`. |
|
||||
| `langbot-agent-runner/*/components/agent_runner/*`, external runner daemon/client code, ACP/Codex/Claude runner command wrappers | Repo-local targeted tests; `rtk bin/lbs test run agent-runner-runtime-chaos --dry-run`; `rtk bin/lbs test run agent-runner-release-preflight --dry-run` | ACP or external coding runner behavior changed: add `acp-agent-runner-debug-chat`. |
|
||||
| `langbot-plugin-sdk/src/langbot_plugin/api/entities/builtin/runner/*`, `api/proxies/agent_run_api.py`, runtime pull handlers, plugin manager/runtime IO | `rtk bin/lbs test run agent-runner-runtime-chaos --dry-run`; `rtk bin/lbs test run agent-runner-behavior-matrix --dry-run`; targeted SDK pytest | Runtime delivery or tool-call surface changed: add `agent-runner-release-preflight`, then `local-agent-basic-debug-chat`. |
|
||||
| `langbot-agent-runner/*/components/runner/*`, external runner daemon/client code, ACP/Codex/Claude runner command wrappers | Repo-local targeted tests; `rtk bin/lbs test run agent-runner-runtime-chaos --dry-run`; `rtk bin/lbs test run agent-runner-release-preflight --dry-run` | ACP or external coding runner behavior changed: add `acp-agent-runner-debug-chat`. |
|
||||
| Prompt preprocessing, effective prompt, pipeline AI config, runner binding/default runner migration | `rtk bin/lbs test run agent-runner-behavior-matrix --dry-run`; targeted LangBot pipeline/agent tests | The runner reads host-provided prompt or saved runner config: add `local-agent-effective-prompt-debug-chat`. |
|
||||
| Context window, transcript, history/event state, compaction, checkpoint/steering | `rtk bin/lbs test run agent-runner-behavior-matrix --dry-run`; targeted LangBot agent state/context tests | Multi-turn memory, compaction, or steering behavior changed: add `local-agent-context-compaction-debug-chat` and, for steering-specific changes, `local-agent-steering-debug-chat`. |
|
||||
| Plugin tool authorization, host tool listing, MCP tool bridge, function-call conversion | `rtk bin/lbs test run agent-runner-behavior-matrix --dry-run`; targeted plugin/MCP/tool tests | Tool execution is user-visible: add `local-agent-plugin-tool-call-debug-chat`; for MCP-specific changes add `mcp-stdio-register` then `mcp-stdio-tool-call`. |
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# Agent Runner Release Gate
|
||||
# Runner Release Gate
|
||||
|
||||
Use this reference when judging whether runner externalization is release-ready. The goal is not to enumerate every possible prompt. The gate covers product abilities and trust boundaries with deterministic normal-path cases, then leaves rare negative branches to unit and contract tests.
|
||||
|
||||
@@ -35,7 +35,7 @@ For a quick early blocker check, run:
|
||||
rtk bin/lbs test run agent-runner-release-preflight --dry-run
|
||||
```
|
||||
|
||||
For the code-level AgentRunner probes, run:
|
||||
For the code-level Runner probes, run:
|
||||
|
||||
```bash
|
||||
rtk bin/lbs test run agent-runner-behavior-matrix --dry-run
|
||||
@@ -70,7 +70,7 @@ API integration gate, not a Debug Chat execution proof.
|
||||
|
||||
`agent-runner-qa-debug-chat` is the deterministic live execution proof. It uses
|
||||
a pipeline created by `scripts/e2e/ensure-qa-agent-runner-pipeline.mjs` and
|
||||
expects Debug Chat to return `QA_AGENT_RUNNER_OK:<input>` through
|
||||
expects Debug Chat to return `QA_RUNNER_OK:<input>` through
|
||||
`plugin:qa/agent-runner/default`.
|
||||
|
||||
`agent-runner-ledger-invariants` is the fast Host ledger probe. It uses
|
||||
@@ -97,7 +97,7 @@ If it times out before any test result and a direct `aiosqlite.connect()` script
|
||||
also hangs, classify the run with troubleshooting id
|
||||
`aiosqlite-connect-hangs` instead of treating it as a browser E2E failure.
|
||||
|
||||
`agent-runner-runtime-chaos` runs SDK AgentRunner runtime and pull API handler
|
||||
`agent-runner-runtime-chaos` runs SDK Runner runtime and pull API handler
|
||||
tests from `LANGBOT_PLUGIN_SDK_REPO` or `../langbot-plugin-sdk`.
|
||||
Each probe writes `automation-result.json` and probe logs under
|
||||
`LBS_EVIDENCE_DIR`.
|
||||
@@ -108,9 +108,9 @@ Each probe writes `automation-result.json` and probe logs under
|
||||
| --- | --- | --- |
|
||||
| Authenticated WebUI session | `webui-login-state`, `agent-runner-release-preflight` | The browser profile can operate the same backend that later cases use. |
|
||||
| Generic Pipeline Debug Chat | `pipeline-debug-chat` | The WebUI Debug Chat path itself works before runner-specific failures are diagnosed. |
|
||||
| Deterministic QA runner install | `agent-runner-live-install` | A local `.lbpkg` AgentRunner package can install and register a runner. |
|
||||
| Deterministic QA runner install | `agent-runner-live-install` | A local `.lbpkg` Runner package can install and register a runner. |
|
||||
| Deterministic QA runner Debug Chat | `agent-runner-qa-debug-chat` | The installed QA runner executes through WebUI Debug Chat without a model provider. |
|
||||
| Required runner plugins | `agent-runner-release-preflight` | `langbot-team/LocalAgent` and `langbot-team/ACPAgentRunner` are visible to the host. |
|
||||
| Required runner plugins | `agent-runner-release-preflight` | `langbot-team/LocalAgent` and `langbot-team/ACPRunner` are visible to the host. |
|
||||
| Required QA plugin tools | `plugin-e2e-smoke`, `agent-runner-release-preflight`, `qa-plugin-smoke-live-install` | The deterministic `qa_plugin_echo` and `qa_plugin_fail` tools are exposed before tool-loop and tool-error cases start. |
|
||||
| Knowledge base fixture | `langrag-kb-retrieve`, `local-agent-rag-debug-chat` | LangRAG data is queryable and the runner inserts retrieved context. |
|
||||
| Effective prompt bridge | `local-agent-effective-prompt-debug-chat` | Host prompt preprocessing reaches the runner. |
|
||||
@@ -149,7 +149,7 @@ rtk uv run pytest -q
|
||||
# langbot-plugin-sdk
|
||||
rtk uv run pytest -q
|
||||
|
||||
# langbot-skills saved AgentRunner probes
|
||||
# langbot-skills saved Runner probes
|
||||
rtk bin/lbs test run agent-runner-behavior-matrix --dry-run
|
||||
rtk bin/lbs test run agent-runner-ledger-invariants --dry-run
|
||||
rtk bin/lbs test run agent-runner-ledger-stress --dry-run
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# Dify AgentRunner
|
||||
# Dify Runner
|
||||
|
||||
Use this reference when validating `langbot-team/DifyAgent` through LangBot WebUI.
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# Local Agent Runner Coverage
|
||||
# Local Runner Coverage
|
||||
|
||||
Use this matrix when judging whether the external `langbot-team/LocalAgent` plugin still behaves like the old built-in local-agent runner.
|
||||
|
||||
@@ -10,7 +10,7 @@ The QA target is end-to-end behavior. UI cases prove the host, SDK, plugin runti
|
||||
- `LangBot/src/langbot/pkg/agent/runner/pipeline_adapter.py` adapts Pipeline-only fields into `ctx.adapter.extra.prompt`, `ctx.adapter.extra.params`, and optional `ctx.bootstrap.messages`.
|
||||
- `LangBot/src/langbot/pkg/agent/runner/resource_builder.py` authorizes models, fallback models, rerank models, tools, and knowledge bases for the current run.
|
||||
- `LangBot/src/langbot/pkg/plugin/handler.py` validates run-scoped model/tool/rerank access and calls the host model provider or tool manager with the current query.
|
||||
- `langbot-local-agent/components/agent_runner/default.py` selects streaming or non-streaming execution, retrieves RAG context, builds messages, invokes models with fallback, and runs tool loops.
|
||||
- `langbot-local-agent/components/runner/default.py` selects streaming or non-streaming execution, retrieves RAG context, builds messages, invokes models with fallback, and runs tool loops.
|
||||
- `langbot-local-agent/pkg/messages.py` prefers the host effective prompt from `ctx.adapter.extra.prompt`, uses `ctx.bootstrap.messages` only as a small bootstrap window, and preserves structured/multimodal input while inserting RAG context.
|
||||
|
||||
TODO: Treat `ctx.adapter.extra.prompt` as a temporary Pipeline bridge for old
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
# Local Agent Runner
|
||||
# Local Runner
|
||||
|
||||
Use this reference when validating the pluginized `langbot-team/LocalAgent` runner through the WebUI.
|
||||
|
||||
The goal is behavior parity with the old built-in local-agent runner. The code does not need to be identical, but the visible behavior should match: effective prompt, current input, history, model selection and fallback, tool calling, knowledge retrieval, multimodal input, streaming and non-streaming output all have to reach the runner through the host and SDK.
|
||||
|
||||
For path-by-path coverage, read [Local Agent Runner Coverage](local-agent-runner-coverage.md).
|
||||
For path-by-path coverage, read [Local Runner Coverage](local-agent-runner-coverage.md).
|
||||
|
||||
## Main Surface
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ Measure user experience and internal composition separately:
|
||||
|
||||
- WebUI load and interaction latency.
|
||||
- Debug Chat send-to-first-visible-token and send-to-completion latency.
|
||||
- Pipeline, RAG, plugin runtime, MCP, AgentRunner, and persistence segment
|
||||
- Pipeline, RAG, plugin runtime, MCP, Runner, and persistence segment
|
||||
latency.
|
||||
- Queue wait time, concurrency, throughput, timeout rate, and p95/p99 latency.
|
||||
- Startup, plugin install, knowledge-base ingestion, migration, and recovery
|
||||
|
||||
@@ -33,7 +33,7 @@ Both external runners receive the same host-generated gateway `AgentMCPServerCon
|
||||
|
||||
This is a **runner-plugin transport detail, not a host all-tool-branch issue** — proven by **both** runners discovering skills end-to-end with the unmodified branch (see cases below).
|
||||
|
||||
> **Correction (2026-06-22).** An earlier revision of this doc claimed acp was "blocked" on remote-ssh and *required* `langbot-assets-gateway-public-url`, based on a run that returned `PROBEDONE 0 0` / timeout. That was an **environment artifact, not an acp defect**: a duplicate backend instance (a second checkout `LangBot-master/` whose box runtime contended for the same `--ws-control-port 5410`) plus a wedged plugin runtime (host `emit_event` / `list_agent_runners` action calls timing out with `ActionCallTimeoutError`). Re-run on a clean single-instance runtime, **acp passes via the reverse tunnel with no `public-url`** (`PROBEDONE 1 17`, 8–24s).
|
||||
> **Correction (2026-06-22).** An earlier revision of this doc claimed acp was "blocked" on remote-ssh and *required* `langbot-assets-gateway-public-url`, based on a run that returned `PROBEDONE 0 0` / timeout. That was an **environment artifact, not an acp defect**: a duplicate backend instance (a second checkout `LangBot-master/` whose box runtime contended for the same `--ws-control-port 5410`) plus a wedged plugin runtime (host `emit_event` / `list_runners` action calls timing out with `ActionCallTimeoutError`). Re-run on a clean single-instance runtime, **acp passes via the reverse tunnel with no `public-url`** (`PROBEDONE 1 17`, 8–24s).
|
||||
- **Lifecycle**: discover → activate → operate (native exec under the activated mount path) → register.
|
||||
- **Backend**: docker · nsjail · e2b.
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@ Date: 2026-05-16
|
||||
|
||||
### Symptom
|
||||
|
||||
The WebUI can send a Debug Chat message, but the bot response is missing or says `Agent runner temporarily unavailable`. Backend logs may include `Action list_plugins call timed out`, `Action list_agent_runners call timed out`, or `Action invoke_llm_stream call timed out`.
|
||||
The WebUI can send a Debug Chat message, but the bot response is missing or says `Agent runner temporarily unavailable`. Backend logs may include `Action list_plugins call timed out`, `Action list_runners call timed out`, or `Action invoke_llm_stream call timed out`.
|
||||
|
||||
### Likely Cause
|
||||
|
||||
@@ -78,7 +78,7 @@ Structured entry: `../troubleshooting/marketplace-network-flaky.yaml`
|
||||
|
||||
Marketplace icon/tag/recommendation requests can fail while plugin cards are already visible. Retry first, and use backend component endpoints only to confirm installation results.
|
||||
|
||||
## agent-runner-actor-context-fields: AgentRunner reads old actor fields
|
||||
## agent-runner-actor-context-fields: Runner reads old actor fields
|
||||
|
||||
Structured entry: `../troubleshooting/agent-runner-actor-context-fields.yaml`
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Workspace Release Testing
|
||||
|
||||
Use the workspace gates when changes span LangBot core, the plugin SDK, AgentRunner, or multiple first-party plugins.
|
||||
Use the workspace gates when changes span LangBot core, the plugin SDK, Runner, or multiple first-party plugins.
|
||||
|
||||
## Cost Ladder
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
id: langbot-workspace-release-gate
|
||||
title: "LangBot workspace top-down release gate"
|
||||
description: "Broad release gate combining deterministic repository contracts with representative browser workflows, plugin pages, RAG/parser, EBA, external AgentRunner, and one complex LocalAgent task."
|
||||
description: "Broad release gate combining deterministic repository contracts with representative browser workflows, plugin pages, RAG/parser, EBA, external Runner, and one complex LocalAgent task."
|
||||
type: release_gate
|
||||
priority: p0
|
||||
tags:
|
||||
|
||||
+4
-4
@@ -1,9 +1,9 @@
|
||||
id: agent-runner-actor-context-fields
|
||||
title: "AgentRunner reads old actor.type and actor.id fields"
|
||||
title: "Runner reads old actor.type and actor.id fields"
|
||||
date: 2026-05-17
|
||||
symptoms:
|
||||
- "Pipeline Debug Chat shows Agent runner execution failed."
|
||||
- "Backend logs show an AttributeError from an AgentRunner plugin."
|
||||
- "Backend logs show an AttributeError from an Runner plugin."
|
||||
patterns:
|
||||
- "AttributeError: 'ActorContext' object has no attribute 'type'"
|
||||
- "AttributeError: 'ActorContext' object has no attribute 'id'"
|
||||
@@ -15,8 +15,8 @@ fix_steps:
|
||||
- "Update runner code to read actor.actor_type and actor.actor_id."
|
||||
- "Keep getattr fallback to type/id only if compatibility with older host data is required."
|
||||
- "Restart LangBot or the plugin runtime so the updated plugin code is loaded."
|
||||
- "Add a regression test that builds AgentRunContext with ActorContext(actor_type=..., actor_id=...)."
|
||||
verification: "Run dify-agent-debug-chat or another AgentRunner Debug Chat and confirm the assistant/bot message contains the expected sentinel while backend logs show Streaming completed."
|
||||
- "Add a regression test that builds RunnerContext with ActorContext(actor_type=..., actor_id=...)."
|
||||
verification: "Run dify-agent-debug-chat or another Runner Debug Chat and confirm the assistant/bot message contains the expected sentinel while backend logs show Streaming completed."
|
||||
related_cases:
|
||||
- dify-agent-debug-chat
|
||||
- pipeline-debug-chat
|
||||
|
||||
@@ -2,7 +2,7 @@ id: aiosqlite-connect-hangs
|
||||
title: "aiosqlite connect hangs before ledger pytest starts"
|
||||
category: env_issue
|
||||
symptoms:
|
||||
- "AgentRunner ledger pytest probe times out after collecting tests but before reporting a test result."
|
||||
- "Runner ledger pytest probe times out after collecting tests but before reporting a test result."
|
||||
- "pytest stdout stops at a line like tests/unit_tests/agent/test_run_ledger_store.py."
|
||||
- "A direct aiosqlite.connect(':memory:') script prints its first line and then hangs."
|
||||
patterns:
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
id: ambiguous-runner-default-label
|
||||
title: "AgentRunner selector shows multiple Default or 默认 options"
|
||||
title: "Runner selector shows multiple Default or 默认 options"
|
||||
date: 2026-05-17
|
||||
symptoms:
|
||||
- "The Pipeline AI runner selector shows multiple options named Default or 默认."
|
||||
@@ -9,7 +9,7 @@ patterns:
|
||||
- "label.zh_Hans: 默认"
|
||||
- "label.en_US: Default"
|
||||
likely_causes:
|
||||
- "AgentRunner component ids are commonly named default, but the user-facing metadata.label was also left generic."
|
||||
- "Runner component ids are commonly named default, but the user-facing metadata.label was also left generic."
|
||||
- "The frontend displays metadata.label as the primary option label."
|
||||
fix_steps:
|
||||
- "Keep metadata.name as default if the plugin component id is intended to remain stable."
|
||||
|
||||
@@ -8,7 +8,7 @@ symptoms:
|
||||
- "Knowledge sidebar or plugin sidebar loading may hang or time out."
|
||||
patterns:
|
||||
- "Action list_plugins call timed out"
|
||||
- "Action list_agent_runners call timed out"
|
||||
- "Action list_runners call timed out"
|
||||
- "Action invoke_llm_stream call timed out"
|
||||
- "All models failed during streaming setup"
|
||||
- "Failed to fetch plugins for sidebar"
|
||||
|
||||
Reference in New Issue
Block a user