mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-09-02 15:47:16 +00:00
test(workspace): harden release gate automation
This commit is contained in:
huanghuoguoguo
@@ -19,7 +19,8 @@ steps:
|
||||
- "Read the active LangBot database configuration and inspect the selected run and its ordered events."
|
||||
- "Verify completed terminal state, run.completed, paired tool.call.started/completed events, stable tool names, and monotonic ordering."
|
||||
- "Compare called tools with the authorization snapshot and validate each advertised tool has owner/source, description, and parameter schema."
|
||||
- "Reject malformed event JSON and invalid-JSON, timeout, forbidden, permission-denied, or unauthorized signals."
|
||||
- "Reject malformed persisted event JSON, unrecovered invalid tool arguments, timeout, forbidden, permission-denied, or unauthorized signals."
|
||||
- "Record a malformed model tool-argument payload as a recovery warning only when a later tool call succeeds and the run completes."
|
||||
checks:
|
||||
- "ledger-audit.json status is pass."
|
||||
- "metrics has equal tool_call_started and tool_call_completed counts."
|
||||
|
||||
@@ -16,34 +16,32 @@ skills:
|
||||
env:
|
||||
- LANGBOT_FRONTEND_URL
|
||||
- LANGBOT_BACKEND_URL
|
||||
- LANGBOT_LOCAL_AGENT_PIPELINE_URL
|
||||
- LANGBOT_LOCAL_AGENT_PIPELINE_NAME
|
||||
env_optional:
|
||||
- LANGBOT_E2E_FAKE_PROVIDER_BASE_URL
|
||||
- LANGBOT_FAKE_PROVIDER_PIPELINE_URL
|
||||
- LANGBOT_FAKE_PROVIDER_PIPELINE_NAME
|
||||
automation: scripts/e2e/pipeline-debug-chat.mjs
|
||||
automation_env:
|
||||
- LANGBOT_FRONTEND_URL
|
||||
- LANGBOT_BACKEND_URL
|
||||
- LANGBOT_BROWSER_PROFILE
|
||||
- LANGBOT_CHROMIUM_EXECUTABLE
|
||||
- LANGBOT_LOCAL_AGENT_PIPELINE_URL
|
||||
- LANGBOT_LOCAL_AGENT_PIPELINE_NAME
|
||||
- LANGBOT_FAKE_PROVIDER_PIPELINE_URL
|
||||
- LANGBOT_FAKE_PROVIDER_PIPELINE_NAME
|
||||
- LANGBOT_MCP_QA_STDIO_SERVER_UUID
|
||||
automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL
|
||||
automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME
|
||||
automation_pipeline_url_env: LANGBOT_FAKE_PROVIDER_PIPELINE_URL
|
||||
automation_pipeline_name_env: LANGBOT_FAKE_PROVIDER_PIPELINE_NAME
|
||||
automation_expected_runner_id: "plugin:langbot-team/LocalAgent/default"
|
||||
automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot-team","name":"LocalAgent"}],"enable_all_mcp_servers":false,"bound_mcp_servers":["${LANGBOT_MCP_QA_STDIO_SERVER_UUID}"],"enable_all_skills":false,"bound_skills":[]}'
|
||||
automation_restore_extensions: "1"
|
||||
automation_reset_debug_chat: "1"
|
||||
automation_prompt: "Call the qa_mcp_echo MCP tool with exactly this text: mcp-ok-local-agent. Return only the tool result."
|
||||
automation_expected_text: "qa_mcp_echo:mcp-ok-local-agent"
|
||||
automation_response_timeout_ms: "180000"
|
||||
automation_response_timeout_ms: "60000"
|
||||
setup_automation:
|
||||
- "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env"
|
||||
- "node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env"
|
||||
- "case:mcp-stdio-register"
|
||||
setup_provides_env:
|
||||
- LANGBOT_LOCAL_AGENT_PIPELINE_URL
|
||||
- LANGBOT_LOCAL_AGENT_PIPELINE_NAME
|
||||
- LANGBOT_FAKE_PROVIDER_PIPELINE_URL
|
||||
- LANGBOT_FAKE_PROVIDER_PIPELINE_NAME
|
||||
- LANGBOT_MCP_QA_STDIO_SERVER_UUID
|
||||
failure_patterns:
|
||||
- "qa-plugin-smoke:mcp-ok-local-agent"
|
||||
@@ -54,7 +52,7 @@ failure_patterns:
|
||||
- "no available channel for model"
|
||||
preconditions:
|
||||
- "box.local.allowed_mount_roots includes the bundled MCP fixture directory when LangBot runs stdio MCP servers through Box."
|
||||
- "The selected model route supports function/tool calling, or LANGBOT_E2E_FAKE_PROVIDER_BASE_URL points to scripts/e2e/fake-openai-provider.mjs."
|
||||
- "The target is a local test instance where the QA-owned fake provider/model/pipeline may be created or updated."
|
||||
steps:
|
||||
- "Open LANGBOT_FRONTEND_URL."
|
||||
- "Navigate to MCP Servers."
|
||||
@@ -66,7 +64,7 @@ steps:
|
||||
- "Confirm the server detail page shows Tools: 1 and qa_mcp_echo."
|
||||
- "Open the target local-agent pipeline."
|
||||
- "Use runner Default or the pluginized langbot-team/LocalAgent runner."
|
||||
- "Select a model with function-calling ability that is known to work with tools in the current environment."
|
||||
- "Use the QA fake-provider pipeline prepared by setup automation."
|
||||
- "Open Debug Chat."
|
||||
- "Send: Call the qa_mcp_echo tool with exactly this text: mcp-ok-local-agent. Return only the tool result."
|
||||
checks:
|
||||
@@ -84,7 +82,8 @@ evidence_required:
|
||||
- api_diagnostic
|
||||
- metrics
|
||||
diagnostics:
|
||||
- "For token-free deterministic UI coverage, start scripts/e2e/fake-openai-provider.mjs and pass LANGBOT_E2E_FAKE_PROVIDER_BASE_URL to this case; setup will bind the local-agent pipeline to that fake OpenAI-compatible model."
|
||||
- "The required release case uses the QA fake-provider pipeline so model instruction-following variance cannot hide or mimic an MCP bridge regression."
|
||||
- "Run a separate optional live-provider tool smoke when evaluating a specific provider/model route."
|
||||
- "Run node scripts/e2e/mcp-stdio-fixture.mjs to verify the bundled stdio fixture can list and call qa_mcp_echo without involving a model provider."
|
||||
- "Run node scripts/e2e/mcp-stdio-register.mjs to upsert qa-local-stdio in LangBot and verify /api/v1/tools exposes qa_mcp_echo."
|
||||
- "If backend logs show host_path is outside allowed_mount_roots, add the fixture directory to box.local.allowed_mount_roots in the local LangBot data config."
|
||||
|
||||
@@ -44,8 +44,18 @@ def main() -> int:
|
||||
report = (workspace / "AGENT_REPORT.md").read_text(encoding="utf-8")
|
||||
folded_report = report.casefold()
|
||||
assert "initial" in folded_report and "fail" in folded_report, "report missing initial failure section"
|
||||
for heading in ("root causes", "changed files", "verification"):
|
||||
assert heading in folded_report, f"report missing section: {heading}"
|
||||
assert "root causes" in folded_report, "report missing section: root causes"
|
||||
assert any(
|
||||
heading in folded_report
|
||||
for heading in (
|
||||
"changed files",
|
||||
"files changed",
|
||||
"files modified",
|
||||
"modified files",
|
||||
"changes made",
|
||||
)
|
||||
), "report missing section: changed files"
|
||||
assert "verification" in folded_report, "report missing section: verification"
|
||||
assert report.rstrip().endswith("COMPLEX_AGENT_TASK_OK tests=12 acceptance=PASS")
|
||||
print("HOST_VERIFY_PASS tests=12 acceptance=PASS protected=PASS")
|
||||
return 0
|
||||
|
||||
@@ -119,7 +119,7 @@ try {
|
||||
result.pipeline_id = pipeline.id;
|
||||
result.pipeline_name = pipeline.name || pipelineName;
|
||||
if (!result.pipeline_url && env.LANGBOT_FRONTEND_URL) {
|
||||
result.pipeline_url = `${env.LANGBOT_FRONTEND_URL.replace(/\/$/, "")}/home/pipelines?id=${encodeURIComponent(pipeline.id)}`;
|
||||
result.pipeline_url = `${env.LANGBOT_FRONTEND_URL.replace(/\/$/, "")}/home/agents?id=${encodeURIComponent(pipeline.id)}`;
|
||||
}
|
||||
|
||||
if (resetBeforeRun) {
|
||||
|
||||
@@ -121,7 +121,7 @@ Each probe writes `automation-result.json` and probe logs under
|
||||
| Plugin tool error recovery | `local-agent-tool-error-recovery-debug-chat` | Tool execution errors are serialized into model-facing tool results and the model can produce a final answer instead of failing the run. |
|
||||
| Parallel plugin tool batch | `local-agent-parallel-tools-rag-compaction-debug-chat` | Local-agent executes multiple same-turn plugin tool calls and returns both results with RAG and compacted history. |
|
||||
| MCP registration | `mcp-stdio-register` | The deterministic stdio MCP server is registered and exposes `qa_mcp_echo`. |
|
||||
| MCP tool loop | `mcp-stdio-tool-call` | Local-agent can call the registered MCP tool through the same tool loop. |
|
||||
| MCP tool loop | `mcp-stdio-tool-call` | Local-agent can call the registered MCP tool through the same tool loop using the deterministic QA fake provider. |
|
||||
| Multimodal input | `local-agent-multimodal-debug-chat` | Image upload and structured input reach the runner. |
|
||||
| Multimodal plus RAG | `local-agent-rag-multimodal-debug-chat` | RAG still works when structured image input is present. |
|
||||
| ACP external harness execution | `acp-agent-runner-debug-chat` | ACP executes the configured coding agent and returns visible Debug Chat output. |
|
||||
|
||||
@@ -85,10 +85,15 @@ extension binding uses this UUID, not the human-readable server name.
|
||||
|
||||
## Local-Agent Tool Call Check
|
||||
|
||||
1. Open the target pipeline.
|
||||
2. Confirm `Extensions` allows the MCP server, or that all MCP servers are enabled.
|
||||
3. Use runner `Default` or the pluginized `langbot-team/LocalAgent` runner.
|
||||
4. Select a model with function-calling ability that is known to work with tools in the current environment.
|
||||
The required release case prepares and uses the dedicated QA fake-provider
|
||||
pipeline. This keeps the host, LocalAgent, MCP discovery, function-call
|
||||
conversion, tool execution, and result-return path deterministic. A real model
|
||||
that ignores the tool instruction must not be reported as an MCP bridge failure.
|
||||
|
||||
1. Run `node scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env`.
|
||||
2. Open `LANGBOT_FAKE_PROVIDER_PIPELINE_URL`.
|
||||
3. Confirm `Extensions` allows the MCP server, or that all MCP servers are enabled.
|
||||
4. Confirm the pipeline uses the pluginized `langbot-team/LocalAgent` runner.
|
||||
5. Open `Debug Chat`.
|
||||
6. Ask:
|
||||
|
||||
@@ -111,3 +116,7 @@ qa-plugin-smoke:mcp-ok-local-agent
|
||||
That proves a plugin tool was called, not the MCP server.
|
||||
|
||||
If the provider returns `model_not_found` or `no available channel` only when tools are supplied, switch to a known-good function-calling model before diagnosing MCP or local-agent. That failure means the selected model route is unavailable for the requested tool-call shape.
|
||||
|
||||
Run a real-provider tool smoke separately when validating a particular model
|
||||
route. It measures provider tool-use behavior in addition to the LangBot path
|
||||
and is therefore not the deterministic release gate.
|
||||
|
||||
@@ -22,4 +22,5 @@ Do not add Space model concurrency to these gates. Provider concurrency mixes ex
|
||||
- Repository contract failure: fix the owning repository and add the narrowest deterministic regression.
|
||||
- Browser workflow failure: correlate the screenshot, console, network, and backend log from the same run.
|
||||
- Complex Agent failure: inspect `ledger-audit.json` before blaming the model. Tool pairing, authorization, JSON, and timeout failures are product signals.
|
||||
- A provider-origin malformed tool-argument payload is a recovery warning only when the ledger keeps a paired error result, a later tool call succeeds, and the run completes. Malformed persisted event JSON and unrecovered tool-argument errors remain release failures.
|
||||
- Packaging `env_issue`: restore isolated build dependency access, then rerun packaging only. It must not mask passing or failing runtime contracts.
|
||||
|
||||
@@ -6,18 +6,24 @@ symptoms:
|
||||
- "Debug Chat shows Agent runner temporarily unavailable after the user message is sent."
|
||||
- "Basic streaming prompts may work, but tool-call, non-streaming, or multimodal prompts fail with the same runner and pipeline."
|
||||
- "The failure happens after the local-agent runner starts, not during plugin discovery."
|
||||
- "A short model preflight passes, but a sustained Agent run later fails after the provider quota is exhausted."
|
||||
patterns:
|
||||
- "runner.llm_error"
|
||||
- "runner.tool_loop_error"
|
||||
- "model_not_found"
|
||||
- "no available channel for model"
|
||||
- "invalid api key"
|
||||
- "insufficient user quota"
|
||||
- "insufficient_quota"
|
||||
- "quota exceeded"
|
||||
- "余额不足"
|
||||
- "当前分组上游负载已饱和"
|
||||
- "All models failed during streaming setup"
|
||||
likely_causes:
|
||||
- "The selected model route is unavailable in the current LangBot Space or upstream group."
|
||||
- "The selected model works for plain chat but is not available for tool-call, multimodal, or non-streaming request shapes."
|
||||
- "The provider credential or quota is invalid for the non-streaming path."
|
||||
- "The provider account has enough balance for a short preflight but not for the sustained task."
|
||||
- "The selected model has function-call or vision metadata, but the upstream distributor cannot currently serve that model."
|
||||
fix_steps:
|
||||
- "First rerun a basic local-agent Debug Chat prompt on the same pipeline to confirm the runner host path still works."
|
||||
@@ -25,6 +31,7 @@ fix_steps:
|
||||
- "When tool-call cases fail, retest with a model that is known to support function calling in the active environment."
|
||||
- "When multimodal cases fail, retest with a model route that is known to accept image content."
|
||||
- "When non-streaming fails with invalid api key, verify the provider credential used by the non-streaming requester path."
|
||||
- "When logs report insufficient quota, restore provider balance or switch to another authorized route before rerunning the live-model case."
|
||||
- "Do not classify this as an MCP, RAG, or local-agent runner implementation failure until the same model route works for the requested request shape outside the failing case."
|
||||
verification: "The same case produces the expected bot-visible sentinel or tool result, and backend logs show request completion without runner.llm_error, runner.tool_loop_error, or All models failed."
|
||||
related_cases:
|
||||
@@ -33,3 +40,4 @@ related_cases:
|
||||
- mcp-stdio-tool-call
|
||||
- local-agent-multimodal-debug-chat
|
||||
- local-agent-nonstreaming-debug-chat
|
||||
- local-agent-complex-coding-task-debug-chat
|
||||
|
||||
Reference in New Issue
Block a user