diff --git a/skills/package.json b/skills/package.json index 4b9f42be7..e7f3ecdae 100644 --- a/skills/package.json +++ b/skills/package.json @@ -13,7 +13,7 @@ "pretest": "node scripts/bootstrap-lbs.mjs", "precheck": "node scripts/bootstrap-lbs.mjs", "lbs": "node src/lbs.ts", - "test": "node test/lbs-cli.test.ts", + "test": "node test/lbs-cli.test.ts && python3 -m unittest discover -s test -p 'test_*.py'", "validate": "node src/lbs.ts validate", "index": "node src/lbs.ts index", "index:check": "node src/lbs.ts index --check", diff --git a/skills/scripts/e2e/agent-run-ledger-audit.py b/skills/scripts/e2e/agent-run-ledger-audit.py index ec969b8aa..c9ea3ce25 100644 --- a/skills/scripts/e2e/agent-run-ledger-audit.py +++ b/skills/scripts/e2e/agent-run-ledger-audit.py @@ -16,6 +16,8 @@ import sqlalchemy import yaml from sqlalchemy.ext.asyncio import create_async_engine +from agent_run_ledger_policy import classify_invalid_tool_argument_errors, load_ledger_json + def database_url(repo: pathlib.Path) -> str: config = yaml.safe_load((repo / "data/config.yaml").read_text(encoding="utf-8")) or {} @@ -37,16 +39,6 @@ def database_url(repo: pathlib.Path) -> str: raise RuntimeError(f"Unsupported database backend: {kind}") -def load_json(value: str | None, *, field: str, failures: list[dict]) -> object: - if not value: - return {} - try: - return json.loads(value) - except (TypeError, ValueError) as exc: - failures.append({"kind": "invalid_json", "field": field, "reason": str(exc)}) - return {} - - def parse_created_after(value: str | None) -> datetime.datetime | None: if not value: return None @@ -143,7 +135,11 @@ async def audit( finally: await engine.dispose() - authorization = load_json(run_row.get("authorization_json"), field="agent_run.authorization_json", failures=failures) + authorization = load_ledger_json( + run_row.get("authorization_json"), + field="agent_run.authorization_json", + failures=failures, + ) tools = authorization.get("resources", {}).get("tools", []) if isinstance(authorization, dict) else [] allowed_tools: dict[str, dict] = {} incomplete_tool_metadata: list[dict] = [] @@ -173,7 +169,13 @@ async def audit( event_types: list[str] = [] invalid_event_json = 0 suspicious_errors: list[dict] = [] - forbidden_pattern = re.compile(r"invalid json(?: arguments)?|unauthori[sz]ed|permission denied|forbidden|timed?\s*out|timeout", re.I) + invalid_tool_argument_errors: list[dict] = [] + successful_tool_completion_sequences: list[int] = [] + forbidden_pattern = re.compile( + r"invalid json(?! arguments)|unauthori[sz]ed|permission denied|forbidden|timed?\s*out|timeout", + re.I, + ) + invalid_tool_arguments_pattern = re.compile(r"invalid json arguments", re.I) def error_surface(value: object) -> list[str]: """Collect diagnostic fields without treating normal tool parameters as errors.""" @@ -192,7 +194,11 @@ async def audit( event_type = str(row["type"]) event_types.append(event_type) before = len(failures) - data = load_json(row.get("data_json"), field=f"agent_run_event[{row['sequence']}].data_json", failures=failures) + data = load_ledger_json( + row.get("data_json"), + field=f"agent_run_event[{row['sequence']}].data_json", + failures=failures, + ) invalid_event_json += int(len(failures) > before) if not isinstance(data, dict): failures.append({"kind": "invalid_event_payload", "sequence": row["sequence"], "type": event_type}) @@ -206,12 +212,20 @@ async def audit( starts.setdefault(call_id, []).append(item) else: completions.setdefault(call_id, []).append(item) + if not data.get("error") and data.get("result") is not None: + successful_tool_completion_sequences.append(row["sequence"]) diagnostic_text = "\n".join(error_surface(data)) if event_type == "run.failed": diagnostic_text += "\n" + json.dumps(data, ensure_ascii=True) match = forbidden_pattern.search(diagnostic_text) if match: suspicious_errors.append({"sequence": row["sequence"], "type": event_type, "signal": match.group(0)}) + elif event_type == "tool.call.completed": + match = invalid_tool_arguments_pattern.search(diagnostic_text) + if match: + invalid_tool_argument_errors.append( + {"sequence": row["sequence"], "type": event_type, "signal": match.group(0)} + ) if run_row["status"] != "completed": failures.append({"kind": "run_status", "actual": run_row["status"], "expected": "completed"}) @@ -236,6 +250,18 @@ async def audit( unauthorized_calls.append({"tool_call_id": call_id, "tool_name": started[0]["tool_name"]}) if unauthorized_calls: failures.append({"kind": "unauthorized_tool_calls", "calls": unauthorized_calls}) + unrecovered_argument_errors, recovered_argument_warnings = classify_invalid_tool_argument_errors( + invalid_tool_argument_errors, + successful_tool_completion_sequences=successful_tool_completion_sequences, + run_completed=( + run_row["status"] == "completed" + and "run.completed" in event_types + and "run.failed" not in event_types + ), + ) + if unrecovered_argument_errors: + suspicious_errors.extend(unrecovered_argument_errors) + warnings.extend(recovered_argument_warnings) if suspicious_errors: failures.append({"kind": "forbidden_error_signals", "events": suspicious_errors}) if not event_rows: @@ -281,6 +307,7 @@ async def audit( "authorized_tool_count": len(allowed_tools), "invalid_event_json": invalid_event_json, "suspicious_error_count": len(suspicious_errors), + "recovered_tool_argument_error_count": len(recovered_argument_warnings), } return { "status": "pass" if not failures else "fail", diff --git a/skills/scripts/e2e/agent_run_ledger_policy.py b/skills/scripts/e2e/agent_run_ledger_policy.py new file mode 100644 index 000000000..3fc0f3943 --- /dev/null +++ b/skills/scripts/e2e/agent_run_ledger_policy.py @@ -0,0 +1,43 @@ +"""Policy helpers for classifying AgentRunner ledger error signals.""" + +from __future__ import annotations + +import json + + +def load_ledger_json(value: str | None, *, field: str, failures: list[dict]) -> object: + """Decode persisted ledger JSON and retain corruption as an invariant failure.""" + if not value: + return {} + try: + return json.loads(value) + except (TypeError, ValueError) as exc: + failures.append({"kind": "invalid_json", "field": field, "reason": str(exc)}) + return {} + + +def classify_invalid_tool_argument_errors( + events: list[dict], + *, + successful_tool_completion_sequences: list[int], + run_completed: bool, +) -> tuple[list[dict], list[dict]]: + """Split malformed tool arguments into recovered warnings and hard failures.""" + failures: list[dict] = [] + warnings: list[dict] = [] + for event in events: + recovered = run_completed and any( + sequence > event["sequence"] + for sequence in successful_tool_completion_sequences + ) + if recovered: + warnings.append( + { + "kind": "recovered_tool_argument_error", + "event": event, + "reason": "The model continued with a later successful tool call and the run completed.", + } + ) + else: + failures.append(event) + return failures, warnings diff --git a/skills/scripts/e2e/bot-event-routing-product-flow.mjs b/skills/scripts/e2e/bot-event-routing-product-flow.mjs index 62544202a..c0032de83 100644 --- a/skills/scripts/e2e/bot-event-routing-product-flow.mjs +++ b/skills/scripts/e2e/bot-event-routing-product-flow.mjs @@ -111,7 +111,13 @@ try { { exact: true }, ); await noEventRoutes.waitFor(); - await messageBehavior.waitFor(); + try { + await messageBehavior.waitFor({ timeout: 5_000 }); + } catch { + // Async adapter fields can rerender once after the first menu click. + await addBehavior.click(); + await messageBehavior.waitFor(); + } await page.waitForTimeout(250); await safeScreenshot(page, scenarioMenuScreenshot); await messageBehavior.click(); diff --git a/skills/scripts/e2e/ensure-fake-provider-pipeline.mjs b/skills/scripts/e2e/ensure-fake-provider-pipeline.mjs index bf8c1835b..01f762319 100755 --- a/skills/scripts/e2e/ensure-fake-provider-pipeline.mjs +++ b/skills/scripts/e2e/ensure-fake-provider-pipeline.mjs @@ -148,7 +148,7 @@ try { modelUuid: model.uuid, }); Object.assign(result, pipeline); - result.pipeline_url = `${frontendUrl.replace(/\/$/, "")}/home/pipelines?id=${encodeURIComponent(pipeline.pipeline_id)}`; + result.pipeline_url = `${frontendUrl.replace(/\/$/, "")}/home/agents?id=${encodeURIComponent(pipeline.pipeline_id)}`; const runConfig = await configureFakeProvider(fakeProvider.url, targetFakeProviderConfig(), true); result.fake_provider.config = runConfig.config || targetFakeProviderConfig(); @@ -425,7 +425,7 @@ async function ensureModel({ backendUrl, token, providerUuid, name }) { const body = { name, provider_uuid: providerUuid, - abilities: [], + abilities: ["func_call", "vision"], context_length: positiveInteger(env.LANGBOT_FAKE_PROVIDER_CONTEXT_LENGTH, 8192), extra_args: {}, prefered_ranking: 0, diff --git a/skills/scripts/e2e/wizard-onebot-agent-runtime.mjs b/skills/scripts/e2e/wizard-onebot-agent-runtime.mjs index b270634e3..1c6c9b4a7 100644 --- a/skills/scripts/e2e/wizard-onebot-agent-runtime.mjs +++ b/skills/scripts/e2e/wizard-onebot-agent-runtime.mjs @@ -204,9 +204,16 @@ try { result.visible_signals.push("bot-created", "adapter-enabled"); await page.getByRole("button", { name: /Next|下一步|次へ/ }).click(); - await page - .getByText(/Local Agent|本地 Agent/) - .first() + const localAgentTitle = page + .getByText(/^(Local Agent|本地 Agent)$/) + .first(); + const localAgentCard = localAgentTitle.locator( + 'xpath=ancestor::*[@data-slot="card"][1]', + ); + await localAgentCard + .getByRole("button", { + name: /Use This Runner|使用此运行器|この Runner を使用/, + }) .click(); await page.waitForFunction( () => diff --git a/skills/skills.index.json b/skills/skills.index.json index 8f4b89a27..a0e550c26 100644 --- a/skills/skills.index.json +++ b/skills/skills.index.json @@ -1537,12 +1537,12 @@ ], "automation": "scripts/e2e/pipeline-debug-chat.mjs", "setup_automation": [ - "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env", + "node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env", "case:mcp-stdio-register" ], "setup_provides_env": [ - "LANGBOT_LOCAL_AGENT_PIPELINE_URL", - "LANGBOT_LOCAL_AGENT_PIPELINE_NAME", + "LANGBOT_FAKE_PROVIDER_PIPELINE_URL", + "LANGBOT_FAKE_PROVIDER_PIPELINE_NAME", "LANGBOT_MCP_QA_STDIO_SERVER_UUID" ], "evidence_required": [ @@ -2458,7 +2458,8 @@ "local-agent-plugin-tool-call-debug-chat", "mcp-stdio-tool-call", "local-agent-multimodal-debug-chat", - "local-agent-nonstreaming-debug-chat" + "local-agent-nonstreaming-debug-chat", + "local-agent-complex-coding-task-debug-chat" ] }, { diff --git a/skills/skills/langbot-testing/cases/agent-run-ledger-audit.yaml b/skills/skills/langbot-testing/cases/agent-run-ledger-audit.yaml index 394bc74c9..29d58753f 100644 --- a/skills/skills/langbot-testing/cases/agent-run-ledger-audit.yaml +++ b/skills/skills/langbot-testing/cases/agent-run-ledger-audit.yaml @@ -19,7 +19,8 @@ steps: - "Read the active LangBot database configuration and inspect the selected run and its ordered events." - "Verify completed terminal state, run.completed, paired tool.call.started/completed events, stable tool names, and monotonic ordering." - "Compare called tools with the authorization snapshot and validate each advertised tool has owner/source, description, and parameter schema." - - "Reject malformed event JSON and invalid-JSON, timeout, forbidden, permission-denied, or unauthorized signals." + - "Reject malformed persisted event JSON, unrecovered invalid tool arguments, timeout, forbidden, permission-denied, or unauthorized signals." + - "Record a malformed model tool-argument payload as a recovery warning only when a later tool call succeeds and the run completes." checks: - "ledger-audit.json status is pass." - "metrics has equal tool_call_started and tool_call_completed counts." diff --git a/skills/skills/langbot-testing/cases/mcp-stdio-tool-call.yaml b/skills/skills/langbot-testing/cases/mcp-stdio-tool-call.yaml index b2c874c7a..7f2827136 100644 --- a/skills/skills/langbot-testing/cases/mcp-stdio-tool-call.yaml +++ b/skills/skills/langbot-testing/cases/mcp-stdio-tool-call.yaml @@ -16,34 +16,32 @@ skills: env: - LANGBOT_FRONTEND_URL - LANGBOT_BACKEND_URL - - LANGBOT_LOCAL_AGENT_PIPELINE_URL - - LANGBOT_LOCAL_AGENT_PIPELINE_NAME -env_optional: - - LANGBOT_E2E_FAKE_PROVIDER_BASE_URL + - LANGBOT_FAKE_PROVIDER_PIPELINE_URL + - LANGBOT_FAKE_PROVIDER_PIPELINE_NAME automation: scripts/e2e/pipeline-debug-chat.mjs automation_env: - LANGBOT_FRONTEND_URL - LANGBOT_BACKEND_URL - LANGBOT_BROWSER_PROFILE - LANGBOT_CHROMIUM_EXECUTABLE - - LANGBOT_LOCAL_AGENT_PIPELINE_URL - - LANGBOT_LOCAL_AGENT_PIPELINE_NAME + - LANGBOT_FAKE_PROVIDER_PIPELINE_URL + - LANGBOT_FAKE_PROVIDER_PIPELINE_NAME - LANGBOT_MCP_QA_STDIO_SERVER_UUID -automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL -automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME +automation_pipeline_url_env: LANGBOT_FAKE_PROVIDER_PIPELINE_URL +automation_pipeline_name_env: LANGBOT_FAKE_PROVIDER_PIPELINE_NAME automation_expected_runner_id: "plugin:langbot-team/LocalAgent/default" automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot-team","name":"LocalAgent"}],"enable_all_mcp_servers":false,"bound_mcp_servers":["${LANGBOT_MCP_QA_STDIO_SERVER_UUID}"],"enable_all_skills":false,"bound_skills":[]}' automation_restore_extensions: "1" automation_reset_debug_chat: "1" automation_prompt: "Call the qa_mcp_echo MCP tool with exactly this text: mcp-ok-local-agent. Return only the tool result." automation_expected_text: "qa_mcp_echo:mcp-ok-local-agent" -automation_response_timeout_ms: "180000" +automation_response_timeout_ms: "60000" setup_automation: - - "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env" + - "node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env" - "case:mcp-stdio-register" setup_provides_env: - - LANGBOT_LOCAL_AGENT_PIPELINE_URL - - LANGBOT_LOCAL_AGENT_PIPELINE_NAME + - LANGBOT_FAKE_PROVIDER_PIPELINE_URL + - LANGBOT_FAKE_PROVIDER_PIPELINE_NAME - LANGBOT_MCP_QA_STDIO_SERVER_UUID failure_patterns: - "qa-plugin-smoke:mcp-ok-local-agent" @@ -54,7 +52,7 @@ failure_patterns: - "no available channel for model" preconditions: - "box.local.allowed_mount_roots includes the bundled MCP fixture directory when LangBot runs stdio MCP servers through Box." - - "The selected model route supports function/tool calling, or LANGBOT_E2E_FAKE_PROVIDER_BASE_URL points to scripts/e2e/fake-openai-provider.mjs." + - "The target is a local test instance where the QA-owned fake provider/model/pipeline may be created or updated." steps: - "Open LANGBOT_FRONTEND_URL." - "Navigate to MCP Servers." @@ -66,7 +64,7 @@ steps: - "Confirm the server detail page shows Tools: 1 and qa_mcp_echo." - "Open the target local-agent pipeline." - "Use runner Default or the pluginized langbot-team/LocalAgent runner." - - "Select a model with function-calling ability that is known to work with tools in the current environment." + - "Use the QA fake-provider pipeline prepared by setup automation." - "Open Debug Chat." - "Send: Call the qa_mcp_echo tool with exactly this text: mcp-ok-local-agent. Return only the tool result." checks: @@ -84,7 +82,8 @@ evidence_required: - api_diagnostic - metrics diagnostics: - - "For token-free deterministic UI coverage, start scripts/e2e/fake-openai-provider.mjs and pass LANGBOT_E2E_FAKE_PROVIDER_BASE_URL to this case; setup will bind the local-agent pipeline to that fake OpenAI-compatible model." + - "The required release case uses the QA fake-provider pipeline so model instruction-following variance cannot hide or mimic an MCP bridge regression." + - "Run a separate optional live-provider tool smoke when evaluating a specific provider/model route." - "Run node scripts/e2e/mcp-stdio-fixture.mjs to verify the bundled stdio fixture can list and call qa_mcp_echo without involving a model provider." - "Run node scripts/e2e/mcp-stdio-register.mjs to upsert qa-local-stdio in LangBot and verify /api/v1/tools exposes qa_mcp_echo." - "If backend logs show host_path is outside allowed_mount_roots, add the fixture directory to box.local.allowed_mount_roots in the local LangBot data config." diff --git a/skills/skills/langbot-testing/fixtures/complex-agent-task/verify.py b/skills/skills/langbot-testing/fixtures/complex-agent-task/verify.py index 006bfad67..e58a69a92 100644 --- a/skills/skills/langbot-testing/fixtures/complex-agent-task/verify.py +++ b/skills/skills/langbot-testing/fixtures/complex-agent-task/verify.py @@ -44,8 +44,18 @@ def main() -> int: report = (workspace / "AGENT_REPORT.md").read_text(encoding="utf-8") folded_report = report.casefold() assert "initial" in folded_report and "fail" in folded_report, "report missing initial failure section" - for heading in ("root causes", "changed files", "verification"): - assert heading in folded_report, f"report missing section: {heading}" + assert "root causes" in folded_report, "report missing section: root causes" + assert any( + heading in folded_report + for heading in ( + "changed files", + "files changed", + "files modified", + "modified files", + "changes made", + ) + ), "report missing section: changed files" + assert "verification" in folded_report, "report missing section: verification" assert report.rstrip().endswith("COMPLEX_AGENT_TASK_OK tests=12 acceptance=PASS") print("HOST_VERIFY_PASS tests=12 acceptance=PASS protected=PASS") return 0 diff --git a/skills/skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs b/skills/skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs index af5153dbf..5b156c470 100755 --- a/skills/skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs +++ b/skills/skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs @@ -119,7 +119,7 @@ try { result.pipeline_id = pipeline.id; result.pipeline_name = pipeline.name || pipelineName; if (!result.pipeline_url && env.LANGBOT_FRONTEND_URL) { - result.pipeline_url = `${env.LANGBOT_FRONTEND_URL.replace(/\/$/, "")}/home/pipelines?id=${encodeURIComponent(pipeline.id)}`; + result.pipeline_url = `${env.LANGBOT_FRONTEND_URL.replace(/\/$/, "")}/home/agents?id=${encodeURIComponent(pipeline.id)}`; } if (resetBeforeRun) { diff --git a/skills/skills/langbot-testing/references/agent-runner-release-gate.md b/skills/skills/langbot-testing/references/agent-runner-release-gate.md index eb99d5d3f..f9e61cb25 100644 --- a/skills/skills/langbot-testing/references/agent-runner-release-gate.md +++ b/skills/skills/langbot-testing/references/agent-runner-release-gate.md @@ -121,7 +121,7 @@ Each probe writes `automation-result.json` and probe logs under | Plugin tool error recovery | `local-agent-tool-error-recovery-debug-chat` | Tool execution errors are serialized into model-facing tool results and the model can produce a final answer instead of failing the run. | | Parallel plugin tool batch | `local-agent-parallel-tools-rag-compaction-debug-chat` | Local-agent executes multiple same-turn plugin tool calls and returns both results with RAG and compacted history. | | MCP registration | `mcp-stdio-register` | The deterministic stdio MCP server is registered and exposes `qa_mcp_echo`. | -| MCP tool loop | `mcp-stdio-tool-call` | Local-agent can call the registered MCP tool through the same tool loop. | +| MCP tool loop | `mcp-stdio-tool-call` | Local-agent can call the registered MCP tool through the same tool loop using the deterministic QA fake provider. | | Multimodal input | `local-agent-multimodal-debug-chat` | Image upload and structured input reach the runner. | | Multimodal plus RAG | `local-agent-rag-multimodal-debug-chat` | RAG still works when structured image input is present. | | ACP external harness execution | `acp-agent-runner-debug-chat` | ACP executes the configured coding agent and returns visible Debug Chat output. | diff --git a/skills/skills/langbot-testing/references/mcp-stdio-testing.md b/skills/skills/langbot-testing/references/mcp-stdio-testing.md index a92218c40..bd8635b9c 100644 --- a/skills/skills/langbot-testing/references/mcp-stdio-testing.md +++ b/skills/skills/langbot-testing/references/mcp-stdio-testing.md @@ -85,10 +85,15 @@ extension binding uses this UUID, not the human-readable server name. ## Local-Agent Tool Call Check -1. Open the target pipeline. -2. Confirm `Extensions` allows the MCP server, or that all MCP servers are enabled. -3. Use runner `Default` or the pluginized `langbot-team/LocalAgent` runner. -4. Select a model with function-calling ability that is known to work with tools in the current environment. +The required release case prepares and uses the dedicated QA fake-provider +pipeline. This keeps the host, LocalAgent, MCP discovery, function-call +conversion, tool execution, and result-return path deterministic. A real model +that ignores the tool instruction must not be reported as an MCP bridge failure. + +1. Run `node scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env`. +2. Open `LANGBOT_FAKE_PROVIDER_PIPELINE_URL`. +3. Confirm `Extensions` allows the MCP server, or that all MCP servers are enabled. +4. Confirm the pipeline uses the pluginized `langbot-team/LocalAgent` runner. 5. Open `Debug Chat`. 6. Ask: @@ -111,3 +116,7 @@ qa-plugin-smoke:mcp-ok-local-agent That proves a plugin tool was called, not the MCP server. If the provider returns `model_not_found` or `no available channel` only when tools are supplied, switch to a known-good function-calling model before diagnosing MCP or local-agent. That failure means the selected model route is unavailable for the requested tool-call shape. + +Run a real-provider tool smoke separately when validating a particular model +route. It measures provider tool-use behavior in addition to the LangBot path +and is therefore not the deterministic release gate. diff --git a/skills/skills/langbot-testing/references/workspace-release-testing.md b/skills/skills/langbot-testing/references/workspace-release-testing.md index bec6ba638..25e9b4671 100644 --- a/skills/skills/langbot-testing/references/workspace-release-testing.md +++ b/skills/skills/langbot-testing/references/workspace-release-testing.md @@ -22,4 +22,5 @@ Do not add Space model concurrency to these gates. Provider concurrency mixes ex - Repository contract failure: fix the owning repository and add the narrowest deterministic regression. - Browser workflow failure: correlate the screenshot, console, network, and backend log from the same run. - Complex Agent failure: inspect `ledger-audit.json` before blaming the model. Tool pairing, authorization, JSON, and timeout failures are product signals. +- A provider-origin malformed tool-argument payload is a recovery warning only when the ledger keeps a paired error result, a later tool call succeeds, and the run completes. Malformed persisted event JSON and unrecovered tool-argument errors remain release failures. - Packaging `env_issue`: restore isolated build dependency access, then rerun packaging only. It must not mask passing or failing runtime contracts. diff --git a/skills/skills/langbot-testing/troubleshooting/local-agent-model-route-unavailable.yaml b/skills/skills/langbot-testing/troubleshooting/local-agent-model-route-unavailable.yaml index 779654492..f7c2b2960 100644 --- a/skills/skills/langbot-testing/troubleshooting/local-agent-model-route-unavailable.yaml +++ b/skills/skills/langbot-testing/troubleshooting/local-agent-model-route-unavailable.yaml @@ -6,18 +6,24 @@ symptoms: - "Debug Chat shows Agent runner temporarily unavailable after the user message is sent." - "Basic streaming prompts may work, but tool-call, non-streaming, or multimodal prompts fail with the same runner and pipeline." - "The failure happens after the local-agent runner starts, not during plugin discovery." + - "A short model preflight passes, but a sustained Agent run later fails after the provider quota is exhausted." patterns: - "runner.llm_error" - "runner.tool_loop_error" - "model_not_found" - "no available channel for model" - "invalid api key" + - "insufficient user quota" + - "insufficient_quota" + - "quota exceeded" + - "余额不足" - "当前分组上游负载已饱和" - "All models failed during streaming setup" likely_causes: - "The selected model route is unavailable in the current LangBot Space or upstream group." - "The selected model works for plain chat but is not available for tool-call, multimodal, or non-streaming request shapes." - "The provider credential or quota is invalid for the non-streaming path." + - "The provider account has enough balance for a short preflight but not for the sustained task." - "The selected model has function-call or vision metadata, but the upstream distributor cannot currently serve that model." fix_steps: - "First rerun a basic local-agent Debug Chat prompt on the same pipeline to confirm the runner host path still works." @@ -25,6 +31,7 @@ fix_steps: - "When tool-call cases fail, retest with a model that is known to support function calling in the active environment." - "When multimodal cases fail, retest with a model route that is known to accept image content." - "When non-streaming fails with invalid api key, verify the provider credential used by the non-streaming requester path." + - "When logs report insufficient quota, restore provider balance or switch to another authorized route before rerunning the live-model case." - "Do not classify this as an MCP, RAG, or local-agent runner implementation failure until the same model route works for the requested request shape outside the failing case." verification: "The same case produces the expected bot-visible sentinel or tool result, and backend logs show request completion without runner.llm_error, runner.tool_loop_error, or All models failed." related_cases: @@ -33,3 +40,4 @@ related_cases: - mcp-stdio-tool-call - local-agent-multimodal-debug-chat - local-agent-nonstreaming-debug-chat + - local-agent-complex-coding-task-debug-chat diff --git a/skills/src/log-guard.ts b/skills/src/log-guard.ts index 671faccc5..3c0552cb2 100644 --- a/skills/src/log-guard.ts +++ b/skills/src/log-guard.ts @@ -338,9 +338,27 @@ function scanLogTextIntoState( function buildLogGuardResult(scan: LogScanConfig, sources: LogSourceSummary[], state: MutableScanState): LogGuardResult { const scannedCount = sources.filter((source) => source.status === "scanned").length; + const hardFailures = state.findings.filter( + (finding) => finding.severity === "fail" || finding.severity === "missing_input", + ); + const relatedEnvIssues = state.findings.filter( + (finding) => finding.severity === "env_issue" && finding.related_to_case !== false, + ); + const hardFailuresAreExplainedProviderTracebacks = hardFailures.length > 0 + && relatedEnvIssues.length > 0 + && hardFailures.every((failure) => + failure.kind === "python_traceback" + && relatedEnvIssues.some((envIssue) => + envIssue.source === failure.source + && envIssue.path === failure.path + && typeof envIssue.line === "number" + && typeof failure.line === "number" + && Math.abs(envIssue.line - failure.line) <= 100, + ), + ); const status = scannedCount === 0 && state.findings.length === 0 ? "not_run" - : state.findings.some((finding) => finding.severity === "fail" || finding.severity === "missing_input") + : hardFailures.length > 0 && !hardFailuresAreExplainedProviderTracebacks ? "fail" : state.findings.some((finding) => finding.severity === "matched_troubleshooting" && finding.related_to_case !== false) ? "fail" @@ -528,7 +546,7 @@ function scanTroubleshootingPatterns( } function isModelRouteUnavailableText(text: string): boolean { - return /model_not_found|no available channel for model|invalid api key|当前分组上游负载已饱和/i.test(text); + return /model_not_found|no available channel for model|invalid api key|insufficient user quota|insufficient_quota|quota exceeded|余额不足|当前分组上游负载已饱和/i.test(text); } function scanCaseDeclaredPatterns( diff --git a/skills/test/lbs-cli.test.ts b/skills/test/lbs-cli.test.ts index a818a709b..df878648b 100644 --- a/skills/test/lbs-cli.test.ts +++ b/skills/test/lbs-cli.test.ts @@ -3708,7 +3708,7 @@ test("MCP stdio tool-call case setups pipeline and registered MCP server", () => assert.deepEqual( run.setup_automation.map((item: { entry: string }) => item.entry), [ - "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env", + "node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env", "case:mcp-stdio-register", ], ); @@ -3739,8 +3739,8 @@ test("MCP stdio tool-call case setups pipeline and registered MCP server", () => assert.equal(planResult.code, 0); const plan = JSON.parse(planResult.output); assert.deepEqual(plan.setup_provides_env, [ - "LANGBOT_LOCAL_AGENT_PIPELINE_URL", - "LANGBOT_LOCAL_AGENT_PIPELINE_NAME", + "LANGBOT_FAKE_PROVIDER_PIPELINE_URL", + "LANGBOT_FAKE_PROVIDER_PIPELINE_NAME", "LANGBOT_MCP_QA_STDIO_SERVER_UUID", ]); assert.ok( @@ -4588,6 +4588,47 @@ test("test report classifies model route failures as env_issue", () => { } }); +test("test report classifies provider quota tracebacks as env_issue", () => { + const tmp = mkdtempSync(join(tmpdir(), "lbs-report-quota-env-issue-")); + try { + const logPath = join(tmp, "backend.log"); + writeFileSync( + logPath, + [ + "[05-21 10:31:00.000] chat.py (2) - [ERROR] : Request Failed: Traceback (most recent call last):", + " File \"provider.py\", line 1, in invoke", + "openai.PermissionDeniedError: insufficient user quota", + "[05-21 10:31:01.000] pipeline.py (3) - [ERROR] : runner.llm_error All models failed during streaming setup: insufficient user quota", + ].join("\n"), + ); + + const result = capture(() => + commandTestReport( + ctx([ + "test", + "report", + "local-agent-complex-coding-task-debug-chat", + "--backend-log", + logPath, + "--json", + ]), + ), + ); + assert.equal(result.code, 0); + const report = JSON.parse(result.output); + assert.equal(report.log_guard.status, "env_issue"); + assert.ok( + report.log_guard.findings.some( + (finding: { severity?: string; pattern?: string }) => + finding.severity === "env_issue" && + finding.pattern === "insufficient user quota", + ), + ); + } finally { + rmSync(tmp, { recursive: true, force: true }); + } +}); + test("test report infers scan window from automation result evidence", () => { const tmp = mkdtempSync(join(tmpdir(), "lbs-report-evidence-window-")); try { diff --git a/skills/test/test_agent_run_ledger_policy.py b/skills/test/test_agent_run_ledger_policy.py new file mode 100644 index 000000000..df934611e --- /dev/null +++ b/skills/test/test_agent_run_ledger_policy.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts" / "e2e")) + +from agent_run_ledger_policy import ( # noqa: E402 + classify_invalid_tool_argument_errors, + load_ledger_json, +) + + +class AgentRunLedgerPolicyTests(unittest.TestCase): + def setUp(self) -> None: + self.event = { + "sequence": 10, + "type": "tool.call.completed", + "signal": "Invalid JSON arguments", + } + + def test_completed_run_with_later_success_is_warning(self) -> None: + failures, warnings = classify_invalid_tool_argument_errors( + [self.event], + successful_tool_completion_sequences=[12], + run_completed=True, + ) + + self.assertEqual(failures, []) + self.assertEqual(warnings[0]["kind"], "recovered_tool_argument_error") + + def test_completed_run_without_later_success_still_fails(self) -> None: + failures, warnings = classify_invalid_tool_argument_errors( + [self.event], + successful_tool_completion_sequences=[8], + run_completed=True, + ) + + self.assertEqual(failures, [self.event]) + self.assertEqual(warnings, []) + + def test_incomplete_run_still_fails_even_with_later_success(self) -> None: + failures, warnings = classify_invalid_tool_argument_errors( + [self.event], + successful_tool_completion_sequences=[12], + run_completed=False, + ) + + self.assertEqual(failures, [self.event]) + self.assertEqual(warnings, []) + + def test_malformed_persisted_event_json_remains_an_invariant_failure(self) -> None: + failures: list[dict] = [] + + value = load_ledger_json( + '{"tool_call_id":', + field="agent_run_event[10].data_json", + failures=failures, + ) + + self.assertEqual(value, {}) + self.assertEqual(failures[0]["kind"], "invalid_json") + self.assertEqual(failures[0]["field"], "agent_run_event[10].data_json") + + +if __name__ == "__main__": + unittest.main()