test(workspace): harden release gate automation

2026-07-21 11:56:09 +00:00 · 2026-07-20 17:13:19 +08:00
parent 7615c40f73
commit e3df35e7f7
18 changed files with 293 additions and 53 deletions
@@ -13,7 +13,7 @@
    "pretest": "node scripts/bootstrap-lbs.mjs",
    "precheck": "node scripts/bootstrap-lbs.mjs",
    "lbs": "node src/lbs.ts",
-    "test": "node test/lbs-cli.test.ts",
+    "test": "node test/lbs-cli.test.ts && python3 -m unittest discover -s test -p 'test_*.py'",
    "validate": "node src/lbs.ts validate",
    "index": "node src/lbs.ts index",
    "index:check": "node src/lbs.ts index --check",
@@ -16,6 +16,8 @@ import sqlalchemy
 import yaml
 from sqlalchemy.ext.asyncio import create_async_engine

+from agent_run_ledger_policy import classify_invalid_tool_argument_errors, load_ledger_json
+

 def database_url(repo: pathlib.Path) -> str:
    config = yaml.safe_load((repo / "data/config.yaml").read_text(encoding="utf-8")) or {}
@@ -37,16 +39,6 @@ def database_url(repo: pathlib.Path) -> str:
    raise RuntimeError(f"Unsupported database backend: {kind}")


-def load_json(value: str | None, *, field: str, failures: list[dict]) -> object:
-    if not value:
-        return {}
-    try:
-        return json.loads(value)
-    except (TypeError, ValueError) as exc:
-        failures.append({"kind": "invalid_json", "field": field, "reason": str(exc)})
-        return {}
-
-
 def parse_created_after(value: str | None) -> datetime.datetime | None:
    if not value:
        return None
@@ -143,7 +135,11 @@ async def audit(
    finally:
        await engine.dispose()

-    authorization = load_json(run_row.get("authorization_json"), field="agent_run.authorization_json", failures=failures)
+    authorization = load_ledger_json(
+        run_row.get("authorization_json"),
+        field="agent_run.authorization_json",
+        failures=failures,
+    )
    tools = authorization.get("resources", {}).get("tools", []) if isinstance(authorization, dict) else []
    allowed_tools: dict[str, dict] = {}
    incomplete_tool_metadata: list[dict] = []
@@ -173,7 +169,13 @@ async def audit(
    event_types: list[str] = []
    invalid_event_json = 0
    suspicious_errors: list[dict] = []
-    forbidden_pattern = re.compile(r"invalid json(?: arguments)?|unauthori[sz]ed|permission denied|forbidden|timed?\s*out|timeout", re.I)
+    invalid_tool_argument_errors: list[dict] = []
+    successful_tool_completion_sequences: list[int] = []
+    forbidden_pattern = re.compile(
+        r"invalid json(?! arguments)|unauthori[sz]ed|permission denied|forbidden|timed?\s*out|timeout",
+        re.I,
+    )
+    invalid_tool_arguments_pattern = re.compile(r"invalid json arguments", re.I)

    def error_surface(value: object) -> list[str]:
        """Collect diagnostic fields without treating normal tool parameters as errors."""
@@ -192,7 +194,11 @@ async def audit(
        event_type = str(row["type"])
        event_types.append(event_type)
        before = len(failures)
-        data = load_json(row.get("data_json"), field=f"agent_run_event[{row['sequence']}].data_json", failures=failures)
+        data = load_ledger_json(
+            row.get("data_json"),
+            field=f"agent_run_event[{row['sequence']}].data_json",
+            failures=failures,
+        )
        invalid_event_json += int(len(failures) > before)
        if not isinstance(data, dict):
            failures.append({"kind": "invalid_event_payload", "sequence": row["sequence"], "type": event_type})
@@ -206,12 +212,20 @@ async def audit(
                starts.setdefault(call_id, []).append(item)
            else:
                completions.setdefault(call_id, []).append(item)
+                if not data.get("error") and data.get("result") is not None:
+                    successful_tool_completion_sequences.append(row["sequence"])
        diagnostic_text = "\n".join(error_surface(data))
        if event_type == "run.failed":
            diagnostic_text += "\n" + json.dumps(data, ensure_ascii=True)
        match = forbidden_pattern.search(diagnostic_text)
        if match:
            suspicious_errors.append({"sequence": row["sequence"], "type": event_type, "signal": match.group(0)})
+        elif event_type == "tool.call.completed":
+            match = invalid_tool_arguments_pattern.search(diagnostic_text)
+            if match:
+                invalid_tool_argument_errors.append(
+                    {"sequence": row["sequence"], "type": event_type, "signal": match.group(0)}
+                )

    if run_row["status"] != "completed":
        failures.append({"kind": "run_status", "actual": run_row["status"], "expected": "completed"})
@@ -236,6 +250,18 @@ async def audit(
            unauthorized_calls.append({"tool_call_id": call_id, "tool_name": started[0]["tool_name"]})
    if unauthorized_calls:
        failures.append({"kind": "unauthorized_tool_calls", "calls": unauthorized_calls})
+    unrecovered_argument_errors, recovered_argument_warnings = classify_invalid_tool_argument_errors(
+        invalid_tool_argument_errors,
+        successful_tool_completion_sequences=successful_tool_completion_sequences,
+        run_completed=(
+            run_row["status"] == "completed"
+            and "run.completed" in event_types
+            and "run.failed" not in event_types
+        ),
+    )
+    if unrecovered_argument_errors:
+        suspicious_errors.extend(unrecovered_argument_errors)
+    warnings.extend(recovered_argument_warnings)
    if suspicious_errors:
        failures.append({"kind": "forbidden_error_signals", "events": suspicious_errors})
    if not event_rows:
@@ -281,6 +307,7 @@ async def audit(
        "authorized_tool_count": len(allowed_tools),
        "invalid_event_json": invalid_event_json,
        "suspicious_error_count": len(suspicious_errors),
+        "recovered_tool_argument_error_count": len(recovered_argument_warnings),
    }
    return {
        "status": "pass" if not failures else "fail",
@@ -0,0 +1,43 @@
+"""Policy helpers for classifying AgentRunner ledger error signals."""
+
+from __future__ import annotations
+
+import json
+
+
+def load_ledger_json(value: str | None, *, field: str, failures: list[dict]) -> object:
+    """Decode persisted ledger JSON and retain corruption as an invariant failure."""
+    if not value:
+        return {}
+    try:
+        return json.loads(value)
+    except (TypeError, ValueError) as exc:
+        failures.append({"kind": "invalid_json", "field": field, "reason": str(exc)})
+        return {}
+
+
+def classify_invalid_tool_argument_errors(
+    events: list[dict],
+    *,
+    successful_tool_completion_sequences: list[int],
+    run_completed: bool,
+) -> tuple[list[dict], list[dict]]:
+    """Split malformed tool arguments into recovered warnings and hard failures."""
+    failures: list[dict] = []
+    warnings: list[dict] = []
+    for event in events:
+        recovered = run_completed and any(
+            sequence > event["sequence"]
+            for sequence in successful_tool_completion_sequences
+        )
+        if recovered:
+            warnings.append(
+                {
+                    "kind": "recovered_tool_argument_error",
+                    "event": event,
+                    "reason": "The model continued with a later successful tool call and the run completed.",
+                }
+            )
+        else:
+            failures.append(event)
+    return failures, warnings
@@ -111,7 +111,13 @@ try {
    { exact: true },
  );
  await noEventRoutes.waitFor();
-  await messageBehavior.waitFor();
+  try {
+    await messageBehavior.waitFor({ timeout: 5_000 });
+  } catch {
+    // Async adapter fields can rerender once after the first menu click.
+    await addBehavior.click();
+    await messageBehavior.waitFor();
+  }
  await page.waitForTimeout(250);
  await safeScreenshot(page, scenarioMenuScreenshot);
  await messageBehavior.click();
@@ -148,7 +148,7 @@ try {
    modelUuid: model.uuid,
  });
  Object.assign(result, pipeline);
-  result.pipeline_url = `${frontendUrl.replace(/\/$/, "")}/home/pipelines?id=${encodeURIComponent(pipeline.pipeline_id)}`;
+  result.pipeline_url = `${frontendUrl.replace(/\/$/, "")}/home/agents?id=${encodeURIComponent(pipeline.pipeline_id)}`;

  const runConfig = await configureFakeProvider(fakeProvider.url, targetFakeProviderConfig(), true);
  result.fake_provider.config = runConfig.config || targetFakeProviderConfig();
@@ -425,7 +425,7 @@ async function ensureModel({ backendUrl, token, providerUuid, name }) {
  const body = {
    name,
    provider_uuid: providerUuid,
-    abilities: [],
+    abilities: ["func_call", "vision"],
    context_length: positiveInteger(env.LANGBOT_FAKE_PROVIDER_CONTEXT_LENGTH, 8192),
    extra_args: {},
    prefered_ranking: 0,
@@ -204,9 +204,16 @@ try {
  result.visible_signals.push("bot-created", "adapter-enabled");

  await page.getByRole("button", { name: /Next|下一步|次へ/ }).click();
-  await page
-    .getByText(/Local Agent|本地 Agent/)
-    .first()
+  const localAgentTitle = page
+    .getByText(/^(Local Agent|本地 Agent)$/)
+    .first();
+  const localAgentCard = localAgentTitle.locator(
+    'xpath=ancestor::*[@data-slot="card"][1]',
+  );
+  await localAgentCard
+    .getByRole("button", {
+      name: /Use This Runner|使用此运行器|この Runner を使用/,
+    })
    .click();
  await page.waitForFunction(
    () =>
@@ -1537,12 +1537,12 @@
          ],
          "automation": "scripts/e2e/pipeline-debug-chat.mjs",
          "setup_automation": [
-            "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env",
+            "node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env",
            "case:mcp-stdio-register"
          ],
          "setup_provides_env": [
-            "LANGBOT_LOCAL_AGENT_PIPELINE_URL",
-            "LANGBOT_LOCAL_AGENT_PIPELINE_NAME",
+            "LANGBOT_FAKE_PROVIDER_PIPELINE_URL",
+            "LANGBOT_FAKE_PROVIDER_PIPELINE_NAME",
            "LANGBOT_MCP_QA_STDIO_SERVER_UUID"
          ],
          "evidence_required": [
@@ -2458,7 +2458,8 @@
            "local-agent-plugin-tool-call-debug-chat",
            "mcp-stdio-tool-call",
            "local-agent-multimodal-debug-chat",
-            "local-agent-nonstreaming-debug-chat"
+            "local-agent-nonstreaming-debug-chat",
+            "local-agent-complex-coding-task-debug-chat"
          ]
        },
        {
@@ -19,7 +19,8 @@ steps:
  - "Read the active LangBot database configuration and inspect the selected run and its ordered events."
  - "Verify completed terminal state, run.completed, paired tool.call.started/completed events, stable tool names, and monotonic ordering."
  - "Compare called tools with the authorization snapshot and validate each advertised tool has owner/source, description, and parameter schema."
-  - "Reject malformed event JSON and invalid-JSON, timeout, forbidden, permission-denied, or unauthorized signals."
+  - "Reject malformed persisted event JSON, unrecovered invalid tool arguments, timeout, forbidden, permission-denied, or unauthorized signals."
+  - "Record a malformed model tool-argument payload as a recovery warning only when a later tool call succeeds and the run completes."
 checks:
  - "ledger-audit.json status is pass."
  - "metrics has equal tool_call_started and tool_call_completed counts."
@@ -16,34 +16,32 @@ skills:
 env:
  - LANGBOT_FRONTEND_URL
  - LANGBOT_BACKEND_URL
-  - LANGBOT_LOCAL_AGENT_PIPELINE_URL
-  - LANGBOT_LOCAL_AGENT_PIPELINE_NAME
-env_optional:
-  - LANGBOT_E2E_FAKE_PROVIDER_BASE_URL
+  - LANGBOT_FAKE_PROVIDER_PIPELINE_URL
+  - LANGBOT_FAKE_PROVIDER_PIPELINE_NAME
 automation: scripts/e2e/pipeline-debug-chat.mjs
 automation_env:
  - LANGBOT_FRONTEND_URL
  - LANGBOT_BACKEND_URL
  - LANGBOT_BROWSER_PROFILE
  - LANGBOT_CHROMIUM_EXECUTABLE
-  - LANGBOT_LOCAL_AGENT_PIPELINE_URL
-  - LANGBOT_LOCAL_AGENT_PIPELINE_NAME
+  - LANGBOT_FAKE_PROVIDER_PIPELINE_URL
+  - LANGBOT_FAKE_PROVIDER_PIPELINE_NAME
  - LANGBOT_MCP_QA_STDIO_SERVER_UUID
-automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL
-automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME
+automation_pipeline_url_env: LANGBOT_FAKE_PROVIDER_PIPELINE_URL
+automation_pipeline_name_env: LANGBOT_FAKE_PROVIDER_PIPELINE_NAME
 automation_expected_runner_id: "plugin:langbot-team/LocalAgent/default"
 automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot-team","name":"LocalAgent"}],"enable_all_mcp_servers":false,"bound_mcp_servers":["${LANGBOT_MCP_QA_STDIO_SERVER_UUID}"],"enable_all_skills":false,"bound_skills":[]}'
 automation_restore_extensions: "1"
 automation_reset_debug_chat: "1"
 automation_prompt: "Call the qa_mcp_echo MCP tool with exactly this text: mcp-ok-local-agent. Return only the tool result."
 automation_expected_text: "qa_mcp_echo:mcp-ok-local-agent"
-automation_response_timeout_ms: "180000"
+automation_response_timeout_ms: "60000"
 setup_automation:
-  - "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env"
+  - "node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env"
  - "case:mcp-stdio-register"
 setup_provides_env:
-  - LANGBOT_LOCAL_AGENT_PIPELINE_URL
-  - LANGBOT_LOCAL_AGENT_PIPELINE_NAME
+  - LANGBOT_FAKE_PROVIDER_PIPELINE_URL
+  - LANGBOT_FAKE_PROVIDER_PIPELINE_NAME
  - LANGBOT_MCP_QA_STDIO_SERVER_UUID
 failure_patterns:
  - "qa-plugin-smoke:mcp-ok-local-agent"
@@ -54,7 +52,7 @@ failure_patterns:
  - "no available channel for model"
 preconditions:
  - "box.local.allowed_mount_roots includes the bundled MCP fixture directory when LangBot runs stdio MCP servers through Box."
-  - "The selected model route supports function/tool calling, or LANGBOT_E2E_FAKE_PROVIDER_BASE_URL points to scripts/e2e/fake-openai-provider.mjs."
+  - "The target is a local test instance where the QA-owned fake provider/model/pipeline may be created or updated."
 steps:
  - "Open LANGBOT_FRONTEND_URL."
  - "Navigate to MCP Servers."
@@ -66,7 +64,7 @@ steps:
  - "Confirm the server detail page shows Tools: 1 and qa_mcp_echo."
  - "Open the target local-agent pipeline."
  - "Use runner Default or the pluginized langbot-team/LocalAgent runner."
-  - "Select a model with function-calling ability that is known to work with tools in the current environment."
+  - "Use the QA fake-provider pipeline prepared by setup automation."
  - "Open Debug Chat."
  - "Send: Call the qa_mcp_echo tool with exactly this text: mcp-ok-local-agent. Return only the tool result."
 checks:
@@ -84,7 +82,8 @@ evidence_required:
  - api_diagnostic
  - metrics
 diagnostics:
-  - "For token-free deterministic UI coverage, start scripts/e2e/fake-openai-provider.mjs and pass LANGBOT_E2E_FAKE_PROVIDER_BASE_URL to this case; setup will bind the local-agent pipeline to that fake OpenAI-compatible model."
+  - "The required release case uses the QA fake-provider pipeline so model instruction-following variance cannot hide or mimic an MCP bridge regression."
+  - "Run a separate optional live-provider tool smoke when evaluating a specific provider/model route."
  - "Run node scripts/e2e/mcp-stdio-fixture.mjs to verify the bundled stdio fixture can list and call qa_mcp_echo without involving a model provider."
  - "Run node scripts/e2e/mcp-stdio-register.mjs to upsert qa-local-stdio in LangBot and verify /api/v1/tools exposes qa_mcp_echo."
  - "If backend logs show host_path is outside allowed_mount_roots, add the fixture directory to box.local.allowed_mount_roots in the local LangBot data config."
@@ -44,8 +44,18 @@ def main() -> int:
    report = (workspace / "AGENT_REPORT.md").read_text(encoding="utf-8")
    folded_report = report.casefold()
    assert "initial" in folded_report and "fail" in folded_report, "report missing initial failure section"
-    for heading in ("root causes", "changed files", "verification"):
-        assert heading in folded_report, f"report missing section: {heading}"
+    assert "root causes" in folded_report, "report missing section: root causes"
+    assert any(
+        heading in folded_report
+        for heading in (
+            "changed files",
+            "files changed",
+            "files modified",
+            "modified files",
+            "changes made",
+        )
+    ), "report missing section: changed files"
+    assert "verification" in folded_report, "report missing section: verification"
    assert report.rstrip().endswith("COMPLEX_AGENT_TASK_OK tests=12 acceptance=PASS")
    print("HOST_VERIFY_PASS tests=12 acceptance=PASS protected=PASS")
    return 0
@@ -119,7 +119,7 @@ try {
  result.pipeline_id = pipeline.id;
  result.pipeline_name = pipeline.name || pipelineName;
  if (!result.pipeline_url && env.LANGBOT_FRONTEND_URL) {
-    result.pipeline_url = `${env.LANGBOT_FRONTEND_URL.replace(/\/$/, "")}/home/pipelines?id=${encodeURIComponent(pipeline.id)}`;
+    result.pipeline_url = `${env.LANGBOT_FRONTEND_URL.replace(/\/$/, "")}/home/agents?id=${encodeURIComponent(pipeline.id)}`;
  }

  if (resetBeforeRun) {
@@ -121,7 +121,7 @@ Each probe writes `automation-result.json` and probe logs under
 | Plugin tool error recovery | `local-agent-tool-error-recovery-debug-chat` | Tool execution errors are serialized into model-facing tool results and the model can produce a final answer instead of failing the run. |
 | Parallel plugin tool batch | `local-agent-parallel-tools-rag-compaction-debug-chat` | Local-agent executes multiple same-turn plugin tool calls and returns both results with RAG and compacted history. |
 | MCP registration | `mcp-stdio-register` | The deterministic stdio MCP server is registered and exposes `qa_mcp_echo`. |
-| MCP tool loop | `mcp-stdio-tool-call` | Local-agent can call the registered MCP tool through the same tool loop. |
+| MCP tool loop | `mcp-stdio-tool-call` | Local-agent can call the registered MCP tool through the same tool loop using the deterministic QA fake provider. |
 | Multimodal input | `local-agent-multimodal-debug-chat` | Image upload and structured input reach the runner. |
 | Multimodal plus RAG | `local-agent-rag-multimodal-debug-chat` | RAG still works when structured image input is present. |
 | ACP external harness execution | `acp-agent-runner-debug-chat` | ACP executes the configured coding agent and returns visible Debug Chat output. |
@@ -85,10 +85,15 @@ extension binding uses this UUID, not the human-readable server name.

 ## Local-Agent Tool Call Check

-1. Open the target pipeline.
-2. Confirm `Extensions` allows the MCP server, or that all MCP servers are enabled.
-3. Use runner `Default` or the pluginized `langbot-team/LocalAgent` runner.
-4. Select a model with function-calling ability that is known to work with tools in the current environment.
+The required release case prepares and uses the dedicated QA fake-provider
+pipeline. This keeps the host, LocalAgent, MCP discovery, function-call
+conversion, tool execution, and result-return path deterministic. A real model
+that ignores the tool instruction must not be reported as an MCP bridge failure.
+
+1. Run `node scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env`.
+2. Open `LANGBOT_FAKE_PROVIDER_PIPELINE_URL`.
+3. Confirm `Extensions` allows the MCP server, or that all MCP servers are enabled.
+4. Confirm the pipeline uses the pluginized `langbot-team/LocalAgent` runner.
 5. Open `Debug Chat`.
 6. Ask:

@@ -111,3 +116,7 @@ qa-plugin-smoke:mcp-ok-local-agent
 That proves a plugin tool was called, not the MCP server.

 If the provider returns `model_not_found` or `no available channel` only when tools are supplied, switch to a known-good function-calling model before diagnosing MCP or local-agent. That failure means the selected model route is unavailable for the requested tool-call shape.
+
+Run a real-provider tool smoke separately when validating a particular model
+route. It measures provider tool-use behavior in addition to the LangBot path
+and is therefore not the deterministic release gate.
@@ -22,4 +22,5 @@ Do not add Space model concurrency to these gates. Provider concurrency mixes ex
 - Repository contract failure: fix the owning repository and add the narrowest deterministic regression.
 - Browser workflow failure: correlate the screenshot, console, network, and backend log from the same run.
 - Complex Agent failure: inspect `ledger-audit.json` before blaming the model. Tool pairing, authorization, JSON, and timeout failures are product signals.
+- A provider-origin malformed tool-argument payload is a recovery warning only when the ledger keeps a paired error result, a later tool call succeeds, and the run completes. Malformed persisted event JSON and unrecovered tool-argument errors remain release failures.
 - Packaging `env_issue`: restore isolated build dependency access, then rerun packaging only. It must not mask passing or failing runtime contracts.
@@ -6,18 +6,24 @@ symptoms:
  - "Debug Chat shows Agent runner temporarily unavailable after the user message is sent."
  - "Basic streaming prompts may work, but tool-call, non-streaming, or multimodal prompts fail with the same runner and pipeline."
  - "The failure happens after the local-agent runner starts, not during plugin discovery."
+  - "A short model preflight passes, but a sustained Agent run later fails after the provider quota is exhausted."
 patterns:
  - "runner.llm_error"
  - "runner.tool_loop_error"
  - "model_not_found"
  - "no available channel for model"
  - "invalid api key"
+  - "insufficient user quota"
+  - "insufficient_quota"
+  - "quota exceeded"
+  - "余额不足"
  - "当前分组上游负载已饱和"
  - "All models failed during streaming setup"
 likely_causes:
  - "The selected model route is unavailable in the current LangBot Space or upstream group."
  - "The selected model works for plain chat but is not available for tool-call, multimodal, or non-streaming request shapes."
  - "The provider credential or quota is invalid for the non-streaming path."
+  - "The provider account has enough balance for a short preflight but not for the sustained task."
  - "The selected model has function-call or vision metadata, but the upstream distributor cannot currently serve that model."
 fix_steps:
  - "First rerun a basic local-agent Debug Chat prompt on the same pipeline to confirm the runner host path still works."
@@ -25,6 +31,7 @@ fix_steps:
  - "When tool-call cases fail, retest with a model that is known to support function calling in the active environment."
  - "When multimodal cases fail, retest with a model route that is known to accept image content."
  - "When non-streaming fails with invalid api key, verify the provider credential used by the non-streaming requester path."
+  - "When logs report insufficient quota, restore provider balance or switch to another authorized route before rerunning the live-model case."
  - "Do not classify this as an MCP, RAG, or local-agent runner implementation failure until the same model route works for the requested request shape outside the failing case."
 verification: "The same case produces the expected bot-visible sentinel or tool result, and backend logs show request completion without runner.llm_error, runner.tool_loop_error, or All models failed."
 related_cases:
@@ -33,3 +40,4 @@ related_cases:
  - mcp-stdio-tool-call
  - local-agent-multimodal-debug-chat
  - local-agent-nonstreaming-debug-chat
+  - local-agent-complex-coding-task-debug-chat
@@ -338,9 +338,27 @@ function scanLogTextIntoState(

 function buildLogGuardResult(scan: LogScanConfig, sources: LogSourceSummary[], state: MutableScanState): LogGuardResult {
  const scannedCount = sources.filter((source) => source.status === "scanned").length;
+  const hardFailures = state.findings.filter(
+    (finding) => finding.severity === "fail" || finding.severity === "missing_input",
+  );
+  const relatedEnvIssues = state.findings.filter(
+    (finding) => finding.severity === "env_issue" && finding.related_to_case !== false,
+  );
+  const hardFailuresAreExplainedProviderTracebacks = hardFailures.length > 0
+    && relatedEnvIssues.length > 0
+    && hardFailures.every((failure) =>
+      failure.kind === "python_traceback"
+      && relatedEnvIssues.some((envIssue) =>
+        envIssue.source === failure.source
+        && envIssue.path === failure.path
+        && typeof envIssue.line === "number"
+        && typeof failure.line === "number"
+        && Math.abs(envIssue.line - failure.line) <= 100,
+      ),
+    );
  const status = scannedCount === 0 && state.findings.length === 0
    ? "not_run"
-    : state.findings.some((finding) => finding.severity === "fail" || finding.severity === "missing_input")
+    : hardFailures.length > 0 && !hardFailuresAreExplainedProviderTracebacks
      ? "fail"
      : state.findings.some((finding) => finding.severity === "matched_troubleshooting" && finding.related_to_case !== false)
        ? "fail"
@@ -528,7 +546,7 @@ function scanTroubleshootingPatterns(
 }

 function isModelRouteUnavailableText(text: string): boolean {
-  return /model_not_found|no available channel for model|invalid api key|当前分组上游负载已饱和/i.test(text);
+  return /model_not_found|no available channel for model|invalid api key|insufficient user quota|insufficient_quota|quota exceeded|余额不足|当前分组上游负载已饱和/i.test(text);
 }

 function scanCaseDeclaredPatterns(
@@ -3708,7 +3708,7 @@ test("MCP stdio tool-call case setups pipeline and registered MCP server", () =>
    assert.deepEqual(
      run.setup_automation.map((item: { entry: string }) => item.entry),
      [
-        "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env",
+        "node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env",
        "case:mcp-stdio-register",
      ],
    );
@@ -3739,8 +3739,8 @@ test("MCP stdio tool-call case setups pipeline and registered MCP server", () =>
    assert.equal(planResult.code, 0);
    const plan = JSON.parse(planResult.output);
    assert.deepEqual(plan.setup_provides_env, [
-      "LANGBOT_LOCAL_AGENT_PIPELINE_URL",
-      "LANGBOT_LOCAL_AGENT_PIPELINE_NAME",
+      "LANGBOT_FAKE_PROVIDER_PIPELINE_URL",
+      "LANGBOT_FAKE_PROVIDER_PIPELINE_NAME",
      "LANGBOT_MCP_QA_STDIO_SERVER_UUID",
    ]);
    assert.ok(
@@ -4588,6 +4588,47 @@ test("test report classifies model route failures as env_issue", () => {
  }
 });

+test("test report classifies provider quota tracebacks as env_issue", () => {
+  const tmp = mkdtempSync(join(tmpdir(), "lbs-report-quota-env-issue-"));
+  try {
+    const logPath = join(tmp, "backend.log");
+    writeFileSync(
+      logPath,
+      [
+        "[05-21 10:31:00.000] chat.py (2) - [ERROR] : Request Failed: Traceback (most recent call last):",
+        "  File \"provider.py\", line 1, in invoke",
+        "openai.PermissionDeniedError: insufficient user quota",
+        "[05-21 10:31:01.000] pipeline.py (3) - [ERROR] : runner.llm_error All models failed during streaming setup: insufficient user quota",
+      ].join("\n"),
+    );
+
+    const result = capture(() =>
+      commandTestReport(
+        ctx([
+          "test",
+          "report",
+          "local-agent-complex-coding-task-debug-chat",
+          "--backend-log",
+          logPath,
+          "--json",
+        ]),
+      ),
+    );
+    assert.equal(result.code, 0);
+    const report = JSON.parse(result.output);
+    assert.equal(report.log_guard.status, "env_issue");
+    assert.ok(
+      report.log_guard.findings.some(
+        (finding: { severity?: string; pattern?: string }) =>
+          finding.severity === "env_issue" &&
+          finding.pattern === "insufficient user quota",
+      ),
+    );
+  } finally {
+    rmSync(tmp, { recursive: true, force: true });
+  }
+});
+
 test("test report infers scan window from automation result evidence", () => {
  const tmp = mkdtempSync(join(tmpdir(), "lbs-report-evidence-window-"));
  try {
@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+import sys
+import unittest
+from pathlib import Path
+
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts" / "e2e"))
+
+from agent_run_ledger_policy import (  # noqa: E402
+    classify_invalid_tool_argument_errors,
+    load_ledger_json,
+)
+
+
+class AgentRunLedgerPolicyTests(unittest.TestCase):
+    def setUp(self) -> None:
+        self.event = {
+            "sequence": 10,
+            "type": "tool.call.completed",
+            "signal": "Invalid JSON arguments",
+        }
+
+    def test_completed_run_with_later_success_is_warning(self) -> None:
+        failures, warnings = classify_invalid_tool_argument_errors(
+            [self.event],
+            successful_tool_completion_sequences=[12],
+            run_completed=True,
+        )
+
+        self.assertEqual(failures, [])
+        self.assertEqual(warnings[0]["kind"], "recovered_tool_argument_error")
+
+    def test_completed_run_without_later_success_still_fails(self) -> None:
+        failures, warnings = classify_invalid_tool_argument_errors(
+            [self.event],
+            successful_tool_completion_sequences=[8],
+            run_completed=True,
+        )
+
+        self.assertEqual(failures, [self.event])
+        self.assertEqual(warnings, [])
+
+    def test_incomplete_run_still_fails_even_with_later_success(self) -> None:
+        failures, warnings = classify_invalid_tool_argument_errors(
+            [self.event],
+            successful_tool_completion_sequences=[12],
+            run_completed=False,
+        )
+
+        self.assertEqual(failures, [self.event])
+        self.assertEqual(warnings, [])
+
+    def test_malformed_persisted_event_json_remains_an_invariant_failure(self) -> None:
+        failures: list[dict] = []
+
+        value = load_ledger_json(
+            '{"tool_call_id":',
+            field="agent_run_event[10].data_json",
+            failures=failures,
+        )
+
+        self.assertEqual(value, {})
+        self.assertEqual(failures[0]["kind"], "invalid_json")
+        self.assertEqual(failures[0]["field"], "agent_run_event[10].data_json")
+
+
+if __name__ == "__main__":
+    unittest.main()