test(workspace): harden release gate automation

skills/scripts/e2e/agent-run-ledger-audit.py

@@ -16,6 +16,8 @@ import sqlalchemy
 import yaml
 from sqlalchemy.ext.asyncio import create_async_engine
 
+from agent_run_ledger_policy import classify_invalid_tool_argument_errors, load_ledger_json
+
 
 def database_url(repo: pathlib.Path) -> str:
     config = yaml.safe_load((repo / "data/config.yaml").read_text(encoding="utf-8")) or {}
@@ -37,16 +39,6 @@ def database_url(repo: pathlib.Path) -> str:
     raise RuntimeError(f"Unsupported database backend: {kind}")
 
 
-def load_json(value: str | None, *, field: str, failures: list[dict]) -> object:
-    if not value:
-        return {}
-    try:
-        return json.loads(value)
-    except (TypeError, ValueError) as exc:
-        failures.append({"kind": "invalid_json", "field": field, "reason": str(exc)})
-        return {}
-
-
 def parse_created_after(value: str | None) -> datetime.datetime | None:
     if not value:
         return None
@@ -143,7 +135,11 @@ async def audit(
     finally:
         await engine.dispose()
 
-    authorization = load_json(run_row.get("authorization_json"), field="agent_run.authorization_json", failures=failures)
+    authorization = load_ledger_json(
+        run_row.get("authorization_json"),
+        field="agent_run.authorization_json",
+        failures=failures,
+    )
     tools = authorization.get("resources", {}).get("tools", []) if isinstance(authorization, dict) else []
     allowed_tools: dict[str, dict] = {}
     incomplete_tool_metadata: list[dict] = []
@@ -173,7 +169,13 @@ async def audit(
     event_types: list[str] = []
     invalid_event_json = 0
     suspicious_errors: list[dict] = []
-    forbidden_pattern = re.compile(r"invalid json(?: arguments)?|unauthori[sz]ed|permission denied|forbidden|timed?\s*out|timeout", re.I)
+    invalid_tool_argument_errors: list[dict] = []
+    successful_tool_completion_sequences: list[int] = []
+    forbidden_pattern = re.compile(
+        r"invalid json(?! arguments)|unauthori[sz]ed|permission denied|forbidden|timed?\s*out|timeout",
+        re.I,
+    )
+    invalid_tool_arguments_pattern = re.compile(r"invalid json arguments", re.I)
 
     def error_surface(value: object) -> list[str]:
         """Collect diagnostic fields without treating normal tool parameters as errors."""
@@ -192,7 +194,11 @@ async def audit(
         event_type = str(row["type"])
         event_types.append(event_type)
         before = len(failures)
-        data = load_json(row.get("data_json"), field=f"agent_run_event[{row['sequence']}].data_json", failures=failures)
+        data = load_ledger_json(
+            row.get("data_json"),
+            field=f"agent_run_event[{row['sequence']}].data_json",
+            failures=failures,
+        )
         invalid_event_json += int(len(failures) > before)
         if not isinstance(data, dict):
             failures.append({"kind": "invalid_event_payload", "sequence": row["sequence"], "type": event_type})
@@ -206,12 +212,20 @@ async def audit(
                 starts.setdefault(call_id, []).append(item)
             else:
                 completions.setdefault(call_id, []).append(item)
+                if not data.get("error") and data.get("result") is not None:
+                    successful_tool_completion_sequences.append(row["sequence"])
         diagnostic_text = "\n".join(error_surface(data))
         if event_type == "run.failed":
             diagnostic_text += "\n" + json.dumps(data, ensure_ascii=True)
         match = forbidden_pattern.search(diagnostic_text)
         if match:
             suspicious_errors.append({"sequence": row["sequence"], "type": event_type, "signal": match.group(0)})
+        elif event_type == "tool.call.completed":
+            match = invalid_tool_arguments_pattern.search(diagnostic_text)
+            if match:
+                invalid_tool_argument_errors.append(
+                    {"sequence": row["sequence"], "type": event_type, "signal": match.group(0)}
+                )
 
     if run_row["status"] != "completed":
         failures.append({"kind": "run_status", "actual": run_row["status"], "expected": "completed"})
@@ -236,6 +250,18 @@ async def audit(
             unauthorized_calls.append({"tool_call_id": call_id, "tool_name": started[0]["tool_name"]})
     if unauthorized_calls:
         failures.append({"kind": "unauthorized_tool_calls", "calls": unauthorized_calls})
+    unrecovered_argument_errors, recovered_argument_warnings = classify_invalid_tool_argument_errors(
+        invalid_tool_argument_errors,
+        successful_tool_completion_sequences=successful_tool_completion_sequences,
+        run_completed=(
+            run_row["status"] == "completed"
+            and "run.completed" in event_types
+            and "run.failed" not in event_types
+        ),
+    )
+    if unrecovered_argument_errors:
+        suspicious_errors.extend(unrecovered_argument_errors)
+    warnings.extend(recovered_argument_warnings)
     if suspicious_errors:
         failures.append({"kind": "forbidden_error_signals", "events": suspicious_errors})
     if not event_rows:
@@ -281,6 +307,7 @@ async def audit(
         "authorized_tool_count": len(allowed_tools),
         "invalid_event_json": invalid_event_json,
         "suspicious_error_count": len(suspicious_errors),
+        "recovered_tool_argument_error_count": len(recovered_argument_warnings),
     }
     return {
         "status": "pass" if not failures else "fail",

skills/scripts/e2e/agent_run_ledger_policy.py

@@ -0,0 +1,43 @@
+"""Policy helpers for classifying AgentRunner ledger error signals."""
+
+from __future__ import annotations
+
+import json
+
+
+def load_ledger_json(value: str | None, *, field: str, failures: list[dict]) -> object:
+    """Decode persisted ledger JSON and retain corruption as an invariant failure."""
+    if not value:
+        return {}
+    try:
+        return json.loads(value)
+    except (TypeError, ValueError) as exc:
+        failures.append({"kind": "invalid_json", "field": field, "reason": str(exc)})
+        return {}
+
+
+def classify_invalid_tool_argument_errors(
+    events: list[dict],
+    *,
+    successful_tool_completion_sequences: list[int],
+    run_completed: bool,
+) -> tuple[list[dict], list[dict]]:
+    """Split malformed tool arguments into recovered warnings and hard failures."""
+    failures: list[dict] = []
+    warnings: list[dict] = []
+    for event in events:
+        recovered = run_completed and any(
+            sequence > event["sequence"]
+            for sequence in successful_tool_completion_sequences
+        )
+        if recovered:
+            warnings.append(
+                {
+                    "kind": "recovered_tool_argument_error",
+                    "event": event,
+                    "reason": "The model continued with a later successful tool call and the run completed.",
+                }
+            )
+        else:
+            failures.append(event)
+    return failures, warnings

skills/scripts/e2e/bot-event-routing-product-flow.mjs

@@ -111,7 +111,13 @@ try {
     { exact: true },
   );
   await noEventRoutes.waitFor();
-  await messageBehavior.waitFor();
+  try {
+    await messageBehavior.waitFor({ timeout: 5_000 });
+  } catch {
+    // Async adapter fields can rerender once after the first menu click.
+    await addBehavior.click();
+    await messageBehavior.waitFor();
+  }
   await page.waitForTimeout(250);
   await safeScreenshot(page, scenarioMenuScreenshot);
   await messageBehavior.click();

skills/scripts/e2e/ensure-fake-provider-pipeline.mjs

@@ -148,7 +148,7 @@ try {
     modelUuid: model.uuid,
   });
   Object.assign(result, pipeline);
-  result.pipeline_url = `${frontendUrl.replace(/\/$/, "")}/home/pipelines?id=${encodeURIComponent(pipeline.pipeline_id)}`;
+  result.pipeline_url = `${frontendUrl.replace(/\/$/, "")}/home/agents?id=${encodeURIComponent(pipeline.pipeline_id)}`;
 
   const runConfig = await configureFakeProvider(fakeProvider.url, targetFakeProviderConfig(), true);
   result.fake_provider.config = runConfig.config || targetFakeProviderConfig();
@@ -425,7 +425,7 @@ async function ensureModel({ backendUrl, token, providerUuid, name }) {
   const body = {
     name,
     provider_uuid: providerUuid,
-    abilities: [],
+    abilities: ["func_call", "vision"],
     context_length: positiveInteger(env.LANGBOT_FAKE_PROVIDER_CONTEXT_LENGTH, 8192),
     extra_args: {},
     prefered_ranking: 0,

skills/scripts/e2e/wizard-onebot-agent-runtime.mjs

@@ -204,9 +204,16 @@ try {
   result.visible_signals.push("bot-created", "adapter-enabled");
 
   await page.getByRole("button", { name: /Next|下一步|次へ/ }).click();
-  await page
-    .getByText(/Local Agent|本地 Agent/)
-    .first()
+  const localAgentTitle = page
+    .getByText(/^(Local Agent|本地 Agent)$/)
+    .first();
+  const localAgentCard = localAgentTitle.locator(
+    'xpath=ancestor::*[@data-slot="card"][1]',
+  );
+  await localAgentCard
+    .getByRole("button", {
+      name: /Use This Runner|使用此运行器|この Runner を使用/,
+    })
     .click();
   await page.waitForFunction(
     () =>