test(workspace): add top-down release gates

skills/docs/user-guide.md

@@ -98,6 +98,23 @@ message truncation. Do not treat it as a long-term QA contract.
 
 ## 5. Run Gates
 
+Cross-repository contract gate (no model calls):
+
+```bash
+bin/lbs suite plan langbot-workspace-contract-gate
+bin/lbs suite run langbot-workspace-contract-gate --run-id langbot-workspace-contract-local
+```
+
+Top-down workspace release gate (browser workflows plus one complex Agent task):
+
+```bash
+bin/lbs suite plan langbot-workspace-release-gate
+bin/lbs suite run langbot-workspace-release-gate --run-id langbot-workspace-release-local --include-manual-check
+```
+
+Run `agent-run-ledger-audit` immediately after the complex Agent case. Set
+`LANGBOT_AGENT_RUN_ID` when the test instance has concurrent operator traffic.
+
 Fast contract gate, no live service required:
 
 ```bash

skills/scripts/e2e/agent-run-ledger-audit.mjs

@@ -0,0 +1,83 @@
+#!/usr/bin/env node
+
+import { spawn } from "node:child_process";
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import { env } from "node:process";
+import {
+  ensureEvidence,
+  evidencePaths,
+  exitCode,
+  loadEnvFiles,
+  localIsoWithOffset,
+  resolveLangBotRepo,
+  writeResult,
+} from "./lib/langbot-e2e.mjs";
+
+await loadEnvFiles();
+const caseId = env.LBS_CASE_ID || "agent-run-ledger-audit";
+const paths = evidencePaths(caseId);
+await ensureEvidence(paths);
+const startedAt = new Date();
+const auditPath = join(paths.evidenceDir, "ledger-audit.json");
+const result = {
+  source: "automation",
+  case_id: caseId,
+  run_id: paths.runId,
+  started_at: startedAt.toISOString(),
+  started_at_local: localIsoWithOffset(startedAt),
+  finished_at: "",
+  finished_at_local: "",
+  status: "fail",
+  reason: "",
+  audited_run: null,
+  metrics_summary: null,
+  failures: [],
+  warnings: [],
+  evidence: {
+    ledger_audit_json: auditPath,
+    automation_result_json: paths.automationResultJson,
+    result_json: paths.resultJson,
+  },
+  evidence_collected: ["filesystem", "metrics", "api_diagnostic"],
+};
+
+function run(command, args, cwd) {
+  return new Promise((resolvePromise) => {
+    const child = spawn(command, args, { cwd, env, stdio: ["ignore", "pipe", "pipe"] });
+    let stderr = "";
+    child.stderr.on("data", (chunk) => { stderr += chunk; });
+    child.on("error", (error) => resolvePromise({ status: null, stderr, error }));
+    child.on("close", (status) => resolvePromise({ status, stderr, error: null }));
+  });
+}
+
+try {
+  const repo = await resolveLangBotRepo();
+  const python = join(repo, ".venv", "bin", "python");
+  const args = ["scripts/e2e/agent-run-ledger-audit.py", "--repo", repo, "--output", auditPath];
+  if (env.LANGBOT_AGENT_RUN_ID) args.push("--run-id", env.LANGBOT_AGENT_RUN_ID);
+  const execution = await run(python, args, process.cwd());
+  const report = JSON.parse(await readFile(auditPath, "utf8"));
+  result.status = report.status;
+  result.reason = report.reason;
+  result.audited_run = report.run || null;
+  result.metrics_summary = report.metrics || null;
+  result.failures = report.failures || [];
+  result.warnings = report.warnings || [];
+  if (execution.error && result.status === "pass") {
+    result.status = "env_issue";
+    result.reason = execution.error.message;
+  }
+} catch (error) {
+  result.status = /ENOENT|not found|No matching/i.test(error.message) ? "env_issue" : "fail";
+  result.reason = error.message;
+} finally {
+  const finishedAt = new Date();
+  result.finished_at = finishedAt.toISOString();
+  result.finished_at_local = localIsoWithOffset(finishedAt);
+  await writeResult(paths, result);
+  console.log(JSON.stringify(result, null, 2));
+}
+
+process.exit(exitCode(result.status));

skills/scripts/e2e/agent-run-ledger-audit.py

@@ -0,0 +1,217 @@
+#!/usr/bin/env python3
+"""Audit one persisted AgentRunner run without exposing authorization secrets."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import os
+import pathlib
+import re
+import sys
+import urllib.parse
+
+import sqlalchemy
+import yaml
+from sqlalchemy.ext.asyncio import create_async_engine
+
+
+def database_url(repo: pathlib.Path) -> str:
+    config = yaml.safe_load((repo / "data/config.yaml").read_text(encoding="utf-8")) or {}
+    database = config.get("database", {})
+    kind = database.get("use", "sqlite")
+    if kind == "sqlite":
+        path = pathlib.Path(database.get("sqlite", {}).get("path", "data/langbot.db"))
+        if not path.is_absolute():
+            path = repo / path
+        return f"sqlite+aiosqlite:///{path}"
+    if kind in {"postgres", "postgresql"}:
+        values = database.get("postgresql", {})
+        user = urllib.parse.quote_plus(str(values.get("user", "postgres")))
+        password = urllib.parse.quote_plus(str(values.get("password", "postgres")))
+        host = values.get("host", "127.0.0.1")
+        port = values.get("port", 5432)
+        name = values.get("database", "postgres")
+        return f"postgresql+asyncpg://{user}:{password}@{host}:{port}/{name}"
+    raise RuntimeError(f"Unsupported database backend: {kind}")
+
+
+def load_json(value: str | None, *, field: str, failures: list[dict]) -> object:
+    if not value:
+        return {}
+    try:
+        return json.loads(value)
+    except (TypeError, ValueError) as exc:
+        failures.append({"kind": "invalid_json", "field": field, "reason": str(exc)})
+        return {}
+
+
+async def audit(repo: pathlib.Path, run_id: str | None) -> dict:
+    engine = create_async_engine(database_url(repo))
+    failures: list[dict] = []
+    warnings: list[dict] = []
+    try:
+        async with engine.connect() as connection:
+            if run_id:
+                run_row = (await connection.execute(
+                    sqlalchemy.text("SELECT * FROM agent_run WHERE run_id = :run_id"),
+                    {"run_id": run_id},
+                )).mappings().first()
+            else:
+                run_row = (await connection.execute(
+                    sqlalchemy.text("SELECT * FROM agent_run ORDER BY id DESC LIMIT 1")
+                )).mappings().first()
+            if run_row is None:
+                return {"status": "env_issue", "reason": "No matching AgentRunner run exists.", "failures": [], "warnings": []}
+
+            selected_run_id = str(run_row["run_id"])
+            event_rows = (await connection.execute(
+                sqlalchemy.text("SELECT sequence, type, data_json, metadata_json FROM agent_run_event WHERE run_id = :run_id ORDER BY sequence"),
+                {"run_id": selected_run_id},
+            )).mappings().all()
+    finally:
+        await engine.dispose()
+
+    authorization = load_json(run_row.get("authorization_json"), field="agent_run.authorization_json", failures=failures)
+    tools = authorization.get("resources", {}).get("tools", []) if isinstance(authorization, dict) else []
+    allowed_tools: dict[str, dict] = {}
+    incomplete_tool_metadata: list[dict] = []
+    for tool in tools if isinstance(tools, list) else []:
+        if not isinstance(tool, dict):
+            incomplete_tool_metadata.append({"tool_name": "", "missing": ["tool object"]})
+            continue
+        name = str(tool.get("tool_name", ""))
+        missing = []
+        if not name:
+            missing.append("tool_name")
+        if not str(tool.get("description", "")).strip():
+            missing.append("description")
+        if not isinstance(tool.get("parameters"), dict):
+            missing.append("parameters")
+        if not (tool.get("source") or tool.get("tool_type") or tool.get("source_id")):
+            missing.append("owner")
+        if missing:
+            incomplete_tool_metadata.append({"tool_name": name, "missing": missing})
+        if name:
+            allowed_tools[name] = tool
+    if incomplete_tool_metadata:
+        failures.append({"kind": "incomplete_tool_metadata", "tools": incomplete_tool_metadata})
+
+    starts: dict[str, list[dict]] = {}
+    completions: dict[str, list[dict]] = {}
+    event_types: list[str] = []
+    invalid_event_json = 0
+    suspicious_errors: list[dict] = []
+    forbidden_pattern = re.compile(r"invalid json(?: arguments)?|unauthori[sz]ed|permission denied|forbidden|timed?\s*out|timeout", re.I)
+
+    def error_surface(value: object) -> list[str]:
+        """Collect diagnostic fields without treating normal tool parameters as errors."""
+        collected: list[str] = []
+        if not isinstance(value, dict):
+            return collected
+        for key, item in value.items():
+            normalized = str(key).lower()
+            if normalized in {"error", "code", "status", "reason", "error_message"} and item is not None and item != "":
+                collected.append(str(item))
+            if isinstance(item, dict):
+                collected.extend(error_surface(item))
+        return collected
+
+    for row in event_rows:
+        event_type = str(row["type"])
+        event_types.append(event_type)
+        before = len(failures)
+        data = load_json(row.get("data_json"), field=f"agent_run_event[{row['sequence']}].data_json", failures=failures)
+        invalid_event_json += int(len(failures) > before)
+        if not isinstance(data, dict):
+            failures.append({"kind": "invalid_event_payload", "sequence": row["sequence"], "type": event_type})
+            continue
+        if event_type in {"tool.call.started", "tool.call.completed"}:
+            call_id = str(data.get("tool_call_id", ""))
+            item = {"sequence": row["sequence"], "tool_name": str(data.get("tool_name", "")), "data": data}
+            if not call_id:
+                failures.append({"kind": "missing_tool_call_id", "sequence": row["sequence"], "type": event_type})
+            elif event_type == "tool.call.started":
+                starts.setdefault(call_id, []).append(item)
+            else:
+                completions.setdefault(call_id, []).append(item)
+        diagnostic_text = "\n".join(error_surface(data))
+        if event_type == "run.failed":
+            diagnostic_text += "\n" + json.dumps(data, ensure_ascii=True)
+        match = forbidden_pattern.search(diagnostic_text)
+        if match:
+            suspicious_errors.append({"sequence": row["sequence"], "type": event_type, "signal": match.group(0)})
+
+    if run_row["status"] != "completed":
+        failures.append({"kind": "run_status", "actual": run_row["status"], "expected": "completed"})
+    if "run.completed" not in event_types:
+        failures.append({"kind": "missing_run_completed_event"})
+    if "run.failed" in event_types:
+        failures.append({"kind": "run_failed_event"})
+
+    all_call_ids = sorted(set(starts) | set(completions))
+    unauthorized_calls = []
+    for call_id in all_call_ids:
+        started = starts.get(call_id, [])
+        completed = completions.get(call_id, [])
+        if len(started) != 1 or len(completed) != 1:
+            failures.append({"kind": "tool_call_pairing", "tool_call_id": call_id, "started": len(started), "completed": len(completed)})
+            continue
+        if started[0]["tool_name"] != completed[0]["tool_name"]:
+            failures.append({"kind": "tool_name_mismatch", "tool_call_id": call_id})
+        if started[0]["sequence"] >= completed[0]["sequence"]:
+            failures.append({"kind": "tool_call_order", "tool_call_id": call_id})
+        if started[0]["tool_name"] not in allowed_tools:
+            unauthorized_calls.append({"tool_call_id": call_id, "tool_name": started[0]["tool_name"]})
+    if unauthorized_calls:
+        failures.append({"kind": "unauthorized_tool_calls", "calls": unauthorized_calls})
+    if suspicious_errors:
+        failures.append({"kind": "forbidden_error_signals", "events": suspicious_errors})
+    if not event_rows:
+        failures.append({"kind": "missing_run_events"})
+    if not tools:
+        warnings.append({"kind": "no_authorized_tools", "reason": "The run authorization snapshot exposes no tools."})
+
+    metrics = {
+        "event_count": len(event_rows),
+        "tool_call_started": sum(len(items) for items in starts.values()),
+        "tool_call_completed": sum(len(items) for items in completions.values()),
+        "tool_call_ids": len(all_call_ids),
+        "authorized_tool_count": len(allowed_tools),
+        "invalid_event_json": invalid_event_json,
+        "suspicious_error_count": len(suspicious_errors),
+    }
+    return {
+        "status": "pass" if not failures else "fail",
+        "reason": "Agent run ledger audit passed." if not failures else f"Agent run ledger audit found {len(failures)} invariant failure(s).",
+        "run": {
+            "run_id": selected_run_id,
+            "runner_id": run_row["runner_id"],
+            "status": run_row["status"],
+            "created_at": str(run_row["created_at"]),
+            "finished_at": str(run_row["finished_at"]),
+        },
+        "metrics": metrics,
+        "failures": failures,
+        "warnings": warnings,
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--repo", required=True)
+    parser.add_argument("--run-id")
+    parser.add_argument("--output", required=True)
+    args = parser.parse_args()
+    try:
+        report = asyncio.run(audit(pathlib.Path(args.repo).resolve(), args.run_id))
+    except Exception as exc:  # noqa: BLE001 - probe must classify environment failures
+        report = {"status": "env_issue", "reason": str(exc), "failures": [], "warnings": []}
+    pathlib.Path(args.output).write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
+    print(json.dumps(report))
+    return 0 if report["status"] == "pass" else 2 if report["status"] == "env_issue" else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

skills/scripts/e2e/install-qa-plugin-smoke.mjs

@@ -33,6 +33,7 @@ const expectedTools = (env.LANGBOT_E2E_EXPECTED_TOOLS || env.LANGBOT_E2E_EXPECTE
   .map((item) => item.trim())
   .filter(Boolean);
 const expectedRunnerId = env.LANGBOT_E2E_EXPECTED_RUNNER_ID || "";
+const forceReinstall = /^(?:1|true|yes|on)$/i.test(env.LANGBOT_E2E_FORCE_REINSTALL || "");
 
 const result = {
   source: "automation",
@@ -74,8 +75,10 @@ try {
     result.tool_names = await listToolNames(backendUrl, auth.token);
   }
   const missingToolsBefore = expectedTools.filter((tool) => !result.tool_names.includes(tool));
-  if (result.plugin_present_before && missingToolsBefore.length > 0) {
-    result.reinstall_reason = `Installed plugin is missing expected tools: ${missingToolsBefore.join(", ")}`;
+  if (result.plugin_present_before && (forceReinstall || missingToolsBefore.length > 0)) {
+    result.reinstall_reason = forceReinstall
+      ? "Explicit reinstall requested by LANGBOT_E2E_FORCE_REINSTALL."
+      : `Installed plugin is missing expected tools: ${missingToolsBefore.join(", ")}`;
     const removeTask = await removePlugin(backendUrl, auth.token);
     if (!isTaskComplete(removeTask)) {
       throw new Error(`Plugin reinstall cleanup did not complete successfully: ${JSON.stringify(removeTask)}`);

skills/scripts/e2e/lib/debug-chat.mjs

@@ -28,6 +28,11 @@ export function findNewFailureSignal(beforeText, afterText, failureSignals = DEB
   return failureSignals.find((signal) => countOccurrences(afterText, signal) > countOccurrences(beforeText, signal)) || "";
 }
 
+export function hasDebugChatOutcome(text, expectedText, minExpectedCount, failureBaselines = []) {
+  if (countOccurrences(text, expectedText) >= minExpectedCount) return true;
+  return failureBaselines.some(({ signal, count }) => countOccurrences(text, signal) > count);
+}
+
 function findFailureSignalInText(text, failureSignals = DEBUG_CHAT_FAILURE_SIGNALS) {
   return failureSignals.find((signal) => String(text || "").includes(signal)) || "";
 }
@@ -320,12 +325,24 @@ export async function visibleDebugChatMessages(page) {
   });
 }
 
-export async function waitForExpectedDebugChatText(page, { expectedText, minExpectedCount, timeoutMs }) {
+export async function waitForExpectedDebugChatText(page, {
+  expectedText,
+  minExpectedCount,
+  timeoutMs,
+  beforeText = "",
+  failureSignals = DEBUG_CHAT_FAILURE_SIGNALS,
+}) {
+  const failureBaselines = failureSignals.map((signal) => ({
+    signal,
+    count: countOccurrences(beforeText, signal),
+  }));
   await page.waitForFunction(
-    ({ expected, min }) => {
-      return document.body.innerText.split(expected).length - 1 >= min;
+    ({ expected, min, failures }) => {
+      const text = document.body.innerText;
+      if (text.split(expected).length - 1 >= min) return true;
+      return failures.some(({ signal, count }) => text.split(signal).length - 1 > count);
     },
-    { expected: expectedText, min: minExpectedCount },
+    { expected: expectedText, min: minExpectedCount, failures: failureBaselines },
     { timeout: timeoutMs },
   ).catch(() => {});
 }
@@ -397,6 +414,8 @@ export async function runDebugChatPrompt(page, {
     minExpectedCount,
     prompt,
     timeoutMs: responseTimeoutMs,
+    beforeText,
+    failureSignals,
   });
   await waitForDebugChatTextStable(page);
 

skills/scripts/e2e/mcp-stdio-register.mjs

@@ -71,7 +71,7 @@ const result = {
     automation_result_json: paths.automationResultJson,
     result_json: paths.resultJson,
   },
-  evidence_collected: ["api_diagnostic"],
+  evidence_collected: ["screenshot", "console", "network", "api_diagnostic"],
   wrote_env: false,
 };
 

skills/scripts/e2e/workspace-compatibility-preflight.mjs

@@ -0,0 +1,163 @@
+#!/usr/bin/env node
+
+import { spawn } from "node:child_process";
+import { access, readFile, writeFile } from "node:fs/promises";
+import { delimiter, dirname, join, resolve } from "node:path";
+import { env } from "node:process";
+import {
+  ensureEvidence,
+  evidencePaths,
+  exitCode,
+  loadEnvFiles,
+  localIsoWithOffset,
+  resolveLangBotRepo,
+  writeResult,
+} from "./lib/langbot-e2e.mjs";
+
+await loadEnvFiles();
+const caseId = env.LBS_CASE_ID || "workspace-compatibility-preflight";
+const paths = evidencePaths(caseId);
+await ensureEvidence(paths);
+const startedAt = new Date();
+const detailsPath = join(paths.evidenceDir, "workspace-preflight.json");
+
+const result = {
+  source: "automation",
+  case_id: caseId,
+  run_id: paths.runId,
+  started_at: startedAt.toISOString(),
+  started_at_local: localIsoWithOffset(startedAt),
+  finished_at: "",
+  finished_at_local: "",
+  status: "fail",
+  reason: "",
+  checks: [],
+  warnings: [],
+  evidence: {
+    workspace_preflight_json: detailsPath,
+    automation_result_json: paths.automationResultJson,
+    result_json: paths.resultJson,
+  },
+  evidence_collected: ["filesystem", "api_diagnostic"],
+};
+
+const repositories = [
+  { id: "langbot", directory: "LangBot", envKey: "LANGBOT_REPO", manifest: false },
+  { id: "plugin-sdk", directory: "langbot-plugin-sdk", envKey: "LANGBOT_PLUGIN_SDK_REPO", manifest: false },
+  { id: "agent-runner", directory: "langbot-agent-runner", envKey: "LANGBOT_AGENT_RUNNER_REPO", manifest: false },
+  { id: "local-agent", directory: "langbot-local-agent", envKey: "LANGBOT_LOCAL_AGENT_REPO", identity: "langbot-team/LocalAgent" },
+  { id: "control-plane", directory: "langbot-agent-control-plane", envKey: "LANGBOT_AGENT_CONTROL_PLANE_REPO", identity: "langbot/agent-control-plane" },
+  { id: "longterm-memory", directory: "langbot-longterm-memory", envKey: "LANGBOT_LONGTERM_MEMORY_REPO", identity: "langbot-team/LongTermMemory" },
+  { id: "parser", directory: "langbot-parser", envKey: "LANGBOT_PARSER_PLUGIN_REPO", identity: "langbot-team/GeneralParsers" },
+  { id: "rag", directory: "langbot-rag", envKey: "LANGBOT_RAG_PLUGIN_REPO", identity: "langbot-team/LangRAG" },
+  { id: "skill-authoring", directory: "langbot-skill-authoring", envKey: "LANGBOT_SKILL_AUTHORING_REPO", identity: "huanghuoguoguo/skill-authoring" },
+];
+
+function run(command, args, options = {}) {
+  return new Promise((resolvePromise) => {
+    const child = spawn(command, args, { ...options, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] });
+    let stdout = "";
+    let stderr = "";
+    child.stdout.on("data", (chunk) => { stdout += chunk; });
+    child.stderr.on("data", (chunk) => { stderr += chunk; });
+    child.on("error", (error) => resolvePromise({ status: null, stdout, stderr, error }));
+    child.on("close", (status) => resolvePromise({ status, stdout, stderr, error: null }));
+  });
+}
+
+function addCheck(name, status, detail = {}) {
+  result.checks.push({ name, status, ...detail });
+}
+
+try {
+  const langbotRepo = await resolveLangBotRepo();
+  const workspaceRoot = resolve(env.LANGBOT_WORKSPACE_ROOT || dirname(langbotRepo));
+  const resolved = {};
+
+  for (const repository of repositories) {
+    const path = resolve(env[repository.envKey] || (repository.id === "langbot" ? langbotRepo : join(workspaceRoot, repository.directory)));
+    resolved[repository.id] = path;
+    try {
+      await access(join(path, ".git"));
+    } catch {
+      addCheck(`repo:${repository.id}`, "fail", { path, reason: "Git checkout is missing." });
+      continue;
+    }
+
+    const branch = await run("git", ["branch", "--show-current"], { cwd: path });
+    const branchName = branch.stdout.trim();
+    const compatible = branch.status === 0 && /^(?:main|dev\/4\.11\.x)$/.test(branchName);
+    addCheck(`repo:${repository.id}`, compatible ? "pass" : "fail", {
+      path,
+      branch: branchName,
+      reason: compatible ? "" : "Expected main or dev/4.11.x compatibility branch.",
+    });
+
+    const dirty = await run("git", ["status", "--short"], { cwd: path });
+    if (dirty.stdout.trim()) {
+      result.warnings.push({ name: `dirty:${repository.id}`, path, entries: dirty.stdout.trim().split(/\r?\n/).length });
+    }
+
+    if (repository.identity) {
+      const manifestPath = join(path, "manifest.yaml");
+      try {
+        const manifest = await readFile(manifestPath, "utf8");
+        const author = manifest.match(/^\s{2}author:\s*([^\s#]+)/m)?.[1] || "";
+        const name = manifest.match(/^\s{2}name:\s*([^\s#]+)/m)?.[1] || "";
+        const identity = `${author}/${name}`;
+        addCheck(`manifest:${repository.id}`, identity === repository.identity ? "pass" : "fail", {
+          path: manifestPath,
+          identity,
+          expected_identity: repository.identity,
+        });
+      } catch (error) {
+        addCheck(`manifest:${repository.id}`, "fail", { path: manifestPath, reason: error.message });
+      }
+    }
+  }
+
+  const python = join(resolved.langbot, ".venv", "bin", "python");
+  try {
+    await access(python);
+    addCheck("langbot-venv", "pass", { python });
+  } catch {
+    addCheck("langbot-venv", "fail", { python, reason: "LangBot virtualenv Python is missing." });
+  }
+
+  const sdkSrc = join(resolved["plugin-sdk"], "src");
+  const importProbe = await run(python, ["-c", [
+    "import json, pathlib, langbot_plugin",
+    "from langbot_plugin.api.entities.builtin.agent_runner.input import AgentInput",
+    "from langbot_plugin.api.entities.builtin.agent_runner.result import AgentRunResult",
+    "print(json.dumps({'path': str(pathlib.Path(langbot_plugin.__file__).resolve()), 'entities': [AgentInput.__name__, AgentRunResult.__name__]}))",
+  ].join("; ")], {
+    cwd: resolved.langbot,
+    env: { ...env, PYTHONPATH: [sdkSrc, env.PYTHONPATH].filter(Boolean).join(delimiter) },
+  });
+  let importDetail = {};
+  try { importDetail = JSON.parse(importProbe.stdout.trim()); } catch { importDetail = { stderr: importProbe.stderr.trim() }; }
+  const localSdkLoaded = importProbe.status === 0 && resolve(importDetail.path || "").startsWith(resolve(sdkSrc));
+  addCheck("local-sdk-import", localSdkLoaded ? "pass" : "fail", {
+    expected_root: resolve(sdkSrc),
+    ...importDetail,
+    reason: localSdkLoaded ? "" : "langbot_plugin did not load from the workspace SDK source tree.",
+  });
+
+  const failures = result.checks.filter((check) => check.status === "fail");
+  result.status = failures.length === 0 ? "pass" : "fail";
+  result.reason = failures.length === 0
+    ? `Workspace compatibility preflight passed with ${result.warnings.length} non-blocking dirty-worktree warning(s).`
+    : `Workspace compatibility preflight found ${failures.length} blocking check(s).`;
+} catch (error) {
+  result.status = /missing|ENOENT|not found/i.test(error.message) ? "env_issue" : "fail";
+  result.reason = error.message;
+} finally {
+  const finishedAt = new Date();
+  result.finished_at = finishedAt.toISOString();
+  result.finished_at_local = localIsoWithOffset(finishedAt);
+  await writeFile(detailsPath, `${JSON.stringify({ checks: result.checks, warnings: result.warnings }, null, 2)}\n`, "utf8");
+  await writeResult(paths, result);
+  console.log(JSON.stringify(result, null, 2));
+}
+
+process.exit(exitCode(result.status));

skills/scripts/e2e/workspace-plugin-pages-smoke.mjs

@@ -0,0 +1,186 @@
+#!/usr/bin/env node
+
+import { join } from "node:path";
+import { env } from "node:process";
+import {
+  apiJson,
+  createBrowser,
+  ensureAuthenticatedBrowser,
+  ensureEvidence,
+  evidencePaths,
+  exitCode,
+  loadEnvFiles,
+  localIsoWithOffset,
+  safeScreenshot,
+  scanBrowserDiagnostics,
+  writeResult,
+} from "./lib/langbot-e2e.mjs";
+
+await loadEnvFiles();
+const caseId = env.LBS_CASE_ID || "workspace-plugin-pages-smoke";
+const paths = evidencePaths(caseId);
+await ensureEvidence(paths);
+const startedAt = new Date();
+const frontendUrl = env.LANGBOT_FRONTEND_URL || "";
+const backendUrl = env.LANGBOT_BACKEND_URL || "";
+
+const targets = [
+  {
+    id: "control-plane",
+    author: "langbot",
+    plugin: "agent-control-plane",
+    page: "control-plane",
+    signal: "Agent Control Plane",
+    endpoint: "/health",
+    method: "GET",
+  },
+  {
+    id: "memory-console",
+    author: "langbot-team",
+    plugin: "LongTermMemory",
+    page: "memory_console",
+    signal: "Memory Console",
+    endpoint: "/summary",
+    method: "GET",
+  },
+  {
+    id: "langrag-observability",
+    author: "langbot-team",
+    plugin: "LangRAG",
+    page: "observability",
+    signal: "LangRAG Observability",
+    endpoint: "/snapshot",
+    method: "GET",
+  },
+  {
+    id: "skill-authoring",
+    author: "huanghuoguoguo",
+    plugin: "skill-authoring",
+    page: "authoring",
+    signal: "Skill Authoring",
+    endpoint: "/health",
+    method: "GET",
+  },
+];
+
+const result = {
+  source: "automation",
+  case_id: caseId,
+  run_id: paths.runId,
+  started_at: startedAt.toISOString(),
+  started_at_local: localIsoWithOffset(startedAt),
+  finished_at: "",
+  finished_at_local: "",
+  status: "fail",
+  reason: "",
+  frontend_url: frontendUrl,
+  backend_url: backendUrl,
+  pages: [],
+  browser_diagnostics: null,
+  evidence: {
+    console_log: paths.consoleLog,
+    network_log: paths.networkLog,
+    screenshot: paths.screenshot,
+    automation_result_json: paths.automationResultJson,
+    result_json: paths.resultJson,
+  },
+  evidence_collected: ["ui", "screenshot", "console", "network", "api_diagnostic"],
+};
+
+let browser;
+try {
+  if (!frontendUrl || !backendUrl) throw new Error("LANGBOT_FRONTEND_URL and LANGBOT_BACKEND_URL must be configured.");
+  browser = await createBrowser(paths);
+  const { page } = browser;
+  await page.goto(frontendUrl, { waitUntil: "domcontentloaded" });
+  const auth = await ensureAuthenticatedBrowser(page, { frontendUrl, backendUrl });
+  if (auth.status !== "pass") {
+    result.status = auth.status;
+    throw new Error(auth.reason);
+  }
+  const token = await page.evaluate(() => localStorage.getItem("token") || "");
+  const pluginsResponse = await apiJson(backendUrl, "/api/v1/plugins", { token });
+  if (pluginsResponse.status >= 400 || pluginsResponse.json.code !== 0) {
+    throw new Error(pluginsResponse.json.msg || "Failed to list installed plugins.");
+  }
+  const installed = new Map((pluginsResponse.json.data?.plugins || []).map((item) => {
+    const metadata = item.manifest?.manifest?.metadata || item.manifest?.metadata || {};
+    return [`${metadata.author || ""}/${metadata.name || ""}`, item];
+  }));
+  const missing = targets.filter((target) => {
+    const plugin = installed.get(`${target.author}/${target.plugin}`);
+    if (!plugin) return true;
+    const pages = plugin.manifest?.manifest?.spec?.pages || plugin.manifest?.spec?.pages || [];
+    return !pages.some((pageSpec) => pageSpec.id === target.page);
+  });
+  if (missing.length) {
+    result.status = "blocked";
+    throw new Error(`Required plugin page fixture(s) are not installed or do not register the expected Page ID: ${missing.map((item) => item.id).join(", ")}`);
+  }
+
+  for (const target of targets) {
+    const routeId = `${target.author}/${target.plugin}/${target.page}`;
+    const route = `${frontendUrl.replace(/\/$/, "")}/home/plugin-pages?id=${encodeURIComponent(routeId)}`;
+    const pageResult = {
+      id: target.id,
+      route,
+      visible_signal: target.signal,
+      ui_status: "fail",
+      api_status: "fail",
+      screenshot: join(paths.evidenceDir, `${target.id}.png`),
+      page_api: null,
+    };
+    result.pages.push(pageResult);
+
+    await page.goto(route, { waitUntil: "domcontentloaded" });
+    await page.waitForLoadState("networkidle", { timeout: 15_000 }).catch(() => {});
+    const frame = page.frameLocator("iframe").first();
+    await frame.getByText(target.signal, { exact: true }).first().waitFor({ state: "visible", timeout: 30_000 });
+    pageResult.ui_status = "pass";
+    await safeScreenshot(page, pageResult.screenshot);
+
+    const apiResponse = await apiJson(
+      backendUrl,
+      `/api/v1/plugins/${encodeURIComponent(target.author)}/${encodeURIComponent(target.plugin)}/page-api`,
+      {
+        method: "POST",
+        token,
+        body: { page_id: target.page, endpoint: target.endpoint, method: target.method, body: null },
+      },
+    );
+    pageResult.page_api = {
+      endpoint: target.endpoint,
+      method: target.method,
+      http_status: apiResponse.status,
+      code: apiResponse.json.code ?? null,
+      has_data: apiResponse.json.data !== undefined && apiResponse.json.data !== null,
+    };
+    if (apiResponse.status >= 400 || apiResponse.json.code !== 0 || apiResponse.json.data == null) {
+      throw new Error(`${target.id} Page API ${target.method} ${target.endpoint} failed.`);
+    }
+    pageResult.api_status = "pass";
+  }
+
+  await safeScreenshot(page, paths.screenshot);
+  result.browser_diagnostics = await scanBrowserDiagnostics(paths);
+  if (result.browser_diagnostics.status !== "pass") {
+    throw new Error(result.browser_diagnostics.reason);
+  }
+  result.status = "pass";
+  result.reason = "All workspace plugin pages rendered in the LangBot WebUI and their read-only Page APIs passed.";
+} catch (error) {
+  if (result.status === "fail" && /configured|Playwright is not installed|ECONNREFUSED|ERR_CONNECTION/i.test(error.message)) {
+    result.status = "env_issue";
+  }
+  result.reason = error.message;
+} finally {
+  if (browser?.page) await safeScreenshot(browser.page, paths.screenshot);
+  if (browser) await browser.close().catch(() => {});
+  const finishedAt = new Date();
+  result.finished_at = finishedAt.toISOString();
+  result.finished_at_local = localIsoWithOffset(finishedAt);
+  await writeResult(paths, result);
+  console.log(JSON.stringify(result, null, 2));
+}
+
+process.exit(exitCode(result.status));

skills/scripts/e2e/workspace-repository-contracts.mjs

@@ -0,0 +1,136 @@
+#!/usr/bin/env node
+
+import { spawn } from "node:child_process";
+import { access, mkdir, writeFile } from "node:fs/promises";
+import { basename, delimiter, dirname, join, resolve } from "node:path";
+import { env } from "node:process";
+import {
+  ensureEvidence,
+  evidencePaths,
+  exitCode,
+  loadEnvFiles,
+  localIsoWithOffset,
+  resolveLangBotRepo,
+  writeResult,
+} from "./lib/langbot-e2e.mjs";
+
+await loadEnvFiles();
+const caseId = env.LBS_CASE_ID || "workspace-repository-contracts";
+const paths = evidencePaths(caseId);
+await ensureEvidence(paths);
+const startedAt = new Date();
+const logsDir = join(paths.evidenceDir, "repository-contracts");
+await mkdir(logsDir, { recursive: true });
+
+const result = {
+  source: "automation",
+  case_id: caseId,
+  run_id: paths.runId,
+  started_at: startedAt.toISOString(),
+  started_at_local: localIsoWithOffset(startedAt),
+  finished_at: "",
+  finished_at_local: "",
+  status: "fail",
+  reason: "",
+  repositories: [],
+  evidence: {
+    contracts_dir: logsDir,
+    automation_result_json: paths.automationResultJson,
+    result_json: paths.resultJson,
+  },
+  evidence_collected: ["filesystem", "metrics"],
+};
+
+function run(command, args, { cwd, processEnv, timeoutMs = 900_000 }) {
+  return new Promise((resolvePromise) => {
+    const started = Date.now();
+    const child = spawn(command, args, { cwd, env: processEnv, stdio: ["ignore", "pipe", "pipe"] });
+    let stdout = "";
+    let stderr = "";
+    let timedOut = false;
+    const timer = setTimeout(() => {
+      timedOut = true;
+      child.kill("SIGTERM");
+    }, timeoutMs);
+    child.stdout.on("data", (chunk) => { stdout += chunk; });
+    child.stderr.on("data", (chunk) => { stderr += chunk; });
+    child.on("error", (error) => {
+      clearTimeout(timer);
+      resolvePromise({ status: null, stdout, stderr, error, timedOut, durationMs: Date.now() - started });
+    });
+    child.on("close", (status) => {
+      clearTimeout(timer);
+      resolvePromise({ status, stdout, stderr, error: null, timedOut, durationMs: Date.now() - started });
+    });
+  });
+}
+
+try {
+  const langbotRepo = await resolveLangBotRepo();
+  const workspaceRoot = resolve(env.LANGBOT_WORKSPACE_ROOT || dirname(langbotRepo));
+  const sdkRepo = resolve(env.LANGBOT_PLUGIN_SDK_REPO || join(workspaceRoot, "langbot-plugin-sdk"));
+  const python = join(langbotRepo, ".venv", "bin", "python");
+  await access(python);
+  await access(join(sdkRepo, "src", "langbot_plugin"));
+  const processEnv = {
+    ...env,
+    PYTHONPATH: [join(sdkRepo, "src"), env.PYTHONPATH].filter(Boolean).join(delimiter),
+  };
+  const specs = [
+    ["agent-runner", "langbot-agent-runner", ["-m", "pytest", "tests", "-q"]],
+    ["control-plane", "langbot-agent-control-plane", ["-m", "pytest", "tests", "-q"]],
+    ["longterm-memory", "langbot-longterm-memory", ["-m", "pytest", "tests", "-q"]],
+    ["parser", "langbot-parser", ["-m", "pytest", "tests", "-q"]],
+    ["rag", "langbot-rag", ["-m", "pytest", "tests", "-q"]],
+    ["skill-authoring", "langbot-skill-authoring", ["-m", "pytest", "tests", "-q"]],
+    ["local-agent", "langbot-local-agent", ["-m", "pytest", "tests", "-q"]],
+    ["langbot-agent-provider", "LangBot", ["-m", "pytest", "tests/unit_tests/agent", "tests/unit_tests/provider", "-q"]],
+    ["skills-cli", "LangBot/skills", ["--test", "test/lbs-cli.test.ts"]],
+    ["plugin-sdk-runtime", "langbot-plugin-sdk", ["-m", "pytest", "tests", "-q", "--ignore=tests/packaging/test_installed_cli_blackbox.py"]],
+    ["plugin-sdk-packaging", "langbot-plugin-sdk", ["-m", "pytest", "tests/packaging/test_installed_cli_blackbox.py", "-q"]],
+  ];
+
+  for (const [id, directory, args] of specs) {
+    const cwd = resolve(workspaceRoot, directory);
+    const command = id === "skills-cli" ? env.LANGBOT_NODE || "node" : python;
+    const execution = await run(command, args, { cwd, processEnv });
+    const stdoutPath = join(logsDir, `${id}.stdout.log`);
+    const stderrPath = join(logsDir, `${id}.stderr.log`);
+    await writeFile(stdoutPath, execution.stdout, "utf8");
+    await writeFile(stderrPath, execution.stderr, "utf8");
+    const combined = `${execution.stdout}\n${execution.stderr}`;
+    const packagingNetworkIssue = id === "plugin-sdk-packaging"
+      && execution.status !== 0
+      && /hatchling|PyPI|TLS|SSL|network|timed out|Temporary failure|connection|EOF/i.test(combined);
+    const status = execution.status === 0 ? "pass" : packagingNetworkIssue ? "env_issue" : "fail";
+    result.repositories.push({
+      id,
+      repository: basename(cwd),
+      status,
+      exit_status: execution.status,
+      timed_out: execution.timedOut,
+      duration_ms: execution.durationMs,
+      stdout_log: stdoutPath,
+      stderr_log: stderrPath,
+      reason: execution.error?.message || (execution.timedOut ? "Test command timed out." : packagingNetworkIssue ? "Packaging blackbox could not fetch its isolated build dependency." : ""),
+    });
+  }
+
+  const productFailures = result.repositories.filter((item) => item.status === "fail");
+  const envIssues = result.repositories.filter((item) => item.status === "env_issue");
+  result.status = productFailures.length === 0 ? "pass" : "fail";
+  result.reason = productFailures.length === 0
+    ? `All repository product contracts passed; ${envIssues.length} packaging environment issue(s) were classified separately.`
+    : `${productFailures.length} repository product contract group(s) failed.`;
+} catch (error) {
+  result.status = "env_issue";
+  result.reason = error.message;
+} finally {
+  const finishedAt = new Date();
+  result.finished_at = finishedAt.toISOString();
+  result.finished_at_local = localIsoWithOffset(finishedAt);
+  await writeResult(paths, result);
+  console.log(JSON.stringify(result, null, 2));
+}
+
+process.exit(exitCode(result.status));

skills/skills.index.json

@@ -136,10 +136,12 @@
         "references/sandbox-skill-authoring.md",
         "references/skill-all-tool-acceptance.md",
         "references/troubleshooting.md",
-        "references/web-ui-testing.md"
+        "references/web-ui-testing.md",
+        "references/workspace-release-testing.md"
       ],
       "cases": [
         "acp-agent-runner-debug-chat",
+        "agent-run-ledger-audit",
         "agent-runner-async-db-readiness",
         "agent-runner-behavior-matrix",
         "agent-runner-fixture-contract",
@@ -195,7 +197,10 @@
         "webui-login-state",
         "wizard-onebot-agent-runtime",
         "wizard-runner-marketplace-catalog",
-        "wizard-scenario-routing"
+        "wizard-scenario-routing",
+        "workspace-compatibility-preflight",
+        "workspace-plugin-pages-smoke",
+        "workspace-repository-contracts"
       ],
       "case_summaries": [
         {
@@ -228,6 +233,30 @@
             "backend_log"
           ]
         },
+        {
+          "id": "agent-run-ledger-audit",
+          "title": "Persisted AgentRunner run ledger passes end-to-end invariants",
+          "mode": "probe",
+          "area": "agent",
+          "type": "regression",
+          "priority": "p0",
+          "risk": "high",
+          "ci_eligible": false,
+          "tags": [
+            "agent-runner",
+            "ledger",
+            "audit",
+            "tools"
+          ],
+          "automation": "scripts/e2e/agent-run-ledger-audit.mjs",
+          "setup_automation": [],
+          "setup_provides_env": [],
+          "evidence_required": [
+            "filesystem",
+            "metrics",
+            "api_diagnostic"
+          ]
+        },
         {
           "id": "agent-runner-async-db-readiness",
           "title": "AgentRunner async DB readiness probe",
@@ -892,7 +921,8 @@
             "ui",
             "screenshot",
             "console",
-            "backend_log"
+            "network",
+            "api_diagnostic"
           ]
         },
         {
@@ -1040,7 +1070,6 @@
             "screenshot",
             "console",
             "network",
-            "backend_log",
             "api_diagnostic",
             "filesystem",
             "metrics"
@@ -1486,9 +1515,11 @@
           ],
           "evidence_required": [
             "ui",
+            "screenshot",
             "console",
-            "backend_log",
-            "api_diagnostic"
+            "network",
+            "api_diagnostic",
+            "metrics"
           ]
         },
         {
@@ -1797,6 +1828,78 @@
             "console",
             "api_diagnostic"
           ]
+        },
+        {
+          "id": "workspace-compatibility-preflight",
+          "title": "LangBot workspace repositories and local SDK are mutually compatible",
+          "mode": "probe",
+          "area": "release",
+          "type": "smoke",
+          "priority": "p0",
+          "risk": "high",
+          "ci_eligible": true,
+          "tags": [
+            "workspace",
+            "compatibility",
+            "preflight",
+            "sdk"
+          ],
+          "automation": "scripts/e2e/workspace-compatibility-preflight.mjs",
+          "setup_automation": [],
+          "setup_provides_env": [],
+          "evidence_required": [
+            "filesystem",
+            "api_diagnostic"
+          ]
+        },
+        {
+          "id": "workspace-plugin-pages-smoke",
+          "title": "Major workspace plugin pages render and answer read-only Page APIs",
+          "mode": "agent-browser",
+          "area": "plugin",
+          "type": "smoke",
+          "priority": "p0",
+          "risk": "high",
+          "ci_eligible": false,
+          "tags": [
+            "workspace",
+            "plugin",
+            "page",
+            "browser"
+          ],
+          "automation": "scripts/e2e/workspace-plugin-pages-smoke.mjs",
+          "setup_automation": [],
+          "setup_provides_env": [],
+          "evidence_required": [
+            "ui",
+            "screenshot",
+            "console",
+            "network",
+            "api_diagnostic"
+          ]
+        },
+        {
+          "id": "workspace-repository-contracts",
+          "title": "Changed LangBot workspace repositories pass their contract suites",
+          "mode": "probe",
+          "area": "release",
+          "type": "regression",
+          "priority": "p0",
+          "risk": "high",
+          "ci_eligible": true,
+          "tags": [
+            "workspace",
+            "contracts",
+            "pytest",
+            "sdk"
+          ],
+          "automation": "scripts/e2e/workspace-repository-contracts.mjs",
+          "setup_automation": [],
+          "setup_provides_env": [],
+          "evidence_required": [
+            "filesystem",
+            "metrics"
+          ]
         }
       ],
       "suites": [
@@ -1809,6 +1912,8 @@
         "langbot-performance-contract-gate",
         "langbot-performance-reliability-gate",
         "langbot-user-path-performance-gate",
+        "langbot-workspace-contract-gate",
+        "langbot-workspace-release-gate",
         "local-agent-gate"
       ],
       "suite_summaries": [
@@ -2003,6 +2108,57 @@
             "pipeline-debug-chat-performance"
           ]
         },
+        {
+          "id": "langbot-workspace-contract-gate",
+          "title": "LangBot workspace contract gate",
+          "description": "Low-cost cross-repository gate for branch/SDK compatibility and deterministic product contracts before live workflow testing.",
+          "type": "regression",
+          "priority": "p0",
+          "tags": [
+            "workspace",
+            "contracts",
+            "pr-gate"
+          ],
+          "cases": [
+            "workspace-compatibility-preflight",
+            "agent-runner-fixture-contract",
+            "agent-runner-behavior-matrix",
+            "agent-runner-ledger-invariants",
+            "langbot-fault-taxonomy-contract",
+            "langbot-overhead-accounting-contract",
+            "workspace-repository-contracts"
+          ]
+        },
+        {
+          "id": "langbot-workspace-release-gate",
+          "title": "LangBot workspace top-down release gate",
+          "description": "Broad release gate combining deterministic repository contracts with representative browser workflows, plugin pages, RAG/parser, EBA, external AgentRunner, and one complex LocalAgent task.",
+          "type": "release_gate",
+          "priority": "p0",
+          "tags": [
+            "workspace",
+            "release-gate",
+            "browser",
+            "agent"
+          ],
+          "cases": [
+            "workspace-compatibility-preflight",
+            "workspace-repository-contracts",
+            "agent-runner-release-preflight",
+            "webui-login-state",
+            "wizard-scenario-routing",
+            "bot-event-routing-product-flow",
+            "agent-runner-health-visibility",
+            "agent-runner-qa-debug-chat",
+            "local-agent-complex-coding-task-debug-chat",
+            "agent-run-ledger-audit",
+            "langrag-kb-retrieve",
+            "workspace-plugin-pages-smoke",
+            "mcp-stdio-register",
+            "mcp-stdio-tool-call",
+            "wizard-onebot-agent-runtime"
+          ]
+        },
         {
           "id": "local-agent-gate",
           "title": "Local Agent runner regression gate",
@@ -2145,10 +2301,12 @@
         "aiosqlite-connect-hangs",
         "ambiguous-runner-default-label",
         "backend-not-listening",
+        "box-runtime-silent-action-timeout",
         "box-session-conflict-logical-metadata",
         "debug-chat-history-contaminates-automation",
         "dynamic-form-missing-config-id",
         "e2b-extra-mount-sync-missing",
+        "local-agent-mcp-text-result-empty-follow-up",
         "local-agent-model-route-unavailable",
         "marketplace-network-flaky",
         "mcp-stdio-args-not-applied",
@@ -2203,6 +2361,16 @@
             "local-agent-basic-debug-chat"
           ]
         },
+        {
+          "id": "box-runtime-silent-action-timeout",
+          "title": "Box runtime process stays alive while all action RPC calls time out",
+          "category": "product",
+          "related_cases": [
+            "local-agent-complex-coding-task-debug-chat",
+            "agent-run-ledger-audit",
+            "mcp-stdio-tool-call"
+          ]
+        },
         {
           "id": "box-session-conflict-logical-metadata",
           "title": "BoxSessionConflictError after a successful first exec",
@@ -2238,6 +2406,16 @@
             "sandbox-skill-authoring-e2e"
           ]
         },
+        {
+          "id": "local-agent-mcp-text-result-empty-follow-up",
+          "title": "MCP tool succeeds but Local Agent completes without a visible reply",
+          "category": "product",
+          "related_cases": [
+            "mcp-stdio-register",
+            "mcp-stdio-tool-call",
+            "agent-run-ledger-audit"
+          ]
+        },
         {
           "id": "local-agent-model-route-unavailable",
           "title": "Local Agent model route is unavailable for the requested run shape",

skills/skills/langbot-testing/SKILL.md

@@ -22,6 +22,7 @@ Use this skill when an agent needs to verify LangBot behavior through the WebUI
 - **LangRAG knowledge bases**: read `references/langrag-knowledge-base.md`.
 - **MCP stdio tool testing**: read `references/mcp-stdio-testing.md`.
 - **Performance, reliability, or chaos probes**: read `references/performance-reliability-testing.md`.
+- **Cross-repository workspace and release gates**: read `references/workspace-release-testing.md`.
 - **Drive a live instance over MCP (not raw HTTP)**: use the `langbot-mcp-ops` skill — the instance exposes an MCP server at `http://<host>:5300/mcp` (reuses API keys). Useful for setting up bots/pipelines/models as test fixtures programmatically.
 - **Known failures and fixes**: read `references/troubleshooting.md`.
 - **Reusable test groups**: run `bin/lbs suite list` and `bin/lbs suite plan <suite-id>` before manually assembling a case set.

skills/skills/langbot-testing/cases/agent-run-ledger-audit.yaml

@@ -0,0 +1,41 @@
+id: agent-run-ledger-audit
+title: "Persisted AgentRunner run ledger passes end-to-end invariants"
+mode: probe
+area: agent
+type: regression
+priority: p0
+risk: high
+ci_eligible: false
+tags:
+  - agent-runner
+  - ledger
+  - audit
+  - tools
+skills:
+  - langbot-testing
+automation: scripts/e2e/agent-run-ledger-audit.mjs
+steps:
+  - "Set LANGBOT_AGENT_RUN_ID to audit a specific run, or leave it unset to audit the latest persisted AgentRunner run."
+  - "Read the active LangBot database configuration and inspect the selected run and its ordered events."
+  - "Verify completed terminal state, run.completed, paired tool.call.started/completed events, stable tool names, and monotonic ordering."
+  - "Compare called tools with the authorization snapshot and validate each advertised tool has owner/source, description, and parameter schema."
+  - "Reject malformed event JSON and invalid-JSON, timeout, forbidden, permission-denied, or unauthorized signals."
+checks:
+  - "ledger-audit.json status is pass."
+  - "metrics has equal tool_call_started and tool_call_completed counts."
+  - "failures is empty and no unauthorized tool call is present."
+evidence_required:
+  - filesystem
+  - metrics
+  - api_diagnostic
+diagnostics:
+  - "Run this immediately after a complex UI Agent task so latest-run selection cannot drift to unrelated traffic."
+  - "Use LANGBOT_AGENT_RUN_ID when multiple operators share the test instance."
+success_patterns:
+  - "Agent run ledger audit passed"
+failure_patterns:
+  - "tool_call_pairing"
+  - "forbidden_error_signals"
+  - "unauthorized_tool_calls"
+troubleshooting:
+  - box-runtime-silent-action-timeout

skills/skills/langbot-testing/cases/langrag-kb-retrieve.yaml

@@ -47,7 +47,8 @@ evidence_required:
   - ui
   - screenshot
   - console
-  - backend_log
+  - network
+  - api_diagnostic
 diagnostics:
   - "If no LangRAG engine is available, check /api/v1/knowledge/engines and install langbot-team/LangRAG."
   - "If the embedding selector does not show a local Chroma model, confirm the model exists under embedding_models, not llm_models."

skills/skills/langbot-testing/cases/local-agent-complex-coding-task-debug-chat.yaml

@@ -72,7 +72,6 @@ evidence_required:
   - screenshot
   - console
   - network
-  - backend_log
   - api_diagnostic
   - filesystem
   - metrics
@@ -91,5 +90,6 @@ failure_patterns:
 troubleshooting:
   - local-agent-model-route-unavailable
   - sandbox-native-tools-unavailable
+  - box-runtime-silent-action-timeout
   - plugin-runtime-timeout
   - debug-chat-history-contaminates-automation

skills/skills/langbot-testing/cases/mcp-stdio-tool-call.yaml

@@ -74,14 +74,15 @@ checks:
   - "UI: The MCP detail page shows qa_mcp_echo."
   - "API diagnostic: /api/v1/tools contains qa_mcp_echo."
   - "UI: Debug Chat bot response contains qa_mcp_echo:mcp-ok-local-agent."
-  - "Logs: Backend logs show the MCP tool call was executed, not only listed."
   - "API diagnostic: pipeline-extensions-diagnostic.json shows the qa-local-stdio server UUID is the only bound MCP server for the run."
   - "Console: No unexpected frontend errors appear during MCP form use or Debug Chat."
 evidence_required:
   - ui
+  - screenshot
   - console
-  - backend_log
+  - network
   - api_diagnostic
+  - metrics
 diagnostics:
   - "For token-free deterministic UI coverage, start scripts/e2e/fake-openai-provider.mjs and pass LANGBOT_E2E_FAKE_PROVIDER_BASE_URL to this case; setup will bind the local-agent pipeline to that fake OpenAI-compatible model."
   - "Run node scripts/e2e/mcp-stdio-fixture.mjs to verify the bundled stdio fixture can list and call qa_mcp_echo without involving a model provider."
@@ -95,5 +96,6 @@ troubleshooting:
   - mcp-stdio-args-not-applied
   - tool-name-collision-between-mcp-and-plugin
   - local-agent-model-route-unavailable
+  - local-agent-mcp-text-result-empty-follow-up
   - survey-widget-blocks-debug-chat
   - plugin-runtime-timeout

skills/skills/langbot-testing/cases/workspace-compatibility-preflight.yaml

@@ -0,0 +1,38 @@
+id: workspace-compatibility-preflight
+title: "LangBot workspace repositories and local SDK are mutually compatible"
+mode: probe
+area: release
+type: smoke
+priority: p0
+risk: high
+ci_eligible: true
+tags:
+  - workspace
+  - compatibility
+  - preflight
+  - sdk
+skills:
+  - langbot-testing
+automation: scripts/e2e/workspace-compatibility-preflight.mjs
+steps:
+  - "Discover the active LangBot checkout and sibling workspace repositories, with LANGBOT_WORKSPACE_ROOT and repository-specific env overrides available for non-default layouts."
+  - "Verify every checkout is on main or dev/4.11.x and record dirty worktrees as warnings only."
+  - "Validate plugin manifest identities for LocalAgent, Control Plane, LongTermMemory, GeneralParsers, LangRAG, and Skill Authoring."
+  - "Use the LangBot virtualenv to import langbot_plugin and AgentRunner Protocol entities from the workspace SDK source tree."
+checks:
+  - "workspace-preflight.json contains no failed checks."
+  - "The langbot_plugin import path is inside the discovered langbot-plugin-sdk/src directory."
+  - "Dirty worktrees are visible as warnings and do not hide compatibility failures."
+evidence_required:
+  - filesystem
+  - api_diagnostic
+diagnostics:
+  - "A branch failure means the checkout is outside the supported main/dev/4.11.x workspace combination."
+  - "A local-sdk-import failure usually means uv restored the pinned PyPI wheel or the workspace SDK path is wrong."
+success_patterns:
+  - "Workspace compatibility preflight passed"
+failure_patterns:
+  - "blocking check"
+  - "did not load from the workspace SDK"
+troubleshooting:
+  - uv-run-resyncs-local-sdk

skills/skills/langbot-testing/cases/workspace-plugin-pages-smoke.yaml

@@ -0,0 +1,55 @@
+id: workspace-plugin-pages-smoke
+title: "Major workspace plugin pages render and answer read-only Page APIs"
+mode: agent-browser
+area: plugin
+type: smoke
+priority: p0
+risk: high
+ci_eligible: false
+tags:
+  - workspace
+  - plugin
+  - page
+  - browser
+skills:
+  - langbot-env-setup
+  - langbot-testing
+env:
+  - LANGBOT_FRONTEND_URL
+  - LANGBOT_BACKEND_URL
+automation: scripts/e2e/workspace-plugin-pages-smoke.mjs
+automation_env:
+  - LANGBOT_FRONTEND_URL
+  - LANGBOT_BACKEND_URL
+  - LANGBOT_BROWSER_PROFILE
+  - LANGBOT_CHROMIUM_EXECUTABLE
+preconditions:
+  - "Control Plane, LongTermMemory, LangRAG, and Skill Authoring are installed and initialized in the selected local test instance."
+  - "The browser profile is authenticated for the same LangBot backend, or LANGBOT_E2E_LOGIN_USER is configured for local recovery login."
+steps:
+  - "Open each /home/plugin-pages route through the LangBot WebUI."
+  - "Verify the iframe visibly renders Agent Control Plane, Memory Console, LangRAG Observability, and Skill Authoring."
+  - "Using the same browser login, call the read-only Page APIs /health, /summary, /snapshot, and /health respectively."
+  - "Capture one screenshot per page plus browser console/network diagnostics."
+checks:
+  - "Every page has ui_status pass and api_status pass."
+  - "No blank iframe, Page API error, frontend uncaught exception, or HTTP 5xx is recorded."
+  - "No plugin is silently skipped when missing; fixture absence is blocked."
+evidence_required:
+  - ui
+  - screenshot
+  - console
+  - network
+  - api_diagnostic
+diagnostics:
+  - "blocked means one or more required plugins are not installed in the selected test instance."
+  - "A UI-only failure with a passing Page API usually points to plugin asset routing, iframe loading, or frontend page discovery."
+success_patterns:
+  - "All workspace plugin pages rendered"
+failure_patterns:
+  - "Required plugin page fixture"
+  - "Page API"
+  - "Browser diagnostics found"
+troubleshooting:
+  - plugin-runtime-timeout
+  - backend-not-listening

skills/skills/langbot-testing/cases/workspace-repository-contracts.yaml

@@ -0,0 +1,36 @@
+id: workspace-repository-contracts
+title: "Changed LangBot workspace repositories pass their contract suites"
+mode: probe
+area: release
+type: regression
+priority: p0
+risk: high
+ci_eligible: true
+tags:
+  - workspace
+  - contracts
+  - pytest
+  - sdk
+skills:
+  - langbot-testing
+automation: scripts/e2e/workspace-repository-contracts.mjs
+steps:
+  - "Resolve the active LangBot virtualenv and workspace SDK source path."
+  - "Run tests independently for AgentRunner, Control Plane, LongTermMemory, Parser, RAG, Skill Authoring, LocalAgent, LangBot Agent/Provider, the skills CLI, and SDK runtime contracts."
+  - "Run SDK packaging blackbox separately so an isolated build dependency network failure is classified as env_issue without masking product test failures."
+  - "Write per-repository stdout, stderr, status, and duration under repository-contracts/."
+checks:
+  - "Every product contract group has status pass."
+  - "A packaging env_issue is recorded separately and does not convert a product failure into a pass."
+  - "Every command uses the LangBot virtualenv and local SDK PYTHONPATH."
+evidence_required:
+  - filesystem
+  - metrics
+diagnostics:
+  - "Inspect the named repository stdout/stderr logs before rerunning a broader suite."
+  - "Packaging dependency fetch errors belong to environment readiness; assertion/import/runtime failures belong to the product."
+success_patterns:
+  - "All repository product contracts passed"
+failure_patterns:
+  - "repository product contract group"
+  - "Test command timed out"

skills/skills/langbot-testing/references/workspace-release-testing.md

@@ -0,0 +1,25 @@
+# Workspace Release Testing
+
+Use the workspace gates when changes span LangBot core, the plugin SDK, AgentRunner, or multiple first-party plugins.
+
+## Cost Ladder
+
+1. Run `langbot-workspace-contract-gate` after each coherent change. It catches wrong branches, wrong SDK imports, protocol drift, and repository regressions without model calls.
+2. Run the narrow browser case for the changed surface, such as `workspace-plugin-pages-smoke`, before broad release testing.
+3. Run `langbot-workspace-release-gate` before merging or publishing coordinated releases. It deliberately uses one complex LocalAgent coding task rather than many low-value model conversations.
+4. Audit the persisted run with `agent-run-ledger-audit`. Set `LANGBOT_AGENT_RUN_ID` when other traffic may have created a newer run.
+
+Run the browser-driven `langrag-parser-golden-e2e` separately when Parser,
+LangRAG ingestion, document upload, or parser selection changes. It intentionally
+remains a manual deep case until its full WebUI setup path is automated; it is
+not listed in the automated release gate where it could be silently skipped.
+
+Do not add Space model concurrency to these gates. Provider concurrency mixes external quota and latency into product results and is covered separately by optional performance cases.
+
+## Failure Triage
+
+- Compatibility preflight failure: correct the checkout, virtualenv, or local SDK import before testing behavior.
+- Repository contract failure: fix the owning repository and add the narrowest deterministic regression.
+- Browser workflow failure: correlate the screenshot, console, network, and backend log from the same run.
+- Complex Agent failure: inspect `ledger-audit.json` before blaming the model. Tool pairing, authorization, JSON, and timeout failures are product signals.
+- Packaging `env_issue`: restore isolated build dependency access, then rerun packaging only. It must not mask passing or failing runtime contracts.

skills/skills/langbot-testing/suites/langbot-workspace-contract-gate.yaml

@@ -0,0 +1,17 @@
+id: langbot-workspace-contract-gate
+title: "LangBot workspace contract gate"
+description: "Low-cost cross-repository gate for branch/SDK compatibility and deterministic product contracts before live workflow testing."
+type: regression
+priority: p0
+tags:
+  - workspace
+  - contracts
+  - pr-gate
+cases:
+  - workspace-compatibility-preflight
+  - agent-runner-fixture-contract
+  - agent-runner-behavior-matrix
+  - agent-runner-ledger-invariants
+  - langbot-fault-taxonomy-contract
+  - langbot-overhead-accounting-contract
+  - workspace-repository-contracts

skills/skills/langbot-testing/suites/langbot-workspace-release-gate.yaml

@@ -0,0 +1,26 @@
+id: langbot-workspace-release-gate
+title: "LangBot workspace top-down release gate"
+description: "Broad release gate combining deterministic repository contracts with representative browser workflows, plugin pages, RAG/parser, EBA, external AgentRunner, and one complex LocalAgent task."
+type: release_gate
+priority: p0
+tags:
+  - workspace
+  - release-gate
+  - browser
+  - agent
+cases:
+  - workspace-compatibility-preflight
+  - workspace-repository-contracts
+  - agent-runner-release-preflight
+  - webui-login-state
+  - wizard-scenario-routing
+  - bot-event-routing-product-flow
+  - agent-runner-health-visibility
+  - agent-runner-qa-debug-chat
+  - local-agent-complex-coding-task-debug-chat
+  - agent-run-ledger-audit
+  - langrag-kb-retrieve
+  - workspace-plugin-pages-smoke
+  - mcp-stdio-register
+  - mcp-stdio-tool-call
+  - wizard-onebot-agent-runtime

skills/skills/langbot-testing/troubleshooting/box-runtime-silent-action-timeout.yaml

@@ -0,0 +1,25 @@
+id: box-runtime-silent-action-timeout
+title: "Box runtime process stays alive while all action RPC calls time out"
+date: 2026-07-19
+symptoms:
+  - "The Box child process and Docker backend are still running, but native exec/read/write actions never return."
+  - "The backend repeatedly falls back to an empty skill list while Box status previously reported available."
+  - "A LocalAgent run records tool.call.started followed by run.failed with runner.timeout and no tool.call.completed."
+patterns:
+  - "Action box_list_skills call timed out"
+  - "Action call_tool call timed out"
+  - "Action box_get_managed_process call timed out"
+  - "Heartbeat to Box runtime failed; reconnecting"
+likely_causes:
+  - "The local stdio Box action channel became silently unresponsive without closing its child process or pipe."
+  - "Before the heartbeat recovery fix, ping failures were only logged at debug level and BoxService remained falsely available."
+fix_steps:
+  - "Use a LangBot build where Box heartbeat failures call the runtime disconnect callback."
+  - "Restart LangBot once to replace an already-stale connector created by an older process."
+  - "Wait for Box runtime disconnected followed by Box runtime reconnected, sandbox features restored."
+  - "Do not attribute the failure to the model until a direct native exec path succeeds."
+verification: "Run a complex LocalAgent task, then agent-run-ledger-audit. The run must complete with every tool.call.started paired with tool.call.completed and no timeout signal."
+related_cases:
+  - local-agent-complex-coding-task-debug-chat
+  - agent-run-ledger-audit
+  - mcp-stdio-tool-call

skills/skills/langbot-testing/troubleshooting/local-agent-mcp-text-result-empty-follow-up.yaml

@@ -0,0 +1,25 @@
+id: local-agent-mcp-text-result-empty-follow-up
+title: "MCP tool succeeds but Local Agent completes without a visible reply"
+date: 2026-07-19
+category: product
+symptoms:
+  - "The run ledger records matching tool.call.started and tool.call.completed events for the MCP tool, followed by run.completed."
+  - "Debug Chat shows no new assistant message and waits until its response timeout."
+  - "The provider records a successful follow-up model call, while the backend reports a stream with zero visible chunks or characters."
+patterns:
+  - "Streaming completed: 0 chunks, 0 chars"
+  - "tool.call.completed"
+  - "run.completed"
+  - "Expected text did not appear in a new assistant message"
+likely_causes:
+  - "Local Agent serialized a text-only MCP ContentElement list as JSON instead of plain tool-result text, and the selected model returned no visible follow-up content."
+  - "The streaming caller accepted an empty final transport chunk as a successful model response, allowing a zero-output run to complete."
+fix_steps:
+  - "Use a LocalAgent build that flattens text-only MCP ContentElement lists before the tool follow-up model turn."
+  - "Use a LocalAgent build that rejects empty final chunks and fails or falls back instead of completing a zero-output run."
+  - "Rebuild and reinstall the local LocalAgent package so the running plugin runtime contains the fix."
+verification: "Run mcp-stdio-tool-call with only qa-local-stdio bound. Debug Chat must show qa_mcp_echo:mcp-ok-local-agent, and an empty follow-up regression must end as run.failed rather than run.completed."
+related_cases:
+  - mcp-stdio-register
+  - mcp-stdio-tool-call
+  - agent-run-ledger-audit

skills/test/lbs-cli.test.ts

@@ -54,6 +54,7 @@ import { repoRoot } from "../src/cli.ts";
 import {
   classifyDebugChatResult,
   findNewFailureSignal,
+  hasDebugChatOutcome,
   minExpectedOccurrences,
 } from "../scripts/e2e/lib/debug-chat.mjs";
 import {
@@ -1913,6 +1914,17 @@ test("debug chat classifier distinguishes new failure signals from old history",
   assert.equal(custom.failure_signal, "qa-plugin-smoke:mcp-ok-local-agent");
 });
 
+test("debug chat outcome wait can stop on a new failure signal", () => {
+  const baselines = [{ signal: "runner.timeout", count: 1 }];
+
+  assert.equal(hasDebugChatOutcome("old runner.timeout", "EXPECTED", 1, baselines), false);
+  assert.equal(
+    hasDebugChatOutcome("old runner.timeout\nnew runner.timeout", "EXPECTED", 1, baselines),
+    true,
+  );
+  assert.equal(hasDebugChatOutcome("EXPECTED", "EXPECTED", 1, baselines), true);
+});
+
 test("debug chat classifier lets new failure signals override stale expected history", () => {
   const result = classifyDebugChatResult({
     beforeText: "Bot: qa_mcp_echo:mcp-ok-local-agent",

src/langbot/pkg/box/connector.py

@@ -166,8 +166,13 @@ class BoxRuntimeConnector(ManagedRuntimeConnector):
             try:
                 await self.ping()
                 self.ap.logger.debug('Heartbeat to Box runtime success.')
+            except asyncio.CancelledError:
+                raise
             except Exception as e:
-                self.ap.logger.debug(f'Failed to heartbeat to Box runtime: {e}')
+                self.ap.logger.warning(f'Heartbeat to Box runtime failed; reconnecting: {e}')
+                if self.runtime_disconnect_callback is not None:
+                    await self.runtime_disconnect_callback(self)
+                    return
 
     async def ping(self) -> None:
         if self._handler is None:

tests/unit_tests/box/test_box_connector.py

@@ -1,7 +1,8 @@
 from __future__ import annotations
 
+import asyncio
 from types import SimpleNamespace
-from unittest.mock import Mock
+from unittest.mock import AsyncMock, Mock
 
 import pytest
 
@@ -104,3 +105,23 @@ def test_box_runtime_connector_dispose_terminates_subprocess(monkeypatch: pytest
     ctrl_task.cancel.assert_called_once()
     assert connector._handler_task is None
     assert connector._ctrl_task is None
+
+
+@pytest.mark.asyncio
+async def test_box_runtime_connector_heartbeat_failure_requests_reconnect(monkeypatch: pytest.MonkeyPatch):
+    monkeypatch.setattr('langbot.pkg.box.connector._HEARTBEAT_INTERVAL_SEC', 0)
+    disconnected = asyncio.Event()
+    callbacks = []
+
+    async def on_disconnect(connector):
+        callbacks.append(connector)
+        disconnected.set()
+
+    connector = BoxRuntimeConnector(make_app(Mock()), runtime_disconnect_callback=on_disconnect)
+    connector._handler = SimpleNamespace(call_action=AsyncMock(side_effect=TimeoutError('silent runtime')))
+
+    heartbeat = asyncio.create_task(connector._heartbeat_loop())
+    await asyncio.wait_for(disconnected.wait(), timeout=1)
+    await heartbeat
+
+    assert callbacks == [connector]