diff --git a/skills/docs/user-guide.md b/skills/docs/user-guide.md index 124d3af36..19eed56b6 100644 --- a/skills/docs/user-guide.md +++ b/skills/docs/user-guide.md @@ -98,6 +98,23 @@ message truncation. Do not treat it as a long-term QA contract. ## 5. Run Gates +Cross-repository contract gate (no model calls): + +```bash +bin/lbs suite plan langbot-workspace-contract-gate +bin/lbs suite run langbot-workspace-contract-gate --run-id langbot-workspace-contract-local +``` + +Top-down workspace release gate (browser workflows plus one complex Agent task): + +```bash +bin/lbs suite plan langbot-workspace-release-gate +bin/lbs suite run langbot-workspace-release-gate --run-id langbot-workspace-release-local --include-manual-check +``` + +Run `agent-run-ledger-audit` immediately after the complex Agent case. Set +`LANGBOT_AGENT_RUN_ID` when the test instance has concurrent operator traffic. + Fast contract gate, no live service required: ```bash diff --git a/skills/scripts/e2e/agent-run-ledger-audit.mjs b/skills/scripts/e2e/agent-run-ledger-audit.mjs new file mode 100644 index 000000000..0d335d786 --- /dev/null +++ b/skills/scripts/e2e/agent-run-ledger-audit.mjs @@ -0,0 +1,83 @@ +#!/usr/bin/env node + +import { spawn } from "node:child_process"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { env } from "node:process"; +import { + ensureEvidence, + evidencePaths, + exitCode, + loadEnvFiles, + localIsoWithOffset, + resolveLangBotRepo, + writeResult, +} from "./lib/langbot-e2e.mjs"; + +await loadEnvFiles(); +const caseId = env.LBS_CASE_ID || "agent-run-ledger-audit"; +const paths = evidencePaths(caseId); +await ensureEvidence(paths); +const startedAt = new Date(); +const auditPath = join(paths.evidenceDir, "ledger-audit.json"); +const result = { + source: "automation", + case_id: caseId, + run_id: paths.runId, + started_at: startedAt.toISOString(), + started_at_local: localIsoWithOffset(startedAt), + finished_at: "", + finished_at_local: "", + status: "fail", + reason: "", + audited_run: null, + metrics_summary: null, + failures: [], + warnings: [], + evidence: { + ledger_audit_json: auditPath, + automation_result_json: paths.automationResultJson, + result_json: paths.resultJson, + }, + evidence_collected: ["filesystem", "metrics", "api_diagnostic"], +}; + +function run(command, args, cwd) { + return new Promise((resolvePromise) => { + const child = spawn(command, args, { cwd, env, stdio: ["ignore", "pipe", "pipe"] }); + let stderr = ""; + child.stderr.on("data", (chunk) => { stderr += chunk; }); + child.on("error", (error) => resolvePromise({ status: null, stderr, error })); + child.on("close", (status) => resolvePromise({ status, stderr, error: null })); + }); +} + +try { + const repo = await resolveLangBotRepo(); + const python = join(repo, ".venv", "bin", "python"); + const args = ["scripts/e2e/agent-run-ledger-audit.py", "--repo", repo, "--output", auditPath]; + if (env.LANGBOT_AGENT_RUN_ID) args.push("--run-id", env.LANGBOT_AGENT_RUN_ID); + const execution = await run(python, args, process.cwd()); + const report = JSON.parse(await readFile(auditPath, "utf8")); + result.status = report.status; + result.reason = report.reason; + result.audited_run = report.run || null; + result.metrics_summary = report.metrics || null; + result.failures = report.failures || []; + result.warnings = report.warnings || []; + if (execution.error && result.status === "pass") { + result.status = "env_issue"; + result.reason = execution.error.message; + } +} catch (error) { + result.status = /ENOENT|not found|No matching/i.test(error.message) ? "env_issue" : "fail"; + result.reason = error.message; +} finally { + const finishedAt = new Date(); + result.finished_at = finishedAt.toISOString(); + result.finished_at_local = localIsoWithOffset(finishedAt); + await writeResult(paths, result); + console.log(JSON.stringify(result, null, 2)); +} + +process.exit(exitCode(result.status)); diff --git a/skills/scripts/e2e/agent-run-ledger-audit.py b/skills/scripts/e2e/agent-run-ledger-audit.py new file mode 100644 index 000000000..a001e03d0 --- /dev/null +++ b/skills/scripts/e2e/agent-run-ledger-audit.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +"""Audit one persisted AgentRunner run without exposing authorization secrets.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import pathlib +import re +import sys +import urllib.parse + +import sqlalchemy +import yaml +from sqlalchemy.ext.asyncio import create_async_engine + + +def database_url(repo: pathlib.Path) -> str: + config = yaml.safe_load((repo / "data/config.yaml").read_text(encoding="utf-8")) or {} + database = config.get("database", {}) + kind = database.get("use", "sqlite") + if kind == "sqlite": + path = pathlib.Path(database.get("sqlite", {}).get("path", "data/langbot.db")) + if not path.is_absolute(): + path = repo / path + return f"sqlite+aiosqlite:///{path}" + if kind in {"postgres", "postgresql"}: + values = database.get("postgresql", {}) + user = urllib.parse.quote_plus(str(values.get("user", "postgres"))) + password = urllib.parse.quote_plus(str(values.get("password", "postgres"))) + host = values.get("host", "127.0.0.1") + port = values.get("port", 5432) + name = values.get("database", "postgres") + return f"postgresql+asyncpg://{user}:{password}@{host}:{port}/{name}" + raise RuntimeError(f"Unsupported database backend: {kind}") + + +def load_json(value: str | None, *, field: str, failures: list[dict]) -> object: + if not value: + return {} + try: + return json.loads(value) + except (TypeError, ValueError) as exc: + failures.append({"kind": "invalid_json", "field": field, "reason": str(exc)}) + return {} + + +async def audit(repo: pathlib.Path, run_id: str | None) -> dict: + engine = create_async_engine(database_url(repo)) + failures: list[dict] = [] + warnings: list[dict] = [] + try: + async with engine.connect() as connection: + if run_id: + run_row = (await connection.execute( + sqlalchemy.text("SELECT * FROM agent_run WHERE run_id = :run_id"), + {"run_id": run_id}, + )).mappings().first() + else: + run_row = (await connection.execute( + sqlalchemy.text("SELECT * FROM agent_run ORDER BY id DESC LIMIT 1") + )).mappings().first() + if run_row is None: + return {"status": "env_issue", "reason": "No matching AgentRunner run exists.", "failures": [], "warnings": []} + + selected_run_id = str(run_row["run_id"]) + event_rows = (await connection.execute( + sqlalchemy.text("SELECT sequence, type, data_json, metadata_json FROM agent_run_event WHERE run_id = :run_id ORDER BY sequence"), + {"run_id": selected_run_id}, + )).mappings().all() + finally: + await engine.dispose() + + authorization = load_json(run_row.get("authorization_json"), field="agent_run.authorization_json", failures=failures) + tools = authorization.get("resources", {}).get("tools", []) if isinstance(authorization, dict) else [] + allowed_tools: dict[str, dict] = {} + incomplete_tool_metadata: list[dict] = [] + for tool in tools if isinstance(tools, list) else []: + if not isinstance(tool, dict): + incomplete_tool_metadata.append({"tool_name": "", "missing": ["tool object"]}) + continue + name = str(tool.get("tool_name", "")) + missing = [] + if not name: + missing.append("tool_name") + if not str(tool.get("description", "")).strip(): + missing.append("description") + if not isinstance(tool.get("parameters"), dict): + missing.append("parameters") + if not (tool.get("source") or tool.get("tool_type") or tool.get("source_id")): + missing.append("owner") + if missing: + incomplete_tool_metadata.append({"tool_name": name, "missing": missing}) + if name: + allowed_tools[name] = tool + if incomplete_tool_metadata: + failures.append({"kind": "incomplete_tool_metadata", "tools": incomplete_tool_metadata}) + + starts: dict[str, list[dict]] = {} + completions: dict[str, list[dict]] = {} + event_types: list[str] = [] + invalid_event_json = 0 + suspicious_errors: list[dict] = [] + forbidden_pattern = re.compile(r"invalid json(?: arguments)?|unauthori[sz]ed|permission denied|forbidden|timed?\s*out|timeout", re.I) + + def error_surface(value: object) -> list[str]: + """Collect diagnostic fields without treating normal tool parameters as errors.""" + collected: list[str] = [] + if not isinstance(value, dict): + return collected + for key, item in value.items(): + normalized = str(key).lower() + if normalized in {"error", "code", "status", "reason", "error_message"} and item is not None and item != "": + collected.append(str(item)) + if isinstance(item, dict): + collected.extend(error_surface(item)) + return collected + + for row in event_rows: + event_type = str(row["type"]) + event_types.append(event_type) + before = len(failures) + data = load_json(row.get("data_json"), field=f"agent_run_event[{row['sequence']}].data_json", failures=failures) + invalid_event_json += int(len(failures) > before) + if not isinstance(data, dict): + failures.append({"kind": "invalid_event_payload", "sequence": row["sequence"], "type": event_type}) + continue + if event_type in {"tool.call.started", "tool.call.completed"}: + call_id = str(data.get("tool_call_id", "")) + item = {"sequence": row["sequence"], "tool_name": str(data.get("tool_name", "")), "data": data} + if not call_id: + failures.append({"kind": "missing_tool_call_id", "sequence": row["sequence"], "type": event_type}) + elif event_type == "tool.call.started": + starts.setdefault(call_id, []).append(item) + else: + completions.setdefault(call_id, []).append(item) + diagnostic_text = "\n".join(error_surface(data)) + if event_type == "run.failed": + diagnostic_text += "\n" + json.dumps(data, ensure_ascii=True) + match = forbidden_pattern.search(diagnostic_text) + if match: + suspicious_errors.append({"sequence": row["sequence"], "type": event_type, "signal": match.group(0)}) + + if run_row["status"] != "completed": + failures.append({"kind": "run_status", "actual": run_row["status"], "expected": "completed"}) + if "run.completed" not in event_types: + failures.append({"kind": "missing_run_completed_event"}) + if "run.failed" in event_types: + failures.append({"kind": "run_failed_event"}) + + all_call_ids = sorted(set(starts) | set(completions)) + unauthorized_calls = [] + for call_id in all_call_ids: + started = starts.get(call_id, []) + completed = completions.get(call_id, []) + if len(started) != 1 or len(completed) != 1: + failures.append({"kind": "tool_call_pairing", "tool_call_id": call_id, "started": len(started), "completed": len(completed)}) + continue + if started[0]["tool_name"] != completed[0]["tool_name"]: + failures.append({"kind": "tool_name_mismatch", "tool_call_id": call_id}) + if started[0]["sequence"] >= completed[0]["sequence"]: + failures.append({"kind": "tool_call_order", "tool_call_id": call_id}) + if started[0]["tool_name"] not in allowed_tools: + unauthorized_calls.append({"tool_call_id": call_id, "tool_name": started[0]["tool_name"]}) + if unauthorized_calls: + failures.append({"kind": "unauthorized_tool_calls", "calls": unauthorized_calls}) + if suspicious_errors: + failures.append({"kind": "forbidden_error_signals", "events": suspicious_errors}) + if not event_rows: + failures.append({"kind": "missing_run_events"}) + if not tools: + warnings.append({"kind": "no_authorized_tools", "reason": "The run authorization snapshot exposes no tools."}) + + metrics = { + "event_count": len(event_rows), + "tool_call_started": sum(len(items) for items in starts.values()), + "tool_call_completed": sum(len(items) for items in completions.values()), + "tool_call_ids": len(all_call_ids), + "authorized_tool_count": len(allowed_tools), + "invalid_event_json": invalid_event_json, + "suspicious_error_count": len(suspicious_errors), + } + return { + "status": "pass" if not failures else "fail", + "reason": "Agent run ledger audit passed." if not failures else f"Agent run ledger audit found {len(failures)} invariant failure(s).", + "run": { + "run_id": selected_run_id, + "runner_id": run_row["runner_id"], + "status": run_row["status"], + "created_at": str(run_row["created_at"]), + "finished_at": str(run_row["finished_at"]), + }, + "metrics": metrics, + "failures": failures, + "warnings": warnings, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--repo", required=True) + parser.add_argument("--run-id") + parser.add_argument("--output", required=True) + args = parser.parse_args() + try: + report = asyncio.run(audit(pathlib.Path(args.repo).resolve(), args.run_id)) + except Exception as exc: # noqa: BLE001 - probe must classify environment failures + report = {"status": "env_issue", "reason": str(exc), "failures": [], "warnings": []} + pathlib.Path(args.output).write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + print(json.dumps(report)) + return 0 if report["status"] == "pass" else 2 if report["status"] == "env_issue" else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/scripts/e2e/install-qa-plugin-smoke.mjs b/skills/scripts/e2e/install-qa-plugin-smoke.mjs index 523588232..18c2b27d9 100755 --- a/skills/scripts/e2e/install-qa-plugin-smoke.mjs +++ b/skills/scripts/e2e/install-qa-plugin-smoke.mjs @@ -33,6 +33,7 @@ const expectedTools = (env.LANGBOT_E2E_EXPECTED_TOOLS || env.LANGBOT_E2E_EXPECTE .map((item) => item.trim()) .filter(Boolean); const expectedRunnerId = env.LANGBOT_E2E_EXPECTED_RUNNER_ID || ""; +const forceReinstall = /^(?:1|true|yes|on)$/i.test(env.LANGBOT_E2E_FORCE_REINSTALL || ""); const result = { source: "automation", @@ -74,8 +75,10 @@ try { result.tool_names = await listToolNames(backendUrl, auth.token); } const missingToolsBefore = expectedTools.filter((tool) => !result.tool_names.includes(tool)); - if (result.plugin_present_before && missingToolsBefore.length > 0) { - result.reinstall_reason = `Installed plugin is missing expected tools: ${missingToolsBefore.join(", ")}`; + if (result.plugin_present_before && (forceReinstall || missingToolsBefore.length > 0)) { + result.reinstall_reason = forceReinstall + ? "Explicit reinstall requested by LANGBOT_E2E_FORCE_REINSTALL." + : `Installed plugin is missing expected tools: ${missingToolsBefore.join(", ")}`; const removeTask = await removePlugin(backendUrl, auth.token); if (!isTaskComplete(removeTask)) { throw new Error(`Plugin reinstall cleanup did not complete successfully: ${JSON.stringify(removeTask)}`); diff --git a/skills/scripts/e2e/lib/debug-chat.mjs b/skills/scripts/e2e/lib/debug-chat.mjs index 3f2903410..177a796d4 100755 --- a/skills/scripts/e2e/lib/debug-chat.mjs +++ b/skills/scripts/e2e/lib/debug-chat.mjs @@ -28,6 +28,11 @@ export function findNewFailureSignal(beforeText, afterText, failureSignals = DEB return failureSignals.find((signal) => countOccurrences(afterText, signal) > countOccurrences(beforeText, signal)) || ""; } +export function hasDebugChatOutcome(text, expectedText, minExpectedCount, failureBaselines = []) { + if (countOccurrences(text, expectedText) >= minExpectedCount) return true; + return failureBaselines.some(({ signal, count }) => countOccurrences(text, signal) > count); +} + function findFailureSignalInText(text, failureSignals = DEBUG_CHAT_FAILURE_SIGNALS) { return failureSignals.find((signal) => String(text || "").includes(signal)) || ""; } @@ -320,12 +325,24 @@ export async function visibleDebugChatMessages(page) { }); } -export async function waitForExpectedDebugChatText(page, { expectedText, minExpectedCount, timeoutMs }) { +export async function waitForExpectedDebugChatText(page, { + expectedText, + minExpectedCount, + timeoutMs, + beforeText = "", + failureSignals = DEBUG_CHAT_FAILURE_SIGNALS, +}) { + const failureBaselines = failureSignals.map((signal) => ({ + signal, + count: countOccurrences(beforeText, signal), + })); await page.waitForFunction( - ({ expected, min }) => { - return document.body.innerText.split(expected).length - 1 >= min; + ({ expected, min, failures }) => { + const text = document.body.innerText; + if (text.split(expected).length - 1 >= min) return true; + return failures.some(({ signal, count }) => text.split(signal).length - 1 > count); }, - { expected: expectedText, min: minExpectedCount }, + { expected: expectedText, min: minExpectedCount, failures: failureBaselines }, { timeout: timeoutMs }, ).catch(() => {}); } @@ -397,6 +414,8 @@ export async function runDebugChatPrompt(page, { minExpectedCount, prompt, timeoutMs: responseTimeoutMs, + beforeText, + failureSignals, }); await waitForDebugChatTextStable(page); diff --git a/skills/scripts/e2e/mcp-stdio-register.mjs b/skills/scripts/e2e/mcp-stdio-register.mjs index 07747ac55..017b14305 100755 --- a/skills/scripts/e2e/mcp-stdio-register.mjs +++ b/skills/scripts/e2e/mcp-stdio-register.mjs @@ -71,7 +71,7 @@ const result = { automation_result_json: paths.automationResultJson, result_json: paths.resultJson, }, - evidence_collected: ["api_diagnostic"], + evidence_collected: ["screenshot", "console", "network", "api_diagnostic"], wrote_env: false, }; diff --git a/skills/scripts/e2e/workspace-compatibility-preflight.mjs b/skills/scripts/e2e/workspace-compatibility-preflight.mjs new file mode 100644 index 000000000..951e17a38 --- /dev/null +++ b/skills/scripts/e2e/workspace-compatibility-preflight.mjs @@ -0,0 +1,163 @@ +#!/usr/bin/env node + +import { spawn } from "node:child_process"; +import { access, readFile, writeFile } from "node:fs/promises"; +import { delimiter, dirname, join, resolve } from "node:path"; +import { env } from "node:process"; +import { + ensureEvidence, + evidencePaths, + exitCode, + loadEnvFiles, + localIsoWithOffset, + resolveLangBotRepo, + writeResult, +} from "./lib/langbot-e2e.mjs"; + +await loadEnvFiles(); +const caseId = env.LBS_CASE_ID || "workspace-compatibility-preflight"; +const paths = evidencePaths(caseId); +await ensureEvidence(paths); +const startedAt = new Date(); +const detailsPath = join(paths.evidenceDir, "workspace-preflight.json"); + +const result = { + source: "automation", + case_id: caseId, + run_id: paths.runId, + started_at: startedAt.toISOString(), + started_at_local: localIsoWithOffset(startedAt), + finished_at: "", + finished_at_local: "", + status: "fail", + reason: "", + checks: [], + warnings: [], + evidence: { + workspace_preflight_json: detailsPath, + automation_result_json: paths.automationResultJson, + result_json: paths.resultJson, + }, + evidence_collected: ["filesystem", "api_diagnostic"], +}; + +const repositories = [ + { id: "langbot", directory: "LangBot", envKey: "LANGBOT_REPO", manifest: false }, + { id: "plugin-sdk", directory: "langbot-plugin-sdk", envKey: "LANGBOT_PLUGIN_SDK_REPO", manifest: false }, + { id: "agent-runner", directory: "langbot-agent-runner", envKey: "LANGBOT_AGENT_RUNNER_REPO", manifest: false }, + { id: "local-agent", directory: "langbot-local-agent", envKey: "LANGBOT_LOCAL_AGENT_REPO", identity: "langbot-team/LocalAgent" }, + { id: "control-plane", directory: "langbot-agent-control-plane", envKey: "LANGBOT_AGENT_CONTROL_PLANE_REPO", identity: "langbot/agent-control-plane" }, + { id: "longterm-memory", directory: "langbot-longterm-memory", envKey: "LANGBOT_LONGTERM_MEMORY_REPO", identity: "langbot-team/LongTermMemory" }, + { id: "parser", directory: "langbot-parser", envKey: "LANGBOT_PARSER_PLUGIN_REPO", identity: "langbot-team/GeneralParsers" }, + { id: "rag", directory: "langbot-rag", envKey: "LANGBOT_RAG_PLUGIN_REPO", identity: "langbot-team/LangRAG" }, + { id: "skill-authoring", directory: "langbot-skill-authoring", envKey: "LANGBOT_SKILL_AUTHORING_REPO", identity: "huanghuoguoguo/skill-authoring" }, +]; + +function run(command, args, options = {}) { + return new Promise((resolvePromise) => { + const child = spawn(command, args, { ...options, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { stdout += chunk; }); + child.stderr.on("data", (chunk) => { stderr += chunk; }); + child.on("error", (error) => resolvePromise({ status: null, stdout, stderr, error })); + child.on("close", (status) => resolvePromise({ status, stdout, stderr, error: null })); + }); +} + +function addCheck(name, status, detail = {}) { + result.checks.push({ name, status, ...detail }); +} + +try { + const langbotRepo = await resolveLangBotRepo(); + const workspaceRoot = resolve(env.LANGBOT_WORKSPACE_ROOT || dirname(langbotRepo)); + const resolved = {}; + + for (const repository of repositories) { + const path = resolve(env[repository.envKey] || (repository.id === "langbot" ? langbotRepo : join(workspaceRoot, repository.directory))); + resolved[repository.id] = path; + try { + await access(join(path, ".git")); + } catch { + addCheck(`repo:${repository.id}`, "fail", { path, reason: "Git checkout is missing." }); + continue; + } + + const branch = await run("git", ["branch", "--show-current"], { cwd: path }); + const branchName = branch.stdout.trim(); + const compatible = branch.status === 0 && /^(?:main|dev\/4\.11\.x)$/.test(branchName); + addCheck(`repo:${repository.id}`, compatible ? "pass" : "fail", { + path, + branch: branchName, + reason: compatible ? "" : "Expected main or dev/4.11.x compatibility branch.", + }); + + const dirty = await run("git", ["status", "--short"], { cwd: path }); + if (dirty.stdout.trim()) { + result.warnings.push({ name: `dirty:${repository.id}`, path, entries: dirty.stdout.trim().split(/\r?\n/).length }); + } + + if (repository.identity) { + const manifestPath = join(path, "manifest.yaml"); + try { + const manifest = await readFile(manifestPath, "utf8"); + const author = manifest.match(/^\s{2}author:\s*([^\s#]+)/m)?.[1] || ""; + const name = manifest.match(/^\s{2}name:\s*([^\s#]+)/m)?.[1] || ""; + const identity = `${author}/${name}`; + addCheck(`manifest:${repository.id}`, identity === repository.identity ? "pass" : "fail", { + path: manifestPath, + identity, + expected_identity: repository.identity, + }); + } catch (error) { + addCheck(`manifest:${repository.id}`, "fail", { path: manifestPath, reason: error.message }); + } + } + } + + const python = join(resolved.langbot, ".venv", "bin", "python"); + try { + await access(python); + addCheck("langbot-venv", "pass", { python }); + } catch { + addCheck("langbot-venv", "fail", { python, reason: "LangBot virtualenv Python is missing." }); + } + + const sdkSrc = join(resolved["plugin-sdk"], "src"); + const importProbe = await run(python, ["-c", [ + "import json, pathlib, langbot_plugin", + "from langbot_plugin.api.entities.builtin.agent_runner.input import AgentInput", + "from langbot_plugin.api.entities.builtin.agent_runner.result import AgentRunResult", + "print(json.dumps({'path': str(pathlib.Path(langbot_plugin.__file__).resolve()), 'entities': [AgentInput.__name__, AgentRunResult.__name__]}))", + ].join("; ")], { + cwd: resolved.langbot, + env: { ...env, PYTHONPATH: [sdkSrc, env.PYTHONPATH].filter(Boolean).join(delimiter) }, + }); + let importDetail = {}; + try { importDetail = JSON.parse(importProbe.stdout.trim()); } catch { importDetail = { stderr: importProbe.stderr.trim() }; } + const localSdkLoaded = importProbe.status === 0 && resolve(importDetail.path || "").startsWith(resolve(sdkSrc)); + addCheck("local-sdk-import", localSdkLoaded ? "pass" : "fail", { + expected_root: resolve(sdkSrc), + ...importDetail, + reason: localSdkLoaded ? "" : "langbot_plugin did not load from the workspace SDK source tree.", + }); + + const failures = result.checks.filter((check) => check.status === "fail"); + result.status = failures.length === 0 ? "pass" : "fail"; + result.reason = failures.length === 0 + ? `Workspace compatibility preflight passed with ${result.warnings.length} non-blocking dirty-worktree warning(s).` + : `Workspace compatibility preflight found ${failures.length} blocking check(s).`; +} catch (error) { + result.status = /missing|ENOENT|not found/i.test(error.message) ? "env_issue" : "fail"; + result.reason = error.message; +} finally { + const finishedAt = new Date(); + result.finished_at = finishedAt.toISOString(); + result.finished_at_local = localIsoWithOffset(finishedAt); + await writeFile(detailsPath, `${JSON.stringify({ checks: result.checks, warnings: result.warnings }, null, 2)}\n`, "utf8"); + await writeResult(paths, result); + console.log(JSON.stringify(result, null, 2)); +} + +process.exit(exitCode(result.status)); diff --git a/skills/scripts/e2e/workspace-plugin-pages-smoke.mjs b/skills/scripts/e2e/workspace-plugin-pages-smoke.mjs new file mode 100644 index 000000000..6c238c0a4 --- /dev/null +++ b/skills/scripts/e2e/workspace-plugin-pages-smoke.mjs @@ -0,0 +1,186 @@ +#!/usr/bin/env node + +import { join } from "node:path"; +import { env } from "node:process"; +import { + apiJson, + createBrowser, + ensureAuthenticatedBrowser, + ensureEvidence, + evidencePaths, + exitCode, + loadEnvFiles, + localIsoWithOffset, + safeScreenshot, + scanBrowserDiagnostics, + writeResult, +} from "./lib/langbot-e2e.mjs"; + +await loadEnvFiles(); +const caseId = env.LBS_CASE_ID || "workspace-plugin-pages-smoke"; +const paths = evidencePaths(caseId); +await ensureEvidence(paths); +const startedAt = new Date(); +const frontendUrl = env.LANGBOT_FRONTEND_URL || ""; +const backendUrl = env.LANGBOT_BACKEND_URL || ""; + +const targets = [ + { + id: "control-plane", + author: "langbot", + plugin: "agent-control-plane", + page: "control-plane", + signal: "Agent Control Plane", + endpoint: "/health", + method: "GET", + }, + { + id: "memory-console", + author: "langbot-team", + plugin: "LongTermMemory", + page: "memory_console", + signal: "Memory Console", + endpoint: "/summary", + method: "GET", + }, + { + id: "langrag-observability", + author: "langbot-team", + plugin: "LangRAG", + page: "observability", + signal: "LangRAG Observability", + endpoint: "/snapshot", + method: "GET", + }, + { + id: "skill-authoring", + author: "huanghuoguoguo", + plugin: "skill-authoring", + page: "authoring", + signal: "Skill Authoring", + endpoint: "/health", + method: "GET", + }, +]; + +const result = { + source: "automation", + case_id: caseId, + run_id: paths.runId, + started_at: startedAt.toISOString(), + started_at_local: localIsoWithOffset(startedAt), + finished_at: "", + finished_at_local: "", + status: "fail", + reason: "", + frontend_url: frontendUrl, + backend_url: backendUrl, + pages: [], + browser_diagnostics: null, + evidence: { + console_log: paths.consoleLog, + network_log: paths.networkLog, + screenshot: paths.screenshot, + automation_result_json: paths.automationResultJson, + result_json: paths.resultJson, + }, + evidence_collected: ["ui", "screenshot", "console", "network", "api_diagnostic"], +}; + +let browser; +try { + if (!frontendUrl || !backendUrl) throw new Error("LANGBOT_FRONTEND_URL and LANGBOT_BACKEND_URL must be configured."); + browser = await createBrowser(paths); + const { page } = browser; + await page.goto(frontendUrl, { waitUntil: "domcontentloaded" }); + const auth = await ensureAuthenticatedBrowser(page, { frontendUrl, backendUrl }); + if (auth.status !== "pass") { + result.status = auth.status; + throw new Error(auth.reason); + } + const token = await page.evaluate(() => localStorage.getItem("token") || ""); + const pluginsResponse = await apiJson(backendUrl, "/api/v1/plugins", { token }); + if (pluginsResponse.status >= 400 || pluginsResponse.json.code !== 0) { + throw new Error(pluginsResponse.json.msg || "Failed to list installed plugins."); + } + const installed = new Map((pluginsResponse.json.data?.plugins || []).map((item) => { + const metadata = item.manifest?.manifest?.metadata || item.manifest?.metadata || {}; + return [`${metadata.author || ""}/${metadata.name || ""}`, item]; + })); + const missing = targets.filter((target) => { + const plugin = installed.get(`${target.author}/${target.plugin}`); + if (!plugin) return true; + const pages = plugin.manifest?.manifest?.spec?.pages || plugin.manifest?.spec?.pages || []; + return !pages.some((pageSpec) => pageSpec.id === target.page); + }); + if (missing.length) { + result.status = "blocked"; + throw new Error(`Required plugin page fixture(s) are not installed or do not register the expected Page ID: ${missing.map((item) => item.id).join(", ")}`); + } + + for (const target of targets) { + const routeId = `${target.author}/${target.plugin}/${target.page}`; + const route = `${frontendUrl.replace(/\/$/, "")}/home/plugin-pages?id=${encodeURIComponent(routeId)}`; + const pageResult = { + id: target.id, + route, + visible_signal: target.signal, + ui_status: "fail", + api_status: "fail", + screenshot: join(paths.evidenceDir, `${target.id}.png`), + page_api: null, + }; + result.pages.push(pageResult); + + await page.goto(route, { waitUntil: "domcontentloaded" }); + await page.waitForLoadState("networkidle", { timeout: 15_000 }).catch(() => {}); + const frame = page.frameLocator("iframe").first(); + await frame.getByText(target.signal, { exact: true }).first().waitFor({ state: "visible", timeout: 30_000 }); + pageResult.ui_status = "pass"; + await safeScreenshot(page, pageResult.screenshot); + + const apiResponse = await apiJson( + backendUrl, + `/api/v1/plugins/${encodeURIComponent(target.author)}/${encodeURIComponent(target.plugin)}/page-api`, + { + method: "POST", + token, + body: { page_id: target.page, endpoint: target.endpoint, method: target.method, body: null }, + }, + ); + pageResult.page_api = { + endpoint: target.endpoint, + method: target.method, + http_status: apiResponse.status, + code: apiResponse.json.code ?? null, + has_data: apiResponse.json.data !== undefined && apiResponse.json.data !== null, + }; + if (apiResponse.status >= 400 || apiResponse.json.code !== 0 || apiResponse.json.data == null) { + throw new Error(`${target.id} Page API ${target.method} ${target.endpoint} failed.`); + } + pageResult.api_status = "pass"; + } + + await safeScreenshot(page, paths.screenshot); + result.browser_diagnostics = await scanBrowserDiagnostics(paths); + if (result.browser_diagnostics.status !== "pass") { + throw new Error(result.browser_diagnostics.reason); + } + result.status = "pass"; + result.reason = "All workspace plugin pages rendered in the LangBot WebUI and their read-only Page APIs passed."; +} catch (error) { + if (result.status === "fail" && /configured|Playwright is not installed|ECONNREFUSED|ERR_CONNECTION/i.test(error.message)) { + result.status = "env_issue"; + } + result.reason = error.message; +} finally { + if (browser?.page) await safeScreenshot(browser.page, paths.screenshot); + if (browser) await browser.close().catch(() => {}); + const finishedAt = new Date(); + result.finished_at = finishedAt.toISOString(); + result.finished_at_local = localIsoWithOffset(finishedAt); + await writeResult(paths, result); + console.log(JSON.stringify(result, null, 2)); +} + +process.exit(exitCode(result.status)); diff --git a/skills/scripts/e2e/workspace-repository-contracts.mjs b/skills/scripts/e2e/workspace-repository-contracts.mjs new file mode 100644 index 000000000..ac8a64e55 --- /dev/null +++ b/skills/scripts/e2e/workspace-repository-contracts.mjs @@ -0,0 +1,136 @@ +#!/usr/bin/env node + +import { spawn } from "node:child_process"; +import { access, mkdir, writeFile } from "node:fs/promises"; +import { basename, delimiter, dirname, join, resolve } from "node:path"; +import { env } from "node:process"; +import { + ensureEvidence, + evidencePaths, + exitCode, + loadEnvFiles, + localIsoWithOffset, + resolveLangBotRepo, + writeResult, +} from "./lib/langbot-e2e.mjs"; + +await loadEnvFiles(); +const caseId = env.LBS_CASE_ID || "workspace-repository-contracts"; +const paths = evidencePaths(caseId); +await ensureEvidence(paths); +const startedAt = new Date(); +const logsDir = join(paths.evidenceDir, "repository-contracts"); +await mkdir(logsDir, { recursive: true }); + +const result = { + source: "automation", + case_id: caseId, + run_id: paths.runId, + started_at: startedAt.toISOString(), + started_at_local: localIsoWithOffset(startedAt), + finished_at: "", + finished_at_local: "", + status: "fail", + reason: "", + repositories: [], + evidence: { + contracts_dir: logsDir, + automation_result_json: paths.automationResultJson, + result_json: paths.resultJson, + }, + evidence_collected: ["filesystem", "metrics"], +}; + +function run(command, args, { cwd, processEnv, timeoutMs = 900_000 }) { + return new Promise((resolvePromise) => { + const started = Date.now(); + const child = spawn(command, args, { cwd, env: processEnv, stdio: ["ignore", "pipe", "pipe"] }); + let stdout = ""; + let stderr = ""; + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + child.kill("SIGTERM"); + }, timeoutMs); + child.stdout.on("data", (chunk) => { stdout += chunk; }); + child.stderr.on("data", (chunk) => { stderr += chunk; }); + child.on("error", (error) => { + clearTimeout(timer); + resolvePromise({ status: null, stdout, stderr, error, timedOut, durationMs: Date.now() - started }); + }); + child.on("close", (status) => { + clearTimeout(timer); + resolvePromise({ status, stdout, stderr, error: null, timedOut, durationMs: Date.now() - started }); + }); + }); +} + +try { + const langbotRepo = await resolveLangBotRepo(); + const workspaceRoot = resolve(env.LANGBOT_WORKSPACE_ROOT || dirname(langbotRepo)); + const sdkRepo = resolve(env.LANGBOT_PLUGIN_SDK_REPO || join(workspaceRoot, "langbot-plugin-sdk")); + const python = join(langbotRepo, ".venv", "bin", "python"); + await access(python); + await access(join(sdkRepo, "src", "langbot_plugin")); + const processEnv = { + ...env, + PYTHONPATH: [join(sdkRepo, "src"), env.PYTHONPATH].filter(Boolean).join(delimiter), + }; + const specs = [ + ["agent-runner", "langbot-agent-runner", ["-m", "pytest", "tests", "-q"]], + ["control-plane", "langbot-agent-control-plane", ["-m", "pytest", "tests", "-q"]], + ["longterm-memory", "langbot-longterm-memory", ["-m", "pytest", "tests", "-q"]], + ["parser", "langbot-parser", ["-m", "pytest", "tests", "-q"]], + ["rag", "langbot-rag", ["-m", "pytest", "tests", "-q"]], + ["skill-authoring", "langbot-skill-authoring", ["-m", "pytest", "tests", "-q"]], + ["local-agent", "langbot-local-agent", ["-m", "pytest", "tests", "-q"]], + ["langbot-agent-provider", "LangBot", ["-m", "pytest", "tests/unit_tests/agent", "tests/unit_tests/provider", "-q"]], + ["skills-cli", "LangBot/skills", ["--test", "test/lbs-cli.test.ts"]], + ["plugin-sdk-runtime", "langbot-plugin-sdk", ["-m", "pytest", "tests", "-q", "--ignore=tests/packaging/test_installed_cli_blackbox.py"]], + ["plugin-sdk-packaging", "langbot-plugin-sdk", ["-m", "pytest", "tests/packaging/test_installed_cli_blackbox.py", "-q"]], + ]; + + for (const [id, directory, args] of specs) { + const cwd = resolve(workspaceRoot, directory); + const command = id === "skills-cli" ? env.LANGBOT_NODE || "node" : python; + const execution = await run(command, args, { cwd, processEnv }); + const stdoutPath = join(logsDir, `${id}.stdout.log`); + const stderrPath = join(logsDir, `${id}.stderr.log`); + await writeFile(stdoutPath, execution.stdout, "utf8"); + await writeFile(stderrPath, execution.stderr, "utf8"); + const combined = `${execution.stdout}\n${execution.stderr}`; + const packagingNetworkIssue = id === "plugin-sdk-packaging" + && execution.status !== 0 + && /hatchling|PyPI|TLS|SSL|network|timed out|Temporary failure|connection|EOF/i.test(combined); + const status = execution.status === 0 ? "pass" : packagingNetworkIssue ? "env_issue" : "fail"; + result.repositories.push({ + id, + repository: basename(cwd), + status, + exit_status: execution.status, + timed_out: execution.timedOut, + duration_ms: execution.durationMs, + stdout_log: stdoutPath, + stderr_log: stderrPath, + reason: execution.error?.message || (execution.timedOut ? "Test command timed out." : packagingNetworkIssue ? "Packaging blackbox could not fetch its isolated build dependency." : ""), + }); + } + + const productFailures = result.repositories.filter((item) => item.status === "fail"); + const envIssues = result.repositories.filter((item) => item.status === "env_issue"); + result.status = productFailures.length === 0 ? "pass" : "fail"; + result.reason = productFailures.length === 0 + ? `All repository product contracts passed; ${envIssues.length} packaging environment issue(s) were classified separately.` + : `${productFailures.length} repository product contract group(s) failed.`; +} catch (error) { + result.status = "env_issue"; + result.reason = error.message; +} finally { + const finishedAt = new Date(); + result.finished_at = finishedAt.toISOString(); + result.finished_at_local = localIsoWithOffset(finishedAt); + await writeResult(paths, result); + console.log(JSON.stringify(result, null, 2)); +} + +process.exit(exitCode(result.status)); diff --git a/skills/skills.index.json b/skills/skills.index.json index e82a794b0..48298764c 100644 --- a/skills/skills.index.json +++ b/skills/skills.index.json @@ -136,10 +136,12 @@ "references/sandbox-skill-authoring.md", "references/skill-all-tool-acceptance.md", "references/troubleshooting.md", - "references/web-ui-testing.md" + "references/web-ui-testing.md", + "references/workspace-release-testing.md" ], "cases": [ "acp-agent-runner-debug-chat", + "agent-run-ledger-audit", "agent-runner-async-db-readiness", "agent-runner-behavior-matrix", "agent-runner-fixture-contract", @@ -195,7 +197,10 @@ "webui-login-state", "wizard-onebot-agent-runtime", "wizard-runner-marketplace-catalog", - "wizard-scenario-routing" + "wizard-scenario-routing", + "workspace-compatibility-preflight", + "workspace-plugin-pages-smoke", + "workspace-repository-contracts" ], "case_summaries": [ { @@ -228,6 +233,30 @@ "backend_log" ] }, + { + "id": "agent-run-ledger-audit", + "title": "Persisted AgentRunner run ledger passes end-to-end invariants", + "mode": "probe", + "area": "agent", + "type": "regression", + "priority": "p0", + "risk": "high", + "ci_eligible": false, + "tags": [ + "agent-runner", + "ledger", + "audit", + "tools" + ], + "automation": "scripts/e2e/agent-run-ledger-audit.mjs", + "setup_automation": [], + "setup_provides_env": [], + "evidence_required": [ + "filesystem", + "metrics", + "api_diagnostic" + ] + }, { "id": "agent-runner-async-db-readiness", "title": "AgentRunner async DB readiness probe", @@ -892,7 +921,8 @@ "ui", "screenshot", "console", - "backend_log" + "network", + "api_diagnostic" ] }, { @@ -1040,7 +1070,6 @@ "screenshot", "console", "network", - "backend_log", "api_diagnostic", "filesystem", "metrics" @@ -1486,9 +1515,11 @@ ], "evidence_required": [ "ui", + "screenshot", "console", - "backend_log", - "api_diagnostic" + "network", + "api_diagnostic", + "metrics" ] }, { @@ -1797,6 +1828,78 @@ "console", "api_diagnostic" ] + }, + { + "id": "workspace-compatibility-preflight", + "title": "LangBot workspace repositories and local SDK are mutually compatible", + "mode": "probe", + "area": "release", + "type": "smoke", + "priority": "p0", + "risk": "high", + "ci_eligible": true, + "tags": [ + "workspace", + "compatibility", + "preflight", + "sdk" + ], + "automation": "scripts/e2e/workspace-compatibility-preflight.mjs", + "setup_automation": [], + "setup_provides_env": [], + "evidence_required": [ + "filesystem", + "api_diagnostic" + ] + }, + { + "id": "workspace-plugin-pages-smoke", + "title": "Major workspace plugin pages render and answer read-only Page APIs", + "mode": "agent-browser", + "area": "plugin", + "type": "smoke", + "priority": "p0", + "risk": "high", + "ci_eligible": false, + "tags": [ + "workspace", + "plugin", + "page", + "browser" + ], + "automation": "scripts/e2e/workspace-plugin-pages-smoke.mjs", + "setup_automation": [], + "setup_provides_env": [], + "evidence_required": [ + "ui", + "screenshot", + "console", + "network", + "api_diagnostic" + ] + }, + { + "id": "workspace-repository-contracts", + "title": "Changed LangBot workspace repositories pass their contract suites", + "mode": "probe", + "area": "release", + "type": "regression", + "priority": "p0", + "risk": "high", + "ci_eligible": true, + "tags": [ + "workspace", + "contracts", + "pytest", + "sdk" + ], + "automation": "scripts/e2e/workspace-repository-contracts.mjs", + "setup_automation": [], + "setup_provides_env": [], + "evidence_required": [ + "filesystem", + "metrics" + ] } ], "suites": [ @@ -1809,6 +1912,8 @@ "langbot-performance-contract-gate", "langbot-performance-reliability-gate", "langbot-user-path-performance-gate", + "langbot-workspace-contract-gate", + "langbot-workspace-release-gate", "local-agent-gate" ], "suite_summaries": [ @@ -2003,6 +2108,57 @@ "pipeline-debug-chat-performance" ] }, + { + "id": "langbot-workspace-contract-gate", + "title": "LangBot workspace contract gate", + "description": "Low-cost cross-repository gate for branch/SDK compatibility and deterministic product contracts before live workflow testing.", + "type": "regression", + "priority": "p0", + "tags": [ + "workspace", + "contracts", + "pr-gate" + ], + "cases": [ + "workspace-compatibility-preflight", + "agent-runner-fixture-contract", + "agent-runner-behavior-matrix", + "agent-runner-ledger-invariants", + "langbot-fault-taxonomy-contract", + "langbot-overhead-accounting-contract", + "workspace-repository-contracts" + ] + }, + { + "id": "langbot-workspace-release-gate", + "title": "LangBot workspace top-down release gate", + "description": "Broad release gate combining deterministic repository contracts with representative browser workflows, plugin pages, RAG/parser, EBA, external AgentRunner, and one complex LocalAgent task.", + "type": "release_gate", + "priority": "p0", + "tags": [ + "workspace", + "release-gate", + "browser", + "agent" + ], + "cases": [ + "workspace-compatibility-preflight", + "workspace-repository-contracts", + "agent-runner-release-preflight", + "webui-login-state", + "wizard-scenario-routing", + "bot-event-routing-product-flow", + "agent-runner-health-visibility", + "agent-runner-qa-debug-chat", + "local-agent-complex-coding-task-debug-chat", + "agent-run-ledger-audit", + "langrag-kb-retrieve", + "workspace-plugin-pages-smoke", + "mcp-stdio-register", + "mcp-stdio-tool-call", + "wizard-onebot-agent-runtime" + ] + }, { "id": "local-agent-gate", "title": "Local Agent runner regression gate", @@ -2145,10 +2301,12 @@ "aiosqlite-connect-hangs", "ambiguous-runner-default-label", "backend-not-listening", + "box-runtime-silent-action-timeout", "box-session-conflict-logical-metadata", "debug-chat-history-contaminates-automation", "dynamic-form-missing-config-id", "e2b-extra-mount-sync-missing", + "local-agent-mcp-text-result-empty-follow-up", "local-agent-model-route-unavailable", "marketplace-network-flaky", "mcp-stdio-args-not-applied", @@ -2203,6 +2361,16 @@ "local-agent-basic-debug-chat" ] }, + { + "id": "box-runtime-silent-action-timeout", + "title": "Box runtime process stays alive while all action RPC calls time out", + "category": "product", + "related_cases": [ + "local-agent-complex-coding-task-debug-chat", + "agent-run-ledger-audit", + "mcp-stdio-tool-call" + ] + }, { "id": "box-session-conflict-logical-metadata", "title": "BoxSessionConflictError after a successful first exec", @@ -2238,6 +2406,16 @@ "sandbox-skill-authoring-e2e" ] }, + { + "id": "local-agent-mcp-text-result-empty-follow-up", + "title": "MCP tool succeeds but Local Agent completes without a visible reply", + "category": "product", + "related_cases": [ + "mcp-stdio-register", + "mcp-stdio-tool-call", + "agent-run-ledger-audit" + ] + }, { "id": "local-agent-model-route-unavailable", "title": "Local Agent model route is unavailable for the requested run shape", diff --git a/skills/skills/langbot-testing/SKILL.md b/skills/skills/langbot-testing/SKILL.md index 748ae9b81..bf3e4c271 100644 --- a/skills/skills/langbot-testing/SKILL.md +++ b/skills/skills/langbot-testing/SKILL.md @@ -22,6 +22,7 @@ Use this skill when an agent needs to verify LangBot behavior through the WebUI - **LangRAG knowledge bases**: read `references/langrag-knowledge-base.md`. - **MCP stdio tool testing**: read `references/mcp-stdio-testing.md`. - **Performance, reliability, or chaos probes**: read `references/performance-reliability-testing.md`. +- **Cross-repository workspace and release gates**: read `references/workspace-release-testing.md`. - **Drive a live instance over MCP (not raw HTTP)**: use the `langbot-mcp-ops` skill — the instance exposes an MCP server at `http://:5300/mcp` (reuses API keys). Useful for setting up bots/pipelines/models as test fixtures programmatically. - **Known failures and fixes**: read `references/troubleshooting.md`. - **Reusable test groups**: run `bin/lbs suite list` and `bin/lbs suite plan ` before manually assembling a case set. diff --git a/skills/skills/langbot-testing/cases/agent-run-ledger-audit.yaml b/skills/skills/langbot-testing/cases/agent-run-ledger-audit.yaml new file mode 100644 index 000000000..394bc74c9 --- /dev/null +++ b/skills/skills/langbot-testing/cases/agent-run-ledger-audit.yaml @@ -0,0 +1,41 @@ +id: agent-run-ledger-audit +title: "Persisted AgentRunner run ledger passes end-to-end invariants" +mode: probe +area: agent +type: regression +priority: p0 +risk: high +ci_eligible: false +tags: + - agent-runner + - ledger + - audit + - tools +skills: + - langbot-testing +automation: scripts/e2e/agent-run-ledger-audit.mjs +steps: + - "Set LANGBOT_AGENT_RUN_ID to audit a specific run, or leave it unset to audit the latest persisted AgentRunner run." + - "Read the active LangBot database configuration and inspect the selected run and its ordered events." + - "Verify completed terminal state, run.completed, paired tool.call.started/completed events, stable tool names, and monotonic ordering." + - "Compare called tools with the authorization snapshot and validate each advertised tool has owner/source, description, and parameter schema." + - "Reject malformed event JSON and invalid-JSON, timeout, forbidden, permission-denied, or unauthorized signals." +checks: + - "ledger-audit.json status is pass." + - "metrics has equal tool_call_started and tool_call_completed counts." + - "failures is empty and no unauthorized tool call is present." +evidence_required: + - filesystem + - metrics + - api_diagnostic +diagnostics: + - "Run this immediately after a complex UI Agent task so latest-run selection cannot drift to unrelated traffic." + - "Use LANGBOT_AGENT_RUN_ID when multiple operators share the test instance." +success_patterns: + - "Agent run ledger audit passed" +failure_patterns: + - "tool_call_pairing" + - "forbidden_error_signals" + - "unauthorized_tool_calls" +troubleshooting: + - box-runtime-silent-action-timeout diff --git a/skills/skills/langbot-testing/cases/langrag-kb-retrieve.yaml b/skills/skills/langbot-testing/cases/langrag-kb-retrieve.yaml index d7dbd884e..49fdec8ed 100644 --- a/skills/skills/langbot-testing/cases/langrag-kb-retrieve.yaml +++ b/skills/skills/langbot-testing/cases/langrag-kb-retrieve.yaml @@ -47,7 +47,8 @@ evidence_required: - ui - screenshot - console - - backend_log + - network + - api_diagnostic diagnostics: - "If no LangRAG engine is available, check /api/v1/knowledge/engines and install langbot-team/LangRAG." - "If the embedding selector does not show a local Chroma model, confirm the model exists under embedding_models, not llm_models." diff --git a/skills/skills/langbot-testing/cases/local-agent-complex-coding-task-debug-chat.yaml b/skills/skills/langbot-testing/cases/local-agent-complex-coding-task-debug-chat.yaml index c27d30367..0bed19c51 100644 --- a/skills/skills/langbot-testing/cases/local-agent-complex-coding-task-debug-chat.yaml +++ b/skills/skills/langbot-testing/cases/local-agent-complex-coding-task-debug-chat.yaml @@ -72,7 +72,6 @@ evidence_required: - screenshot - console - network - - backend_log - api_diagnostic - filesystem - metrics @@ -91,5 +90,6 @@ failure_patterns: troubleshooting: - local-agent-model-route-unavailable - sandbox-native-tools-unavailable + - box-runtime-silent-action-timeout - plugin-runtime-timeout - debug-chat-history-contaminates-automation diff --git a/skills/skills/langbot-testing/cases/mcp-stdio-tool-call.yaml b/skills/skills/langbot-testing/cases/mcp-stdio-tool-call.yaml index 093e23d50..b2c874c7a 100644 --- a/skills/skills/langbot-testing/cases/mcp-stdio-tool-call.yaml +++ b/skills/skills/langbot-testing/cases/mcp-stdio-tool-call.yaml @@ -74,14 +74,15 @@ checks: - "UI: The MCP detail page shows qa_mcp_echo." - "API diagnostic: /api/v1/tools contains qa_mcp_echo." - "UI: Debug Chat bot response contains qa_mcp_echo:mcp-ok-local-agent." - - "Logs: Backend logs show the MCP tool call was executed, not only listed." - "API diagnostic: pipeline-extensions-diagnostic.json shows the qa-local-stdio server UUID is the only bound MCP server for the run." - "Console: No unexpected frontend errors appear during MCP form use or Debug Chat." evidence_required: - ui + - screenshot - console - - backend_log + - network - api_diagnostic + - metrics diagnostics: - "For token-free deterministic UI coverage, start scripts/e2e/fake-openai-provider.mjs and pass LANGBOT_E2E_FAKE_PROVIDER_BASE_URL to this case; setup will bind the local-agent pipeline to that fake OpenAI-compatible model." - "Run node scripts/e2e/mcp-stdio-fixture.mjs to verify the bundled stdio fixture can list and call qa_mcp_echo without involving a model provider." @@ -95,5 +96,6 @@ troubleshooting: - mcp-stdio-args-not-applied - tool-name-collision-between-mcp-and-plugin - local-agent-model-route-unavailable + - local-agent-mcp-text-result-empty-follow-up - survey-widget-blocks-debug-chat - plugin-runtime-timeout diff --git a/skills/skills/langbot-testing/cases/workspace-compatibility-preflight.yaml b/skills/skills/langbot-testing/cases/workspace-compatibility-preflight.yaml new file mode 100644 index 000000000..0f1202586 --- /dev/null +++ b/skills/skills/langbot-testing/cases/workspace-compatibility-preflight.yaml @@ -0,0 +1,38 @@ +id: workspace-compatibility-preflight +title: "LangBot workspace repositories and local SDK are mutually compatible" +mode: probe +area: release +type: smoke +priority: p0 +risk: high +ci_eligible: true +tags: + - workspace + - compatibility + - preflight + - sdk +skills: + - langbot-testing +automation: scripts/e2e/workspace-compatibility-preflight.mjs +steps: + - "Discover the active LangBot checkout and sibling workspace repositories, with LANGBOT_WORKSPACE_ROOT and repository-specific env overrides available for non-default layouts." + - "Verify every checkout is on main or dev/4.11.x and record dirty worktrees as warnings only." + - "Validate plugin manifest identities for LocalAgent, Control Plane, LongTermMemory, GeneralParsers, LangRAG, and Skill Authoring." + - "Use the LangBot virtualenv to import langbot_plugin and AgentRunner Protocol entities from the workspace SDK source tree." +checks: + - "workspace-preflight.json contains no failed checks." + - "The langbot_plugin import path is inside the discovered langbot-plugin-sdk/src directory." + - "Dirty worktrees are visible as warnings and do not hide compatibility failures." +evidence_required: + - filesystem + - api_diagnostic +diagnostics: + - "A branch failure means the checkout is outside the supported main/dev/4.11.x workspace combination." + - "A local-sdk-import failure usually means uv restored the pinned PyPI wheel or the workspace SDK path is wrong." +success_patterns: + - "Workspace compatibility preflight passed" +failure_patterns: + - "blocking check" + - "did not load from the workspace SDK" +troubleshooting: + - uv-run-resyncs-local-sdk diff --git a/skills/skills/langbot-testing/cases/workspace-plugin-pages-smoke.yaml b/skills/skills/langbot-testing/cases/workspace-plugin-pages-smoke.yaml new file mode 100644 index 000000000..1129599eb --- /dev/null +++ b/skills/skills/langbot-testing/cases/workspace-plugin-pages-smoke.yaml @@ -0,0 +1,55 @@ +id: workspace-plugin-pages-smoke +title: "Major workspace plugin pages render and answer read-only Page APIs" +mode: agent-browser +area: plugin +type: smoke +priority: p0 +risk: high +ci_eligible: false +tags: + - workspace + - plugin + - page + - browser +skills: + - langbot-env-setup + - langbot-testing +env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL +automation: scripts/e2e/workspace-plugin-pages-smoke.mjs +automation_env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_BROWSER_PROFILE + - LANGBOT_CHROMIUM_EXECUTABLE +preconditions: + - "Control Plane, LongTermMemory, LangRAG, and Skill Authoring are installed and initialized in the selected local test instance." + - "The browser profile is authenticated for the same LangBot backend, or LANGBOT_E2E_LOGIN_USER is configured for local recovery login." +steps: + - "Open each /home/plugin-pages route through the LangBot WebUI." + - "Verify the iframe visibly renders Agent Control Plane, Memory Console, LangRAG Observability, and Skill Authoring." + - "Using the same browser login, call the read-only Page APIs /health, /summary, /snapshot, and /health respectively." + - "Capture one screenshot per page plus browser console/network diagnostics." +checks: + - "Every page has ui_status pass and api_status pass." + - "No blank iframe, Page API error, frontend uncaught exception, or HTTP 5xx is recorded." + - "No plugin is silently skipped when missing; fixture absence is blocked." +evidence_required: + - ui + - screenshot + - console + - network + - api_diagnostic +diagnostics: + - "blocked means one or more required plugins are not installed in the selected test instance." + - "A UI-only failure with a passing Page API usually points to plugin asset routing, iframe loading, or frontend page discovery." +success_patterns: + - "All workspace plugin pages rendered" +failure_patterns: + - "Required plugin page fixture" + - "Page API" + - "Browser diagnostics found" +troubleshooting: + - plugin-runtime-timeout + - backend-not-listening diff --git a/skills/skills/langbot-testing/cases/workspace-repository-contracts.yaml b/skills/skills/langbot-testing/cases/workspace-repository-contracts.yaml new file mode 100644 index 000000000..296b6f94f --- /dev/null +++ b/skills/skills/langbot-testing/cases/workspace-repository-contracts.yaml @@ -0,0 +1,36 @@ +id: workspace-repository-contracts +title: "Changed LangBot workspace repositories pass their contract suites" +mode: probe +area: release +type: regression +priority: p0 +risk: high +ci_eligible: true +tags: + - workspace + - contracts + - pytest + - sdk +skills: + - langbot-testing +automation: scripts/e2e/workspace-repository-contracts.mjs +steps: + - "Resolve the active LangBot virtualenv and workspace SDK source path." + - "Run tests independently for AgentRunner, Control Plane, LongTermMemory, Parser, RAG, Skill Authoring, LocalAgent, LangBot Agent/Provider, the skills CLI, and SDK runtime contracts." + - "Run SDK packaging blackbox separately so an isolated build dependency network failure is classified as env_issue without masking product test failures." + - "Write per-repository stdout, stderr, status, and duration under repository-contracts/." +checks: + - "Every product contract group has status pass." + - "A packaging env_issue is recorded separately and does not convert a product failure into a pass." + - "Every command uses the LangBot virtualenv and local SDK PYTHONPATH." +evidence_required: + - filesystem + - metrics +diagnostics: + - "Inspect the named repository stdout/stderr logs before rerunning a broader suite." + - "Packaging dependency fetch errors belong to environment readiness; assertion/import/runtime failures belong to the product." +success_patterns: + - "All repository product contracts passed" +failure_patterns: + - "repository product contract group" + - "Test command timed out" diff --git a/skills/skills/langbot-testing/references/workspace-release-testing.md b/skills/skills/langbot-testing/references/workspace-release-testing.md new file mode 100644 index 000000000..bec6ba638 --- /dev/null +++ b/skills/skills/langbot-testing/references/workspace-release-testing.md @@ -0,0 +1,25 @@ +# Workspace Release Testing + +Use the workspace gates when changes span LangBot core, the plugin SDK, AgentRunner, or multiple first-party plugins. + +## Cost Ladder + +1. Run `langbot-workspace-contract-gate` after each coherent change. It catches wrong branches, wrong SDK imports, protocol drift, and repository regressions without model calls. +2. Run the narrow browser case for the changed surface, such as `workspace-plugin-pages-smoke`, before broad release testing. +3. Run `langbot-workspace-release-gate` before merging or publishing coordinated releases. It deliberately uses one complex LocalAgent coding task rather than many low-value model conversations. +4. Audit the persisted run with `agent-run-ledger-audit`. Set `LANGBOT_AGENT_RUN_ID` when other traffic may have created a newer run. + +Run the browser-driven `langrag-parser-golden-e2e` separately when Parser, +LangRAG ingestion, document upload, or parser selection changes. It intentionally +remains a manual deep case until its full WebUI setup path is automated; it is +not listed in the automated release gate where it could be silently skipped. + +Do not add Space model concurrency to these gates. Provider concurrency mixes external quota and latency into product results and is covered separately by optional performance cases. + +## Failure Triage + +- Compatibility preflight failure: correct the checkout, virtualenv, or local SDK import before testing behavior. +- Repository contract failure: fix the owning repository and add the narrowest deterministic regression. +- Browser workflow failure: correlate the screenshot, console, network, and backend log from the same run. +- Complex Agent failure: inspect `ledger-audit.json` before blaming the model. Tool pairing, authorization, JSON, and timeout failures are product signals. +- Packaging `env_issue`: restore isolated build dependency access, then rerun packaging only. It must not mask passing or failing runtime contracts. diff --git a/skills/skills/langbot-testing/suites/langbot-workspace-contract-gate.yaml b/skills/skills/langbot-testing/suites/langbot-workspace-contract-gate.yaml new file mode 100644 index 000000000..14d65a54c --- /dev/null +++ b/skills/skills/langbot-testing/suites/langbot-workspace-contract-gate.yaml @@ -0,0 +1,17 @@ +id: langbot-workspace-contract-gate +title: "LangBot workspace contract gate" +description: "Low-cost cross-repository gate for branch/SDK compatibility and deterministic product contracts before live workflow testing." +type: regression +priority: p0 +tags: + - workspace + - contracts + - pr-gate +cases: + - workspace-compatibility-preflight + - agent-runner-fixture-contract + - agent-runner-behavior-matrix + - agent-runner-ledger-invariants + - langbot-fault-taxonomy-contract + - langbot-overhead-accounting-contract + - workspace-repository-contracts diff --git a/skills/skills/langbot-testing/suites/langbot-workspace-release-gate.yaml b/skills/skills/langbot-testing/suites/langbot-workspace-release-gate.yaml new file mode 100644 index 000000000..ae342208e --- /dev/null +++ b/skills/skills/langbot-testing/suites/langbot-workspace-release-gate.yaml @@ -0,0 +1,26 @@ +id: langbot-workspace-release-gate +title: "LangBot workspace top-down release gate" +description: "Broad release gate combining deterministic repository contracts with representative browser workflows, plugin pages, RAG/parser, EBA, external AgentRunner, and one complex LocalAgent task." +type: release_gate +priority: p0 +tags: + - workspace + - release-gate + - browser + - agent +cases: + - workspace-compatibility-preflight + - workspace-repository-contracts + - agent-runner-release-preflight + - webui-login-state + - wizard-scenario-routing + - bot-event-routing-product-flow + - agent-runner-health-visibility + - agent-runner-qa-debug-chat + - local-agent-complex-coding-task-debug-chat + - agent-run-ledger-audit + - langrag-kb-retrieve + - workspace-plugin-pages-smoke + - mcp-stdio-register + - mcp-stdio-tool-call + - wizard-onebot-agent-runtime diff --git a/skills/skills/langbot-testing/troubleshooting/box-runtime-silent-action-timeout.yaml b/skills/skills/langbot-testing/troubleshooting/box-runtime-silent-action-timeout.yaml new file mode 100644 index 000000000..788eff36a --- /dev/null +++ b/skills/skills/langbot-testing/troubleshooting/box-runtime-silent-action-timeout.yaml @@ -0,0 +1,25 @@ +id: box-runtime-silent-action-timeout +title: "Box runtime process stays alive while all action RPC calls time out" +date: 2026-07-19 +symptoms: + - "The Box child process and Docker backend are still running, but native exec/read/write actions never return." + - "The backend repeatedly falls back to an empty skill list while Box status previously reported available." + - "A LocalAgent run records tool.call.started followed by run.failed with runner.timeout and no tool.call.completed." +patterns: + - "Action box_list_skills call timed out" + - "Action call_tool call timed out" + - "Action box_get_managed_process call timed out" + - "Heartbeat to Box runtime failed; reconnecting" +likely_causes: + - "The local stdio Box action channel became silently unresponsive without closing its child process or pipe." + - "Before the heartbeat recovery fix, ping failures were only logged at debug level and BoxService remained falsely available." +fix_steps: + - "Use a LangBot build where Box heartbeat failures call the runtime disconnect callback." + - "Restart LangBot once to replace an already-stale connector created by an older process." + - "Wait for Box runtime disconnected followed by Box runtime reconnected, sandbox features restored." + - "Do not attribute the failure to the model until a direct native exec path succeeds." +verification: "Run a complex LocalAgent task, then agent-run-ledger-audit. The run must complete with every tool.call.started paired with tool.call.completed and no timeout signal." +related_cases: + - local-agent-complex-coding-task-debug-chat + - agent-run-ledger-audit + - mcp-stdio-tool-call diff --git a/skills/skills/langbot-testing/troubleshooting/local-agent-mcp-text-result-empty-follow-up.yaml b/skills/skills/langbot-testing/troubleshooting/local-agent-mcp-text-result-empty-follow-up.yaml new file mode 100644 index 000000000..ab98dba30 --- /dev/null +++ b/skills/skills/langbot-testing/troubleshooting/local-agent-mcp-text-result-empty-follow-up.yaml @@ -0,0 +1,25 @@ +id: local-agent-mcp-text-result-empty-follow-up +title: "MCP tool succeeds but Local Agent completes without a visible reply" +date: 2026-07-19 +category: product +symptoms: + - "The run ledger records matching tool.call.started and tool.call.completed events for the MCP tool, followed by run.completed." + - "Debug Chat shows no new assistant message and waits until its response timeout." + - "The provider records a successful follow-up model call, while the backend reports a stream with zero visible chunks or characters." +patterns: + - "Streaming completed: 0 chunks, 0 chars" + - "tool.call.completed" + - "run.completed" + - "Expected text did not appear in a new assistant message" +likely_causes: + - "Local Agent serialized a text-only MCP ContentElement list as JSON instead of plain tool-result text, and the selected model returned no visible follow-up content." + - "The streaming caller accepted an empty final transport chunk as a successful model response, allowing a zero-output run to complete." +fix_steps: + - "Use a LocalAgent build that flattens text-only MCP ContentElement lists before the tool follow-up model turn." + - "Use a LocalAgent build that rejects empty final chunks and fails or falls back instead of completing a zero-output run." + - "Rebuild and reinstall the local LocalAgent package so the running plugin runtime contains the fix." +verification: "Run mcp-stdio-tool-call with only qa-local-stdio bound. Debug Chat must show qa_mcp_echo:mcp-ok-local-agent, and an empty follow-up regression must end as run.failed rather than run.completed." +related_cases: + - mcp-stdio-register + - mcp-stdio-tool-call + - agent-run-ledger-audit diff --git a/skills/test/lbs-cli.test.ts b/skills/test/lbs-cli.test.ts index 84a2b4001..4d20faf62 100644 --- a/skills/test/lbs-cli.test.ts +++ b/skills/test/lbs-cli.test.ts @@ -54,6 +54,7 @@ import { repoRoot } from "../src/cli.ts"; import { classifyDebugChatResult, findNewFailureSignal, + hasDebugChatOutcome, minExpectedOccurrences, } from "../scripts/e2e/lib/debug-chat.mjs"; import { @@ -1913,6 +1914,17 @@ test("debug chat classifier distinguishes new failure signals from old history", assert.equal(custom.failure_signal, "qa-plugin-smoke:mcp-ok-local-agent"); }); +test("debug chat outcome wait can stop on a new failure signal", () => { + const baselines = [{ signal: "runner.timeout", count: 1 }]; + + assert.equal(hasDebugChatOutcome("old runner.timeout", "EXPECTED", 1, baselines), false); + assert.equal( + hasDebugChatOutcome("old runner.timeout\nnew runner.timeout", "EXPECTED", 1, baselines), + true, + ); + assert.equal(hasDebugChatOutcome("EXPECTED", "EXPECTED", 1, baselines), true); +}); + test("debug chat classifier lets new failure signals override stale expected history", () => { const result = classifyDebugChatResult({ beforeText: "Bot: qa_mcp_echo:mcp-ok-local-agent", diff --git a/src/langbot/pkg/box/connector.py b/src/langbot/pkg/box/connector.py index 2257910d1..adb1e2244 100644 --- a/src/langbot/pkg/box/connector.py +++ b/src/langbot/pkg/box/connector.py @@ -166,8 +166,13 @@ class BoxRuntimeConnector(ManagedRuntimeConnector): try: await self.ping() self.ap.logger.debug('Heartbeat to Box runtime success.') + except asyncio.CancelledError: + raise except Exception as e: - self.ap.logger.debug(f'Failed to heartbeat to Box runtime: {e}') + self.ap.logger.warning(f'Heartbeat to Box runtime failed; reconnecting: {e}') + if self.runtime_disconnect_callback is not None: + await self.runtime_disconnect_callback(self) + return async def ping(self) -> None: if self._handler is None: diff --git a/tests/unit_tests/box/test_box_connector.py b/tests/unit_tests/box/test_box_connector.py index ddd4899b0..e160b0cab 100644 --- a/tests/unit_tests/box/test_box_connector.py +++ b/tests/unit_tests/box/test_box_connector.py @@ -1,7 +1,8 @@ from __future__ import annotations +import asyncio from types import SimpleNamespace -from unittest.mock import Mock +from unittest.mock import AsyncMock, Mock import pytest @@ -104,3 +105,23 @@ def test_box_runtime_connector_dispose_terminates_subprocess(monkeypatch: pytest ctrl_task.cancel.assert_called_once() assert connector._handler_task is None assert connector._ctrl_task is None + + +@pytest.mark.asyncio +async def test_box_runtime_connector_heartbeat_failure_requests_reconnect(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr('langbot.pkg.box.connector._HEARTBEAT_INTERVAL_SEC', 0) + disconnected = asyncio.Event() + callbacks = [] + + async def on_disconnect(connector): + callbacks.append(connector) + disconnected.set() + + connector = BoxRuntimeConnector(make_app(Mock()), runtime_disconnect_callback=on_disconnect) + connector._handler = SimpleNamespace(call_action=AsyncMock(side_effect=TimeoutError('silent runtime'))) + + heartbeat = asyncio.create_task(connector._heartbeat_loop()) + await asyncio.wait_for(disconnected.wait(), timeout=1) + await heartbeat + + assert callbacks == [connector]