test(agent-runner): strengthen local agent e2e gate

skills/src/commands/log.ts

@@ -117,7 +117,7 @@ function patternContextFromOptions(root: string, options: Record<string, string
 function latestOrExplicitBackendLog(root: string, options: Record<string, string | boolean>): string {
   const explicit = optionString(options, "backend-log");
   if (explicit) return resolve(explicit);
-  const auto = latestLangBotLogPath(loadEnv(root));
+  const auto = latestLangBotLogPath(loadEnv(root), root);
   return auto ? resolve(auto) : "";
 }
 

skills/src/commands/test.ts

@@ -343,6 +343,78 @@ function manualEvidenceTemplate(mode: string): ManualEvidenceTemplate {
   };
 }
 
+function existingEvidencePath(evidence: Record<string, string> | undefined, key: string): string {
+  const path = evidence?.[key];
+  return path && existsSync(path) ? path : "";
+}
+
+function renderAutomationEvidence(
+  mode: string,
+  automation: AutomationResultEvidence,
+  logGuard: LogGuardResult,
+): ManualEvidenceTemplate {
+  if (automation.status !== "loaded") return manualEvidenceTemplate(mode);
+
+  const isProbe = isProbeMode(mode);
+  const screenshot = existingEvidencePath(automation.evidence, "screenshot");
+  const consoleLog = existingEvidencePath(automation.evidence, "console_log");
+  const networkLog = existingEvidencePath(automation.evidence, "network_log");
+  const apiDiagnostics = Object.entries(automation.evidence ?? {})
+    .filter(([key, value]) => key.endsWith("_diagnostic_json") && value && existsSync(value))
+    .map(([, value]) => value);
+  const successLines = logGuard.success_signals
+    .map((signal) => signal.excerpt || `${signal.source}:${signal.line ?? ""} ${signal.pattern}`.trim())
+    .slice(0, 3);
+  const findings = logGuard.findings
+    .map((finding) => `${finding.severity}/${finding.kind}${finding.excerpt ? `: ${finding.excerpt}` : ""}`)
+    .slice(0, 3);
+  const chatSummary = automation.chat_results?.length
+    ? automation.chat_results
+      .map((item) => `#${(item.index ?? 0) + 1} ${item.status ?? "unknown"} ${item.expected_text ?? ""}`.trim())
+      .join("; ")
+    : automation.reason || "";
+  const diagnostics = [
+    automation.browser_diagnostics?.reason,
+    apiDiagnostics.length ? `api diagnostics: ${apiDiagnostics.join(", ")}` : "",
+  ].filter(Boolean).join("; ") || "No extra diagnostics recorded.";
+
+  if (isProbe) {
+    return {
+      result: automation.result || "unknown",
+      target_tested: String(automation.url || automation.path || "automation target"),
+      execution_path: "automation script",
+      probe_result: automation.reason || chatSummary || "Automation completed.",
+      logs_or_artifacts: [
+        consoleLog ? `console=${consoleLog}` : "",
+        networkLog ? `network=${networkLog}` : "",
+        automation.path ? `automation=${automation.path}` : "",
+      ].filter(Boolean).join(", ") || "No log artifacts recorded.",
+      diagnostics,
+      matched_troubleshooting: findings.length ? findings.join(" | ") : "None.",
+      assets_to_update: automation.result === "pass" ? "None." : "Review case/troubleshooting assets if this failure is expected to recur.",
+    };
+  }
+
+  return {
+    result: automation.result || "unknown",
+    url_tested: automation.url || "Not recorded.",
+    browser_path: "direct Playwright automation",
+    ui_result: chatSummary || automation.reason || "Automation completed.",
+    console_errors: automation.browser_diagnostics?.status === "fail"
+      ? automation.browser_diagnostics.reason || "Browser diagnostics failed."
+      : "None found by browser diagnostics.",
+    network_symptoms: networkLog ? `Captured in ${networkLog}` : "No network log artifact recorded.",
+    backend_logs: successLines.length
+      ? successLines.join(" | ")
+      : `Log Guard status: ${logGuard.status}${findings.length ? `; ${findings.join(" | ")}` : ""}`,
+    frontend_logs: consoleLog ? `Captured in ${consoleLog}` : "No frontend console log artifact recorded.",
+    screenshots: screenshot || "No screenshot artifact recorded.",
+    diagnostics,
+    matched_troubleshooting: findings.length ? findings.join(" | ") : "None.",
+    assets_to_update: automation.result === "pass" ? "None." : "Review case/troubleshooting assets if this failure is expected to recur.",
+  };
+}
+
 function envSummary(item: StructuredItem, env: Record<string, string>): Record<string, string> {
   const keys = [
     ...listValue(item.fields, "env"),
@@ -1321,6 +1393,8 @@ export function commandTestRun(ctx: CommandContext): number {
 function buildReport(root: string, item: StructuredItem, options: Record<string, string | boolean>): TestReport {
   const env = loadEnv(root);
   const mode = caseMode(item);
+  const automationResult = readAutomationResultEvidence(options);
+  const logGuard = scanStructuredLogSources(root, item, options);
   const related = relatedTroubleshooting(root, item).map((entry) => ({
     id: scalar(entry.fields, "id"),
     title: scalar(entry.fields, "title"),
@@ -1332,8 +1406,8 @@ function buildReport(root: string, item: StructuredItem, options: Record<string,
     generated_at: new Date().toISOString(),
     case: caseSummary(item),
     result_options: ["pass", "fail", "blocked", "env_issue", "flaky"],
-    automation_result: readAutomationResultEvidence(options),
-    manual_evidence: manualEvidenceTemplate(mode),
+    automation_result: automationResult,
+    manual_evidence: renderAutomationEvidence(mode, automationResult, logGuard),
     environment: envSummary(item, env),
     required_skills: listValue(item.fields, "skills"),
     steps: listValue(item.fields, "steps"),
@@ -1344,7 +1418,7 @@ function buildReport(root: string, item: StructuredItem, options: Record<string,
     failure_patterns: listValue(item.fields, "failure_patterns"),
     expected_failures: listValue(item.fields, "expected_failures"),
     troubleshooting: related,
-    log_guard: scanStructuredLogSources(root, item, options),
+    log_guard: logGuard,
   };
 }
 

skills/src/commands/validate.ts

@@ -125,6 +125,8 @@ function validateCaseItem(root: string, item: StructuredItem, skillNames: Set<st
     ...validateEnvKeyList(item, "setup_provides_env"),
     ...validateEnvKeyScalar(item, "automation_pipeline_url_env"),
     ...validateEnvKeyScalar(item, "automation_pipeline_name_env"),
+    ...validateJsonScalar(item, "automation_runner_config_patch_json"),
+    ...validateJsonScalar(item, "automation_extensions_patch_json"),
     ...validateJsonScalar(item, "automation_filesystem_checks_json"),
     ...validateJsonScalar(item, "metrics_thresholds_json"),
     ...validateJsonScalar(item, "load_profile_json"),

skills/src/log-guard.ts

@@ -1,5 +1,5 @@
 import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
-import { dirname, join, resolve } from "node:path";
+import { basename, dirname, join, resolve } from "node:path";
 import type { StructuredItem } from "./types.ts";
 import { listValue, loadEnv, loadStructuredItems, scalar } from "./fs.ts";
 
@@ -99,6 +99,22 @@ export type AutomationResultEvidence = {
   url?: string;
   prompt?: string;
   expected_text?: string;
+  stream_output?: string;
+  prompt_count?: number;
+  image_fixture?: string;
+  evidence?: Record<string, string>;
+  evidence_collected?: string[];
+  browser_diagnostics?: {
+    status?: string;
+    reason?: string;
+    finding_count?: number;
+  };
+  chat_results?: Array<{
+    index?: number;
+    expected_text?: string;
+    status?: string;
+    reason?: string;
+  }>;
   metrics_summary?: Record<string, unknown>;
   thresholds_summary?: Record<string, unknown>;
   artifacts?: Record<string, unknown>;
@@ -216,13 +232,13 @@ export function scanLogSources(
   for (const configured of configuredSources) {
     const explicitPath = options[configured.option];
     const autoPath = configured.source === "backend" && options["no-auto-log"] !== true
-      ? latestLangBotLogPath(env)
+      ? latestLangBotLogPath(env, root)
       : null;
     const rawPath = typeof explicitPath === "string" ? explicitPath : autoPath;
     const autoDetected = typeof explicitPath !== "string" && rawPath === autoPath;
     if (!rawPath) {
       if (configured.source === "backend" && options["no-auto-log"] !== true) {
-        const logsDir = env.LANGBOT_REPO ? join(env.LANGBOT_REPO, "data", "logs") : "LANGBOT_REPO/data/logs";
+        const logsDir = langBotLogsDir(env, root) ?? "LANGBOT_REPO/data/logs";
         sources.push({ source: "backend", path: join(logsDir, "langbot-*.log"), status: "auto_not_found", line_count: 0, auto_detected: true });
       }
       continue;
@@ -610,6 +626,47 @@ function objectField(data: Record<string, unknown>, key: string): Record<string,
     : undefined;
 }
 
+function stringMapField(data: Record<string, unknown>, key: string): Record<string, string> | undefined {
+  const value = data[key];
+  if (!value || typeof value !== "object" || Array.isArray(value)) return undefined;
+  const entries = Object.entries(value as Record<string, unknown>)
+    .filter((entry): entry is [string, string] => typeof entry[1] === "string" && entry[1].trim().length > 0);
+  return entries.length ? Object.fromEntries(entries) : undefined;
+}
+
+function stringListField(data: Record<string, unknown>, key: string): string[] | undefined {
+  const value = data[key];
+  if (!Array.isArray(value)) return undefined;
+  const items = value.map((item) => typeof item === "string" ? item : "").filter(Boolean);
+  return items.length ? items : undefined;
+}
+
+function chatResultsField(data: Record<string, unknown>): AutomationResultEvidence["chat_results"] {
+  const value = data.chat_results;
+  if (!Array.isArray(value)) return undefined;
+  const results = value
+    .filter((item): item is Record<string, unknown> => Boolean(item) && typeof item === "object" && !Array.isArray(item))
+    .map((item) => ({
+      index: numberField(item, "index"),
+      expected_text: stringField(item, "expected_text"),
+      status: stringField(item, "status"),
+      reason: stringField(item, "reason"),
+    }));
+  return results.length ? results : undefined;
+}
+
+function browserDiagnosticsField(data: Record<string, unknown>): AutomationResultEvidence["browser_diagnostics"] {
+  const value = data.browser_diagnostics;
+  if (!value || typeof value !== "object" || Array.isArray(value)) return undefined;
+  const diagnostics = value as Record<string, unknown>;
+  const findings = Array.isArray(diagnostics.findings) ? diagnostics.findings.length : undefined;
+  return {
+    status: stringField(diagnostics, "status"),
+    reason: stringField(diagnostics, "reason"),
+    finding_count: findings,
+  };
+}
+
 function evidenceDirFromOptions(options: Record<string, string | boolean>): string | undefined {
   const explicit = typeof options["evidence-dir"] === "string" ? options["evidence-dir"] : undefined;
   if (explicit) return resolve(explicit);
@@ -652,6 +709,13 @@ export function readAutomationResultEvidence(options: Record<string, string | bo
       url: stringField(result, "url"),
       prompt: redactSecrets(stringField(result, "prompt") ?? ""),
       expected_text: stringField(result, "expected_text"),
+      stream_output: typeof result.stream_output === "boolean" ? String(result.stream_output) : stringField(result, "stream_output"),
+      prompt_count: numberField(result, "prompt_count"),
+      image_fixture: stringField(result, "image_fixture"),
+      evidence: stringMapField(result, "evidence"),
+      evidence_collected: stringListField(result, "evidence_collected"),
+      browser_diagnostics: browserDiagnosticsField(result),
+      chat_results: chatResultsField(result),
       metrics_summary: objectField(result, "metrics_summary"),
       thresholds_summary: objectField(result, "thresholds_summary"),
       artifacts: objectField(result, "artifacts"),
@@ -661,10 +725,31 @@ export function readAutomationResultEvidence(options: Record<string, string | bo
   }
 }
 
-export function latestLangBotLogPath(env: Record<string, string>): string | null {
-  const repo = env.LANGBOT_REPO;
-  if (!repo) return null;
-  const logsDir = join(repo, "data", "logs");
+function langBotRepoFromRoot(root: string): string | null {
+  const candidates = [
+    resolve(root),
+    basename(resolve(root)) === "skills" ? resolve(root, "..") : "",
+    resolve(root, "../LangBot"),
+    resolve(root, "LangBot"),
+  ].filter(Boolean);
+
+  const seen = new Set<string>();
+  for (const candidate of candidates) {
+    if (seen.has(candidate)) continue;
+    seen.add(candidate);
+    if (existsSync(join(candidate, "data", "config.yaml"))) return candidate;
+  }
+  return null;
+}
+
+function langBotLogsDir(env: Record<string, string>, root: string): string | null {
+  const repo = env.LANGBOT_REPO ? resolve(env.LANGBOT_REPO) : langBotRepoFromRoot(root);
+  return repo ? join(repo, "data", "logs") : null;
+}
+
+export function latestLangBotLogPath(env: Record<string, string>, root = process.cwd()): string | null {
+  const logsDir = langBotLogsDir(env, root);
+  if (!logsDir) return null;
   if (!existsSync(logsDir)) return null;
 
   const candidates = readdirSync(logsDir)

skills/src/readiness.ts

@@ -111,6 +111,8 @@ export function automationEnvDefaults(item: StructuredItem, env: EnvSource = pro
     ["automation_image_base64_fixture", "LANGBOT_E2E_IMAGE_BASE64_PATH"],
     ["automation_runner_config_patch_json", "LANGBOT_E2E_RUNNER_CONFIG_PATCH_JSON"],
     ["automation_restore_runner_config", "LANGBOT_E2E_RESTORE_RUNNER_CONFIG"],
+    ["automation_extensions_patch_json", "LANGBOT_E2E_EXTENSIONS_PATCH_JSON"],
+    ["automation_restore_extensions", "LANGBOT_E2E_RESTORE_EXTENSIONS"],
     ["automation_expected_runner_id", "LANGBOT_E2E_EXPECTED_RUNNER_ID"],
     ["automation_reset_debug_chat", "LANGBOT_E2E_RESET_DEBUG_CHAT"],
     ["automation_debug_chat_session_type", "LANGBOT_E2E_DEBUG_CHAT_SESSION_TYPE"],