Add performance and reliability QA gates (#2283)

* Add performance and reliability QA gates * test(skills): prepare user path performance gate * test(skills): add debug chat load gate * test(skills): extend fake provider load profiles * test(skills): add debug chat timing and isolation probes * test(skills): clarify manual QA perf gates
2026-06-26 07:24:20 +00:00 · 2026-06-25 13:02:44 +00:00
parent 20636ac432
commit 5b2826fa49
52 changed files with 6974 additions and 42 deletions
@@ -0,0 +1,205 @@
+#!/usr/bin/env node
+
+import { spawn } from "node:child_process";
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { dirname, resolve } from "node:path";
+import { env } from "node:process";
+import {
+  appendLine,
+  ensureEvidence,
+  evidencePaths,
+  loadEnvFiles,
+  redact,
+  writeResult,
+} from "./lib/langbot-e2e.mjs";
+
+const caseId = "ensure-fake-provider-cross-pipelines";
+const DEFAULT_PIPELINE_A_NAME = "LangBot QA Fake Provider Debug Chat A";
+const DEFAULT_PIPELINE_B_NAME = "LangBot QA Fake Provider Debug Chat B";
+
+await loadEnvFiles();
+const paths = evidencePaths(caseId);
+await ensureEvidence(paths);
+
+const writeEnv = process.argv.includes("--write-env");
+const envLocalPath = resolve("skills/.env.local");
+const pipelineAName = env.LANGBOT_FAKE_PROVIDER_PIPELINE_A_NAME || DEFAULT_PIPELINE_A_NAME;
+const pipelineBName = env.LANGBOT_FAKE_PROVIDER_PIPELINE_B_NAME || DEFAULT_PIPELINE_B_NAME;
+
+const result = {
+  source: "setup_automation",
+  case_id: caseId,
+  run_id: paths.runId,
+  status: "fail",
+  reason: "",
+  pipeline_a: {
+    name: pipelineAName,
+    id: "",
+    url: "",
+  },
+  pipeline_b: {
+    name: pipelineBName,
+    id: "",
+    url: "",
+  },
+  fake_provider: {
+    url: "",
+    base_url: "",
+    pid: null,
+  },
+  wrote_env: false,
+  evidence: {
+    console_log: paths.consoleLog,
+    automation_result_json: paths.automationResultJson,
+    result_json: paths.resultJson,
+  },
+  evidence_collected: ["api_diagnostic", "filesystem"],
+};
+
+try {
+  console.error(`[langbot-qa] configuring cross-pipeline QA fixtures: pipeline_a=\"${pipelineAName}\", pipeline_b=\"${pipelineBName}\"`);
+  console.error("[langbot-qa] run these fake-provider setup/probe commands serially when they share LANGBOT_FAKE_PROVIDER_URL.");
+  if (pipelineAName === pipelineBName) {
+    throw new Error("LANGBOT_FAKE_PROVIDER_PIPELINE_A_NAME and LANGBOT_FAKE_PROVIDER_PIPELINE_B_NAME must be different.");
+  }
+
+  const setupA = await runPipelineSetup(pipelineAName, "A");
+  const setupB = await runPipelineSetup(pipelineBName, "B");
+  result.pipeline_a = {
+    name: setupA.pipeline_name || pipelineAName,
+    id: setupA.pipeline_id || "",
+    url: setupA.pipeline_url || "",
+  };
+  result.pipeline_b = {
+    name: setupB.pipeline_name || pipelineBName,
+    id: setupB.pipeline_id || "",
+    url: setupB.pipeline_url || "",
+  };
+  result.fake_provider = {
+    url: setupB.fake_provider?.url || setupA.fake_provider?.url || "",
+    base_url: setupB.fake_provider?.base_url || setupA.fake_provider?.base_url || "",
+    pid: setupB.fake_provider?.pid ?? setupA.fake_provider?.pid ?? null,
+  };
+
+  if (!result.pipeline_a.url || !result.pipeline_b.url || !result.fake_provider.url) {
+    throw new Error("Cross-pipeline fake provider setup did not return both pipeline URLs and provider URL.");
+  }
+
+  if (writeEnv) {
+    await upsertEnvLocal(envLocalPath, {
+      LANGBOT_FAKE_PROVIDER_URL: result.fake_provider.url,
+      LANGBOT_FAKE_PROVIDER_BASE_URL: result.fake_provider.base_url,
+      LANGBOT_FAKE_PROVIDER_PID: result.fake_provider.pid ? String(result.fake_provider.pid) : "",
+      LANGBOT_FAKE_PROVIDER_PIPELINE_A_URL: result.pipeline_a.url,
+      LANGBOT_FAKE_PROVIDER_PIPELINE_A_NAME: result.pipeline_a.name,
+      LANGBOT_FAKE_PROVIDER_PIPELINE_B_URL: result.pipeline_b.url,
+      LANGBOT_FAKE_PROVIDER_PIPELINE_B_NAME: result.pipeline_b.name,
+    });
+    result.wrote_env = true;
+  }
+
+  result.status = "pass";
+  result.reason = "Fake provider cross-pipeline fixtures are configured.";
+} catch (error) {
+  result.status = looksLikeEnvIssue(error) ? "env_issue" : "fail";
+  result.reason = safeReason(error.message);
+} finally {
+  await writeResult(paths, result);
+  console.log(JSON.stringify(result, null, 2));
+}
+
+process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1);
+
+function runPipelineSetup(pipelineName, label) {
+  return new Promise((resolvePromise, rejectPromise) => {
+    const child = spawn(process.execPath, ["scripts/e2e/ensure-fake-provider-pipeline.mjs"], {
+      cwd: resolve("."),
+      env: {
+        ...env,
+        LANGBOT_FAKE_PROVIDER_PIPELINE_NAME: pipelineName,
+        LANGBOT_FAKE_PROVIDER_FIRST_TOKEN_DELAY_MS: env.LANGBOT_FAKE_PROVIDER_FIRST_TOKEN_DELAY_MS || "25",
+        LANGBOT_FAKE_PROVIDER_CHUNK_DELAY_MS: env.LANGBOT_FAKE_PROVIDER_CHUNK_DELAY_MS || "10",
+        LANGBOT_FAKE_PROVIDER_CHUNK_COUNT: env.LANGBOT_FAKE_PROVIDER_CHUNK_COUNT || "0",
+        LANGBOT_FAKE_PROVIDER_FAIL_FIRST_N: "0",
+        LANGBOT_FAKE_PROVIDER_FAIL_EVERY_N: "0",
+        LANGBOT_FAKE_PROVIDER_FAULT_STATUS: env.LANGBOT_FAKE_PROVIDER_FAULT_STATUS || "500",
+        LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK: "false",
+        LANGBOT_FAKE_PROVIDER_DYNAMIC_RESPONSE: "true",
+      },
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+
+    let stdout = "";
+    let stderr = "";
+    child.stdout.on("data", (chunk) => {
+      const text = chunk.toString();
+      stdout += text;
+      appendLine(paths.consoleLog, `[setup ${label} stdout] ${text.trimEnd()}`).catch(() => {});
+    });
+    child.stderr.on("data", (chunk) => {
+      const text = chunk.toString();
+      stderr += text;
+      appendLine(paths.consoleLog, `[setup ${label} stderr] ${text.trimEnd()}`).catch(() => {});
+    });
+    child.on("error", rejectPromise);
+    child.on("close", (code) => {
+      const parsed = parseJsonOutput(stdout);
+      if (code !== 0 || parsed.status !== "pass") {
+        rejectPromise(new Error(parsed.reason || stderr || `Fake provider pipeline setup ${label} exited with ${code}.`));
+        return;
+      }
+      resolvePromise(parsed);
+    });
+  });
+}
+
+function parseJsonOutput(text) {
+  const trimmed = String(text || "").trim();
+  if (!trimmed) return {};
+  try {
+    return JSON.parse(trimmed);
+  } catch {
+    const start = trimmed.indexOf("{");
+    const end = trimmed.lastIndexOf("}");
+    if (start >= 0 && end > start) {
+      try {
+        return JSON.parse(trimmed.slice(start, end + 1));
+      } catch {
+        return {};
+      }
+    }
+    return {};
+  }
+}
+
+async function upsertEnvLocal(path, updates) {
+  await mkdir(dirname(path), { recursive: true });
+  let text = "";
+  try {
+    text = await readFile(path, "utf8");
+  } catch {
+    text = "";
+  }
+  const lines = text.split(/\r?\n/);
+  const seen = new Set();
+  const next = lines.map((line) => {
+    const trimmed = line.trim();
+    const match = trimmed.match(/^([A-Z][A-Z0-9_]*)=/);
+    if (!match || updates[match[1]] === undefined) return line;
+    seen.add(match[1]);
+    return `${match[1]}=${updates[match[1]]}`;
+  });
+  for (const [key, value] of Object.entries(updates)) {
+    if (!seen.has(key)) next.push(`${key}=${value}`);
+  }
+  await writeFile(path, `${next.join("\n").replace(/\n+$/, "")}\n`, "utf8");
+}
+
+function looksLikeEnvIssue(error) {
+  const message = String(error?.message || error || "");
+  return /fetch failed|ECONNREFUSED|ENOTFOUND|LANGBOT_.*not configured|Could not read recovery_key|Backend did not respond/i.test(message);
+}
+
+function safeReason(value) {
+  return redact(String(value || "")).slice(0, 1000);
+}
@@ -0,0 +1,635 @@
+#!/usr/bin/env node
+
+import { spawn } from "node:child_process";
+import { open, readFile, mkdir, writeFile } from "node:fs/promises";
+import { dirname, resolve } from "node:path";
+import { env } from "node:process";
+import {
+  apiJson,
+  ensureEvidence,
+  evidencePaths,
+  loadEnvFiles,
+  redact,
+  resetAndAuthLocalUser,
+  writeResult,
+} from "./lib/langbot-e2e.mjs";
+
+const RUNNER_ID = "local-agent";
+const DEFAULT_LOCAL_PASSWORD = "LangBotE2ELocalPass!2026";
+const DEFAULT_PIPELINE_NAME = "LangBot QA Fake Provider Debug Chat";
+const DEFAULT_PROVIDER_NAME = "LangBot QA Fake OpenAI Provider";
+const QA_RESOURCE_DESCRIPTION = "Managed by LangBot skills QA automation for controlled fake-provider Debug Chat tests. Safe to delete when local QA fixtures are no longer needed.";
+const DEFAULT_MODEL_NAME = "gpt-4o-mini";
+const DEFAULT_REQUESTER = "openai-chat-completions";
+
+const caseId = "ensure-fake-provider-pipeline";
+
+await loadEnvFiles();
+const paths = evidencePaths(caseId);
+await ensureEvidence(paths);
+
+const writeEnv = process.argv.includes("--write-env");
+const frontendUrl = env.LANGBOT_FRONTEND_URL || "";
+const backendUrl = env.LANGBOT_BACKEND_URL || "";
+const envLocalPath = resolve("skills/.env.local");
+const repoRoot = resolve(env.LANGBOT_REPO || "..");
+const fakeStateDir = resolve(env.LANGBOT_FAKE_PROVIDER_STATE_DIR || resolve(repoRoot, ".qa/fake-provider"));
+const fakeStatePath = resolve(fakeStateDir, "state.json");
+const fakeStdoutPath = resolve(fakeStateDir, "fake-provider.stdout.log");
+const fakeStderrPath = resolve(fakeStateDir, "fake-provider.stderr.log");
+const pipelineName = env.LANGBOT_FAKE_PROVIDER_PIPELINE_NAME || DEFAULT_PIPELINE_NAME;
+const providerName = env.LANGBOT_FAKE_PROVIDER_NAME || DEFAULT_PROVIDER_NAME;
+const requester = env.LANGBOT_FAKE_PROVIDER_REQUESTER || DEFAULT_REQUESTER;
+const modelName = env.LANGBOT_FAKE_PROVIDER_MODEL_NAME || DEFAULT_MODEL_NAME;
+
+const result = {
+  source: "automation",
+  case_id: caseId,
+  run_id: paths.runId,
+  status: "fail",
+  reason: "",
+  frontend_url: frontendUrl,
+  backend_url: backendUrl,
+  fake_provider: {
+    url: "",
+    base_url: "",
+    pid: null,
+    reused: false,
+    config: {},
+    state_file: fakeStatePath,
+    stdout_log: fakeStdoutPath,
+    stderr_log: fakeStderrPath,
+  },
+  provider: {
+    uuid: "",
+    name: providerName,
+    requester,
+    created: false,
+    updated: false,
+  },
+  model: {
+    uuid: "",
+    name: modelName,
+    created: false,
+    updated: false,
+    test_status: "not_run",
+    test_reason: "",
+  },
+  pipeline_id: "",
+  pipeline_name: pipelineName,
+  pipeline_url: "",
+  created: false,
+  updated: false,
+  wrote_env: false,
+  evidence: {
+    console_log: paths.consoleLog,
+    network_log: paths.networkLog,
+    automation_result_json: paths.automationResultJson,
+    result_json: paths.resultJson,
+  },
+  evidence_collected: ["api_diagnostic", "network", "filesystem"],
+};
+
+try {
+  console.error(`[langbot-qa] configuring QA-owned fake-provider fixtures: provider=\"${providerName}\", pipeline=\"${pipelineName}\"`);
+  console.error("[langbot-qa] this setup may create or update local QA provider/model/pipeline resources on the selected backend.");
+  if (!backendUrl) {
+    result.status = "env_issue";
+    throw new Error("LANGBOT_BACKEND_URL is not configured.");
+  }
+  if (!frontendUrl) {
+    result.status = "env_issue";
+    throw new Error("LANGBOT_FRONTEND_URL is not configured.");
+  }
+
+  const fakeProvider = await ensureFakeProvider();
+  const setupConfig = await configureFakeProvider(fakeProvider.url, healthyFakeProviderConfig(), true);
+  result.fake_provider = {
+    ...result.fake_provider,
+    ...fakeProvider,
+    config: setupConfig.config || healthyFakeProviderConfig(),
+  };
+
+  const user = env.LANGBOT_E2E_LOGIN_USER || "";
+  const password = env.LANGBOT_E2E_LOGIN_PASSWORD || DEFAULT_LOCAL_PASSWORD;
+  if (!user) {
+    result.status = "env_issue";
+    throw new Error("LANGBOT_E2E_LOGIN_USER is required so this setup can create/update the fake provider pipeline.");
+  }
+
+  const auth = await resetAndAuthLocalUser({ backendUrl, user, password });
+  const wizard = await skipWizard({ backendUrl, token: auth.token });
+  if (wizard.status !== "pass") {
+    result.status = "fail";
+    throw new Error(wizard.reason || "Failed to mark the local QA wizard as skipped.");
+  }
+
+  const provider = await ensureProvider({
+    backendUrl,
+    token: auth.token,
+    name: providerName,
+    requester,
+    baseUrl: fakeProvider.base_url,
+  });
+  result.provider = provider;
+
+  const model = await ensureModel({
+    backendUrl,
+    token: auth.token,
+    providerUuid: provider.uuid,
+    name: modelName,
+  });
+  result.model = model;
+
+  const pipeline = await ensurePipeline({
+    backendUrl,
+    token: auth.token,
+    name: pipelineName,
+    modelUuid: model.uuid,
+  });
+  Object.assign(result, pipeline);
+  result.pipeline_url = `${frontendUrl.replace(/\/$/, "")}/home/pipelines?id=${encodeURIComponent(pipeline.pipeline_id)}`;
+
+  const runConfig = await configureFakeProvider(fakeProvider.url, targetFakeProviderConfig(), true);
+  result.fake_provider.config = runConfig.config || targetFakeProviderConfig();
+
+  if (writeEnv) {
+    await upsertEnvLocal(envLocalPath, {
+      LANGBOT_E2E_LOGIN_USER: user,
+      LANGBOT_FAKE_PROVIDER_URL: fakeProvider.url,
+      LANGBOT_FAKE_PROVIDER_BASE_URL: fakeProvider.base_url,
+      LANGBOT_FAKE_PROVIDER_PID: fakeProvider.pid ? String(fakeProvider.pid) : "",
+      LANGBOT_FAKE_PROVIDER_PROVIDER_UUID: provider.uuid,
+      LANGBOT_FAKE_PROVIDER_MODEL_UUID: model.uuid,
+      LANGBOT_FAKE_PROVIDER_PIPELINE_URL: result.pipeline_url,
+      LANGBOT_FAKE_PROVIDER_PIPELINE_NAME: pipelineName,
+    });
+    result.wrote_env = true;
+  }
+
+  result.status = "pass";
+  result.reason = `Fake provider pipeline is configured with ${requester}/${modelName}.`;
+} catch (error) {
+  result.status = result.status === "env_issue" ? "env_issue" : "fail";
+  result.reason = result.reason || safeReason(error.message);
+} finally {
+  await writeResult(paths, result);
+  console.log(JSON.stringify(result, null, 2));
+}
+
+process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1);
+
+async function ensureFakeProvider() {
+  const envUrl = normalizeProviderRootUrl(env.LANGBOT_FAKE_PROVIDER_URL || "");
+  if (envUrl && await fakeProviderHealthy(envUrl) && await fakeProviderConfigurable(envUrl)) {
+    return {
+      url: envUrl,
+      base_url: `${envUrl}/v1`,
+      pid: null,
+      reused: true,
+    };
+  }
+
+  const state = await readState(fakeStatePath);
+  const stateUrl = normalizeProviderRootUrl(state.url || "");
+  if (stateUrl && await fakeProviderHealthy(stateUrl)) {
+    if (await fakeProviderConfigurable(stateUrl)) {
+      return {
+        url: stateUrl,
+        base_url: state.base_url || `${stateUrl}/v1`,
+        pid: Number.isInteger(state.pid) ? state.pid : null,
+        reused: true,
+      };
+    }
+    if (Number.isInteger(state.pid)) await stopProcess(state.pid);
+  }
+
+  await mkdir(fakeStateDir, { recursive: true });
+  await writeFile(fakeStatePath, `${JSON.stringify({ status: "starting", started_at: new Date().toISOString() }, null, 2)}\n`, "utf8");
+  const stdout = await open(fakeStdoutPath, "a");
+  const stderr = await open(fakeStderrPath, "a");
+  const scriptPath = resolve("scripts/e2e/fake-openai-provider.mjs");
+  const host = env.LANGBOT_FAKE_PROVIDER_HOST || "127.0.0.1";
+  const port = env.LANGBOT_FAKE_PROVIDER_PORT || "0";
+  const child = spawn(process.execPath, [
+    scriptPath,
+    `--host=${host}`,
+    `--port=${port}`,
+    `--state-file=${fakeStatePath}`,
+  ], {
+    cwd: resolve("."),
+    detached: true,
+    env: {
+      ...env,
+      LANGBOT_FAKE_PROVIDER_MODEL_NAME: modelName,
+    },
+    stdio: ["ignore", stdout.fd, stderr.fd],
+  });
+  child.unref();
+  await stdout.close();
+  await stderr.close();
+
+  const started = await waitForFakeProviderState(fakeStatePath, child.pid, 10_000);
+  if (!started.url || !await fakeProviderHealthy(started.url) || !await fakeProviderConfigurable(started.url)) {
+    throw new Error(`Fake provider did not become healthy. See ${fakeStderrPath}`);
+  }
+
+  return {
+    url: started.url,
+    base_url: started.base_url || `${started.url}/v1`,
+    pid: child.pid ?? started.pid ?? null,
+    reused: false,
+  };
+}
+
+async function configureFakeProvider(rootUrl, config, resetRequestCount) {
+  const response = await fetch(`${normalizeProviderRootUrl(rootUrl)}/__qa/config`, {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify({
+      config,
+      reset_request_count: resetRequestCount,
+    }),
+    signal: AbortSignal.timeout(3000),
+  });
+  const json = await response.json().catch(() => ({}));
+  if (!response.ok || json.ok !== true) {
+    throw new Error(`Fake provider config failed with HTTP ${response.status}.`);
+  }
+  return json;
+}
+
+async function fakeProviderHealthy(rootUrl) {
+  try {
+    const response = await fetch(`${rootUrl.replace(/\/$/, "")}/healthz`, {
+      signal: AbortSignal.timeout(2000),
+    });
+    if (!response.ok) return false;
+    const json = await response.json().catch(() => ({}));
+    return json.ok === true;
+  } catch {
+    return false;
+  }
+}
+
+async function fakeProviderConfigurable(rootUrl) {
+  try {
+    const response = await fetch(`${rootUrl.replace(/\/$/, "")}/__qa/config`, {
+      signal: AbortSignal.timeout(2000),
+    });
+    if (!response.ok) return false;
+    const json = await response.json().catch(() => ({}));
+    return json.ok === true && json.config && typeof json.config === "object";
+  } catch {
+    return false;
+  }
+}
+
+async function stopProcess(pid) {
+  try {
+    process.kill(pid, "SIGTERM");
+  } catch {
+    return;
+  }
+  await sleep(500);
+}
+
+async function waitForFakeProviderState(path, expectedPid, timeoutMs) {
+  const startedAt = Date.now();
+  let lastState = {};
+  while (Date.now() - startedAt < timeoutMs) {
+    const state = await readState(path);
+    if (state.url && (!expectedPid || state.pid === expectedPid)) return state;
+    lastState = state;
+    await sleep(150);
+  }
+  return lastState;
+}
+
+async function readState(path) {
+  try {
+    return JSON.parse(await readFile(path, "utf8"));
+  } catch {
+    return {};
+  }
+}
+
+function normalizeProviderRootUrl(value) {
+  const trimmed = String(value || "").trim().replace(/\/$/, "");
+  return trimmed.endsWith("/v1") ? trimmed.slice(0, -3) : trimmed;
+}
+
+function healthyFakeProviderConfig() {
+  return {
+    response_text: "OK",
+    first_token_delay_ms: 25,
+    chunk_delay_ms: 10,
+    chunk_count: 0,
+    fault_status: 500,
+    fail_first_n: 0,
+    fail_every_n: 0,
+    fail_after_first_chunk: false,
+    dynamic_response: true,
+  };
+}
+
+function targetFakeProviderConfig() {
+  return {
+    response_text: env.LANGBOT_FAKE_PROVIDER_RESPONSE_TEXT || "OK",
+    first_token_delay_ms: nonNegativeInteger(env.LANGBOT_FAKE_PROVIDER_FIRST_TOKEN_DELAY_MS, 25),
+    chunk_delay_ms: nonNegativeInteger(env.LANGBOT_FAKE_PROVIDER_CHUNK_DELAY_MS, 10),
+    chunk_count: nonNegativeInteger(env.LANGBOT_FAKE_PROVIDER_CHUNK_COUNT, 0),
+    fault_status: httpFaultStatus(env.LANGBOT_FAKE_PROVIDER_FAULT_STATUS, 500),
+    fail_first_n: nonNegativeInteger(env.LANGBOT_FAKE_PROVIDER_FAIL_FIRST_N, 0),
+    fail_every_n: nonNegativeInteger(env.LANGBOT_FAKE_PROVIDER_FAIL_EVERY_N, 0),
+    fail_after_first_chunk: envBool(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK, false),
+    dynamic_response: envBool(env.LANGBOT_FAKE_PROVIDER_DYNAMIC_RESPONSE, true),
+  };
+}
+
+async function skipWizard({ backendUrl, token }) {
+  const response = await apiJson(backendUrl, "/api/v1/system/wizard/completed", {
+    method: "POST",
+    token,
+    body: { status: "skipped" },
+  });
+  const ok = response.status < 400 && response.json.code === 0;
+  return {
+    status: ok ? "pass" : "fail",
+    http_status: response.status,
+    code: response.json.code ?? null,
+    reason: ok ? "Wizard marked skipped for local QA." : response.json.msg || "Wizard status update failed.",
+  };
+}
+
+async function ensureProvider({ backendUrl, token, name, requester, baseUrl }) {
+  const list = await apiJson(backendUrl, "/api/v1/provider/providers", { token });
+  if (isApiFailure(list)) {
+    throw new Error(list.json.msg || "Failed to list providers.");
+  }
+  const providers = list.json.data?.providers || [];
+  const existing = providers.find((provider) => (
+    provider.name === name
+      || (provider.requester === requester && String(provider.base_url || "").replace(/\/$/, "") === baseUrl.replace(/\/$/, ""))
+  ));
+  const body = {
+    name,
+    requester,
+    base_url: baseUrl,
+    api_keys: [env.LANGBOT_FAKE_PROVIDER_API_KEY || "langbot-fake-provider-key"],
+  };
+
+  if (existing?.uuid) {
+    const update = await apiJson(backendUrl, `/api/v1/provider/providers/${encodeURIComponent(existing.uuid)}`, {
+      method: "PUT",
+      token,
+      body,
+    });
+    if (isApiFailure(update)) {
+      throw new Error(update.json.msg || "Failed to update fake provider.");
+    }
+    return {
+      uuid: existing.uuid,
+      name,
+      requester,
+      created: false,
+      updated: true,
+    };
+  }
+
+  const create = await apiJson(backendUrl, "/api/v1/provider/providers", {
+    method: "POST",
+    token,
+    body,
+  });
+  const uuid = create.json.data?.uuid || "";
+  if (isApiFailure(create) || !uuid) {
+    throw new Error(create.json.msg || "Failed to create fake provider.");
+  }
+  return {
+    uuid,
+    name,
+    requester,
+    created: true,
+    updated: false,
+  };
+}
+
+async function ensureModel({ backendUrl, token, providerUuid, name }) {
+  const list = await apiJson(backendUrl, `/api/v1/provider/models/llm?provider_uuid=${encodeURIComponent(providerUuid)}`, { token });
+  if (isApiFailure(list)) {
+    throw new Error(list.json.msg || "Failed to list fake provider models.");
+  }
+  const models = list.json.data?.models || [];
+  const existing = models.find((model) => model.name === name);
+  const body = {
+    name,
+    provider_uuid: providerUuid,
+    abilities: [],
+    context_length: positiveInteger(env.LANGBOT_FAKE_PROVIDER_CONTEXT_LENGTH, 8192),
+    extra_args: {},
+    prefered_ranking: 0,
+  };
+  let modelUuid = existing?.uuid || "";
+  let created = false;
+  let updated = false;
+
+  if (modelUuid) {
+    const update = await apiJson(backendUrl, `/api/v1/provider/models/llm/${encodeURIComponent(modelUuid)}`, {
+      method: "PUT",
+      token,
+      body,
+    });
+    if (isApiFailure(update)) {
+      throw new Error(update.json.msg || "Failed to update fake provider model.");
+    }
+    updated = true;
+  } else {
+    const create = await apiJson(backendUrl, "/api/v1/provider/models/llm", {
+      method: "POST",
+      token,
+      body,
+    });
+    modelUuid = create.json.data?.uuid || "";
+    if (isApiFailure(create) || !modelUuid) {
+      throw new Error(create.json.msg || "Failed to create fake provider model.");
+    }
+    created = true;
+  }
+
+  const test = await apiJson(backendUrl, `/api/v1/provider/models/llm/${encodeURIComponent(modelUuid)}/test`, {
+    method: "POST",
+    token,
+    body: { extra_args: {} },
+  });
+  if (isApiFailure(test)) {
+    throw new Error(safeReason(test.json.msg || test.json.message || "Fake provider model test failed."));
+  }
+
+  return {
+    uuid: modelUuid,
+    name,
+    created,
+    updated,
+    test_status: "pass",
+    test_reason: "",
+  };
+}
+
+async function ensurePipeline({ backendUrl, token, name, modelUuid }) {
+  const list = await apiJson(backendUrl, "/api/v1/pipelines", { token });
+  if (isApiFailure(list)) {
+    throw new Error(list.json.msg || "Failed to list pipelines.");
+  }
+  const pipelines = list.json.data?.pipelines || [];
+  let pipeline = pipelines.find((item) => item.name === name) || null;
+  let created = false;
+
+  if (!pipeline) {
+    const create = await apiJson(backendUrl, "/api/v1/pipelines", {
+      method: "POST",
+      token,
+      body: {
+        name,
+        description: QA_RESOURCE_DESCRIPTION,
+        emoji: "QA",
+      },
+    });
+    const pipelineId = create.json.data?.uuid || "";
+    if (isApiFailure(create) || !pipelineId) {
+      throw new Error(create.json.msg || "Failed to create fake provider pipeline.");
+    }
+    created = true;
+    pipeline = { uuid: pipelineId };
+  }
+
+  const loaded = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, { token });
+  pipeline = loaded.json.data?.pipeline || null;
+  if (isApiFailure(loaded) || !pipeline?.uuid) {
+    throw new Error(loaded.json.msg || "Failed to load fake provider pipeline.");
+  }
+
+  const config = pipeline.config && typeof pipeline.config === "object" ? pipeline.config : {};
+  const ai = config.ai && typeof config.ai === "object" ? config.ai : {};
+  const existingLocalAgentConfig = ai["local-agent"] && typeof ai["local-agent"] === "object"
+    ? ai["local-agent"]
+    : {};
+  const localAgentConfig = {
+    timeout: 60,
+    prompt: [{ role: "system", content: "You are a deterministic QA assistant. Reply exactly as instructed." }],
+    "remove-think": false,
+    "knowledge-bases": [],
+    "box-session-id-template": "{launcher_type}_{launcher_id}",
+    "retrieval-top-k": 5,
+    "rerank-model": "",
+    "rerank-top-k": 5,
+    "max-tool-iterations": 20,
+    "tool-execution-mode": "parallel",
+    "max-tool-result-chars": 20000,
+    "context-history-fetch-limit": 20,
+    "context-window-tokens": 8192,
+    "context-reserve-tokens": 1024,
+    "context-keep-recent-tokens": 2048,
+    "context-summary-tokens": 1024,
+    ...existingLocalAgentConfig,
+    // Current backend truncation still reads this field directly.
+    "max-round": positiveInteger(existingLocalAgentConfig["max-round"], 10),
+    model: {
+      primary: modelUuid,
+      fallbacks: [],
+    },
+  };
+  const updatedConfig = {
+    ...config,
+    ai: {
+      ...ai,
+      runner: {
+        ...(ai.runner && typeof ai.runner === "object" ? ai.runner : {}),
+        id: RUNNER_ID,
+        runner: RUNNER_ID,
+        "expire-time": 0,
+      },
+      "local-agent": localAgentConfig,
+    },
+  };
+
+  const update = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, {
+    method: "PUT",
+    token,
+    body: {
+      name,
+      description: QA_RESOURCE_DESCRIPTION,
+      emoji: "QA",
+      config: updatedConfig,
+    },
+  });
+  if (isApiFailure(update)) {
+    throw new Error(update.json.msg || "Failed to update fake provider pipeline.");
+  }
+
+  return {
+    pipeline_id: pipeline.uuid,
+    pipeline_name: name,
+    created,
+    updated: true,
+  };
+}
+
+function isApiFailure(response) {
+  return response.status >= 400 || (response.json.code !== undefined && response.json.code !== 0);
+}
+
+function positiveInteger(value, fallback) {
+  const parsed = Number(value);
+  return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
+}
+
+function nonNegativeInteger(value, fallback) {
+  const parsed = Number(value);
+  return Number.isInteger(parsed) && parsed >= 0 ? parsed : fallback;
+}
+
+function httpFaultStatus(value, fallback) {
+  const parsed = Number(value);
+  return Number.isInteger(parsed) && parsed >= 400 && parsed <= 599 ? parsed : fallback;
+}
+
+function envBool(value, fallback) {
+  if (value === undefined || value === "") return fallback;
+  if (/^(1|true|yes|on)$/i.test(String(value))) return true;
+  if (/^(0|false|no|off)$/i.test(String(value))) return false;
+  return fallback;
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function safeReason(value) {
+  return redact(String(value || "")).slice(0, 1000);
+}
+
+async function upsertEnvLocal(path, updates) {
+  await mkdir(dirname(path), { recursive: true });
+  let text = "";
+  try {
+    text = await readFile(path, "utf8");
+  } catch {
+    text = "";
+  }
+  const lines = text.split(/\r?\n/);
+  const seen = new Set();
+  const next = lines.map((line) => {
+    const trimmed = line.trim();
+    const equals = trimmed.indexOf("=");
+    if (equals <= 0 || trimmed.startsWith("#")) return line;
+    const key = trimmed.slice(0, equals).trim();
+    if (!(key in updates)) return line;
+    seen.add(key);
+    return `${key}=${updates[key]}`;
+  });
+  for (const [key, value] of Object.entries(updates)) {
+    if (!seen.has(key)) next.push(`${key}=${value}`);
+  }
+  await writeFile(path, `${next.filter((line, index) => line !== "" || index < next.length - 1).join("\n")}\n`, "utf8");
+}
@@ -10,6 +10,7 @@ import {
  ensureEvidence,
  evidencePaths,
  loadEnvFiles,
+  redact,
  resetAndAuthLocalUser,
  safeScreenshot,
  setBrowserToken,
@@ -17,9 +18,12 @@ import {
  writeResult,
 } from "./lib/langbot-e2e.mjs";

-const RUNNER_ID = "plugin:langbot/local-agent/default";
+const RUNNER_ID = "local-agent";
+const SPACE_PROVIDER_UUID = "00000000-0000-0000-0000-000000000000";
 const DEFAULT_PIPELINE_NAME = "Agent QA Local Agent Debug Chat";
 const DEFAULT_LOCAL_PASSWORD = "LangBotE2ELocalPass!2026";
+const DEFAULT_MODEL_TEST_LIMIT = 8;
+const DEFAULT_MODEL_FALLBACK_COUNT = 3;
 const caseId = "ensure-local-agent-pipeline";

 await loadEnvFiles();
@@ -45,11 +49,18 @@ const result = {
  pipeline_url: "",
  runner_id: RUNNER_ID,
  selected_model_id: "",
+  selected_model_name: "",
+  fallback_model_ids: [],
  model_count: 0,
+  space_model_count: 0,
+  scanned_space_model_count: 0,
+  tested_model_count: 0,
+  model_tests: [],
  created: false,
  updated: false,
  wrote_env: false,
  auth: null,
+  wizard: null,
  browser_token_check: null,
  page_signal: "",
  evidence: {
@@ -71,6 +82,7 @@ try {
  const user = env.LANGBOT_E2E_LOGIN_USER || "";
  const password = env.LANGBOT_E2E_LOGIN_PASSWORD || DEFAULT_LOCAL_PASSWORD;
  if (!user) {
+    result.status = "env_issue";
    throw new Error("LANGBOT_E2E_LOGIN_USER is required so this setup can create/update the pipeline via backend API.");
  }

@@ -81,6 +93,13 @@ try {
    backend_token_check: auth.check,
  };

+  const wizard = await skipWizard({ backendUrl, token: auth.token });
+  result.wizard = wizard;
+  if (wizard.status !== "pass") {
+    result.status = "fail";
+    throw new Error(wizard.reason || "Failed to mark the local QA wizard as skipped.");
+  }
+
  const prepared = await ensureLocalAgentPipeline({
    backendUrl,
    token: auth.token,
@@ -99,6 +118,10 @@ try {
      LANGBOT_PIPELINE_NAME: result.pipeline_name || pipelineName,
      LANGBOT_LOCAL_AGENT_PIPELINE_URL: result.pipeline_url,
      LANGBOT_LOCAL_AGENT_PIPELINE_NAME: result.pipeline_name || pipelineName,
+      ...(result.selected_model_id ? {
+        LANGBOT_LOCAL_AGENT_MODEL_UUID: result.selected_model_id,
+        LANGBOT_E2E_MODEL_UUID: result.selected_model_id,
+      } : {}),
    });
    result.wrote_env = true;
  }
@@ -127,6 +150,21 @@ try {

 process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1);

+async function skipWizard({ backendUrl, token }) {
+  const response = await apiJson(backendUrl, "/api/v1/system/wizard/completed", {
+    method: "POST",
+    token,
+    body: { status: "skipped" },
+  });
+  const ok = response.status < 400 && response.json.code === 0;
+  return {
+    status: ok ? "pass" : "fail",
+    http_status: response.status,
+    code: response.json.code ?? null,
+    reason: ok ? "Wizard marked skipped for local QA." : response.json.msg || "Wizard status update failed.",
+  };
+}
+
 async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runnerId }) {
  const [pipelineList, modelList] = await Promise.all([
    apiJson(backendUrl, "/api/v1/pipelines", { token }),
@@ -149,7 +187,19 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne
  }

  const models = modelList.json.data?.models || [];
-  const selectedModel = models.find((model) => model.uuid) || null;
+  const skippedModelIds = new Set(
+    String(env.LANGBOT_E2E_SKIP_MODEL_UUIDS || "")
+      .split(",")
+      .map((item) => item.trim())
+      .filter(Boolean),
+  );
+  const skippedModelNames = new Set(
+    String(env.LANGBOT_E2E_SKIP_MODEL_NAMES || "")
+      .split(",")
+      .map((item) => item.trim())
+      .filter(Boolean),
+  );
+  const spaceModels = models.filter((model) => isSpaceModel(model) && !skippedModelIds.has(model.uuid));
  const pipelines = pipelineList.json.data?.pipelines || [];
  let pipeline = pipelines.find((item) => item.name === pipelineName) || null;
  let created = false;
@@ -170,6 +220,7 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne
        reason: createdResponse.json.msg || "Failed to create pipeline.",
        create_status: createdResponse.status,
        model_count: models.length,
+        space_model_count: spaceModels.length,
      };
    }
    const pipelineId = createdResponse.json.data?.uuid || "";
@@ -183,6 +234,7 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne
      status: "fail",
      reason: "Pipeline was not created or resolved.",
      model_count: models.length,
+      space_model_count: spaceModels.length,
    };
  }

@@ -194,27 +246,37 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne
      get_status: loaded.status,
      pipeline_id: pipeline.uuid,
      model_count: models.length,
+      space_model_count: spaceModels.length,
    };
  }
  pipeline = loaded.json.data.pipeline;

  const config = pipeline.config && typeof pipeline.config === "object" ? pipeline.config : {};
  const ai = config.ai && typeof config.ai === "object" ? config.ai : {};
-  const runnerConfig = ai.runner_config && typeof ai.runner_config === "object" ? ai.runner_config : {};
-  const rawExistingLocalAgentConfig = runnerConfig[runnerId] && typeof runnerConfig[runnerId] === "object"
-    ? runnerConfig[runnerId]
+  const rawExistingLocalAgentConfig = ai["local-agent"] && typeof ai["local-agent"] === "object"
+    ? ai["local-agent"]
    : {};
  const existingLocalAgentConfig = rawExistingLocalAgentConfig;
  const existingModel = existingLocalAgentConfig.model && typeof existingLocalAgentConfig.model === "object"
    ? existingLocalAgentConfig.model
    : {};
  const requestedModelId = env.LANGBOT_LOCAL_AGENT_MODEL_UUID || env.LANGBOT_E2E_MODEL_UUID || "";
-  const selectedModelId = requestedModelId || existingModel.primary || selectedModel?.uuid || "";
+  const selected = await selectWorkingSpaceModel({
+    backendUrl,
+    token,
+    models,
+    skippedModelIds,
+    skippedModelNames,
+    requestedModelId,
+    existingModelId: existingModel.primary || "",
+  });
+  const selectedModelId = selected.selected_model_id || "";
  const localAgentConfig = {
    timeout: 300,
    prompt: [{ role: "system", content: "You are a helpful assistant." }],
    "remove-think": false,
    "knowledge-bases": [],
+    "box-session-id-template": "{launcher_type}_{launcher_id}",
    "retrieval-top-k": 5,
    "rerank-model": "",
    "rerank-top-k": 5,
@@ -227,9 +289,11 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne
    "context-keep-recent-tokens": 20000,
    "context-summary-tokens": 8000,
    ...existingLocalAgentConfig,
+    // Current backend truncation still reads this field directly.
+    "max-round": positiveInteger(existingLocalAgentConfig["max-round"], 10),
    model: {
      primary: selectedModelId,
-      fallbacks: requestedModelId ? [] : Array.isArray(existingModel.fallbacks) ? existingModel.fallbacks : [],
+      fallbacks: selected.fallback_model_ids || [],
    },
  };
  const updatedConfig = {
@@ -239,12 +303,10 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne
      runner: {
        ...(ai.runner && typeof ai.runner === "object" ? ai.runner : {}),
        id: runnerId,
+        runner: runnerId,
        "expire-time": 0,
      },
-      runner_config: {
-        ...runnerConfig,
-        [runnerId]: localAgentConfig,
-      },
+      "local-agent": localAgentConfig,
    },
  };

@@ -265,19 +327,31 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne
      update_status: updateResponse.status,
      pipeline_id: pipeline.uuid,
      model_count: models.length,
+      space_model_count: spaceModels.length,
+      scanned_space_model_count: selected.scanned_space_model_count,
+      tested_model_count: selected.tested_model_count,
+      model_tests: selected.model_tests,
      selected_model_id: selectedModelId,
+      selected_model_name: selected.selected_model_name,
+      fallback_model_ids: selected.fallback_model_ids,
    };
  }

  return {
    status: selectedModelId ? "pass" : "env_issue",
    reason: selectedModelId
-      ? "Local-agent pipeline is configured for Debug Chat."
-      : "Pipeline was created but no LLM model is configured in this LangBot instance.",
+      ? `Local-agent pipeline is configured for Debug Chat with Space model ${selected.selected_model_name || selectedModelId} and ${selected.fallback_model_ids.length} fallback(s).`
+      : selected.reason || "No working Space LLM model is configured in this LangBot instance.",
    pipeline_id: pipeline.uuid,
-    pipeline_name: pipeline.name,
+    pipeline_name: pipelineName,
    model_count: models.length,
+    space_model_count: spaceModels.length,
+    scanned_space_model_count: selected.scanned_space_model_count,
+    tested_model_count: selected.tested_model_count,
+    model_tests: selected.model_tests,
    selected_model_id: selectedModelId,
+    selected_model_name: selected.selected_model_name,
+    fallback_model_ids: selected.fallback_model_ids,
    created,
    updated: true,
  };
@@ -287,6 +361,229 @@ function isApiFailure(response) {
  return response.status >= 400 || (response.json.code !== undefined && response.json.code !== 0);
 }

+function isSpaceModel(model) {
+  const provider = model?.provider && typeof model.provider === "object" ? model.provider : {};
+  return model?.provider_uuid === SPACE_PROVIDER_UUID
+    || provider.uuid === SPACE_PROVIDER_UUID
+    || provider.requester === "space-chat-completions"
+    || provider.name === "LangBot Models";
+}
+
+async function selectWorkingSpaceModel({
+  backendUrl,
+  token,
+  models,
+  skippedModelIds,
+  skippedModelNames,
+  requestedModelId,
+  existingModelId,
+}) {
+  const modelTests = [];
+  const testLimit = positiveInteger(env.LANGBOT_E2E_MODEL_TEST_LIMIT, DEFAULT_MODEL_TEST_LIMIT);
+  const fallbackCount = positiveInteger(env.LANGBOT_E2E_MODEL_FALLBACK_COUNT, DEFAULT_MODEL_FALLBACK_COUNT);
+  const workingModels = [];
+  const spaceModels = rankModels(models.filter((model) => (
+    model.uuid
+      && isSpaceModel(model)
+      && !skippedModelIds.has(model.uuid)
+      && !skippedModelNames.has(model.name)
+  )));
+  const requestedModel = requestedModelId
+    ? spaceModels.find((model) => model.uuid === requestedModelId) || null
+    : null;
+  const existingModel = existingModelId
+    ? spaceModels.find((model) => model.uuid === existingModelId) || null
+    : null;
+  const candidates = uniqueCandidates([
+    ...(requestedModel ? [existingCandidate(requestedModel, "requested")] : []),
+    ...(existingModel ? [existingCandidate(existingModel, "existing-pipeline")] : []),
+    ...spaceModels.map((model) => existingCandidate(model, "configured-space")),
+  ]);
+
+  let scanResult = { status: "skipped", models: [], reason: "" };
+  if (env.LANGBOT_E2E_SCAN_SPACE_MODELS !== "false") {
+    scanResult = await scanSpaceModels({ backendUrl, token });
+    if (scanResult.status === "pass") {
+      const knownNames = new Set(spaceModels.map((model) => model.name));
+      candidates.push(...scanResult.models
+        .filter((model) => model.name && !knownNames.has(model.name) && !skippedModelNames.has(model.name))
+        .map((model) => scannedCandidate(model)));
+    }
+  }
+
+  const unique = uniqueCandidates(candidates);
+  for (const candidate of unique.slice(0, testLimit)) {
+    const test = await ensureAndTestModel({ backendUrl, token, candidate });
+    modelTests.push(test);
+    if (test.status === "pass" && test.model_uuid) {
+      workingModels.push(test);
+      if (workingModels.length >= fallbackCount + 1) break;
+    }
+  }
+
+  if (workingModels.length > 0) {
+    const [primary, ...fallbacks] = workingModels;
+    return {
+      status: "pass",
+      reason: "",
+      selected_model_id: primary.model_uuid,
+      selected_model_name: primary.model_name,
+      fallback_model_ids: fallbacks.map((model) => model.model_uuid),
+      scanned_space_model_count: scanResult.models.length,
+      tested_model_count: modelTests.length,
+      model_tests: modelTests,
+    };
+  }
+
+  const baseReason = unique.length === 0
+    ? scanResult.reason || "No Space LLM model candidates are available."
+    : `No working Space LLM model found after testing ${modelTests.length} candidate(s).`;
+  return {
+    status: "env_issue",
+    reason: requestedModelId && !requestedModel
+      ? `Requested Space LLM model ${requestedModelId} is missing or skipped; ${baseReason}`
+      : baseReason,
+    selected_model_id: "",
+    selected_model_name: "",
+    fallback_model_ids: [],
+    scanned_space_model_count: scanResult.models.length,
+    tested_model_count: modelTests.length,
+    model_tests: modelTests,
+  };
+}
+
+async function scanSpaceModels({ backendUrl, token }) {
+  const response = await apiJson(
+    backendUrl,
+    `/api/v1/provider/providers/${encodeURIComponent(SPACE_PROVIDER_UUID)}/scan-models?type=llm`,
+    { token },
+  );
+  if (isApiFailure(response)) {
+    return {
+      status: "env_issue",
+      models: [],
+      reason: safeReason(response.json.msg || response.json.message || "Failed to scan Space LLM models."),
+    };
+  }
+  return {
+    status: "pass",
+    models: response.json.data?.models || [],
+    reason: "",
+  };
+}
+
+async function ensureAndTestModel({ backendUrl, token, candidate }) {
+  let modelUuid = candidate.uuid || "";
+  let created = false;
+  if (!modelUuid) {
+    const create = await apiJson(backendUrl, "/api/v1/provider/models/llm", {
+      method: "POST",
+      token,
+      body: {
+        name: candidate.name,
+        provider_uuid: SPACE_PROVIDER_UUID,
+        abilities: candidate.abilities || [],
+        context_length: candidate.context_length ?? null,
+        extra_args: {},
+        prefered_ranking: positiveInteger(candidate.prefered_ranking, 0),
+      },
+    });
+    modelUuid = create.json.data?.uuid || "";
+    if (isApiFailure(create) || !modelUuid) {
+      return modelTestResult(candidate, {
+        status: "fail",
+        reason: safeReason(create.json.msg || "Failed to create scanned Space model."),
+        http_status: create.status,
+      });
+    }
+    created = true;
+  }
+
+  const test = await apiJson(backendUrl, `/api/v1/provider/models/llm/${encodeURIComponent(modelUuid)}/test`, {
+    method: "POST",
+    token,
+    body: { extra_args: {} },
+  });
+  const passed = !isApiFailure(test);
+  if (!passed && created) {
+    await apiJson(backendUrl, `/api/v1/provider/models/llm/${encodeURIComponent(modelUuid)}`, {
+      method: "DELETE",
+      token,
+    }).catch(() => {});
+  }
+  return modelTestResult(candidate, {
+    status: passed ? "pass" : "fail",
+    reason: passed ? "" : safeReason(test.json.msg || test.json.message || "Space model test failed."),
+    http_status: test.status,
+    model_uuid: modelUuid,
+    created,
+  });
+}
+
+function modelTestResult(candidate, details) {
+  return {
+    source: candidate.source,
+    model_uuid: details.model_uuid || candidate.uuid || "",
+    model_name: candidate.name,
+    status: details.status,
+    reason: details.reason || "",
+    http_status: details.http_status ?? null,
+    created: Boolean(details.created),
+  };
+}
+
+function existingCandidate(model, source) {
+  return {
+    source,
+    uuid: model.uuid,
+    name: model.name,
+    abilities: model.abilities || [],
+    context_length: model.context_length,
+    prefered_ranking: model.prefered_ranking,
+  };
+}
+
+function scannedCandidate(model) {
+  return {
+    source: "scanned-space",
+    uuid: "",
+    name: model.name || model.id,
+    abilities: model.abilities || [],
+    context_length: model.context_length,
+    prefered_ranking: model.prefered_ranking,
+  };
+}
+
+function uniqueCandidates(candidates) {
+  const seen = new Set();
+  const result = [];
+  for (const candidate of candidates) {
+    const key = candidate.uuid ? `uuid:${candidate.uuid}` : `name:${candidate.name}`;
+    if (!candidate.name || seen.has(key)) continue;
+    seen.add(key);
+    result.push(candidate);
+  }
+  return result;
+}
+
+function rankModels(models) {
+  return [...models].sort((left, right) => {
+    const leftRank = Number.isFinite(Number(left.prefered_ranking)) ? Number(left.prefered_ranking) : 9999;
+    const rightRank = Number.isFinite(Number(right.prefered_ranking)) ? Number(right.prefered_ranking) : 9999;
+    if (leftRank !== rightRank) return leftRank - rightRank;
+    return String(left.name || "").localeCompare(String(right.name || ""));
+  });
+}
+
+function positiveInteger(value, fallback) {
+  const parsed = Number(value);
+  return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
+}
+
+function safeReason(value) {
+  return redact(String(value || "")).slice(0, 1000);
+}
+
 async function upsertEnvLocal(path, updates) {
  let text = "";
  try {
@@ -0,0 +1,496 @@
+#!/usr/bin/env node
+
+import { createServer } from "node:http";
+import { mkdir, writeFile } from "node:fs/promises";
+import { dirname, resolve } from "node:path";
+import { env, exit } from "node:process";
+
+const args = parseArgs(process.argv.slice(2));
+const host = args.host || env.LANGBOT_FAKE_PROVIDER_HOST || "127.0.0.1";
+const port = integer(args.port ?? env.LANGBOT_FAKE_PROVIDER_PORT, 0);
+const stateFile = args["state-file"] || env.LANGBOT_FAKE_PROVIDER_STATE_FILE || "";
+const modelName = env.LANGBOT_FAKE_PROVIDER_MODEL_NAME || "gpt-4o-mini";
+const config = {
+  response_text: env.LANGBOT_FAKE_PROVIDER_RESPONSE_TEXT || "OK",
+  first_token_delay_ms: integer(env.LANGBOT_FAKE_PROVIDER_FIRST_TOKEN_DELAY_MS, 25),
+  chunk_delay_ms: integer(env.LANGBOT_FAKE_PROVIDER_CHUNK_DELAY_MS, 10),
+  chunk_count: integer(env.LANGBOT_FAKE_PROVIDER_CHUNK_COUNT, 0),
+  fault_status: integer(env.LANGBOT_FAKE_PROVIDER_FAULT_STATUS, 500),
+  fail_first_n: integer(env.LANGBOT_FAKE_PROVIDER_FAIL_FIRST_N, 0),
+  fail_every_n: integer(env.LANGBOT_FAKE_PROVIDER_FAIL_EVERY_N, 0),
+  fail_after_first_chunk: bool(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK, false),
+  dynamic_response: !/^(0|false|no|off)$/i.test(env.LANGBOT_FAKE_PROVIDER_DYNAMIC_RESPONSE || ""),
+  request_log_limit: integer(env.LANGBOT_FAKE_PROVIDER_REQUEST_LOG_LIMIT, 500),
+};
+
+let requestCount = 0;
+const recentRequests = [];
+
+const server = createServer(async (request, response) => {
+  const startedAt = Date.now();
+  const startedPerf = performance.now();
+  let requestRecord = null;
+  const url = new URL(request.url || "/", `http://${request.headers.host || `${host}:${port}`}`);
+  try {
+    if (request.method === "GET" && url.pathname === "/healthz") {
+      sendJson(response, 200, {
+        ok: true,
+        model: modelName,
+        config,
+        request_count: requestCount,
+        recent_request_count: recentRequests.length,
+      });
+      return;
+    }
+
+    if (request.method === "GET" && url.pathname === "/__qa/config") {
+      sendJson(response, 200, {
+        ok: true,
+        model: modelName,
+        config,
+        request_count: requestCount,
+        recent_requests: recentRequests,
+      });
+      return;
+    }
+
+    if (request.method === "POST" && url.pathname === "/__qa/config") {
+      const body = await readJson(request);
+      applyConfig(body.config && typeof body.config === "object" ? body.config : body);
+      if (body.reset_request_count !== false) resetRequestState();
+      sendJson(response, 200, {
+        ok: true,
+        model: modelName,
+        config,
+        request_count: requestCount,
+      });
+      return;
+    }
+
+    if (request.method === "POST" && url.pathname === "/__qa/reset") {
+      resetRequestState();
+      sendJson(response, 200, {
+        ok: true,
+        model: modelName,
+        config,
+        request_count: requestCount,
+      });
+      return;
+    }
+
+    if (request.method === "GET" && ["/models", "/v1/models"].includes(url.pathname)) {
+      sendJson(response, 200, {
+        object: "list",
+        data: [
+          {
+            id: modelName,
+            object: "model",
+            created: 1,
+            owned_by: "langbot-qa",
+            type: "llm",
+          },
+        ],
+      });
+      return;
+    }
+
+    if (request.method === "POST" && ["/chat/completions", "/v1/chat/completions"].includes(url.pathname)) {
+      requestCount += 1;
+      const body = await readJson(request);
+      const requestId = `chatcmpl-langbot-fake-${requestCount}`;
+      const shouldFail = requestCount <= config.fail_first_n
+        || (config.fail_every_n > 0 && requestCount % config.fail_every_n === 0);
+      const replyText = responseTextForBody(body);
+      requestRecord = recordRequest({
+        id: requestId,
+        request_number: requestCount,
+        path: url.pathname,
+        stream: Boolean(body.stream),
+        model: body.model || "",
+        message_count: Array.isArray(body.messages) ? body.messages.length : 0,
+        should_fail: shouldFail,
+        status: "running",
+        http_status: null,
+        expected_text: replyText,
+        response_text_preview: previewText(replyText),
+        started_at: new Date(startedAt).toISOString(),
+        started_epoch_ms: startedAt,
+        configured_first_token_delay_ms: config.first_token_delay_ms,
+        configured_chunk_delay_ms: config.chunk_delay_ms,
+        configured_chunk_count: config.chunk_count,
+      });
+
+      if (shouldFail) {
+        await sleep(config.first_token_delay_ms);
+        sendJson(response, config.fault_status, {
+          error: {
+            message: `LangBot fake provider injected HTTP ${config.fault_status}`,
+            type: "fake_provider_fault",
+            code: "fake_provider_fault",
+          },
+        });
+        finishRequestRecord(requestRecord, startedPerf, {
+          status: "http_fault",
+          http_status: config.fault_status,
+        });
+        return;
+      }
+
+      if (body.stream) {
+        await streamCompletion(response, {
+          requestId,
+          model: body.model || modelName,
+          content: replyText,
+          failAfterFirstChunk: config.fail_after_first_chunk,
+          requestRecord,
+          startedPerf,
+        });
+      } else {
+        await sleep(config.first_token_delay_ms + config.chunk_delay_ms);
+        sendJson(response, 200, completionPayload({
+          requestId,
+          model: body.model || modelName,
+          content: replyText,
+        }));
+        markRequestTiming(requestRecord, "first_chunk", startedPerf);
+        markRequestTiming(requestRecord, "first_content_chunk", startedPerf);
+        requestRecord.content_chunk_count = 1;
+        finishRequestRecord(requestRecord, startedPerf, {
+          status: "ok",
+          http_status: 200,
+        });
+      }
+      return;
+    }
+
+    sendJson(response, 404, {
+      error: {
+        message: `No fake provider route for ${request.method} ${url.pathname}`,
+        type: "not_found",
+      },
+    });
+  } catch (error) {
+    if (requestRecord) {
+      finishRequestRecord(requestRecord, startedPerf, {
+        status: "fake_provider_error",
+        http_status: 500,
+        error: error instanceof Error ? error.message : String(error),
+      });
+    }
+    sendJson(response, 500, {
+      error: {
+        message: error instanceof Error ? error.message : String(error),
+        type: "fake_provider_error",
+      },
+    });
+  } finally {
+    const durationMs = Date.now() - startedAt;
+    if (url.pathname !== "/healthz") {
+      console.log(JSON.stringify({
+        at: new Date().toISOString(),
+        method: request.method,
+        path: url.pathname,
+        duration_ms: durationMs,
+      }));
+    }
+  }
+});
+
+server.listen(port, host, async () => {
+  const address = server.address();
+  const selectedPort = typeof address === "object" && address ? address.port : port;
+  const url = `http://${host}:${selectedPort}`;
+  const state = {
+    status: "ready",
+    pid: process.pid,
+    url,
+    base_url: `${url}/v1`,
+    model: modelName,
+    started_at: new Date().toISOString(),
+  };
+  if (stateFile) {
+    const path = resolve(stateFile);
+    await mkdir(dirname(path), { recursive: true });
+    await writeFile(path, `${JSON.stringify(state, null, 2)}\n`, "utf8");
+  }
+  console.log(JSON.stringify(state));
+});
+
+server.on("error", (error) => {
+  console.error(JSON.stringify({
+    status: "error",
+    reason: error instanceof Error ? error.message : String(error),
+  }));
+  exit(1);
+});
+
+process.on("SIGTERM", () => {
+  server.close(() => exit(0));
+});
+
+function parseArgs(argv) {
+  const result = {};
+  for (const item of argv) {
+    const match = item.match(/^--([^=]+)(?:=(.*))?$/);
+    if (!match) continue;
+    result[match[1]] = match[2] ?? "1";
+  }
+  return result;
+}
+
+function integer(value, fallback) {
+  const parsed = Number.parseInt(String(value ?? ""), 10);
+  return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback;
+}
+
+function bool(value, fallback) {
+  if (value === undefined || value === "") return fallback;
+  if (/^(1|true|yes|on)$/i.test(String(value))) return true;
+  if (/^(0|false|no|off)$/i.test(String(value))) return false;
+  return fallback;
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, Math.max(0, ms)));
+}
+
+async function readJson(request) {
+  let text = "";
+  for await (const chunk of request) text += chunk.toString();
+  if (!text) return {};
+  return JSON.parse(text);
+}
+
+function sendJson(response, status, payload) {
+  const text = `${JSON.stringify(payload)}\n`;
+  response.writeHead(status, {
+    "content-type": "application/json",
+    "content-length": Buffer.byteLength(text),
+  });
+  response.end(text);
+}
+
+function completionPayload({ requestId, model, content }) {
+  const completionTokens = tokenEstimate(content);
+  return {
+    id: requestId,
+    object: "chat.completion",
+    created: Math.floor(Date.now() / 1000),
+    model,
+    choices: [
+      {
+        index: 0,
+        message: {
+          role: "assistant",
+          content,
+        },
+        finish_reason: "stop",
+      },
+    ],
+    usage: {
+      prompt_tokens: 8,
+      completion_tokens: completionTokens,
+      total_tokens: 8 + completionTokens,
+    },
+  };
+}
+
+async function streamCompletion(response, {
+  requestId,
+  model,
+  content,
+  failAfterFirstChunk: failMidStream,
+  requestRecord,
+  startedPerf,
+}) {
+  response.writeHead(200, {
+    "content-type": "text/event-stream; charset=utf-8",
+    "cache-control": "no-cache",
+    "connection": "keep-alive",
+  });
+
+  await sleep(config.first_token_delay_ms);
+  markRequestTiming(requestRecord, "first_chunk", startedPerf);
+  writeSse(response, {
+    id: requestId,
+    object: "chat.completion.chunk",
+    created: Math.floor(Date.now() / 1000),
+    model,
+    choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }],
+  });
+
+  const chunks = splitContent(content);
+  for (let index = 0; index < chunks.length; index += 1) {
+    await sleep(config.chunk_delay_ms);
+    if (index === 0) markRequestTiming(requestRecord, "first_content_chunk", startedPerf);
+    requestRecord.content_chunk_count = (requestRecord.content_chunk_count || 0) + 1;
+    writeSse(response, {
+      id: requestId,
+      object: "chat.completion.chunk",
+      created: Math.floor(Date.now() / 1000),
+      model,
+      choices: [{ index: 0, delta: { content: chunks[index] }, finish_reason: null }],
+    });
+    if (failMidStream && index === 0) {
+      finishRequestRecord(requestRecord, startedPerf, {
+        status: "mid_stream_disconnect",
+        http_status: 200,
+      });
+      response.destroy(new Error("LangBot fake provider injected mid-stream disconnect"));
+      return;
+    }
+  }
+
+  await sleep(config.chunk_delay_ms);
+  const completionTokens = tokenEstimate(content);
+  writeSse(response, {
+    id: requestId,
+    object: "chat.completion.chunk",
+    created: Math.floor(Date.now() / 1000),
+    model,
+    choices: [{ index: 0, delta: {}, finish_reason: "stop" }],
+    usage: {
+      prompt_tokens: 8,
+      completion_tokens: completionTokens,
+      total_tokens: 8 + completionTokens,
+    },
+  });
+  response.write("data: [DONE]\n\n");
+  response.end();
+  finishRequestRecord(requestRecord, startedPerf, {
+    status: "ok",
+    http_status: 200,
+  });
+}
+
+function writeSse(response, payload) {
+  response.write(`data: ${JSON.stringify(payload)}\n\n`);
+}
+
+function splitContent(content) {
+  const text = String(content);
+  const requested = config.chunk_count;
+  if (requested <= 1 || text.length <= 1) return [text];
+  const chunkSize = Math.max(1, Math.ceil(text.length / requested));
+  const chunks = [];
+  for (let index = 0; index < text.length; index += chunkSize) {
+    chunks.push(text.slice(index, index + chunkSize));
+  }
+  return chunks;
+}
+
+function tokenEstimate(content) {
+  return Math.max(1, Math.ceil(String(content || "").length / 4));
+}
+
+function responseTextForBody(body) {
+  if (!config.dynamic_response) {
+    return config.response_text;
+  }
+  const messages = Array.isArray(body.messages) ? body.messages : [];
+  const lastUser = [...messages].reverse().find((message) => message?.role === "user");
+  const text = flattenContent(lastUser?.content || "");
+  const quoted = text.match(/["'“”](.{1,80}?)["'“”]/);
+  if (quoted?.[1]) return quoted[1].trim();
+  const exact = text.match(/(?:reply|回复|输出|return)\s+(?:exactly\s+)?([A-Za-z0-9_.:@-]{1,80})/i);
+  if (exact?.[1]) return exact[1].trim().replace(/[。.!?]+$/, "");
+  const only = text.match(/只回复\s*([A-Za-z0-9_.:@-]{1,80})/);
+  if (only?.[1]) return only[1].trim().replace(/[。.!?]+$/, "");
+  return config.response_text;
+}
+
+function flattenContent(content) {
+  if (typeof content === "string") return content;
+  if (Array.isArray(content)) {
+    return content
+      .map((item) => {
+        if (typeof item === "string") return item;
+        if (item && typeof item === "object") return item.text || "";
+        return "";
+      })
+      .join("\n");
+  }
+  return "";
+}
+
+function recordRequest(entry) {
+  const item = {
+    ...entry,
+    at: new Date().toISOString(),
+    finished_at: null,
+    finished_epoch_ms: null,
+    duration_ms: null,
+    first_chunk_at: null,
+    first_chunk_epoch_ms: null,
+    first_chunk_ms: null,
+    first_content_chunk_at: null,
+    first_content_chunk_epoch_ms: null,
+    first_content_chunk_ms: null,
+    content_chunk_count: 0,
+  };
+  recentRequests.push(item);
+  while (recentRequests.length > config.request_log_limit) recentRequests.shift();
+  return item;
+}
+
+function markRequestTiming(entry, key, startedPerf) {
+  if (!entry || entry[`${key}_at`]) return;
+  const now = Date.now();
+  entry[`${key}_at`] = new Date(now).toISOString();
+  entry[`${key}_epoch_ms`] = now;
+  entry[`${key}_ms`] = rounded(performance.now() - startedPerf);
+}
+
+function finishRequestRecord(entry, startedPerf, updates = {}) {
+  if (!entry || entry.finished_at) return;
+  const now = Date.now();
+  Object.assign(entry, updates);
+  entry.finished_at = new Date(now).toISOString();
+  entry.finished_epoch_ms = now;
+  entry.duration_ms = rounded(performance.now() - startedPerf);
+}
+
+function rounded(value) {
+  return Number(value.toFixed(3));
+}
+
+function previewText(value) {
+  return String(value || "").slice(0, 120);
+}
+
+function resetRequestState() {
+  requestCount = 0;
+  recentRequests.length = 0;
+}
+
+function applyConfig(updates) {
+  if (!updates || typeof updates !== "object") return;
+  assignString(updates, "response_text");
+  assignNonNegativeInteger(updates, "first_token_delay_ms");
+  assignNonNegativeInteger(updates, "chunk_delay_ms");
+  assignNonNegativeInteger(updates, "chunk_count");
+  assignNonNegativeInteger(updates, "fail_first_n");
+  assignNonNegativeInteger(updates, "fail_every_n");
+  assignNonNegativeInteger(updates, "request_log_limit");
+  if (updates.fault_status !== undefined) {
+    const parsed = Number.parseInt(String(updates.fault_status), 10);
+    if (Number.isInteger(parsed) && parsed >= 400 && parsed <= 599) config.fault_status = parsed;
+  }
+  assignBoolean(updates, "fail_after_first_chunk");
+  assignBoolean(updates, "dynamic_response");
+}
+
+function assignString(updates, key) {
+  if (updates[key] !== undefined) config[key] = String(updates[key]);
+}
+
+function assignNonNegativeInteger(updates, key) {
+  if (updates[key] === undefined) return;
+  const parsed = Number.parseInt(String(updates[key]), 10);
+  if (Number.isInteger(parsed) && parsed >= 0) config[key] = parsed;
+}
+
+function assignBoolean(updates, key) {
+  if (updates[key] === undefined) return;
+  config[key] = bool(updates[key], config[key]);
+}
@@ -72,6 +72,7 @@ export async function writeResult(paths, result) {
 }

 export async function loadEnvFiles(paths = ["skills/.env", "skills/.env.local"]) {
+  const processEnvKeys = new Set(Object.keys(env));
  for (const path of paths) {
    let text = "";
    try {
@@ -86,7 +87,7 @@ export async function loadEnvFiles(paths = ["skills/.env", "skills/.env.local"])
      if (equals <= 0) continue;
      const key = trimmed.slice(0, equals).trim();
      const value = trimmed.slice(equals + 1).trim().replace(/^["']|["']$/g, "");
-      if (!(key in env)) env[key] = value;
+      if (!processEnvKeys.has(key)) env[key] = value;
    }
  }
 }
@@ -54,6 +54,7 @@ const debugChatSessionType = env.LANGBOT_E2E_DEBUG_CHAT_SESSION_TYPE || "person"
 const pipelineConfigDiagnosticPath = resolve(paths.evidenceDir, "pipeline-config-diagnostic.json");
 const debugChatResetDiagnosticPath = resolve(paths.evidenceDir, "debug-chat-reset-diagnostic.json");
 const pipelineConfigRestoreDiagnosticPath = resolve(paths.evidenceDir, "pipeline-config-restore-diagnostic.json");
+const metricsPath = resolve(paths.evidenceDir, "metrics.json");
 const startedAt = new Date();

 let browser;
@@ -80,10 +81,11 @@ let result = {
    console_log: paths.consoleLog,
    network_log: paths.networkLog,
    screenshot: paths.screenshot,
+    metrics_json: metricsPath,
    automation_result_json: paths.automationResultJson,
    result_json: paths.resultJson,
  },
-  evidence_collected: ["ui", "screenshot", "console", "network"],
+  evidence_collected: ["ui", "screenshot", "console", "network", "metrics"],
 };

 function boolFromEnv(value, defaultValue) {
@@ -103,6 +105,29 @@ function parseJsonEnv(key, fallback) {
  }
 }

+function positiveNumberEnv(key, fallback) {
+  const value = Number(env[key] || "");
+  return Number.isFinite(value) && value >= 0 ? value : fallback;
+}
+
+function percentile(values, percentileValue) {
+  if (values.length === 0) return 0;
+  const sorted = [...values].sort((a, b) => a - b);
+  const index = Math.min(sorted.length - 1, Math.ceil((percentileValue / 100) * sorted.length) - 1);
+  return Number(sorted[index].toFixed(3));
+}
+
+function stats(values) {
+  if (values.length === 0) return { min: 0, p50: 0, p95: 0, p99: 0, max: 0 };
+  return {
+    min: Number(Math.min(...values).toFixed(3)),
+    p50: percentile(values, 50),
+    p95: percentile(values, 95),
+    p99: percentile(values, 99),
+    max: Number(Math.max(...values).toFixed(3)),
+  };
+}
+
 function promptStepsFromEnv() {
  const rawSteps = parseJsonEnv("LANGBOT_E2E_PROMPTS_JSON", null);
  if (rawSteps === null) {
@@ -658,6 +683,7 @@ try {
      } else {
        for (let index = 0; index < promptSteps.length; index += 1) {
          const step = promptSteps[index];
+          const promptStartedAt = Date.now();
          const chatResult = await runDebugChatPrompt(page, {
            prompt: step.prompt,
            expectedText: step.expectedText,
@@ -665,11 +691,13 @@ try {
            imagePath: index === 0 ? imagePath : "",
            failureSignals: failureSignals.length > 0 ? failureSignals : undefined,
          });
+          const promptDurationMs = Date.now() - promptStartedAt;
          result.chat_results.push({
            index,
            expected_text: step.expectedText,
            status: chatResult.status,
            reason: chatResult.reason,
+            response_duration_ms: promptDurationMs,
            min_expected_count: chatResult.min_expected_count,
            final_count: chatResult.final_count,
            before_assistant_expected_count: chatResult.before_assistant_expected_count,
@@ -714,6 +742,56 @@ try {
  const finishedAt = new Date();
  result.finished_at = finishedAt.toISOString();
  result.finished_at_local = localIsoWithOffset(finishedAt);
+  result.duration_ms = finishedAt.getTime() - startedAt.getTime();
+  const responseDurations = result.chat_results
+    .map((item) => item.response_duration_ms)
+    .filter((value) => Number.isFinite(value));
+  const passedPrompts = result.chat_results.filter((item) => item.status === "pass").length;
+  const attemptedPrompts = result.chat_results.length;
+  const errorRate = attemptedPrompts === 0 ? 1 : Number(((attemptedPrompts - passedPrompts) / attemptedPrompts).toFixed(4));
+  const responseStats = stats(responseDurations);
+  const responseP95BudgetMs = positiveNumberEnv(
+    "LANGBOT_E2E_DEBUG_CHAT_RESPONSE_P95_MS",
+    positiveNumberEnv("LANGBOT_DEBUG_CHAT_RESPONSE_P95_MS", safeResponseTimeoutMs),
+  );
+  const maxErrorRate = positiveNumberEnv("LANGBOT_E2E_DEBUG_CHAT_MAX_ERROR_RATE", 0);
+  const metrics = {
+    probe: caseId,
+    url: result.url,
+    prompt_count: result.prompt_count,
+    attempted_prompt_count: attemptedPrompts,
+    passed_prompt_count: passedPrompts,
+    error_rate: errorRate,
+    response_duration_ms: responseStats,
+    total_duration_ms: result.duration_ms,
+    chat_results: result.chat_results,
+  };
+  result.metrics_summary = {
+    prompt_count: metrics.prompt_count,
+    attempted_prompt_count: metrics.attempted_prompt_count,
+    passed_prompt_count: metrics.passed_prompt_count,
+    error_rate: metrics.error_rate,
+    response_p50_ms: metrics.response_duration_ms.p50,
+    response_p95_ms: metrics.response_duration_ms.p95,
+    total_duration_ms: metrics.total_duration_ms,
+  };
+  result.thresholds_summary = {
+    response_p95_ms: {
+      actual: metrics.response_duration_ms.p95,
+      max: responseP95BudgetMs,
+      pass: attemptedPrompts > 0 && metrics.response_duration_ms.p95 <= responseP95BudgetMs,
+    },
+    error_rate: {
+      actual: metrics.error_rate,
+      max: maxErrorRate,
+      pass: metrics.error_rate <= maxErrorRate,
+    },
+  };
+  await writeFile(metricsPath, `${JSON.stringify(metrics, null, 2)}\n`, "utf8");
+  if (result.status === "pass" && !Object.values(result.thresholds_summary).every((item) => item.pass)) {
+    result.status = "fail";
+    result.reason = "Debug Chat performance breached response latency or error-rate thresholds.";
+  }
  const existingEvidence = {};
  for (const [key, value] of Object.entries(result.evidence)) {
    if (typeof value !== "string") continue;