mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-08-09 20:50:58 +00:00
feat: MCP server + in-repo skills (agent-friendly platform) (#2269)
* feat(api): support global API key from config.yaml (api.global_api_key) Accept a config-defined global API key anywhere a web-UI key is accepted (X-API-Key / Bearer), with no login session and no DB record. Useful for automated deployments and AI agents (HTTP API + MCP). Defaults to empty (disabled); does not require the lbk_ prefix. - templates/config.yaml: add api.global_api_key with security notes - service/apikey.py: verify_api_key checks global key first (constant-time) - docs/API_KEY_AUTH.md: document the global key + security guidance - tests: cover global-key match, prefix-free, fallback-to-db, disabled * feat(mcp): expose LangBot management as an MCP server at /mcp Add an MCP (Model Context Protocol) server so external AI agents can manage a LangBot instance. Reuses the same API-key auth as the HTTP API (including the config.yaml global API key). - pkg/api/mcp/server.py: FastMCP server wrapping the service layer; 21 curated tools across system/bots/pipelines/models/knowledge/mcp-servers/skills - pkg/api/mcp/mount.py: ASGI dispatcher fronting Quart; authenticates /mcp requests with an API key, runs the streamable-HTTP session manager lifespan - controller/main.py: serve the wrapped ASGI app via hypercorn (was run_task) - web: new 'MCP' tab in the API integration dialog showing endpoint, auth, and client config; i18n for 8 locales - tests/manual/mcp_smoke.py: e2e check (401 unauth, list tools, call tools) Tool surface is intentionally curated (not all ~25 route groups) to keep the agent surface small, safe, and maintainable. Extend deliberately. * feat(skills): add in-repo skills/ as the single source of truth Migrate the agent skills + QA/e2e test harness from the (now archived) langbot-app/langbot-skills repo into LangBot/skills/, and add four new skills. Migrated: - langbot-plugin-dev, langbot-testing (e2e), langbot-env-setup, langbot-skills-maintenance, langbot-eba-adapter-dev - the bin/lbs CLI (src/, test/, scripts/, schemas/, qa-agent-docs/) New: - langbot-dev core backend + web development - langbot-deploy Docker/K8s deployment + config.yaml + global API key - langbot-mcp-ops operating the LangBot MCP server (/mcp) - langbot-space-ops operating the Space marketplace MCP server - src/cli.ts repoRoot(): recognize the skills assets root (skills.index.json + bin/lbs) so the CLI works when nested inside the LangBot repo - README.md: unified skill catalog; skills.index.json regenerated Parity with source verified: bin/lbs validate + node test suite match the source repo (only the uncommitted .lbpkg build-artifact fixture differs). * docs(agents): document agent-facing surfaces + API/MCP/skills sync rule * docs(readme): add 'Built for AI Agents' section across all locales Highlight MCP server, in-repo skills (single source of truth), AGENTS.md sync rule, and llms.txt. Cross-link LangBot Space MCP marketplace. * style(mcp): fix ruff format + prettier lint in MCP server and API panel * style(web): prettier format MCP i18n locale entries * docs(skills): note MCP instance control in dev/testing skills All development-guidance skills now point to the LangBot instance MCP server (/mcp) and the Space marketplace MCP server, reusing API keys.
This commit is contained in:
@@ -0,0 +1,476 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { resolve } from "node:path";
|
||||
import { env } from "node:process";
|
||||
import {
|
||||
createBrowser,
|
||||
ensureEvidence,
|
||||
evidencePaths,
|
||||
exitCode,
|
||||
localIsoWithOffset,
|
||||
safeScreenshot,
|
||||
writeResult,
|
||||
} from "./lib/langbot-e2e.mjs";
|
||||
|
||||
function loadEnvDefaults(path) {
|
||||
if (!existsSync(path)) return;
|
||||
for (const rawLine of readFileSync(path, "utf8").split(/\r?\n/)) {
|
||||
const line = rawLine.trim();
|
||||
if (!line || line.startsWith("#")) continue;
|
||||
const sep = line.indexOf("=");
|
||||
if (sep === -1) continue;
|
||||
const key = line.slice(0, sep).trim();
|
||||
if (env[key]) continue;
|
||||
env[key] = line.slice(sep + 1).trim().replace(/^["']|["']$/g, "");
|
||||
}
|
||||
}
|
||||
|
||||
function boolFromEnv(value, defaultValue) {
|
||||
if (value === undefined || value === "") return defaultValue;
|
||||
if (/^(0|false|no|off)$/i.test(value)) return false;
|
||||
if (/^(1|true|yes|on)$/i.test(value)) return true;
|
||||
return defaultValue;
|
||||
}
|
||||
|
||||
function firstEnv(...keys) {
|
||||
for (const key of keys) {
|
||||
if (env[key]) return env[key];
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
function redactMessage(text) {
|
||||
return String(text ?? "")
|
||||
.replace(/\bbearer\s+[A-Za-z0-9._~+/=-]{8,}/gi, "Bearer [redacted]")
|
||||
.replace(/\bsk-[A-Za-z0-9_-]{6,}\b/g, "[redacted]")
|
||||
.replace(/(api[_-]?key|authorization|credential|jwt|oauth|password|secret|token)\s*[:=]\s*["']?[^"',\s]+/gi, "$1=[redacted]");
|
||||
}
|
||||
|
||||
function isEnvironmentError(message) {
|
||||
return /Playwright is not installed|LANGBOT_FRONTEND_URL|LANGBOT_BACKEND_URL|ERR_CONNECTION_REFUSED|ECONNREFUSED|net::ERR_|fetch failed|timed out/i
|
||||
.test(message);
|
||||
}
|
||||
|
||||
loadEnvDefaults("skills/.env");
|
||||
loadEnvDefaults("skills/.env.local");
|
||||
|
||||
const caseId = env.LBS_CASE_ID || "agent-runner-release-preflight";
|
||||
const paths = evidencePaths(caseId);
|
||||
await ensureEvidence(paths);
|
||||
|
||||
const backendUrl = env.LANGBOT_BACKEND_URL || "";
|
||||
const frontendUrl = env.LANGBOT_FRONTEND_URL || backendUrl;
|
||||
const testModels = boolFromEnv(env.LANGBOT_PREFLIGHT_TEST_MODELS, true);
|
||||
const requireVision = boolFromEnv(env.LANGBOT_PREFLIGHT_REQUIRE_VISION, true);
|
||||
const diagnosticPath = resolve(paths.evidenceDir, "api-diagnostic.json");
|
||||
const startedAt = new Date();
|
||||
|
||||
const targets = [
|
||||
{
|
||||
id: "local-agent",
|
||||
expected_runner_id: "plugin:langbot/local-agent/default",
|
||||
pipeline_url: firstEnv("LANGBOT_LOCAL_AGENT_PIPELINE_URL"),
|
||||
pipeline_name: firstEnv("LANGBOT_LOCAL_AGENT_PIPELINE_NAME"),
|
||||
require_func_call_model: true,
|
||||
require_vision_model: requireVision,
|
||||
require_langbot_mcp: false,
|
||||
},
|
||||
{
|
||||
id: "acp-agent-runner",
|
||||
expected_runner_id: "plugin:langbot/acp-agent-runner/default",
|
||||
pipeline_url: firstEnv("LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL", "LANGBOT_AGENT_RUNNER_PIPELINE_URL"),
|
||||
pipeline_name: firstEnv("LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME", "LANGBOT_AGENT_RUNNER_PIPELINE_NAME"),
|
||||
require_func_call_model: false,
|
||||
require_vision_model: false,
|
||||
},
|
||||
];
|
||||
|
||||
let browser;
|
||||
const result = {
|
||||
source: "automation",
|
||||
case_id: caseId,
|
||||
run_id: paths.runId,
|
||||
started_at: startedAt.toISOString(),
|
||||
started_at_local: localIsoWithOffset(startedAt),
|
||||
finished_at: "",
|
||||
finished_at_local: "",
|
||||
status: "fail",
|
||||
reason: "",
|
||||
frontend_url: frontendUrl,
|
||||
backend_url: backendUrl,
|
||||
test_models: testModels,
|
||||
require_vision_model: requireVision,
|
||||
evidence: {
|
||||
console_log: paths.consoleLog,
|
||||
network_log: paths.networkLog,
|
||||
screenshot: paths.screenshot,
|
||||
api_diagnostic_json: diagnosticPath,
|
||||
automation_result_json: paths.automationResultJson,
|
||||
result_json: paths.resultJson,
|
||||
},
|
||||
evidence_collected: ["ui", "screenshot", "console", "network", "api_diagnostic"],
|
||||
};
|
||||
|
||||
async function run() {
|
||||
if (!backendUrl || !frontendUrl) {
|
||||
result.status = "env_issue";
|
||||
result.reason = "LANGBOT_FRONTEND_URL and LANGBOT_BACKEND_URL must be configured.";
|
||||
return;
|
||||
}
|
||||
|
||||
browser = await createBrowser(paths);
|
||||
const { page } = browser;
|
||||
await page.goto(frontendUrl, { waitUntil: "domcontentloaded" });
|
||||
await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {});
|
||||
|
||||
const diagnostic = await page.evaluate(async ({ backendUrl, targets, testModels }) => {
|
||||
const blockers = [];
|
||||
const envIssues = [];
|
||||
const warnings = [];
|
||||
const checks = [];
|
||||
|
||||
const addCheck = (name, status, detail = {}) => {
|
||||
checks.push({ name, status, ...detail });
|
||||
if (status === "blocked") blockers.push({ name, ...detail });
|
||||
if (status === "env_issue") envIssues.push({ name, ...detail });
|
||||
};
|
||||
const safeMessage = (value) => String(value ?? "")
|
||||
.replace(/\bbearer\s+[A-Za-z0-9._~+/=-]{8,}/gi, "Bearer [redacted]")
|
||||
.replace(/\bsk-[A-Za-z0-9_-]{6,}\b/g, "[redacted]")
|
||||
.replace(/(api[_-]?key|authorization|credential|jwt|oauth|password|secret|token)\s*[:=]\s*["']?[^"',\s]+/gi, "$1=[redacted]");
|
||||
|
||||
const token = localStorage.getItem("token");
|
||||
if (!token) {
|
||||
addCheck("browser-auth", "blocked", { reason: "Browser profile has no localStorage token." });
|
||||
return { authenticated: false, blockers, env_issues: envIssues, warnings, checks };
|
||||
}
|
||||
|
||||
const headers = {
|
||||
Authorization: `Bearer ${token}`,
|
||||
"Content-Type": "application/json",
|
||||
};
|
||||
const getJson = async (path) => {
|
||||
const response = await fetch(`${backendUrl}${path}`, { headers });
|
||||
return {
|
||||
status: response.status,
|
||||
json: await response.json().catch(() => ({})),
|
||||
};
|
||||
};
|
||||
const postJson = async (path, body) => {
|
||||
const response = await fetch(`${backendUrl}${path}`, {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
return {
|
||||
status: response.status,
|
||||
json: await response.json().catch(() => ({})),
|
||||
};
|
||||
};
|
||||
|
||||
const tokenCheck = await getJson("/api/v1/user/check-token");
|
||||
addCheck(
|
||||
"browser-auth",
|
||||
tokenCheck.status < 400 && (tokenCheck.json.code ?? 0) === 0 ? "pass" : "blocked",
|
||||
{ http_status: tokenCheck.status, code: tokenCheck.json.code ?? null, reason: safeMessage(tokenCheck.json.msg || "") },
|
||||
);
|
||||
|
||||
const systemInfo = await getJson("/api/v1/system/info");
|
||||
addCheck(
|
||||
"backend-system-info",
|
||||
systemInfo.status < 400 ? "pass" : "env_issue",
|
||||
{
|
||||
http_status: systemInfo.status,
|
||||
version: systemInfo.json.data?.version || systemInfo.json.data?.system?.version || "",
|
||||
},
|
||||
);
|
||||
|
||||
const pluginSystem = await getJson("/api/v1/system/status/plugin-system");
|
||||
addCheck(
|
||||
"plugin-system",
|
||||
pluginSystem.status < 400 && (pluginSystem.json.code ?? 0) === 0 ? "pass" : "env_issue",
|
||||
{
|
||||
http_status: pluginSystem.status,
|
||||
code: pluginSystem.json.code ?? null,
|
||||
status: pluginSystem.json.data?.status || pluginSystem.json.data?.state || "",
|
||||
reason: safeMessage(pluginSystem.json.msg || ""),
|
||||
},
|
||||
);
|
||||
|
||||
const boxStatus = await getJson("/api/v1/box/status");
|
||||
addCheck(
|
||||
"box-runtime",
|
||||
boxStatus.status < 400 && (boxStatus.json.code ?? 0) === 0 ? "pass" : "env_issue",
|
||||
{
|
||||
http_status: boxStatus.status,
|
||||
code: boxStatus.json.code ?? null,
|
||||
status: boxStatus.json.data?.status || "",
|
||||
backend: boxStatus.json.data?.backend || "",
|
||||
reason: safeMessage(boxStatus.json.msg || ""),
|
||||
},
|
||||
);
|
||||
|
||||
const plugins = await getJson("/api/v1/plugins");
|
||||
const installedPluginIds = (plugins.json.data?.plugins || [])
|
||||
.map((plugin) => {
|
||||
const metadata = plugin.manifest?.manifest?.metadata || plugin.manifest?.metadata || plugin.metadata || {};
|
||||
return metadata.author && metadata.name ? `${metadata.author}/${metadata.name}` : "";
|
||||
})
|
||||
.filter(Boolean);
|
||||
const requiredPlugins = ["langbot/local-agent", "langbot/acp-agent-runner", "qa/plugin-smoke"];
|
||||
const pluginPresence = Object.fromEntries(requiredPlugins.map((id) => [id, installedPluginIds.includes(id)]));
|
||||
for (const [id, present] of Object.entries(pluginPresence)) {
|
||||
addCheck(`plugin:${id}`, present ? "pass" : "blocked", { plugin_id: id, reason: present ? "" : "Required plugin is not listed by /api/v1/plugins." });
|
||||
}
|
||||
|
||||
const tools = await getJson("/api/v1/tools");
|
||||
const toolNames = (tools.json.data?.tools || [])
|
||||
.map((tool) => tool.name || tool.tool_name || tool.function?.name || "")
|
||||
.filter(Boolean)
|
||||
.sort();
|
||||
addCheck(
|
||||
"tool:qa_plugin_echo",
|
||||
toolNames.includes("qa_plugin_echo") ? "pass" : "blocked",
|
||||
{ reason: toolNames.includes("qa_plugin_echo") ? "" : "qa-plugin-smoke tool qa_plugin_echo is not exposed through /api/v1/tools." },
|
||||
);
|
||||
if (!toolNames.includes("qa_mcp_echo")) {
|
||||
warnings.push({
|
||||
name: "tool:qa_mcp_echo",
|
||||
reason: "qa_mcp_echo is not currently exposed. This is acceptable before mcp-stdio-register, but mcp-stdio-tool-call must run after registration.",
|
||||
});
|
||||
}
|
||||
|
||||
const modelResponse = await getJson("/api/v1/provider/models/llm");
|
||||
const models = (modelResponse.json.data?.models || []).map((model) => ({
|
||||
uuid: model.uuid,
|
||||
name: model.name,
|
||||
abilities: Array.isArray(model.abilities) ? model.abilities : [],
|
||||
provider_uuid: model.provider_uuid || model.provider?.uuid || "",
|
||||
provider_name: model.provider_name || model.provider?.name || "",
|
||||
requester: model.requester || model.provider?.requester || "",
|
||||
}));
|
||||
addCheck(
|
||||
"llm-model-list",
|
||||
modelResponse.status < 400 && (modelResponse.json.code ?? 0) === 0 ? "pass" : "env_issue",
|
||||
{ http_status: modelResponse.status, model_count: models.length, reason: safeMessage(modelResponse.json.msg || "") },
|
||||
);
|
||||
const modelById = new Map(models.map((model) => [model.uuid, model]));
|
||||
|
||||
const pipelineList = await getJson("/api/v1/pipelines");
|
||||
const pipelines = pipelineList.json.data?.pipelines || [];
|
||||
addCheck(
|
||||
"pipeline-list",
|
||||
pipelineList.status < 400 && (pipelineList.json.code ?? 0) === 0 ? "pass" : "blocked",
|
||||
{ http_status: pipelineList.status, pipeline_count: pipelines.length, reason: safeMessage(pipelineList.json.msg || "") },
|
||||
);
|
||||
|
||||
const resolvedPipelines = [];
|
||||
const modelTested = new Set();
|
||||
for (const target of targets) {
|
||||
let pipelineId = "";
|
||||
let matchedBy = "";
|
||||
if (target.pipeline_url) {
|
||||
try {
|
||||
pipelineId = new URL(target.pipeline_url).searchParams.get("id") || "";
|
||||
matchedBy = pipelineId ? "url" : "";
|
||||
} catch {
|
||||
pipelineId = "";
|
||||
}
|
||||
}
|
||||
if (!pipelineId && target.pipeline_name) {
|
||||
const match = pipelines.find((pipeline) => pipeline.name === target.pipeline_name);
|
||||
if (match) {
|
||||
pipelineId = match.uuid;
|
||||
matchedBy = "name";
|
||||
}
|
||||
}
|
||||
if (!pipelineId) {
|
||||
addCheck(`pipeline:${target.id}`, "blocked", {
|
||||
target: target.id,
|
||||
reason: "Required pipeline env is missing or could not resolve to a pipeline id.",
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const response = await getJson(`/api/v1/pipelines/${encodeURIComponent(pipelineId)}`);
|
||||
const pipeline = response.json.data?.pipeline;
|
||||
if (response.status >= 400 || !pipeline) {
|
||||
addCheck(`pipeline:${target.id}`, "blocked", {
|
||||
target: target.id,
|
||||
pipeline_id: pipelineId,
|
||||
http_status: response.status,
|
||||
reason: safeMessage(response.json.msg || "Could not load pipeline."),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const config = pipeline.config || {};
|
||||
const aiConfig = config.ai && typeof config.ai === "object" ? config.ai : {};
|
||||
const runner = aiConfig.runner && typeof aiConfig.runner === "object" ? aiConfig.runner : {};
|
||||
const runnerId = runner.id || runner.runner || "";
|
||||
const runnerConfigs = aiConfig.runner_config && typeof aiConfig.runner_config === "object" ? aiConfig.runner_config : {};
|
||||
const runnerConfig = runnerConfigs[runnerId] && typeof runnerConfigs[runnerId] === "object" ? runnerConfigs[runnerId] : {};
|
||||
const pipelineSummary = {
|
||||
target: target.id,
|
||||
pipeline_id: pipelineId,
|
||||
pipeline_name: pipeline.name,
|
||||
matched_by: matchedBy,
|
||||
runner_id: runnerId,
|
||||
expected_runner_id: target.expected_runner_id,
|
||||
runner_config_keys: Object.keys(runnerConfig).sort(),
|
||||
};
|
||||
resolvedPipelines.push(pipelineSummary);
|
||||
|
||||
addCheck(
|
||||
`pipeline:${target.id}:runner`,
|
||||
runnerId === target.expected_runner_id ? "pass" : "blocked",
|
||||
{
|
||||
...pipelineSummary,
|
||||
reason: runnerId === target.expected_runner_id ? "" : `Expected ${target.expected_runner_id}, got ${runnerId || "<missing>"}.`,
|
||||
},
|
||||
);
|
||||
|
||||
if (target.require_func_call_model || target.require_vision_model || (testModels && target.id === "local-agent")) {
|
||||
const modelConfig = runnerConfig.model;
|
||||
const primaryModelId = typeof modelConfig === "string"
|
||||
? modelConfig
|
||||
: modelConfig && typeof modelConfig === "object"
|
||||
? modelConfig.primary || ""
|
||||
: "";
|
||||
if (!primaryModelId) {
|
||||
addCheck(`pipeline:${target.id}:primary-model`, "blocked", {
|
||||
...pipelineSummary,
|
||||
reason: "Local-agent runner config has no primary model.",
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const model = modelById.get(primaryModelId);
|
||||
if (!model) {
|
||||
addCheck(`pipeline:${target.id}:primary-model`, "blocked", {
|
||||
...pipelineSummary,
|
||||
model_uuid: primaryModelId,
|
||||
reason: "Primary model is not listed by /api/v1/provider/models/llm.",
|
||||
});
|
||||
continue;
|
||||
}
|
||||
addCheck(`pipeline:${target.id}:primary-model`, "pass", {
|
||||
...pipelineSummary,
|
||||
model: {
|
||||
uuid: model.uuid,
|
||||
name: model.name,
|
||||
abilities: model.abilities,
|
||||
provider_name: model.provider_name,
|
||||
requester: model.requester,
|
||||
},
|
||||
});
|
||||
if (target.require_func_call_model) {
|
||||
addCheck(
|
||||
`pipeline:${target.id}:func-call-model`,
|
||||
model.abilities.includes("func_call") ? "pass" : "env_issue",
|
||||
{
|
||||
model_uuid: model.uuid,
|
||||
model_name: model.name,
|
||||
abilities: model.abilities,
|
||||
reason: model.abilities.includes("func_call") ? "" : "Release gate includes tool-call cases; the local-agent primary model must advertise func_call.",
|
||||
},
|
||||
);
|
||||
}
|
||||
if (target.require_vision_model) {
|
||||
addCheck(
|
||||
`pipeline:${target.id}:vision-model`,
|
||||
model.abilities.includes("vision") ? "pass" : "env_issue",
|
||||
{
|
||||
model_uuid: model.uuid,
|
||||
model_name: model.name,
|
||||
abilities: model.abilities,
|
||||
reason: model.abilities.includes("vision") ? "" : "Release gate includes multimodal cases; the local-agent primary model must advertise vision.",
|
||||
},
|
||||
);
|
||||
}
|
||||
if (testModels && !modelTested.has(model.uuid)) {
|
||||
modelTested.add(model.uuid);
|
||||
const modelTest = await postJson(`/api/v1/provider/models/llm/${encodeURIComponent(model.uuid)}/test`, { extra_args: {} });
|
||||
const passed = modelTest.status < 400 && (modelTest.json.code ?? 0) === 0;
|
||||
addCheck(
|
||||
`model-test:${model.name}`,
|
||||
passed ? "pass" : "env_issue",
|
||||
{
|
||||
model_uuid: model.uuid,
|
||||
model_name: model.name,
|
||||
http_status: modelTest.status,
|
||||
code: modelTest.json.code ?? null,
|
||||
reason: passed ? "" : safeMessage(modelTest.json.msg || modelTest.json.message || "Model test failed."),
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
authenticated: true,
|
||||
blockers,
|
||||
env_issues: envIssues,
|
||||
warnings,
|
||||
checks,
|
||||
resolved_pipelines: resolvedPipelines,
|
||||
tools: {
|
||||
required: ["qa_plugin_echo"],
|
||||
optional_before_register: ["qa_mcp_echo"],
|
||||
present: toolNames.filter((name) => ["qa_plugin_echo", "qa_mcp_echo"].includes(name)),
|
||||
},
|
||||
models,
|
||||
};
|
||||
}, { backendUrl, targets, testModels });
|
||||
|
||||
diagnostic.blockers = (diagnostic.blockers || []).map((item) => ({ ...item, reason: redactMessage(item.reason || "") }));
|
||||
diagnostic.env_issues = (diagnostic.env_issues || []).map((item) => ({ ...item, reason: redactMessage(item.reason || "") }));
|
||||
await writeFile(diagnosticPath, `${JSON.stringify(diagnostic, null, 2)}\n`, "utf8");
|
||||
await safeScreenshot(page, paths.screenshot);
|
||||
|
||||
const blockers = diagnostic.blockers || [];
|
||||
const envIssues = diagnostic.env_issues || [];
|
||||
if (blockers.length > 0) {
|
||||
result.status = "blocked";
|
||||
result.reason = `Preflight blocked: ${blockers.map((item) => item.name).join(", ")}`;
|
||||
} else if (envIssues.length > 0) {
|
||||
result.status = "env_issue";
|
||||
result.reason = `Preflight environment issue: ${envIssues.map((item) => item.name).join(", ")}`;
|
||||
} else {
|
||||
result.status = "pass";
|
||||
result.reason = "Release gate preflight passed: auth, plugin runtime, required pipelines, runner ids, tools, and local-agent model checks are ready.";
|
||||
}
|
||||
result.check_count = Array.isArray(diagnostic.checks) ? diagnostic.checks.length : 0;
|
||||
result.warning_count = Array.isArray(diagnostic.warnings) ? diagnostic.warnings.length : 0;
|
||||
}
|
||||
|
||||
try {
|
||||
await run();
|
||||
} catch (error) {
|
||||
const message = redactMessage(error instanceof Error ? error.message : String(error));
|
||||
result.status = isEnvironmentError(message) ? "env_issue" : "fail";
|
||||
result.reason = message;
|
||||
await writeFile(diagnosticPath, `${JSON.stringify({
|
||||
authenticated: false,
|
||||
blockers: [],
|
||||
env_issues: result.status === "env_issue" ? [{ name: "preflight-runtime", reason: message }] : [],
|
||||
warnings: [],
|
||||
checks: [
|
||||
{
|
||||
name: "preflight-runtime",
|
||||
status: result.status,
|
||||
reason: message,
|
||||
},
|
||||
],
|
||||
}, null, 2)}\n`, "utf8").catch(() => {});
|
||||
} finally {
|
||||
if (browser) await browser.close().catch(() => {});
|
||||
const finishedAt = new Date();
|
||||
result.finished_at = finishedAt.toISOString();
|
||||
result.finished_at_local = localIsoWithOffset(finishedAt);
|
||||
await writeResult(paths, result);
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
}
|
||||
|
||||
process.exit(exitCode(result.status));
|
||||
@@ -0,0 +1,263 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import { resolve } from "node:path";
|
||||
import { env } from "node:process";
|
||||
import {
|
||||
apiJson,
|
||||
ensureEvidence,
|
||||
evidencePaths,
|
||||
loadEnvFiles,
|
||||
resetAndAuthLocalUser,
|
||||
writeResult,
|
||||
} from "./lib/langbot-e2e.mjs";
|
||||
|
||||
const RUNNER_ID = "plugin:langbot/acp-agent-runner/default";
|
||||
const DEFAULT_PIPELINE_NAME = "Agent QA ACP Claude Debug Chat";
|
||||
const DEFAULT_LOCAL_PASSWORD = "LangBotE2ELocalPass!2026";
|
||||
const caseId = "ensure-acp-agent-runner-pipeline";
|
||||
|
||||
await loadEnvFiles();
|
||||
const paths = evidencePaths(caseId);
|
||||
await ensureEvidence(paths);
|
||||
|
||||
const writeEnv = process.argv.includes("--write-env");
|
||||
const frontendUrl = env.LANGBOT_FRONTEND_URL || "";
|
||||
const backendUrl = env.LANGBOT_BACKEND_URL || "";
|
||||
const pipelineName = env.LANGBOT_E2E_CREATE_PIPELINE_NAME || env.LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME || DEFAULT_PIPELINE_NAME;
|
||||
const sshTarget = env.LANGBOT_ACP_AGENT_RUNNER_SSH_TARGET || "yhh@101.34.71.12";
|
||||
const sshConnectTimeout = env.LANGBOT_ACP_AGENT_RUNNER_SSH_CONNECT_TIMEOUT || "8";
|
||||
const sshPort = env.LANGBOT_ACP_AGENT_RUNNER_SSH_PORT || "22";
|
||||
const sshIdentityFile = env.LANGBOT_ACP_AGENT_RUNNER_SSH_IDENTITY_FILE || "";
|
||||
const sshExtraOptions = env.LANGBOT_ACP_AGENT_RUNNER_SSH_EXTRA_OPTIONS || "";
|
||||
const remoteWorkspace = env.LANGBOT_ACP_AGENT_RUNNER_REMOTE_WORKSPACE || "/home/yhh/langbot-e2e/acp-workspace";
|
||||
const envLocalPath = resolve("skills/.env.local");
|
||||
|
||||
const result = {
|
||||
source: "automation",
|
||||
case_id: caseId,
|
||||
run_id: paths.runId,
|
||||
status: "fail",
|
||||
reason: "",
|
||||
frontend_url: frontendUrl,
|
||||
backend_url: backendUrl,
|
||||
pipeline_name: pipelineName,
|
||||
pipeline_id: "",
|
||||
pipeline_url: "",
|
||||
runner_id: RUNNER_ID,
|
||||
ssh_target: sshTarget,
|
||||
ssh_port: sshPort,
|
||||
remote_workspace: remoteWorkspace,
|
||||
wrote_env: false,
|
||||
auth: null,
|
||||
evidence: {
|
||||
automation_result_json: paths.automationResultJson,
|
||||
result_json: paths.resultJson,
|
||||
},
|
||||
evidence_collected: ["api_diagnostic"],
|
||||
};
|
||||
|
||||
try {
|
||||
if (!frontendUrl) throw new Error("LANGBOT_FRONTEND_URL is not configured.");
|
||||
if (!backendUrl) throw new Error("LANGBOT_BACKEND_URL is not configured.");
|
||||
|
||||
const user = env.LANGBOT_E2E_LOGIN_USER || "";
|
||||
const password = env.LANGBOT_E2E_LOGIN_PASSWORD || DEFAULT_LOCAL_PASSWORD;
|
||||
if (!user) {
|
||||
throw new Error("LANGBOT_E2E_LOGIN_USER is required so this setup can create/update the pipeline via backend API.");
|
||||
}
|
||||
|
||||
const auth = await resetAndAuthLocalUser({ backendUrl, user, password });
|
||||
result.auth = {
|
||||
source: "local_recovery_login",
|
||||
user,
|
||||
backend_token_check: auth.check,
|
||||
};
|
||||
|
||||
const runnerConfig = {
|
||||
provider: "claude-code",
|
||||
location: "remote-ssh",
|
||||
workspace: remoteWorkspace,
|
||||
"ssh-target": sshTarget,
|
||||
"ssh-port": Number.parseInt(sshPort, 10),
|
||||
"ssh-identity-file": sshIdentityFile,
|
||||
"ssh-connect-timeout": Number.parseInt(sshConnectTimeout, 10),
|
||||
"ssh-extra-options": sshExtraOptions,
|
||||
"langbot-assets-enabled": true,
|
||||
"mcp-bridge-request-timeout": 90,
|
||||
"reuse-session": false,
|
||||
"create-session-if-missing": true,
|
||||
"append-run-scope-prompt": true,
|
||||
"startup-timeout": 30,
|
||||
"initialize-timeout": 120,
|
||||
timeout: 300,
|
||||
};
|
||||
|
||||
const prepared = await ensurePipeline({
|
||||
backendUrl,
|
||||
token: auth.token,
|
||||
pipelineName,
|
||||
runnerId: RUNNER_ID,
|
||||
runnerConfig,
|
||||
});
|
||||
Object.assign(result, prepared);
|
||||
if (result.pipeline_id) {
|
||||
result.pipeline_url = `${frontendUrl.replace(/\/$/, "")}/home/pipelines?id=${encodeURIComponent(result.pipeline_id)}`;
|
||||
}
|
||||
|
||||
if (writeEnv && result.pipeline_id) {
|
||||
await upsertEnvLocal(envLocalPath, {
|
||||
LANGBOT_E2E_LOGIN_USER: user,
|
||||
LANGBOT_ACP_AGENT_RUNNER_SSH_TARGET: sshTarget,
|
||||
LANGBOT_ACP_AGENT_RUNNER_SSH_PORT: sshPort,
|
||||
LANGBOT_ACP_AGENT_RUNNER_SSH_IDENTITY_FILE: sshIdentityFile,
|
||||
LANGBOT_ACP_AGENT_RUNNER_SSH_EXTRA_OPTIONS: sshExtraOptions,
|
||||
LANGBOT_ACP_AGENT_RUNNER_REMOTE_WORKSPACE: remoteWorkspace,
|
||||
LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL: result.pipeline_url,
|
||||
LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME: result.pipeline_name || pipelineName,
|
||||
});
|
||||
result.wrote_env = true;
|
||||
}
|
||||
} catch (error) {
|
||||
result.reason = result.reason || error.message;
|
||||
} finally {
|
||||
await writeResult(paths, result);
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
}
|
||||
|
||||
process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1);
|
||||
|
||||
async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runnerConfig }) {
|
||||
const pipelineList = await apiJson(backendUrl, "/api/v1/pipelines", { token });
|
||||
if (isApiFailure(pipelineList)) {
|
||||
return {
|
||||
status: "fail",
|
||||
reason: pipelineList.json.msg || "Failed to list pipelines.",
|
||||
list_status: pipelineList.status,
|
||||
};
|
||||
}
|
||||
|
||||
const pipelines = pipelineList.json.data?.pipelines || [];
|
||||
let pipeline = pipelines.find((item) => item.name === pipelineName) || null;
|
||||
let created = false;
|
||||
|
||||
if (!pipeline) {
|
||||
const createdResponse = await apiJson(backendUrl, "/api/v1/pipelines", {
|
||||
method: "POST",
|
||||
token,
|
||||
body: {
|
||||
name: pipelineName,
|
||||
description: "Local QA pipeline for real ACP Claude AgentRunner Debug Chat smoke tests.",
|
||||
emoji: "QA",
|
||||
},
|
||||
});
|
||||
if (isApiFailure(createdResponse)) {
|
||||
return {
|
||||
status: "fail",
|
||||
reason: createdResponse.json.msg || "Failed to create pipeline.",
|
||||
create_status: createdResponse.status,
|
||||
};
|
||||
}
|
||||
const pipelineId = createdResponse.json.data?.uuid || "";
|
||||
const loaded = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipelineId)}`, { token });
|
||||
pipeline = loaded.json.data?.pipeline || null;
|
||||
created = true;
|
||||
}
|
||||
|
||||
if (!pipeline?.uuid) {
|
||||
return {
|
||||
status: "fail",
|
||||
reason: "Pipeline was not created or resolved.",
|
||||
};
|
||||
}
|
||||
|
||||
const loaded = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, { token });
|
||||
if (isApiFailure(loaded) || !loaded.json.data?.pipeline) {
|
||||
return {
|
||||
status: "fail",
|
||||
reason: loaded.json.msg || "Failed to load pipeline.",
|
||||
get_status: loaded.status,
|
||||
pipeline_id: pipeline.uuid,
|
||||
};
|
||||
}
|
||||
pipeline = loaded.json.data.pipeline;
|
||||
|
||||
const config = pipeline.config && typeof pipeline.config === "object" ? pipeline.config : {};
|
||||
const ai = config.ai && typeof config.ai === "object" ? config.ai : {};
|
||||
const runnerConfigs = ai.runner_config && typeof ai.runner_config === "object" ? ai.runner_config : {};
|
||||
const updatedConfig = {
|
||||
...config,
|
||||
ai: {
|
||||
...ai,
|
||||
runner: {
|
||||
...(ai.runner && typeof ai.runner === "object" ? ai.runner : {}),
|
||||
id: runnerId,
|
||||
"expire-time": 0,
|
||||
},
|
||||
runner_config: {
|
||||
...runnerConfigs,
|
||||
[runnerId]: runnerConfig,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const updateResponse = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, {
|
||||
method: "PUT",
|
||||
token,
|
||||
body: {
|
||||
name: pipelineName,
|
||||
description: "Local QA pipeline for real ACP Claude AgentRunner Debug Chat smoke tests.",
|
||||
emoji: "QA",
|
||||
config: updatedConfig,
|
||||
},
|
||||
});
|
||||
if (isApiFailure(updateResponse)) {
|
||||
return {
|
||||
status: "fail",
|
||||
reason: updateResponse.json.msg || "Failed to update pipeline.",
|
||||
update_status: updateResponse.status,
|
||||
pipeline_id: pipeline.uuid,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
status: "pass",
|
||||
reason: created ? "ACP AgentRunner pipeline created and configured." : "ACP AgentRunner pipeline updated.",
|
||||
pipeline_id: pipeline.uuid,
|
||||
pipeline_name: pipelineName,
|
||||
created,
|
||||
updated: true,
|
||||
};
|
||||
}
|
||||
|
||||
function isApiFailure(response) {
|
||||
return response.status >= 400 || (response.json && response.json.code !== undefined && response.json.code !== 0);
|
||||
}
|
||||
|
||||
async function upsertEnvLocal(path, values) {
|
||||
let text = "";
|
||||
try {
|
||||
text = await readFile(path, "utf8");
|
||||
} catch {
|
||||
text = "";
|
||||
}
|
||||
const lines = text.split(/\r?\n/);
|
||||
const keys = new Set(Object.keys(values));
|
||||
const output = [];
|
||||
for (const line of lines) {
|
||||
const match = line.match(/^([A-Z][A-Z0-9_]*)=/);
|
||||
if (match && keys.has(match[1])) {
|
||||
output.push(`${match[1]}=${values[match[1]]}`);
|
||||
keys.delete(match[1]);
|
||||
} else if (line !== "" || output.length > 0) {
|
||||
output.push(line);
|
||||
}
|
||||
}
|
||||
if (keys.size > 0 && output.length > 0 && output[output.length - 1] !== "") {
|
||||
output.push("");
|
||||
}
|
||||
for (const key of keys) {
|
||||
output.push(`${key}=${values[key]}`);
|
||||
}
|
||||
await writeFile(path, `${output.join("\n").replace(/\n+$/, "")}\n`, "utf8");
|
||||
}
|
||||
@@ -0,0 +1,293 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import { resolve } from "node:path";
|
||||
import { env } from "node:process";
|
||||
import {
|
||||
apiJson,
|
||||
ensureEvidence,
|
||||
evidencePaths,
|
||||
loadEnvFiles,
|
||||
resetAndAuthLocalUser,
|
||||
writeResult,
|
||||
} from "./lib/langbot-e2e.mjs";
|
||||
|
||||
const caseId = env.LBS_CASE_ID || "ensure-langrag-sentinel-kb";
|
||||
|
||||
await loadEnvFiles();
|
||||
const paths = evidencePaths(caseId);
|
||||
await ensureEvidence(paths);
|
||||
|
||||
const backendUrl = env.LANGBOT_BACKEND_URL || "";
|
||||
const user = env.LANGBOT_E2E_LOGIN_USER || "";
|
||||
const password = env.LANGBOT_E2E_LOGIN_PASSWORD || "LangBotE2ELocalPass!2026";
|
||||
const expectedText = env.LANGBOT_E2E_EXPECTED_TEXT || "azalea-cobalt-7421";
|
||||
const query = env.LANGBOT_E2E_RETRIEVE_QUERY || "What is the local agent runner retrieval sentinel?";
|
||||
const writeEnv = process.argv.includes("--write-env");
|
||||
const checkOnly = process.argv.includes("--check-only");
|
||||
const envLocalPath = resolve("skills/.env.local");
|
||||
const kbName = env.LANGBOT_E2E_RAG_KB_NAME || "qa-local-agent-rag";
|
||||
const sentinelPath = resolve(env.LANGBOT_E2E_RAG_SENTINEL_DOC || "skills/langbot-testing/fixtures/rag/sentinel-doc.txt");
|
||||
const waitMs = Number(env.LANGBOT_E2E_RAG_WAIT_MS || 180_000);
|
||||
|
||||
const result = {
|
||||
source: "automation",
|
||||
case_id: caseId,
|
||||
run_id: paths.runId,
|
||||
status: "fail",
|
||||
reason: "",
|
||||
backend_url: backendUrl,
|
||||
expected_text: expectedText,
|
||||
query,
|
||||
kb_uuid: "",
|
||||
kb_name: "",
|
||||
kb_created: false,
|
||||
uploaded_file_id: "",
|
||||
store_task_id: "",
|
||||
embedding_model_uuid: "",
|
||||
engine_plugin_id: "",
|
||||
checked_bases: [],
|
||||
file_statuses: [],
|
||||
wrote_env: false,
|
||||
evidence: {
|
||||
automation_result_json: paths.automationResultJson,
|
||||
result_json: paths.resultJson,
|
||||
},
|
||||
evidence_collected: ["api_diagnostic"],
|
||||
};
|
||||
|
||||
try {
|
||||
if (!backendUrl) throw new Error("LANGBOT_BACKEND_URL is not configured.");
|
||||
if (!user) throw new Error("LANGBOT_E2E_LOGIN_USER is required.");
|
||||
|
||||
const auth = await resetAndAuthLocalUser({ backendUrl, user, password });
|
||||
const basesResponse = await apiJson(backendUrl, "/api/v1/knowledge/bases", { token: auth.token });
|
||||
if (basesResponse.status >= 400 || basesResponse.json.code !== 0) {
|
||||
throw new Error(basesResponse.json.msg || `Failed to list knowledge bases: HTTP ${basesResponse.status}.`);
|
||||
}
|
||||
|
||||
let bases = basesResponse.json.data?.bases || [];
|
||||
await findSentinelBase(backendUrl, auth.token, bases, result);
|
||||
|
||||
if (!result.kb_uuid && !checkOnly) {
|
||||
const targetBase = bases.find((base) => {
|
||||
const uuid = base.uuid || base.id || "";
|
||||
return (base.name || "") === kbName && !hasRetrieveFailure(result.checked_bases, uuid);
|
||||
});
|
||||
result.kb_uuid = targetBase?.uuid || targetBase?.id || "";
|
||||
result.kb_name = targetBase?.name || kbName;
|
||||
|
||||
if (!result.kb_uuid) {
|
||||
const setup = await createKnowledgeBase(backendUrl, auth.token, kbName);
|
||||
result.kb_uuid = setup.kbUuid;
|
||||
result.kb_name = kbName;
|
||||
result.kb_created = true;
|
||||
result.embedding_model_uuid = setup.embeddingModelUuid;
|
||||
result.engine_plugin_id = setup.enginePluginId;
|
||||
}
|
||||
|
||||
const upload = await uploadDocument(backendUrl, auth.token, sentinelPath);
|
||||
result.uploaded_file_id = upload.fileId;
|
||||
|
||||
const store = await apiJson(backendUrl, `/api/v1/knowledge/bases/${encodeURIComponent(result.kb_uuid)}/files`, {
|
||||
method: "POST",
|
||||
token: auth.token,
|
||||
body: { file_id: upload.fileId },
|
||||
});
|
||||
if (store.status >= 400 || store.json.code !== 0) {
|
||||
throw new Error(store.json.msg || `Failed to store file in knowledge base: HTTP ${store.status}.`);
|
||||
}
|
||||
result.store_task_id = store.json.data?.task_id || "";
|
||||
|
||||
const ready = await waitForSentinel(backendUrl, auth.token, result.kb_uuid, query, expectedText, waitMs);
|
||||
result.file_statuses = ready.fileStatuses;
|
||||
if (ready.matched) {
|
||||
result.checked_bases.push(ready.checked);
|
||||
}
|
||||
}
|
||||
|
||||
if (!result.kb_uuid) {
|
||||
result.status = "env_issue";
|
||||
result.reason = checkOnly
|
||||
? `No existing knowledge base retrieved expected sentinel: ${expectedText}`
|
||||
: `Could not create or verify LangRAG sentinel knowledge base: ${expectedText}`;
|
||||
} else {
|
||||
if (writeEnv) {
|
||||
await upsertEnvLocal(envLocalPath, {
|
||||
LANGBOT_LOCAL_AGENT_RAG_KB_UUID: result.kb_uuid,
|
||||
});
|
||||
result.wrote_env = true;
|
||||
}
|
||||
result.status = "pass";
|
||||
result.reason = `Found LangRAG sentinel knowledge base: ${result.kb_uuid}`;
|
||||
}
|
||||
} catch (error) {
|
||||
result.status = /not configured|required|No existing knowledge base/.test(error.message) ? "env_issue" : "fail";
|
||||
result.reason = error.message;
|
||||
} finally {
|
||||
await writeResult(paths, result);
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
}
|
||||
|
||||
process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1);
|
||||
|
||||
async function findSentinelBase(backendUrl, token, bases, result) {
|
||||
for (const base of bases) {
|
||||
const uuid = base.uuid || base.id || "";
|
||||
if (!uuid) continue;
|
||||
const checked = await retrieveSentinel(backendUrl, token, uuid, base.name || "", result.query, result.expected_text);
|
||||
result.checked_bases.push(checked);
|
||||
if (checked.matched) {
|
||||
result.kb_uuid = uuid;
|
||||
result.kb_name = checked.name;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function createKnowledgeBase(backendUrl, token, name) {
|
||||
const enginesResponse = await apiJson(backendUrl, "/api/v1/knowledge/engines", { token });
|
||||
if (enginesResponse.status >= 400 || enginesResponse.json.code !== 0) {
|
||||
throw new Error(enginesResponse.json.msg || `Failed to list knowledge engines: HTTP ${enginesResponse.status}.`);
|
||||
}
|
||||
const engines = enginesResponse.json.data?.engines || [];
|
||||
const engine = engines.find((item) => item.plugin_id === "langbot-team/LangRAG")
|
||||
|| engines.find((item) => JSON.stringify(item.name || item.label || "").includes("LangRAG"));
|
||||
const enginePluginId = engine?.plugin_id || "";
|
||||
if (!enginePluginId) throw new Error("LangRAG knowledge engine is not installed.");
|
||||
|
||||
const embeddingModelUuid = await pickEmbeddingModel(backendUrl, token);
|
||||
const create = await apiJson(backendUrl, "/api/v1/knowledge/bases", {
|
||||
method: "POST",
|
||||
token,
|
||||
body: {
|
||||
name,
|
||||
description: "Automated LangBot agent-runner RAG sentinel knowledge base.",
|
||||
knowledge_engine_plugin_id: enginePluginId,
|
||||
creation_settings: {
|
||||
embedding_model_uuid: embeddingModelUuid,
|
||||
index_type: "chunk",
|
||||
chunk_size: 512,
|
||||
overlap: 50,
|
||||
},
|
||||
retrieval_settings: {
|
||||
top_k: 5,
|
||||
search_type: "vector",
|
||||
query_rewrite: "off",
|
||||
rerank: "off",
|
||||
context_window: 0,
|
||||
},
|
||||
},
|
||||
});
|
||||
const kbUuid = create.json.data?.uuid || "";
|
||||
if (create.status >= 400 || create.json.code !== 0 || !kbUuid) {
|
||||
throw new Error(create.json.msg || `Failed to create knowledge base: HTTP ${create.status}.`);
|
||||
}
|
||||
return { kbUuid, embeddingModelUuid, enginePluginId };
|
||||
}
|
||||
|
||||
async function pickEmbeddingModel(backendUrl, token) {
|
||||
const configured = env.LANGBOT_LOCAL_AGENT_RAG_EMBEDDING_MODEL_UUID || env.LANGBOT_RAG_EMBEDDING_MODEL_UUID || "";
|
||||
if (configured) return configured;
|
||||
|
||||
const modelsResponse = await apiJson(backendUrl, "/api/v1/provider/models/embedding", { token });
|
||||
if (modelsResponse.status >= 400 || modelsResponse.json.code !== 0) {
|
||||
throw new Error(modelsResponse.json.msg || `Failed to list embedding models: HTTP ${modelsResponse.status}.`);
|
||||
}
|
||||
const models = modelsResponse.json.data?.models || [];
|
||||
const preferred = models.find((model) => /chroma|MiniLM/i.test(model.name || ""))
|
||||
|| models.find((model) => /text-embedding-3-small/i.test(model.name || ""))
|
||||
|| [...models].sort((a, b) => (a.prefered_ranking ?? 9999) - (b.prefered_ranking ?? 9999))[0];
|
||||
const uuid = preferred?.uuid || "";
|
||||
if (!uuid) throw new Error("No embedding model is configured.");
|
||||
return uuid;
|
||||
}
|
||||
|
||||
async function uploadDocument(backendUrl, token, path) {
|
||||
const bytes = await readFile(path);
|
||||
const form = new FormData();
|
||||
form.append("file", new Blob([bytes], { type: "text/plain" }), "sentinel-doc.txt");
|
||||
const response = await fetch(`${backendUrl.replace(/\/$/, "")}/api/v1/files/documents`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
},
|
||||
body: form,
|
||||
});
|
||||
const json = await response.json().catch(() => ({}));
|
||||
const fileId = json.data?.file_id || "";
|
||||
if (response.status >= 400 || json.code !== 0 || !fileId) {
|
||||
throw new Error(json.msg || `Failed to upload sentinel document: HTTP ${response.status}.`);
|
||||
}
|
||||
return { fileId };
|
||||
}
|
||||
|
||||
async function waitForSentinel(backendUrl, token, kbUuid, query, expectedText, timeoutMs) {
|
||||
const started = Date.now();
|
||||
let fileStatuses = [];
|
||||
let lastChecked = null;
|
||||
while (Date.now() - started < timeoutMs) {
|
||||
const files = await apiJson(backendUrl, `/api/v1/knowledge/bases/${encodeURIComponent(kbUuid)}/files`, { token });
|
||||
fileStatuses = files.json.data?.files || fileStatuses;
|
||||
lastChecked = await retrieveSentinel(backendUrl, token, kbUuid, kbName, query, expectedText);
|
||||
if (lastChecked.matched) {
|
||||
return { matched: true, fileStatuses, checked: lastChecked };
|
||||
}
|
||||
if (fileStatuses.some((item) => item.status === "failed")) break;
|
||||
await sleep(2_000);
|
||||
}
|
||||
result.reason = lastChecked?.msg
|
||||
|| `LangRAG sentinel was not retrievable within ${timeoutMs}ms; file statuses: ${JSON.stringify(fileStatuses)}`;
|
||||
result.kb_uuid = "";
|
||||
return { matched: false, fileStatuses, checked: lastChecked };
|
||||
}
|
||||
|
||||
async function retrieveSentinel(backendUrl, token, uuid, name, query, expectedText) {
|
||||
const retrieve = await apiJson(backendUrl, `/api/v1/knowledge/bases/${encodeURIComponent(uuid)}/retrieve`, {
|
||||
method: "POST",
|
||||
token,
|
||||
body: { query },
|
||||
});
|
||||
const text = JSON.stringify(retrieve.json.data?.results || []);
|
||||
return {
|
||||
uuid,
|
||||
name,
|
||||
http_status: retrieve.status,
|
||||
code: retrieve.json.code ?? null,
|
||||
msg: retrieve.json.msg || "",
|
||||
matched: text.includes(expectedText),
|
||||
};
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function hasRetrieveFailure(checkedBases, uuid) {
|
||||
const checked = checkedBases.find((item) => item.uuid === uuid);
|
||||
return checked && (checked.http_status >= 500 || (typeof checked.code === "number" && checked.code < 0));
|
||||
}
|
||||
|
||||
async function upsertEnvLocal(path, values) {
|
||||
let text = "";
|
||||
try {
|
||||
text = await readFile(path, "utf8");
|
||||
} catch {
|
||||
text = "";
|
||||
}
|
||||
const lines = text.split(/\r?\n/);
|
||||
const keys = new Set(Object.keys(values));
|
||||
const output = [];
|
||||
for (const line of lines) {
|
||||
const match = line.match(/^([A-Z][A-Z0-9_]*)=/);
|
||||
if (match && keys.has(match[1])) {
|
||||
output.push(`${match[1]}=${values[match[1]]}`);
|
||||
keys.delete(match[1]);
|
||||
} else if (line !== "" || output.length > 0) {
|
||||
output.push(line);
|
||||
}
|
||||
}
|
||||
if (keys.size > 0 && output.length > 0 && output[output.length - 1] !== "") output.push("");
|
||||
for (const key of keys) output.push(`${key}=${values[key]}`);
|
||||
await writeFile(path, `${output.join("\n").replace(/\n+$/, "")}\n`, "utf8");
|
||||
}
|
||||
@@ -0,0 +1,312 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import { resolve } from "node:path";
|
||||
import { env } from "node:process";
|
||||
import {
|
||||
apiJson,
|
||||
bodyText,
|
||||
createBrowser,
|
||||
ensureEvidence,
|
||||
evidencePaths,
|
||||
loadEnvFiles,
|
||||
resetAndAuthLocalUser,
|
||||
safeScreenshot,
|
||||
setBrowserToken,
|
||||
verifyBrowserToken,
|
||||
writeResult,
|
||||
} from "./lib/langbot-e2e.mjs";
|
||||
|
||||
const RUNNER_ID = "plugin:langbot/local-agent/default";
|
||||
const DEFAULT_PIPELINE_NAME = "Agent QA Local Agent Debug Chat";
|
||||
const DEFAULT_LOCAL_PASSWORD = "LangBotE2ELocalPass!2026";
|
||||
const caseId = "ensure-local-agent-pipeline";
|
||||
|
||||
await loadEnvFiles();
|
||||
const paths = evidencePaths(caseId);
|
||||
await ensureEvidence(paths);
|
||||
|
||||
const writeEnv = process.argv.includes("--write-env");
|
||||
const pipelineName = env.LANGBOT_E2E_CREATE_PIPELINE_NAME || env.LANGBOT_LOCAL_AGENT_PIPELINE_NAME || DEFAULT_PIPELINE_NAME;
|
||||
const frontendUrl = env.LANGBOT_FRONTEND_URL || "";
|
||||
const backendUrl = env.LANGBOT_BACKEND_URL || "";
|
||||
const envLocalPath = resolve("skills/.env.local");
|
||||
|
||||
const result = {
|
||||
source: "automation",
|
||||
case_id: caseId,
|
||||
run_id: paths.runId,
|
||||
status: "fail",
|
||||
reason: "",
|
||||
frontend_url: frontendUrl,
|
||||
backend_url: backendUrl,
|
||||
pipeline_name: pipelineName,
|
||||
pipeline_id: "",
|
||||
pipeline_url: "",
|
||||
runner_id: RUNNER_ID,
|
||||
selected_model_id: "",
|
||||
model_count: 0,
|
||||
created: false,
|
||||
updated: false,
|
||||
wrote_env: false,
|
||||
auth: null,
|
||||
browser_token_check: null,
|
||||
page_signal: "",
|
||||
evidence: {
|
||||
console_log: paths.consoleLog,
|
||||
network_log: paths.networkLog,
|
||||
screenshot: paths.screenshot,
|
||||
automation_result_json: paths.automationResultJson,
|
||||
result_json: paths.resultJson,
|
||||
},
|
||||
evidence_collected: ["api_diagnostic", "console", "network", "screenshot"],
|
||||
};
|
||||
|
||||
let browser;
|
||||
|
||||
try {
|
||||
if (!frontendUrl) throw new Error("LANGBOT_FRONTEND_URL is not configured.");
|
||||
if (!backendUrl) throw new Error("LANGBOT_BACKEND_URL is not configured.");
|
||||
|
||||
const user = env.LANGBOT_E2E_LOGIN_USER || "";
|
||||
const password = env.LANGBOT_E2E_LOGIN_PASSWORD || DEFAULT_LOCAL_PASSWORD;
|
||||
if (!user) {
|
||||
throw new Error("LANGBOT_E2E_LOGIN_USER is required so this setup can create/update the pipeline via backend API.");
|
||||
}
|
||||
|
||||
const auth = await resetAndAuthLocalUser({ backendUrl, user, password });
|
||||
result.auth = {
|
||||
source: "local_recovery_login",
|
||||
user,
|
||||
backend_token_check: auth.check,
|
||||
};
|
||||
|
||||
const prepared = await ensureLocalAgentPipeline({
|
||||
backendUrl,
|
||||
token: auth.token,
|
||||
pipelineName,
|
||||
runnerId: RUNNER_ID,
|
||||
});
|
||||
Object.assign(result, prepared);
|
||||
if (result.pipeline_id) {
|
||||
result.pipeline_url = `${frontendUrl.replace(/\/$/, "")}/home/pipelines?id=${encodeURIComponent(result.pipeline_id)}`;
|
||||
}
|
||||
|
||||
if (writeEnv && result.pipeline_id) {
|
||||
await upsertEnvLocal(envLocalPath, {
|
||||
LANGBOT_E2E_LOGIN_USER: user,
|
||||
LANGBOT_PIPELINE_URL: result.pipeline_url,
|
||||
LANGBOT_PIPELINE_NAME: result.pipeline_name || pipelineName,
|
||||
LANGBOT_LOCAL_AGENT_PIPELINE_URL: result.pipeline_url,
|
||||
LANGBOT_LOCAL_AGENT_PIPELINE_NAME: result.pipeline_name || pipelineName,
|
||||
});
|
||||
result.wrote_env = true;
|
||||
}
|
||||
|
||||
browser = await createBrowser(paths);
|
||||
const { page } = browser;
|
||||
await setBrowserToken(page, frontendUrl, auth.token);
|
||||
const browserCheck = await verifyBrowserToken(page, backendUrl);
|
||||
result.browser_token_check = browserCheck;
|
||||
if (!browserCheck.authenticated) {
|
||||
throw new Error(browserCheck.reason || "Browser token check failed after setup.");
|
||||
}
|
||||
await page.goto(result.pipeline_url || frontendUrl, { waitUntil: "domcontentloaded" });
|
||||
await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {});
|
||||
const text = await bodyText(page);
|
||||
result.page_signal = ["Pipelines", "流水线", pipelineName].find((signal) => text.includes(signal)) || "";
|
||||
} catch (error) {
|
||||
result.status = result.status === "env_issue" ? "env_issue" : "fail";
|
||||
result.reason = result.reason || error.message;
|
||||
} finally {
|
||||
if (browser?.page) await safeScreenshot(browser.page, paths.screenshot);
|
||||
if (browser) await browser.close().catch(() => {});
|
||||
await writeResult(paths, result);
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
}
|
||||
|
||||
process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1);
|
||||
|
||||
async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runnerId }) {
|
||||
const [pipelineList, modelList] = await Promise.all([
|
||||
apiJson(backendUrl, "/api/v1/pipelines", { token }),
|
||||
apiJson(backendUrl, "/api/v1/provider/models/llm", { token }),
|
||||
]);
|
||||
|
||||
if (isApiFailure(pipelineList)) {
|
||||
return {
|
||||
status: "fail",
|
||||
reason: pipelineList.json.msg || "Failed to list pipelines.",
|
||||
list_status: pipelineList.status,
|
||||
};
|
||||
}
|
||||
if (isApiFailure(modelList)) {
|
||||
return {
|
||||
status: "fail",
|
||||
reason: modelList.json.msg || "Failed to list LLM models.",
|
||||
model_status: modelList.status,
|
||||
};
|
||||
}
|
||||
|
||||
const models = modelList.json.data?.models || [];
|
||||
const selectedModel = models.find((model) => model.uuid) || null;
|
||||
const pipelines = pipelineList.json.data?.pipelines || [];
|
||||
let pipeline = pipelines.find((item) => item.name === pipelineName) || null;
|
||||
let created = false;
|
||||
|
||||
if (!pipeline) {
|
||||
const createdResponse = await apiJson(backendUrl, "/api/v1/pipelines", {
|
||||
method: "POST",
|
||||
token,
|
||||
body: {
|
||||
name: pipelineName,
|
||||
description: "Local QA pipeline for AgentRunner Debug Chat smoke tests.",
|
||||
emoji: "QA",
|
||||
},
|
||||
});
|
||||
if (isApiFailure(createdResponse)) {
|
||||
return {
|
||||
status: "fail",
|
||||
reason: createdResponse.json.msg || "Failed to create pipeline.",
|
||||
create_status: createdResponse.status,
|
||||
model_count: models.length,
|
||||
};
|
||||
}
|
||||
const pipelineId = createdResponse.json.data?.uuid || "";
|
||||
const loaded = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipelineId)}`, { token });
|
||||
pipeline = loaded.json.data?.pipeline || null;
|
||||
created = true;
|
||||
}
|
||||
|
||||
if (!pipeline?.uuid) {
|
||||
return {
|
||||
status: "fail",
|
||||
reason: "Pipeline was not created or resolved.",
|
||||
model_count: models.length,
|
||||
};
|
||||
}
|
||||
|
||||
const loaded = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, { token });
|
||||
if (isApiFailure(loaded) || !loaded.json.data?.pipeline) {
|
||||
return {
|
||||
status: "fail",
|
||||
reason: loaded.json.msg || "Failed to load pipeline.",
|
||||
get_status: loaded.status,
|
||||
pipeline_id: pipeline.uuid,
|
||||
model_count: models.length,
|
||||
};
|
||||
}
|
||||
pipeline = loaded.json.data.pipeline;
|
||||
|
||||
const config = pipeline.config && typeof pipeline.config === "object" ? pipeline.config : {};
|
||||
const ai = config.ai && typeof config.ai === "object" ? config.ai : {};
|
||||
const runnerConfig = ai.runner_config && typeof ai.runner_config === "object" ? ai.runner_config : {};
|
||||
const rawExistingLocalAgentConfig = runnerConfig[runnerId] && typeof runnerConfig[runnerId] === "object"
|
||||
? runnerConfig[runnerId]
|
||||
: {};
|
||||
const existingLocalAgentConfig = rawExistingLocalAgentConfig;
|
||||
const existingModel = existingLocalAgentConfig.model && typeof existingLocalAgentConfig.model === "object"
|
||||
? existingLocalAgentConfig.model
|
||||
: {};
|
||||
const requestedModelId = env.LANGBOT_LOCAL_AGENT_MODEL_UUID || env.LANGBOT_E2E_MODEL_UUID || "";
|
||||
const selectedModelId = requestedModelId || existingModel.primary || selectedModel?.uuid || "";
|
||||
const localAgentConfig = {
|
||||
timeout: 300,
|
||||
prompt: [{ role: "system", content: "You are a helpful assistant." }],
|
||||
"remove-think": false,
|
||||
"knowledge-bases": [],
|
||||
"retrieval-top-k": 5,
|
||||
"rerank-model": "",
|
||||
"rerank-top-k": 5,
|
||||
"max-tool-iterations": 20,
|
||||
"tool-execution-mode": "parallel",
|
||||
"max-tool-result-chars": 20000,
|
||||
"context-history-fetch-limit": 50,
|
||||
"context-window-tokens": 200000,
|
||||
"context-reserve-tokens": 16384,
|
||||
"context-keep-recent-tokens": 20000,
|
||||
"context-summary-tokens": 8000,
|
||||
...existingLocalAgentConfig,
|
||||
model: {
|
||||
primary: selectedModelId,
|
||||
fallbacks: requestedModelId ? [] : Array.isArray(existingModel.fallbacks) ? existingModel.fallbacks : [],
|
||||
},
|
||||
};
|
||||
const updatedConfig = {
|
||||
...config,
|
||||
ai: {
|
||||
...ai,
|
||||
runner: {
|
||||
...(ai.runner && typeof ai.runner === "object" ? ai.runner : {}),
|
||||
id: runnerId,
|
||||
"expire-time": 0,
|
||||
},
|
||||
runner_config: {
|
||||
...runnerConfig,
|
||||
[runnerId]: localAgentConfig,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const updateResponse = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, {
|
||||
method: "PUT",
|
||||
token,
|
||||
body: {
|
||||
name: pipelineName,
|
||||
description: "Local QA pipeline for AgentRunner Debug Chat smoke tests.",
|
||||
emoji: "QA",
|
||||
config: updatedConfig,
|
||||
},
|
||||
});
|
||||
if (isApiFailure(updateResponse)) {
|
||||
return {
|
||||
status: "fail",
|
||||
reason: updateResponse.json.msg || "Failed to update pipeline config.",
|
||||
update_status: updateResponse.status,
|
||||
pipeline_id: pipeline.uuid,
|
||||
model_count: models.length,
|
||||
selected_model_id: selectedModelId,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
status: selectedModelId ? "pass" : "env_issue",
|
||||
reason: selectedModelId
|
||||
? "Local-agent pipeline is configured for Debug Chat."
|
||||
: "Pipeline was created but no LLM model is configured in this LangBot instance.",
|
||||
pipeline_id: pipeline.uuid,
|
||||
pipeline_name: pipeline.name,
|
||||
model_count: models.length,
|
||||
selected_model_id: selectedModelId,
|
||||
created,
|
||||
updated: true,
|
||||
};
|
||||
}
|
||||
|
||||
function isApiFailure(response) {
|
||||
return response.status >= 400 || (response.json.code !== undefined && response.json.code !== 0);
|
||||
}
|
||||
|
||||
async function upsertEnvLocal(path, updates) {
|
||||
let text = "";
|
||||
try {
|
||||
text = await readFile(path, "utf8");
|
||||
} catch {
|
||||
text = "";
|
||||
}
|
||||
const lines = text.split(/\r?\n/);
|
||||
const seen = new Set();
|
||||
const next = lines.map((line) => {
|
||||
const trimmed = line.trim();
|
||||
const equals = trimmed.indexOf("=");
|
||||
if (equals <= 0 || trimmed.startsWith("#")) return line;
|
||||
const key = trimmed.slice(0, equals).trim();
|
||||
if (!(key in updates)) return line;
|
||||
seen.add(key);
|
||||
return `${key}=${updates[key]}`;
|
||||
});
|
||||
for (const [key, value] of Object.entries(updates)) {
|
||||
if (!seen.has(key)) next.push(`${key}=${value}`);
|
||||
}
|
||||
await writeFile(path, `${next.filter((line, index) => line !== "" || index < next.length - 1).join("\n")}\n`, "utf8");
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import { resolve } from "node:path";
|
||||
import { env } from "node:process";
|
||||
import {
|
||||
apiJson,
|
||||
ensureEvidence,
|
||||
evidencePaths,
|
||||
loadEnvFiles,
|
||||
resetAndAuthLocalUser,
|
||||
writeResult,
|
||||
} from "./lib/langbot-e2e.mjs";
|
||||
|
||||
const RUNNER_ID = "plugin:qa/agent-runner/default";
|
||||
const DEFAULT_PIPELINE_NAME = "Agent QA Deterministic Runner Debug Chat";
|
||||
const DEFAULT_LOCAL_PASSWORD = "LangBotE2ELocalPass!2026";
|
||||
const caseId = "ensure-qa-agent-runner-pipeline";
|
||||
|
||||
await loadEnvFiles();
|
||||
const paths = evidencePaths(caseId);
|
||||
await ensureEvidence(paths);
|
||||
|
||||
const writeEnv = process.argv.includes("--write-env");
|
||||
const frontendUrl = env.LANGBOT_FRONTEND_URL || "";
|
||||
const backendUrl = env.LANGBOT_BACKEND_URL || "";
|
||||
const pipelineName = env.LANGBOT_E2E_CREATE_PIPELINE_NAME || env.LANGBOT_QA_AGENT_RUNNER_PIPELINE_NAME || DEFAULT_PIPELINE_NAME;
|
||||
const envLocalPath = resolve("skills/.env.local");
|
||||
|
||||
const result = {
|
||||
source: "automation",
|
||||
case_id: caseId,
|
||||
run_id: paths.runId,
|
||||
status: "fail",
|
||||
reason: "",
|
||||
frontend_url: frontendUrl,
|
||||
backend_url: backendUrl,
|
||||
pipeline_name: pipelineName,
|
||||
pipeline_id: "",
|
||||
pipeline_url: "",
|
||||
runner_id: RUNNER_ID,
|
||||
wrote_env: false,
|
||||
auth: null,
|
||||
evidence: {
|
||||
automation_result_json: paths.automationResultJson,
|
||||
result_json: paths.resultJson,
|
||||
},
|
||||
evidence_collected: ["api_diagnostic"],
|
||||
};
|
||||
|
||||
try {
|
||||
if (!frontendUrl) throw new Error("LANGBOT_FRONTEND_URL is not configured.");
|
||||
if (!backendUrl) throw new Error("LANGBOT_BACKEND_URL is not configured.");
|
||||
|
||||
const user = env.LANGBOT_E2E_LOGIN_USER || "";
|
||||
const password = env.LANGBOT_E2E_LOGIN_PASSWORD || DEFAULT_LOCAL_PASSWORD;
|
||||
if (!user) {
|
||||
throw new Error("LANGBOT_E2E_LOGIN_USER is required so this setup can create/update the pipeline via backend API.");
|
||||
}
|
||||
|
||||
const auth = await resetAndAuthLocalUser({ backendUrl, user, password });
|
||||
result.auth = {
|
||||
source: "local_recovery_login",
|
||||
user,
|
||||
backend_token_check: auth.check,
|
||||
};
|
||||
|
||||
const prepared = await ensurePipeline({
|
||||
backendUrl,
|
||||
token: auth.token,
|
||||
pipelineName,
|
||||
runnerId: RUNNER_ID,
|
||||
runnerConfig: {},
|
||||
});
|
||||
Object.assign(result, prepared);
|
||||
if (result.pipeline_id) {
|
||||
result.pipeline_url = `${frontendUrl.replace(/\/$/, "")}/home/pipelines?id=${encodeURIComponent(result.pipeline_id)}`;
|
||||
}
|
||||
|
||||
if (writeEnv && result.pipeline_id) {
|
||||
await upsertEnvLocal(envLocalPath, {
|
||||
LANGBOT_E2E_LOGIN_USER: user,
|
||||
LANGBOT_QA_AGENT_RUNNER_PIPELINE_URL: result.pipeline_url,
|
||||
LANGBOT_QA_AGENT_RUNNER_PIPELINE_NAME: result.pipeline_name || pipelineName,
|
||||
});
|
||||
result.wrote_env = true;
|
||||
}
|
||||
} catch (error) {
|
||||
result.reason = result.reason || error.message;
|
||||
} finally {
|
||||
await writeResult(paths, result);
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
}
|
||||
|
||||
process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1);
|
||||
|
||||
async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runnerConfig }) {
|
||||
const pipelineList = await apiJson(backendUrl, "/api/v1/pipelines", { token });
|
||||
if (isApiFailure(pipelineList)) {
|
||||
return {
|
||||
status: "fail",
|
||||
reason: pipelineList.json.msg || "Failed to list pipelines.",
|
||||
list_status: pipelineList.status,
|
||||
};
|
||||
}
|
||||
|
||||
const pipelines = pipelineList.json.data?.pipelines || [];
|
||||
let pipeline = pipelines.find((item) => item.name === pipelineName) || null;
|
||||
let created = false;
|
||||
|
||||
if (!pipeline) {
|
||||
const createdResponse = await apiJson(backendUrl, "/api/v1/pipelines", {
|
||||
method: "POST",
|
||||
token,
|
||||
body: {
|
||||
name: pipelineName,
|
||||
description: "Local QA pipeline for deterministic QA AgentRunner Debug Chat smoke tests.",
|
||||
emoji: "QA",
|
||||
},
|
||||
});
|
||||
if (isApiFailure(createdResponse)) {
|
||||
return {
|
||||
status: "fail",
|
||||
reason: createdResponse.json.msg || "Failed to create pipeline.",
|
||||
create_status: createdResponse.status,
|
||||
};
|
||||
}
|
||||
const pipelineId = createdResponse.json.data?.uuid || "";
|
||||
const loaded = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipelineId)}`, { token });
|
||||
pipeline = loaded.json.data?.pipeline || null;
|
||||
created = true;
|
||||
}
|
||||
|
||||
if (!pipeline?.uuid) {
|
||||
return {
|
||||
status: "fail",
|
||||
reason: "Pipeline was not created or resolved.",
|
||||
};
|
||||
}
|
||||
|
||||
const loaded = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, { token });
|
||||
if (isApiFailure(loaded) || !loaded.json.data?.pipeline) {
|
||||
return {
|
||||
status: "fail",
|
||||
reason: loaded.json.msg || "Failed to load pipeline.",
|
||||
get_status: loaded.status,
|
||||
pipeline_id: pipeline.uuid,
|
||||
};
|
||||
}
|
||||
pipeline = loaded.json.data.pipeline;
|
||||
|
||||
const config = pipeline.config && typeof pipeline.config === "object" ? pipeline.config : {};
|
||||
const ai = config.ai && typeof config.ai === "object" ? config.ai : {};
|
||||
const runnerConfigs = ai.runner_config && typeof ai.runner_config === "object" ? ai.runner_config : {};
|
||||
const updatedConfig = {
|
||||
...config,
|
||||
ai: {
|
||||
...ai,
|
||||
runner: {
|
||||
...(ai.runner && typeof ai.runner === "object" ? ai.runner : {}),
|
||||
id: runnerId,
|
||||
"expire-time": 0,
|
||||
},
|
||||
runner_config: {
|
||||
...runnerConfigs,
|
||||
[runnerId]: runnerConfig,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const updateResponse = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, {
|
||||
method: "PUT",
|
||||
token,
|
||||
body: {
|
||||
name: pipelineName,
|
||||
description: "Local QA pipeline for deterministic QA AgentRunner Debug Chat smoke tests.",
|
||||
emoji: "QA",
|
||||
config: updatedConfig,
|
||||
},
|
||||
});
|
||||
if (isApiFailure(updateResponse)) {
|
||||
return {
|
||||
status: "fail",
|
||||
reason: updateResponse.json.msg || "Failed to update pipeline.",
|
||||
update_status: updateResponse.status,
|
||||
pipeline_id: pipeline.uuid,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
status: "pass",
|
||||
reason: created ? "QA AgentRunner pipeline created and configured." : "QA AgentRunner pipeline updated.",
|
||||
pipeline_id: pipeline.uuid,
|
||||
pipeline_name: pipelineName,
|
||||
created,
|
||||
updated: true,
|
||||
};
|
||||
}
|
||||
|
||||
function isApiFailure(response) {
|
||||
return response.status >= 400 || (response.json && response.json.code !== undefined && response.json.code !== 0);
|
||||
}
|
||||
|
||||
async function upsertEnvLocal(path, values) {
|
||||
let text = "";
|
||||
try {
|
||||
text = await readFile(path, "utf8");
|
||||
} catch {
|
||||
text = "";
|
||||
}
|
||||
const lines = text.split(/\r?\n/);
|
||||
const keys = new Set(Object.keys(values));
|
||||
const output = [];
|
||||
for (const line of lines) {
|
||||
const match = line.match(/^([A-Z][A-Z0-9_]*)=/);
|
||||
if (match && keys.has(match[1])) {
|
||||
output.push(`${match[1]}=${values[match[1]]}`);
|
||||
keys.delete(match[1]);
|
||||
} else if (line !== "" || output.length > 0) {
|
||||
output.push(line);
|
||||
}
|
||||
}
|
||||
if (keys.size > 0 && output.length > 0 && output[output.length - 1] !== "") {
|
||||
output.push("");
|
||||
}
|
||||
for (const key of keys) {
|
||||
output.push(`${key}=${values[key]}`);
|
||||
}
|
||||
await writeFile(path, `${output.join("\n").replace(/\n+$/, "")}\n`, "utf8");
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { resolve } from "node:path";
|
||||
import { env } from "node:process";
|
||||
import {
|
||||
apiJson,
|
||||
ensureEvidence,
|
||||
evidencePaths,
|
||||
loadEnvFiles,
|
||||
resetAndAuthLocalUser,
|
||||
writeResult,
|
||||
} from "./lib/langbot-e2e.mjs";
|
||||
|
||||
const caseId = env.LBS_CASE_ID || "install-qa-plugin-smoke";
|
||||
const paths = evidencePaths(caseId);
|
||||
await loadEnvFiles();
|
||||
await ensureEvidence(paths);
|
||||
|
||||
const backendUrl = env.LANGBOT_BACKEND_URL || "";
|
||||
const user = env.LANGBOT_E2E_LOGIN_USER || "";
|
||||
const password = env.LANGBOT_E2E_LOGIN_PASSWORD || "LangBotE2ELocalPass!2026";
|
||||
const packagePath = resolve(
|
||||
env.LANGBOT_E2E_PLUGIN_PACKAGE
|
||||
|| env.LANGBOT_QA_PLUGIN_SMOKE_PACKAGE
|
||||
|| "skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/dist/qa-plugin-smoke-0.1.0.lbpkg",
|
||||
);
|
||||
const expectedPluginId = env.LANGBOT_E2E_EXPECTED_PLUGIN_ID || "qa/plugin-smoke";
|
||||
const expectedTool = env.LANGBOT_E2E_EXPECTED_TOOL || (expectedPluginId === "qa/plugin-smoke" ? "qa_plugin_echo" : "");
|
||||
const expectedRunnerId = env.LANGBOT_E2E_EXPECTED_RUNNER_ID || "";
|
||||
|
||||
const result = {
|
||||
source: "automation",
|
||||
case_id: caseId,
|
||||
run_id: paths.runId,
|
||||
status: "fail",
|
||||
reason: "",
|
||||
backend_url: backendUrl,
|
||||
package_path: packagePath,
|
||||
package_preview: null,
|
||||
task_id: null,
|
||||
task: null,
|
||||
plugin_present_before: false,
|
||||
plugin_present_after: false,
|
||||
tool_names: [],
|
||||
runner_ids: [],
|
||||
evidence: {
|
||||
automation_result_json: paths.automationResultJson,
|
||||
result_json: paths.resultJson,
|
||||
},
|
||||
evidence_collected: ["api_diagnostic", "filesystem"],
|
||||
};
|
||||
|
||||
try {
|
||||
if (!backendUrl) throw new Error("LANGBOT_BACKEND_URL is not configured.");
|
||||
if (!user) throw new Error("LANGBOT_E2E_LOGIN_USER is required.");
|
||||
const bytes = await readFile(packagePath);
|
||||
|
||||
const auth = await resetAndAuthLocalUser({ backendUrl, user, password });
|
||||
result.package_preview = await previewPackage(backendUrl, auth.token, bytes, packagePath);
|
||||
const metadata = result.package_preview.metadata || {};
|
||||
if (`${metadata.author}/${metadata.name}` !== expectedPluginId) {
|
||||
throw new Error(`Fixture package metadata is ${metadata.author}/${metadata.name}, expected ${expectedPluginId}.`);
|
||||
}
|
||||
result.plugin_present_before = await hasPlugin(backendUrl, auth.token);
|
||||
|
||||
if (!result.plugin_present_before) {
|
||||
const form = new FormData();
|
||||
form.set("file", new Blob([bytes]), packagePath.split("/").pop());
|
||||
const response = await fetch(`${backendUrl.replace(/\/$/, "")}/api/v1/plugins/install/local`, {
|
||||
method: "POST",
|
||||
headers: { Authorization: `Bearer ${auth.token}` },
|
||||
body: form,
|
||||
});
|
||||
const json = await response.json().catch(() => ({}));
|
||||
if (response.status >= 400 || json.code !== 0) {
|
||||
throw new Error(json.msg || `Plugin install request failed with HTTP ${response.status}.`);
|
||||
}
|
||||
result.task_id = json.data?.task_id ?? null;
|
||||
if (!result.task_id) throw new Error("Plugin install response did not include task_id.");
|
||||
result.task = await waitForTask(backendUrl, auth.token, result.task_id);
|
||||
if (!isTaskComplete(result.task)) {
|
||||
throw new Error(`Plugin install task did not complete successfully: ${JSON.stringify(result.task)}`);
|
||||
}
|
||||
}
|
||||
|
||||
await sleep(1000);
|
||||
result.plugin_present_after = await hasPlugin(backendUrl, auth.token);
|
||||
if (!result.plugin_present_after) throw new Error(`${expectedPluginId} is not listed by /api/v1/plugins after install.`);
|
||||
if (expectedTool) {
|
||||
result.tool_names = await listToolNames(backendUrl, auth.token);
|
||||
if (!result.tool_names.includes(expectedTool)) {
|
||||
throw new Error(`${expectedTool} is not listed by /api/v1/tools after install.`);
|
||||
}
|
||||
}
|
||||
if (expectedRunnerId) {
|
||||
result.runner_ids = await listRunnerIds(backendUrl, auth.token);
|
||||
if (!result.runner_ids.includes(expectedRunnerId)) {
|
||||
throw new Error(`${expectedRunnerId} is not listed by /api/v1/pipelines/_/metadata after install.`);
|
||||
}
|
||||
}
|
||||
|
||||
result.status = "pass";
|
||||
result.reason = `${expectedPluginId} is installed.`;
|
||||
} catch (error) {
|
||||
result.status = "fail";
|
||||
result.reason = error.message;
|
||||
} finally {
|
||||
await writeResult(paths, result);
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
}
|
||||
|
||||
process.exit(result.status === "pass" ? 0 : 1);
|
||||
|
||||
async function hasPlugin(backendUrl, token) {
|
||||
const response = await apiJson(backendUrl, "/api/v1/plugins", { token });
|
||||
const plugins = response.json.data?.plugins || [];
|
||||
return plugins.some((plugin) => {
|
||||
const metadata = plugin.manifest?.manifest?.metadata || plugin.manifest?.metadata || plugin.metadata || {};
|
||||
return `${metadata.author}/${metadata.name}` === expectedPluginId;
|
||||
});
|
||||
}
|
||||
|
||||
async function previewPackage(backendUrl, token, bytes, packagePath) {
|
||||
const form = new FormData();
|
||||
form.set("file", new Blob([bytes]), packagePath.split("/").pop());
|
||||
const response = await fetch(`${backendUrl.replace(/\/$/, "")}/api/v1/plugins/install/local/preview`, {
|
||||
method: "POST",
|
||||
headers: { Authorization: `Bearer ${token}` },
|
||||
body: form,
|
||||
});
|
||||
const json = await response.json().catch(() => ({}));
|
||||
if (response.status >= 400 || json.code !== 0) {
|
||||
throw new Error(json.msg || `Plugin package preview failed with HTTP ${response.status}.`);
|
||||
}
|
||||
return {
|
||||
metadata: json.data?.metadata || {},
|
||||
component_types: json.data?.component_types || [],
|
||||
file_count: json.data?.file_count ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
async function listToolNames(backendUrl, token) {
|
||||
const response = await apiJson(backendUrl, "/api/v1/tools", { token });
|
||||
return (response.json.data?.tools || [])
|
||||
.map((tool) => tool.name || tool.tool_name || tool.function?.name || "")
|
||||
.filter(Boolean)
|
||||
.sort();
|
||||
}
|
||||
|
||||
async function listRunnerIds(backendUrl, token) {
|
||||
const response = await apiJson(backendUrl, "/api/v1/pipelines/_/metadata", { token });
|
||||
const configs = response.json.data?.configs || [];
|
||||
return configs
|
||||
.flatMap((section) => section.stages || [])
|
||||
.flatMap((stage) => stage.config || [])
|
||||
.filter((item) => item.name === "id")
|
||||
.flatMap((item) => item.options || [])
|
||||
.map((option) => option.name || option.value || option.id || "")
|
||||
.filter(Boolean)
|
||||
.sort();
|
||||
}
|
||||
|
||||
async function waitForTask(backendUrl, token, taskId) {
|
||||
const deadline = Date.now() + Number(env.LANGBOT_PLUGIN_INSTALL_TIMEOUT_MS || 120000);
|
||||
let last = null;
|
||||
while (Date.now() < deadline) {
|
||||
const response = await apiJson(backendUrl, `/api/v1/system/tasks/${encodeURIComponent(taskId)}`, { token });
|
||||
last = response.json.data || response.json;
|
||||
if (isTaskComplete(last) || isTaskFailed(last)) return last;
|
||||
await sleep(1000);
|
||||
}
|
||||
return last;
|
||||
}
|
||||
|
||||
function isTaskComplete(task) {
|
||||
const status = String(task?.status || task?.state || "").toLowerCase();
|
||||
const runtimeStatus = String(task?.runtime?.status || task?.runtime?.state || "").toLowerCase();
|
||||
return ["done", "completed", "success", "succeeded", "finished"].includes(status)
|
||||
|| ["done", "completed", "success", "succeeded", "finished"].includes(runtimeStatus)
|
||||
|| task?.done === true
|
||||
|| task?.completed === true
|
||||
|| (task?.runtime?.done === true && !task?.runtime?.exception);
|
||||
}
|
||||
|
||||
function isTaskFailed(task) {
|
||||
const status = String(task?.status || task?.state || "").toLowerCase();
|
||||
const runtimeStatus = String(task?.runtime?.status || task?.runtime?.state || "").toLowerCase();
|
||||
return ["failed", "error", "cancelled", "canceled"].includes(status)
|
||||
|| ["failed", "error", "cancelled", "canceled"].includes(runtimeStatus)
|
||||
|| task?.failed === true
|
||||
|| Boolean(task?.error)
|
||||
|| Boolean(task?.runtime?.exception);
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import {
|
||||
bodyText,
|
||||
createBrowser,
|
||||
ensureEvidence,
|
||||
evidencePaths,
|
||||
exitCode,
|
||||
isLoginUrl,
|
||||
localIsoWithOffset,
|
||||
safeScreenshot,
|
||||
writeResult,
|
||||
} from "./lib/langbot-e2e.mjs";
|
||||
|
||||
const caseId = process.env.LBS_CASE_ID || "langrag-kb-retrieve";
|
||||
const paths = evidencePaths(caseId);
|
||||
await ensureEvidence(paths);
|
||||
|
||||
const startedAt = new Date();
|
||||
const frontendUrl = process.env.LANGBOT_FRONTEND_URL || "";
|
||||
const backendUrl = process.env.LANGBOT_BACKEND_URL || "";
|
||||
const kbUuid = process.env.LANGBOT_LOCAL_AGENT_RAG_KB_UUID || process.env.LANGBOT_RAG_KB_UUID || "";
|
||||
const query = process.env.LANGBOT_E2E_RETRIEVE_QUERY || "What is the local agent runner retrieval sentinel?";
|
||||
const expectedText = process.env.LANGBOT_E2E_EXPECTED_TEXT || "azalea-cobalt-7421";
|
||||
|
||||
let browser;
|
||||
const result = {
|
||||
source: "automation",
|
||||
case_id: caseId,
|
||||
run_id: paths.runId,
|
||||
started_at: startedAt.toISOString(),
|
||||
started_at_local: localIsoWithOffset(startedAt),
|
||||
finished_at: "",
|
||||
finished_at_local: "",
|
||||
status: "fail",
|
||||
reason: "",
|
||||
url: "",
|
||||
kb_uuid: kbUuid,
|
||||
query,
|
||||
expected_text: expectedText,
|
||||
evidence: {
|
||||
console_log: paths.consoleLog,
|
||||
network_log: paths.networkLog,
|
||||
screenshot: paths.screenshot,
|
||||
automation_result_json: paths.automationResultJson,
|
||||
result_json: paths.resultJson,
|
||||
},
|
||||
evidence_collected: ["ui", "screenshot", "console", "network", "api_diagnostic"],
|
||||
};
|
||||
|
||||
try {
|
||||
if (!frontendUrl) throw new Error("LANGBOT_FRONTEND_URL is not configured.");
|
||||
if (!backendUrl) throw new Error("LANGBOT_BACKEND_URL is not configured.");
|
||||
if (!kbUuid) throw new Error("LANGBOT_LOCAL_AGENT_RAG_KB_UUID or LANGBOT_RAG_KB_UUID is required.");
|
||||
|
||||
browser = await createBrowser(paths);
|
||||
const { page } = browser;
|
||||
await page.goto(`${frontendUrl.replace(/\/$/, "")}/home/knowledge`, { waitUntil: "domcontentloaded" });
|
||||
await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {});
|
||||
result.url = page.url();
|
||||
|
||||
const text = await bodyText(page);
|
||||
if (isLoginUrl(page.url()) || /登录|Login|Sign in/i.test(text)) {
|
||||
result.status = "blocked";
|
||||
result.reason = "Browser profile is not authenticated for LANGBOT_FRONTEND_URL.";
|
||||
} else if (!/Knowledge|知识库|qa-local-agent-rag/i.test(text)) {
|
||||
result.status = "fail";
|
||||
result.reason = "Knowledge page opened, but no Knowledge UI signal or QA KB name was visible.";
|
||||
} else {
|
||||
const retrieve = await page.evaluate(async ({ backendUrl, kbUuid, query }) => {
|
||||
const token = localStorage.getItem("token");
|
||||
if (!token) {
|
||||
return { status: "blocked", authenticated: false, reason: "Browser profile has no localStorage token." };
|
||||
}
|
||||
const response = await fetch(`${backendUrl}/api/v1/knowledge/bases/${encodeURIComponent(kbUuid)}/retrieve`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({ query }),
|
||||
});
|
||||
const json = await response.json().catch(() => ({}));
|
||||
return {
|
||||
status: response.status >= 400 ? "fail" : "ready",
|
||||
authenticated: true,
|
||||
http_status: response.status,
|
||||
code: json.code ?? null,
|
||||
msg: json.msg || "",
|
||||
results: json.data?.results || [],
|
||||
};
|
||||
}, { backendUrl, kbUuid, query });
|
||||
|
||||
result.retrieve = {
|
||||
...retrieve,
|
||||
results: Array.isArray(retrieve.results)
|
||||
? retrieve.results.map((item) => ({
|
||||
score: item.score ?? item.distance ?? null,
|
||||
text: String(item.text || item.content || "").slice(0, 500),
|
||||
metadata: item.metadata || {},
|
||||
}))
|
||||
: [],
|
||||
};
|
||||
|
||||
const resultText = JSON.stringify(result.retrieve.results || []);
|
||||
if (retrieve.status === "blocked") {
|
||||
result.status = "blocked";
|
||||
result.reason = retrieve.reason || "Retrieve API blocked.";
|
||||
} else if (retrieve.status === "fail") {
|
||||
result.status = "fail";
|
||||
result.reason = retrieve.msg || "Retrieve API failed.";
|
||||
} else if (!resultText.includes(expectedText)) {
|
||||
result.status = "fail";
|
||||
result.reason = `Retrieve results did not contain expected text: ${expectedText}`;
|
||||
} else {
|
||||
result.status = "pass";
|
||||
result.reason = `Knowledge retrieve returned expected sentinel: ${expectedText}`;
|
||||
}
|
||||
}
|
||||
|
||||
await safeScreenshot(page, paths.screenshot);
|
||||
} catch (error) {
|
||||
result.status = /Playwright is not installed|not configured|required/.test(error.message) ? "env_issue" : "fail";
|
||||
result.reason = error.message;
|
||||
} finally {
|
||||
if (browser) await browser.close().catch(() => {});
|
||||
const finishedAt = new Date();
|
||||
result.finished_at = finishedAt.toISOString();
|
||||
result.finished_at_local = localIsoWithOffset(finishedAt);
|
||||
await writeResult(paths, result);
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
}
|
||||
|
||||
process.exit(exitCode(result.status));
|
||||
@@ -0,0 +1,416 @@
|
||||
import {
|
||||
bodyText,
|
||||
clickFirstVisible,
|
||||
countOccurrences,
|
||||
gotoFrontend,
|
||||
isLoginUrl,
|
||||
} from "./langbot-e2e.mjs";
|
||||
|
||||
export const DEBUG_CHAT_FAILURE_SIGNALS = [
|
||||
"Agent runner temporarily unavailable",
|
||||
"All models failed during streaming setup",
|
||||
"调用超时",
|
||||
"超时",
|
||||
];
|
||||
|
||||
export function minExpectedOccurrences(beforeText, expectedText, prompt) {
|
||||
const beforeCount = countOccurrences(beforeText, expectedText);
|
||||
return beforeCount + (String(prompt).includes(expectedText) ? 2 : 1);
|
||||
}
|
||||
|
||||
export function latestExpectedLeafMatches(latestExpectedLeaf, prompt) {
|
||||
return Boolean(latestExpectedLeaf)
|
||||
&& latestExpectedLeaf !== prompt
|
||||
&& !String(latestExpectedLeaf).includes(prompt);
|
||||
}
|
||||
|
||||
export function findNewFailureSignal(beforeText, afterText, failureSignals = DEBUG_CHAT_FAILURE_SIGNALS) {
|
||||
return failureSignals.find((signal) => countOccurrences(afterText, signal) > countOccurrences(beforeText, signal)) || "";
|
||||
}
|
||||
|
||||
function findFailureSignalInText(text, failureSignals = DEBUG_CHAT_FAILURE_SIGNALS) {
|
||||
return failureSignals.find((signal) => String(text || "").includes(signal)) || "";
|
||||
}
|
||||
|
||||
function countExpectedInMessages(messages, expectedText) {
|
||||
return messages
|
||||
.filter((message) => message.role === "assistant")
|
||||
.reduce((count, message) => count + countOccurrences(message.text, expectedText), 0);
|
||||
}
|
||||
|
||||
function debugChatInput(page) {
|
||||
return page
|
||||
.locator('input[placeholder*="message"], input[placeholder*="消息"], textarea[placeholder*="message"], textarea[placeholder*="消息"]')
|
||||
.last();
|
||||
}
|
||||
|
||||
async function clickDebugChatTab(page) {
|
||||
const tabByRole = page.getByRole("tab", { name: /Debug Chat|调试聊天|调试对话|Debug|调试/i }).first();
|
||||
if (await tabByRole.isVisible({ timeout: 3_000 }).catch(() => false)) {
|
||||
await tabByRole.click();
|
||||
return true;
|
||||
}
|
||||
|
||||
const tabBySelector = page.locator('[role="tab"]').filter({ hasText: /Debug Chat|调试聊天|调试对话|Debug|调试/i }).first();
|
||||
if (await tabBySelector.isVisible({ timeout: 2_000 }).catch(() => false)) {
|
||||
await tabBySelector.click();
|
||||
return true;
|
||||
}
|
||||
|
||||
return Boolean(await clickFirstVisible(page, ["Debug Chat", "调试聊天", "调试对话"], 2_000));
|
||||
}
|
||||
|
||||
async function waitForDebugChatReady(page, timeout = 20_000) {
|
||||
const input = debugChatInput(page);
|
||||
const visible = await input.isVisible({ timeout }).catch(() => false);
|
||||
if (!visible) {
|
||||
return {
|
||||
ready: false,
|
||||
reason: "Debug Chat tab was clicked, but the Debug Chat input did not become visible.",
|
||||
};
|
||||
}
|
||||
|
||||
const enabled = await input.isEnabled({ timeout }).catch(() => false);
|
||||
if (!enabled) {
|
||||
return {
|
||||
ready: false,
|
||||
reason: "Debug Chat input is visible but disabled; WebSocket may not be connected.",
|
||||
};
|
||||
}
|
||||
|
||||
return { ready: true, reason: "" };
|
||||
}
|
||||
|
||||
export function classifyDebugChatResult({
|
||||
beforeText,
|
||||
afterText,
|
||||
expectedText,
|
||||
prompt,
|
||||
latestExpectedLeaf,
|
||||
latestFailureLeaf,
|
||||
beforeMessages = null,
|
||||
afterMessages = null,
|
||||
latestAssistantText = "",
|
||||
failureSignals = DEBUG_CHAT_FAILURE_SIGNALS,
|
||||
}) {
|
||||
const minExpectedCount = minExpectedOccurrences(beforeText, expectedText, prompt);
|
||||
const finalCount = countOccurrences(afterText, expectedText);
|
||||
const failureText = findNewFailureSignal(beforeText, afterText, failureSignals);
|
||||
const promptContainsExpected = String(prompt).includes(expectedText);
|
||||
const hasMessageEvidence = Array.isArray(beforeMessages) && Array.isArray(afterMessages);
|
||||
const beforeAssistantExpectedCount = hasMessageEvidence
|
||||
? countExpectedInMessages(beforeMessages, expectedText)
|
||||
: null;
|
||||
const afterAssistantExpectedCount = hasMessageEvidence
|
||||
? countExpectedInMessages(afterMessages, expectedText)
|
||||
: null;
|
||||
const assistantExpectedIncreased = hasMessageEvidence
|
||||
? afterAssistantExpectedCount > beforeAssistantExpectedCount
|
||||
: false;
|
||||
|
||||
if (hasMessageEvidence) {
|
||||
const latestAssistantFailure = findFailureSignalInText(latestAssistantText, failureSignals);
|
||||
if (latestAssistantFailure) {
|
||||
return {
|
||||
status: "fail",
|
||||
reason: `Debug Chat displayed a known failure signal in the latest assistant message: ${latestAssistantFailure}`,
|
||||
min_expected_count: minExpectedCount,
|
||||
final_count: finalCount,
|
||||
failure_signal: latestAssistantFailure,
|
||||
before_assistant_expected_count: beforeAssistantExpectedCount,
|
||||
after_assistant_expected_count: afterAssistantExpectedCount,
|
||||
};
|
||||
}
|
||||
if (assistantExpectedIncreased && String(latestAssistantText).includes(expectedText)) {
|
||||
return {
|
||||
status: "pass",
|
||||
reason: `Expected text appeared in a new assistant message: ${expectedText}`,
|
||||
min_expected_count: minExpectedCount,
|
||||
final_count: finalCount,
|
||||
before_assistant_expected_count: beforeAssistantExpectedCount,
|
||||
after_assistant_expected_count: afterAssistantExpectedCount,
|
||||
};
|
||||
}
|
||||
if (failureText) {
|
||||
return {
|
||||
status: "fail",
|
||||
reason: `Debug Chat displayed a known failure signal: ${failureText}`,
|
||||
min_expected_count: minExpectedCount,
|
||||
final_count: finalCount,
|
||||
failure_signal: failureText,
|
||||
before_assistant_expected_count: beforeAssistantExpectedCount,
|
||||
after_assistant_expected_count: afterAssistantExpectedCount,
|
||||
};
|
||||
}
|
||||
return {
|
||||
status: "fail",
|
||||
reason: `Expected text did not appear in a new assistant message. Expected assistant occurrences to increase above ${beforeAssistantExpectedCount}, saw ${afterAssistantExpectedCount}.`,
|
||||
min_expected_count: minExpectedCount,
|
||||
final_count: finalCount,
|
||||
before_assistant_expected_count: beforeAssistantExpectedCount,
|
||||
after_assistant_expected_count: afterAssistantExpectedCount,
|
||||
};
|
||||
}
|
||||
if (failureText) {
|
||||
return {
|
||||
status: "fail",
|
||||
reason: `Debug Chat displayed a known failure signal: ${failureText}`,
|
||||
min_expected_count: minExpectedCount,
|
||||
final_count: finalCount,
|
||||
failure_signal: failureText,
|
||||
before_assistant_expected_count: beforeAssistantExpectedCount,
|
||||
after_assistant_expected_count: afterAssistantExpectedCount,
|
||||
};
|
||||
}
|
||||
if (latestExpectedLeafMatches(latestExpectedLeaf, prompt) && finalCount >= minExpectedCount) {
|
||||
return {
|
||||
status: "pass",
|
||||
reason: `Expected text appeared in the latest visible response leaf: ${expectedText}`,
|
||||
min_expected_count: minExpectedCount,
|
||||
final_count: finalCount,
|
||||
};
|
||||
}
|
||||
if (!promptContainsExpected && finalCount >= minExpectedCount) {
|
||||
return {
|
||||
status: "pass",
|
||||
reason: `Expected text appeared enough times for user prompt plus bot response: ${expectedText}`,
|
||||
min_expected_count: minExpectedCount,
|
||||
final_count: finalCount,
|
||||
};
|
||||
}
|
||||
return {
|
||||
status: "fail",
|
||||
reason: `Bot response did not appear. Expected ${minExpectedCount} occurrences of ${expectedText}, saw ${finalCount}.`,
|
||||
min_expected_count: minExpectedCount,
|
||||
final_count: finalCount,
|
||||
};
|
||||
}
|
||||
|
||||
export async function openPipelineDebugChat(page, { pipelineUrl, pipelineName, envHint = "LANGBOT_PIPELINE_URL or LANGBOT_PIPELINE_NAME" }) {
|
||||
if (pipelineUrl) {
|
||||
await page.goto(pipelineUrl, { waitUntil: "domcontentloaded" });
|
||||
await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {});
|
||||
} else {
|
||||
if (!pipelineName) {
|
||||
return {
|
||||
opened: false,
|
||||
status: "blocked",
|
||||
reason: `Set ${envHint} before running pipeline-debug-chat automation.`,
|
||||
};
|
||||
}
|
||||
await gotoFrontend(page);
|
||||
if (isLoginUrl(page.url())) {
|
||||
return {
|
||||
opened: false,
|
||||
status: "blocked",
|
||||
reason: "Browser profile is not authenticated for LANGBOT_FRONTEND_URL.",
|
||||
};
|
||||
}
|
||||
const clickedPipelines = await clickFirstVisible(page, ["Pipelines", "流水线"], 4_000);
|
||||
if (!clickedPipelines) {
|
||||
return { opened: false, status: "fail", reason: "Could not find Pipelines navigation." };
|
||||
}
|
||||
await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {});
|
||||
const clickedPipeline = await clickFirstVisible(page, [pipelineName], 5_000);
|
||||
if (!clickedPipeline) {
|
||||
return { opened: false, status: "blocked", reason: `Could not find pipeline named ${pipelineName}.` };
|
||||
}
|
||||
}
|
||||
|
||||
if (isLoginUrl(page.url())) {
|
||||
return {
|
||||
opened: false,
|
||||
status: "blocked",
|
||||
reason: "Browser profile is not authenticated for LANGBOT_FRONTEND_URL.",
|
||||
};
|
||||
}
|
||||
|
||||
const clickedDebug = await clickDebugChatTab(page);
|
||||
if (!clickedDebug) {
|
||||
return { opened: false, status: "fail", reason: "Could not find the Debug Chat tab." };
|
||||
}
|
||||
await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {});
|
||||
const ready = await waitForDebugChatReady(page);
|
||||
if (!ready.ready) {
|
||||
return { opened: false, status: "fail", reason: ready.reason };
|
||||
}
|
||||
return { opened: true };
|
||||
}
|
||||
|
||||
export async function latestVisibleLeafText(page, needles) {
|
||||
return await page.evaluate((items) => {
|
||||
const isVisible = (element) => {
|
||||
const style = window.getComputedStyle(element);
|
||||
const rect = element.getBoundingClientRect();
|
||||
return style.visibility !== "hidden"
|
||||
&& style.display !== "none"
|
||||
&& rect.width > 0
|
||||
&& rect.height > 0;
|
||||
};
|
||||
const leaves = [];
|
||||
for (const element of document.body.querySelectorAll("*")) {
|
||||
if (!isVisible(element)) continue;
|
||||
const text = element.innerText?.trim();
|
||||
if (!text || text.length > 4000) continue;
|
||||
const visibleChildHasText = Array.from(element.children).some((child) => (
|
||||
isVisible(child) && child.innerText?.trim()
|
||||
));
|
||||
if (visibleChildHasText) continue;
|
||||
if (!items.some((needle) => text.includes(needle))) continue;
|
||||
leaves.push(text);
|
||||
}
|
||||
return leaves.at(-1) || "";
|
||||
}, needles);
|
||||
}
|
||||
|
||||
export async function visibleDebugChatMessages(page) {
|
||||
return await page.evaluate(() => {
|
||||
const isVisible = (element) => {
|
||||
const style = window.getComputedStyle(element);
|
||||
const rect = element.getBoundingClientRect();
|
||||
return style.visibility !== "hidden"
|
||||
&& style.display !== "none"
|
||||
&& rect.width > 0
|
||||
&& rect.height > 0;
|
||||
};
|
||||
const classText = (element) => String(element.getAttribute("class") || "");
|
||||
return Array.from(document.querySelectorAll("div.max-w-3xl"))
|
||||
.filter((element) => isVisible(element))
|
||||
.map((element) => {
|
||||
const row = element.parentElement;
|
||||
const text = element.innerText?.trim() || "";
|
||||
const isUser = classText(element).includes("user-message-bubble")
|
||||
|| classText(row).includes("justify-end");
|
||||
return {
|
||||
role: isUser ? "user" : "assistant",
|
||||
text,
|
||||
};
|
||||
})
|
||||
.filter((message) => message.text);
|
||||
});
|
||||
}
|
||||
|
||||
export async function waitForExpectedDebugChatText(page, { expectedText, minExpectedCount, timeoutMs }) {
|
||||
await page.waitForFunction(
|
||||
({ expected, min }) => {
|
||||
return document.body.innerText.split(expected).length - 1 >= min;
|
||||
},
|
||||
{ expected: expectedText, min: minExpectedCount },
|
||||
{ timeout: timeoutMs },
|
||||
).catch(() => {});
|
||||
}
|
||||
|
||||
export async function waitForDebugChatTextStable(page, { timeoutMs = 5_000, quietMs = 750 } = {}) {
|
||||
const startedAt = Date.now();
|
||||
let lastText = await bodyText(page);
|
||||
let stableSince = Date.now();
|
||||
while (Date.now() - startedAt < timeoutMs) {
|
||||
await page.waitForTimeout(250);
|
||||
const currentText = await bodyText(page);
|
||||
if (currentText !== lastText) {
|
||||
lastText = currentText;
|
||||
stableSince = Date.now();
|
||||
continue;
|
||||
}
|
||||
if (Date.now() - stableSince >= quietMs) return;
|
||||
}
|
||||
}
|
||||
|
||||
export async function attachDebugChatImage(page, imagePath) {
|
||||
if (!imagePath) return { status: "not_required", reason: "" };
|
||||
const input = page.locator('input[type="file"][accept*="image"], input[type="file"]').first();
|
||||
if (!await input.count()) {
|
||||
return { status: "fail", reason: "Could not find a Debug Chat image upload input." };
|
||||
}
|
||||
await input.setInputFiles(imagePath);
|
||||
await page.locator("img").last().waitFor({ state: "visible", timeout: 10_000 }).catch(() => {});
|
||||
return { status: "ready", reason: `Attached image fixture: ${imagePath}` };
|
||||
}
|
||||
|
||||
export async function sendDebugChatPrompt(page, prompt, imagePath = "") {
|
||||
const imageResult = await attachDebugChatImage(page, imagePath);
|
||||
if (imageResult.status === "fail") return imageResult;
|
||||
|
||||
const input = debugChatInput(page);
|
||||
const inputVisible = await input.isVisible({ timeout: 5_000 }).catch(() => false);
|
||||
const inputEnabled = inputVisible && await input.isEnabled({ timeout: 10_000 }).catch(() => false);
|
||||
if (!inputVisible || !inputEnabled) return false;
|
||||
await input.fill(prompt).catch(async () => {
|
||||
await input.click();
|
||||
await input.pressSequentially(prompt);
|
||||
});
|
||||
const clickedSend = await clickFirstVisible(page, ["Send", "发送", "提交"], 1_500);
|
||||
if (!clickedSend) await page.keyboard.press("Enter");
|
||||
await page.getByText(prompt, { exact: false }).last().waitFor({ state: "visible", timeout: 10_000 }).catch(() => {});
|
||||
return true;
|
||||
}
|
||||
|
||||
export async function runDebugChatPrompt(page, { prompt, expectedText, responseTimeoutMs, imagePath = "", failureSignals = DEBUG_CHAT_FAILURE_SIGNALS }) {
|
||||
const beforeText = await bodyText(page);
|
||||
const beforeMessages = await visibleDebugChatMessages(page);
|
||||
const minExpectedCount = minExpectedOccurrences(beforeText, expectedText, prompt);
|
||||
const sent = await sendDebugChatPrompt(page, prompt, imagePath);
|
||||
if (sent !== true) {
|
||||
if (sent && typeof sent === "object" && typeof sent.reason === "string") return sent;
|
||||
return { status: "fail", reason: "Could not find a Debug Chat text input." };
|
||||
}
|
||||
|
||||
await waitForExpectedDebugChatText(page, {
|
||||
expectedText,
|
||||
minExpectedCount,
|
||||
prompt,
|
||||
timeoutMs: responseTimeoutMs,
|
||||
});
|
||||
await waitForDebugChatTextStable(page);
|
||||
|
||||
const afterText = await bodyText(page);
|
||||
const afterMessages = await visibleDebugChatMessages(page);
|
||||
const latestAssistantText = afterMessages.filter((message) => message.role === "assistant").at(-1)?.text || "";
|
||||
const latestExpectedLeaf = await latestVisibleLeafText(page, [expectedText]);
|
||||
const failureText = findNewFailureSignal(beforeText, afterText, failureSignals);
|
||||
const latestFailureLeaf = failureText ? await latestVisibleLeafText(page, [failureText]) : "";
|
||||
|
||||
return classifyDebugChatResult({
|
||||
beforeText,
|
||||
afterText,
|
||||
expectedText,
|
||||
prompt,
|
||||
latestExpectedLeaf,
|
||||
latestFailureLeaf,
|
||||
beforeMessages,
|
||||
afterMessages,
|
||||
latestAssistantText,
|
||||
failureSignals,
|
||||
});
|
||||
}
|
||||
|
||||
export async function setDebugChatStreamOutput(page, desired) {
|
||||
if (desired === null || desired === undefined) return { status: "not_required", reason: "" };
|
||||
|
||||
const streamSwitch = page.locator('[role="switch"]').first();
|
||||
if (!await streamSwitch.isVisible({ timeout: 5_000 }).catch(() => false)) {
|
||||
return { status: "blocked", reason: "Debug Chat stream switch was not visible." };
|
||||
}
|
||||
if (!await streamSwitch.isEnabled({ timeout: 10_000 }).catch(() => false)) {
|
||||
return { status: "blocked", reason: "Debug Chat stream switch was visible but disabled." };
|
||||
}
|
||||
|
||||
const checked = (await streamSwitch.getAttribute("aria-checked").catch(() => null)) === "true";
|
||||
if (checked !== desired) {
|
||||
await streamSwitch.click();
|
||||
await page.waitForFunction(
|
||||
({ selector, expected }) => document.querySelector(selector)?.getAttribute("aria-checked") === String(expected),
|
||||
{ selector: '[role="switch"]', expected: desired },
|
||||
{ timeout: 5_000 },
|
||||
).catch(() => {});
|
||||
}
|
||||
|
||||
const finalChecked = (await streamSwitch.getAttribute("aria-checked").catch(() => null)) === "true";
|
||||
if (finalChecked !== desired) {
|
||||
return {
|
||||
status: "fail",
|
||||
reason: `Debug Chat stream switch did not reach requested state: ${desired ? "on" : "off"}.`,
|
||||
};
|
||||
}
|
||||
return { status: "ready", reason: `Debug Chat stream switch is ${desired ? "on" : "off"}.` };
|
||||
}
|
||||
@@ -0,0 +1,341 @@
|
||||
import { appendFile, mkdir, readFile, stat, writeFile } from "node:fs/promises";
|
||||
import { join, resolve } from "node:path";
|
||||
import { env } from "node:process";
|
||||
|
||||
const secretRe = /(?:authorization|bearer|token|secret|password|api[_-]?key|jwt|oauth)\s*[:=]\s*["']?[^"',\s]+/gi;
|
||||
|
||||
export function redact(text) {
|
||||
return String(text ?? "")
|
||||
.replace(secretRe, (match) => match.replace(/[:=]\s*["']?.*$/, "=[redacted]"))
|
||||
.replace(/\bbearer\s+[A-Za-z0-9._~+/=-]{8,}/gi, "Bearer [redacted]")
|
||||
.replace(/\bsk-[A-Za-z0-9_-]{6,}\b/g, "[redacted]");
|
||||
}
|
||||
|
||||
export function timestampSlug(date = new Date()) {
|
||||
return date.toISOString().replace(/\.\d{3}Z$/, "Z").replace(/[^0-9A-Za-z]+/g, "-").replace(/^-|-$/g, "");
|
||||
}
|
||||
|
||||
export function localIsoWithOffset(date = new Date()) {
|
||||
const offsetMinutes = -date.getTimezoneOffset();
|
||||
const sign = offsetMinutes >= 0 ? "+" : "-";
|
||||
const absolute = Math.abs(offsetMinutes);
|
||||
const pad = (value) => String(value).padStart(2, "0");
|
||||
const yyyy = date.getFullYear();
|
||||
const mm = pad(date.getMonth() + 1);
|
||||
const dd = pad(date.getDate());
|
||||
const hh = pad(date.getHours());
|
||||
const mi = pad(date.getMinutes());
|
||||
const ss = pad(date.getSeconds());
|
||||
const ms = String(date.getMilliseconds()).padStart(3, "0");
|
||||
return `${yyyy}-${mm}-${dd}T${hh}:${mi}:${ss}.${ms}${sign}${pad(Math.floor(absolute / 60))}:${pad(absolute % 60)}`;
|
||||
}
|
||||
|
||||
export function evidencePaths(caseId) {
|
||||
const runId = env.LBS_RUN_ID || `${timestampSlug()}-${caseId}`;
|
||||
const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join("reports", "evidence", runId));
|
||||
return {
|
||||
runId,
|
||||
evidenceDir,
|
||||
consoleLog: join(evidenceDir, "console.log"),
|
||||
networkLog: join(evidenceDir, "network.log"),
|
||||
screenshot: join(evidenceDir, "screenshot.png"),
|
||||
automationResultJson: join(evidenceDir, "automation-result.json"),
|
||||
resultJson: join(evidenceDir, "result.json"),
|
||||
};
|
||||
}
|
||||
|
||||
export async function ensureEvidence(paths) {
|
||||
await mkdir(paths.evidenceDir, { recursive: true });
|
||||
await appendFile(paths.consoleLog, "", "utf8");
|
||||
await appendFile(paths.networkLog, "", "utf8");
|
||||
}
|
||||
|
||||
export async function pathExists(path) {
|
||||
try {
|
||||
await stat(path);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export async function appendLine(path, line) {
|
||||
await appendFile(path, `[${localIsoWithOffset()}] ${redact(line)}\n`, "utf8");
|
||||
}
|
||||
|
||||
export async function writeResult(paths, result) {
|
||||
const text = `${JSON.stringify(result, null, 2)}\n`;
|
||||
if (paths.automationResultJson) await writeFile(paths.automationResultJson, text, "utf8");
|
||||
if (paths.resultJson && paths.resultJson !== paths.automationResultJson) {
|
||||
await writeFile(paths.resultJson, text, "utf8");
|
||||
}
|
||||
}
|
||||
|
||||
export async function loadEnvFiles(paths = ["skills/.env", "skills/.env.local"]) {
|
||||
for (const path of paths) {
|
||||
let text = "";
|
||||
try {
|
||||
text = await readFile(path, "utf8");
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
for (const line of text.split(/\r?\n/)) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed || trimmed.startsWith("#")) continue;
|
||||
const equals = trimmed.indexOf("=");
|
||||
if (equals <= 0) continue;
|
||||
const key = trimmed.slice(0, equals).trim();
|
||||
const value = trimmed.slice(equals + 1).trim().replace(/^["']|["']$/g, "");
|
||||
if (!(key in env)) env[key] = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function readRecoveryKey(repo = env.LANGBOT_REPO || "../LangBot") {
|
||||
const configPath = resolve(repo, "data/config.yaml");
|
||||
const config = await readFile(configPath, "utf8");
|
||||
const match = config.match(/^\s*recovery_key:\s*['"]?([^'"\s#]+)['"]?\s*$/m);
|
||||
return match?.[1] || "";
|
||||
}
|
||||
|
||||
export async function apiJson(backendUrl, path, { method = "GET", token = "", body } = {}) {
|
||||
const headers = { "Content-Type": "application/json" };
|
||||
if (token) headers.Authorization = `Bearer ${token}`;
|
||||
const response = await fetch(`${backendUrl.replace(/\/$/, "")}${path}`, {
|
||||
method,
|
||||
headers,
|
||||
body: body === undefined ? undefined : JSON.stringify(body),
|
||||
});
|
||||
return {
|
||||
status: response.status,
|
||||
json: await response.json().catch(() => ({})),
|
||||
};
|
||||
}
|
||||
|
||||
export async function checkBackendToken(backendUrl, token) {
|
||||
if (!token) {
|
||||
return { authenticated: false, http_status: 0, code: null, reason: "No token." };
|
||||
}
|
||||
const response = await apiJson(backendUrl, "/api/v1/user/check-token", { token });
|
||||
const code = response.json.code ?? null;
|
||||
const authenticated = response.status < 400 && code === 0;
|
||||
return {
|
||||
authenticated,
|
||||
http_status: response.status,
|
||||
code,
|
||||
reason: authenticated ? "Token accepted by backend." : response.json.msg || "Backend rejected token.",
|
||||
};
|
||||
}
|
||||
|
||||
export async function resetAndAuthLocalUser({ backendUrl, user, password, recoveryKey = "" }) {
|
||||
const key = recoveryKey || await readRecoveryKey();
|
||||
if (!key) throw new Error("Could not read recovery_key from LangBot config.");
|
||||
|
||||
const reset = await apiJson(backendUrl, "/api/v1/user/reset-password", {
|
||||
method: "POST",
|
||||
body: {
|
||||
user,
|
||||
recovery_key: key,
|
||||
new_password: password,
|
||||
},
|
||||
});
|
||||
if (reset.status >= 400 || reset.json.code !== 0) {
|
||||
throw new Error(reset.json.msg || `Password reset failed with HTTP ${reset.status}.`);
|
||||
}
|
||||
|
||||
const auth = await apiJson(backendUrl, "/api/v1/user/auth", {
|
||||
method: "POST",
|
||||
body: { user, password },
|
||||
});
|
||||
const token = auth.json.data?.token || "";
|
||||
if (auth.status >= 400 || auth.json.code !== 0 || !token) {
|
||||
throw new Error(auth.json.msg || `Auth failed with HTTP ${auth.status}.`);
|
||||
}
|
||||
|
||||
const check = await checkBackendToken(backendUrl, token);
|
||||
if (!check.authenticated) {
|
||||
throw new Error(check.reason || "Authenticated token failed backend token check.");
|
||||
}
|
||||
|
||||
return { token, check };
|
||||
}
|
||||
|
||||
export async function setBrowserToken(page, frontendUrl, token) {
|
||||
await page.addInitScript((value) => {
|
||||
localStorage.setItem("token", value);
|
||||
}, token);
|
||||
await page.goto(frontendUrl, { waitUntil: "domcontentloaded" });
|
||||
await page.evaluate((value) => localStorage.setItem("token", value), token);
|
||||
}
|
||||
|
||||
export async function verifyBrowserToken(page, backendUrl) {
|
||||
return await page.evaluate(async (baseUrl) => {
|
||||
const token = localStorage.getItem("token");
|
||||
if (!token) {
|
||||
return { authenticated: false, http_status: 0, code: null, reason: "No localStorage token." };
|
||||
}
|
||||
try {
|
||||
const response = await fetch(`${baseUrl.replace(/\/$/, "")}/api/v1/user/check-token`, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
const json = await response.json().catch(() => ({}));
|
||||
const code = json.code ?? null;
|
||||
const authenticated = response.status < 400 && code === 0;
|
||||
return {
|
||||
authenticated,
|
||||
http_status: response.status,
|
||||
code,
|
||||
reason: authenticated ? "Token accepted by backend." : json.msg || "Backend rejected token.",
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
authenticated: false,
|
||||
http_status: 0,
|
||||
code: null,
|
||||
reason: error.message,
|
||||
};
|
||||
}
|
||||
}, backendUrl);
|
||||
}
|
||||
|
||||
export function exitCode(status) {
|
||||
if (status === "pass") return 0;
|
||||
if (status === "blocked" || status === "env_issue") return 2;
|
||||
return 1;
|
||||
}
|
||||
|
||||
export async function loadPlaywright() {
|
||||
try {
|
||||
return await import("playwright");
|
||||
} catch {
|
||||
throw new Error(
|
||||
"Playwright is not installed. Install it in this repo with `npm install --save-dev playwright`, then run `npx playwright install chromium`.",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export async function createBrowser(paths) {
|
||||
const { chromium } = await loadPlaywright();
|
||||
const headed = env.LBS_HEADED === "1";
|
||||
const launchOptions = {
|
||||
headless: !headed,
|
||||
};
|
||||
if (env.LANGBOT_CHROMIUM_EXECUTABLE && await pathExists(env.LANGBOT_CHROMIUM_EXECUTABLE)) {
|
||||
launchOptions.executablePath = env.LANGBOT_CHROMIUM_EXECUTABLE;
|
||||
}
|
||||
|
||||
let browser;
|
||||
let context;
|
||||
if (env.LANGBOT_BROWSER_PROFILE) {
|
||||
context = await chromium.launchPersistentContext(resolve(env.LANGBOT_BROWSER_PROFILE), {
|
||||
...launchOptions,
|
||||
viewport: { width: 1440, height: 960 },
|
||||
});
|
||||
} else {
|
||||
browser = await chromium.launch(launchOptions);
|
||||
context = await browser.newContext({ viewport: { width: 1440, height: 960 } });
|
||||
}
|
||||
const page = context.pages()[0] || await context.newPage();
|
||||
|
||||
page.on("console", (message) => {
|
||||
appendLine(paths.consoleLog, `[${message.type()}] ${message.text()}`).catch(() => {});
|
||||
});
|
||||
page.on("pageerror", (error) => {
|
||||
appendLine(paths.consoleLog, `[pageerror] ${error.message}`).catch(() => {});
|
||||
});
|
||||
page.on("requestfailed", (request) => {
|
||||
appendLine(paths.networkLog, `[requestfailed] ${request.method()} ${request.url()} ${request.failure()?.errorText ?? ""}`).catch(() => {});
|
||||
});
|
||||
page.on("response", (response) => {
|
||||
if (response.status() < 400) return;
|
||||
appendLine(paths.networkLog, `[response] ${response.status()} ${response.url()}`).catch(() => {});
|
||||
});
|
||||
|
||||
return {
|
||||
page,
|
||||
context,
|
||||
async close() {
|
||||
await context.close();
|
||||
if (browser) await browser.close();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function safeScreenshot(page, path) {
|
||||
try {
|
||||
await page.screenshot({ path, fullPage: true });
|
||||
} catch {
|
||||
// Screenshot evidence is useful, but a screenshot failure should not hide the real test result.
|
||||
}
|
||||
}
|
||||
|
||||
export async function gotoFrontend(page) {
|
||||
const frontendUrl = env.LANGBOT_FRONTEND_URL;
|
||||
if (!frontendUrl) {
|
||||
throw new Error("LANGBOT_FRONTEND_URL is not configured.");
|
||||
}
|
||||
await page.goto(frontendUrl, { waitUntil: "domcontentloaded" });
|
||||
await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {});
|
||||
}
|
||||
|
||||
export function isLoginUrl(url) {
|
||||
return /\/login(?:[/?#]|$)/.test(url);
|
||||
}
|
||||
|
||||
export async function bodyText(page) {
|
||||
return await page.locator("body").innerText({ timeout: 5_000 }).catch(() => "");
|
||||
}
|
||||
|
||||
export function countOccurrences(haystack, needle) {
|
||||
if (!needle) return 0;
|
||||
return String(haystack).split(needle).length - 1;
|
||||
}
|
||||
|
||||
export async function clickFirstVisible(page, labels, timeout = 2_000) {
|
||||
for (const label of labels) {
|
||||
const roleButton = page.getByRole("button", { name: label }).first();
|
||||
if (await roleButton.isVisible({ timeout }).catch(() => false)) {
|
||||
await roleButton.click();
|
||||
return label;
|
||||
}
|
||||
|
||||
const roleLink = page.getByRole("link", { name: label }).first();
|
||||
if (await roleLink.isVisible({ timeout }).catch(() => false)) {
|
||||
await roleLink.click();
|
||||
return label;
|
||||
}
|
||||
|
||||
const text = page.getByText(label, { exact: false }).first();
|
||||
if (await text.isVisible({ timeout }).catch(() => false)) {
|
||||
await text.click();
|
||||
return label;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function fillFirstTextInput(page, value) {
|
||||
const candidates = [
|
||||
page.getByRole("textbox").last(),
|
||||
page.locator("textarea").last(),
|
||||
page.locator("[contenteditable=true]").last(),
|
||||
page.locator("input[type=text]").last(),
|
||||
];
|
||||
|
||||
for (const locator of candidates) {
|
||||
if (!await locator.isVisible({ timeout: 2_000 }).catch(() => false)) continue;
|
||||
await locator.fill(value).catch(async () => {
|
||||
await locator.click();
|
||||
await locator.pressSequentially(value);
|
||||
});
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export async function waitForVisibleText(page, text, timeout = 20_000) {
|
||||
await page.getByText(text, { exact: false }).last().waitFor({ state: "visible", timeout });
|
||||
}
|
||||
@@ -0,0 +1,565 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { env } from "node:process";
|
||||
import {
|
||||
DEBUG_CHAT_FAILURE_SIGNALS,
|
||||
openPipelineDebugChat,
|
||||
setDebugChatStreamOutput,
|
||||
visibleDebugChatMessages,
|
||||
waitForDebugChatTextStable,
|
||||
} from "./lib/debug-chat.mjs";
|
||||
import {
|
||||
createBrowser,
|
||||
ensureEvidence,
|
||||
evidencePaths,
|
||||
exitCode,
|
||||
localIsoWithOffset,
|
||||
loadEnvFiles,
|
||||
pathExists,
|
||||
safeScreenshot,
|
||||
writeResult,
|
||||
} from "./lib/langbot-e2e.mjs";
|
||||
|
||||
await loadEnvFiles();
|
||||
|
||||
const caseId = env.LBS_CASE_ID || "local-agent-steering-debug-chat";
|
||||
const paths = evidencePaths(caseId);
|
||||
await ensureEvidence(paths);
|
||||
|
||||
const backendUrl = (env.LANGBOT_BACKEND_URL || "").replace(/\/$/, "");
|
||||
const pipelineUrl = env.LANGBOT_E2E_PIPELINE_URL || env.LANGBOT_LOCAL_AGENT_PIPELINE_URL || env.LANGBOT_PIPELINE_URL || "";
|
||||
const pipelineName = env.LANGBOT_E2E_PIPELINE_NAME || env.LANGBOT_LOCAL_AGENT_PIPELINE_NAME || env.LANGBOT_PIPELINE_NAME || "";
|
||||
const expectedRunnerId = env.LANGBOT_E2E_EXPECTED_RUNNER_ID || "plugin:langbot/local-agent/default";
|
||||
const expectedText = env.LANGBOT_E2E_EXPECTED_TEXT || "qa_steering_sentinel_6194";
|
||||
const responseTimeoutMs = positiveInt(env.LANGBOT_E2E_RESPONSE_TIMEOUT_MS, 240000);
|
||||
const followupDelayMs = 1000;
|
||||
const followupEnabledTimeoutMs = 1500;
|
||||
const firstPrompt = env.LANGBOT_E2E_PROMPT || [
|
||||
"You are running the LangBot steering E2E test.",
|
||||
"First call the qa_plugin_sleep tool with seconds=8 and text=steering-e2e-anchor.",
|
||||
"Do not answer before the tool result is available.",
|
||||
"After the tool returns, answer the latest user follow-up.",
|
||||
"If no follow-up was injected, reply only STEERING_NO_FOLLOWUP.",
|
||||
].join(" ");
|
||||
const followupPrompt = [
|
||||
"This is a steering follow-up sent while the first tool call is still active.",
|
||||
`Return only ${expectedText}.`,
|
||||
].join(" ");
|
||||
|
||||
const pipelineConfigDiagnosticPath = `${paths.evidenceDir}/pipeline-config-diagnostic.json`;
|
||||
const debugChatResetDiagnosticPath = `${paths.evidenceDir}/debug-chat-reset-diagnostic.json`;
|
||||
const toolDiagnosticPath = `${paths.evidenceDir}/tool-diagnostic.json`;
|
||||
|
||||
let browser;
|
||||
const result = {
|
||||
source: "automation",
|
||||
case_id: caseId,
|
||||
run_id: paths.runId,
|
||||
status: "fail",
|
||||
reason: "",
|
||||
started_at: new Date().toISOString(),
|
||||
started_at_local: localIsoWithOffset(new Date()),
|
||||
url: "",
|
||||
backend_url: backendUrl,
|
||||
pipeline_url: pipelineUrl,
|
||||
pipeline_name: pipelineName,
|
||||
expected_runner_id: expectedRunnerId,
|
||||
first_prompt: firstPrompt,
|
||||
followup_prompt: followupPrompt,
|
||||
expected_text: expectedText,
|
||||
followup_delay_ms: followupDelayMs,
|
||||
followup_enabled_timeout_ms: followupEnabledTimeoutMs,
|
||||
response_timeout_ms: responseTimeoutMs,
|
||||
pipeline_config: null,
|
||||
debug_chat_reset: null,
|
||||
tool_diagnostic: null,
|
||||
steering: null,
|
||||
evidence: {
|
||||
console_log: paths.consoleLog,
|
||||
network_log: paths.networkLog,
|
||||
screenshot: paths.screenshot,
|
||||
automation_result_json: paths.automationResultJson,
|
||||
result_json: paths.resultJson,
|
||||
},
|
||||
evidence_collected: ["ui", "console", "network", "screenshot"],
|
||||
};
|
||||
|
||||
try {
|
||||
if (!backendUrl) {
|
||||
result.status = "env_issue";
|
||||
result.reason = "LANGBOT_BACKEND_URL is required.";
|
||||
throw new Error(result.reason);
|
||||
}
|
||||
|
||||
browser = await createBrowser(paths);
|
||||
const { page } = browser;
|
||||
|
||||
const openResult = await openPipelineDebugChat(page, {
|
||||
pipelineUrl,
|
||||
pipelineName,
|
||||
envHint: "case-specific pipeline env mapped to LANGBOT_E2E_PIPELINE_URL or LANGBOT_E2E_PIPELINE_NAME",
|
||||
});
|
||||
result.url = page.url();
|
||||
if (!openResult.opened) {
|
||||
result.status = openResult.status;
|
||||
result.reason = openResult.reason;
|
||||
} else {
|
||||
const pipelineDiagnostic = await inspectPipeline(page, {
|
||||
backendUrl,
|
||||
pipelineUrl,
|
||||
pipelineName,
|
||||
expectedRunnerId,
|
||||
});
|
||||
await writeFile(pipelineConfigDiagnosticPath, `${JSON.stringify(pipelineDiagnostic, null, 2)}\n`, "utf8");
|
||||
result.evidence.pipeline_config_diagnostic_json = pipelineConfigDiagnosticPath;
|
||||
result.pipeline_config = pipelineDiagnostic;
|
||||
if (!result.evidence_collected.includes("api_diagnostic")) result.evidence_collected.push("api_diagnostic");
|
||||
|
||||
const toolDiagnostic = await inspectToolNames(page, { backendUrl });
|
||||
await writeFile(toolDiagnosticPath, `${JSON.stringify(toolDiagnostic, null, 2)}\n`, "utf8");
|
||||
result.evidence.tool_diagnostic_json = toolDiagnosticPath;
|
||||
result.tool_diagnostic = toolDiagnostic;
|
||||
|
||||
if (pipelineDiagnostic.status === "fail" || pipelineDiagnostic.status === "blocked") {
|
||||
result.status = pipelineDiagnostic.status;
|
||||
result.reason = pipelineDiagnostic.reason || "Pipeline diagnostic failed.";
|
||||
} else if (toolDiagnostic.status === "fail" || toolDiagnostic.status === "blocked") {
|
||||
result.status = toolDiagnostic.status;
|
||||
result.reason = toolDiagnostic.reason || "Tool diagnostic failed.";
|
||||
} else if (!toolDiagnostic.tool_names.includes("qa_plugin_sleep")) {
|
||||
result.status = "blocked";
|
||||
result.reason = "qa_plugin_sleep is not exposed by /api/v1/tools; rebuild/reinstall qa-plugin-smoke before running steering E2E.";
|
||||
} else {
|
||||
const resetDiagnostic = await resetPipelineDebugChat(page, {
|
||||
backendUrl,
|
||||
pipelineId: pipelineDiagnostic.pipeline_id,
|
||||
sessionType: "person",
|
||||
});
|
||||
await writeFile(debugChatResetDiagnosticPath, `${JSON.stringify(resetDiagnostic, null, 2)}\n`, "utf8");
|
||||
result.evidence.debug_chat_reset_diagnostic_json = debugChatResetDiagnosticPath;
|
||||
result.debug_chat_reset = resetDiagnostic;
|
||||
|
||||
if (resetDiagnostic.status === "fail" || resetDiagnostic.status === "blocked") {
|
||||
result.status = resetDiagnostic.status;
|
||||
result.reason = resetDiagnostic.reason || "Debug Chat reset failed.";
|
||||
} else {
|
||||
await page.waitForTimeout(1000);
|
||||
const reopenResult = await openPipelineDebugChat(page, {
|
||||
pipelineUrl,
|
||||
pipelineName,
|
||||
envHint: "case-specific pipeline env mapped to LANGBOT_E2E_PIPELINE_URL or LANGBOT_E2E_PIPELINE_NAME",
|
||||
});
|
||||
result.url = page.url();
|
||||
if (!reopenResult.opened) {
|
||||
result.status = reopenResult.status;
|
||||
result.reason = reopenResult.reason;
|
||||
} else {
|
||||
const streamResult = await setDebugChatStreamOutput(page, true);
|
||||
if (streamResult.status === "blocked" || streamResult.status === "fail") {
|
||||
result.status = streamResult.status;
|
||||
result.reason = streamResult.reason;
|
||||
} else {
|
||||
result.steering = await runSteeringProbe(page);
|
||||
result.status = result.steering.status;
|
||||
result.reason = result.steering.reason;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (!["env_issue", "blocked", "fail", "pass"].includes(result.status) || !result.reason) {
|
||||
result.status = /Playwright is not installed|LANGBOT_FRONTEND_URL/.test(error.message) ? "env_issue" : "fail";
|
||||
}
|
||||
result.reason = result.reason || error.message;
|
||||
} finally {
|
||||
if (browser?.page) await safeScreenshot(browser.page, paths.screenshot);
|
||||
if (browser) await browser.close().catch(() => {});
|
||||
const finishedAt = new Date();
|
||||
result.finished_at = finishedAt.toISOString();
|
||||
result.finished_at_local = localIsoWithOffset(finishedAt);
|
||||
const existingEvidence = {};
|
||||
for (const [key, value] of Object.entries(result.evidence)) {
|
||||
if (typeof value !== "string") continue;
|
||||
const isResultFile = value === paths.automationResultJson || value === paths.resultJson;
|
||||
if (isResultFile || await pathExists(value)) existingEvidence[key] = value;
|
||||
}
|
||||
result.evidence = existingEvidence;
|
||||
await writeResult(paths, result);
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
}
|
||||
|
||||
process.exit(exitCode(result.status));
|
||||
|
||||
async function runSteeringProbe(page) {
|
||||
const beforeMessages = await visibleDebugChatMessages(page);
|
||||
const beforeAssistantCount = countRole(beforeMessages, "assistant");
|
||||
const beforeUserCount = countRole(beforeMessages, "user");
|
||||
const firstStartedAt = Date.now();
|
||||
const firstSend = await sendPrompt(page, firstPrompt, { enabledTimeoutMs: 5000 });
|
||||
if (!firstSend.sent) {
|
||||
return {
|
||||
status: "fail",
|
||||
reason: firstSend.reason || "Could not send first Debug Chat prompt.",
|
||||
first_send: firstSend,
|
||||
before_assistant_count: beforeAssistantCount,
|
||||
before_user_count: beforeUserCount,
|
||||
};
|
||||
}
|
||||
|
||||
await page.waitForTimeout(followupDelayMs);
|
||||
const preFollowupMessages = await visibleDebugChatMessages(page);
|
||||
const preFollowupAssistantCount = countRole(preFollowupMessages, "assistant");
|
||||
const followupStartedAt = Date.now();
|
||||
const followupSend = await sendPrompt(page, followupPrompt, { enabledTimeoutMs: followupEnabledTimeoutMs });
|
||||
const followupSentAt = Date.now();
|
||||
if (!followupSend.sent) {
|
||||
return {
|
||||
status: "fail",
|
||||
reason: followupSend.reason || "Could not send steering follow-up while the first run was active.",
|
||||
first_send: firstSend,
|
||||
followup_send: followupSend,
|
||||
first_to_followup_attempt_ms: followupStartedAt - firstStartedAt,
|
||||
followup_send_latency_ms: followupSentAt - followupStartedAt,
|
||||
before_assistant_count: beforeAssistantCount,
|
||||
pre_followup_assistant_count: preFollowupAssistantCount,
|
||||
before_user_count: beforeUserCount,
|
||||
};
|
||||
}
|
||||
|
||||
const waitResult = await waitForLatestAssistantContaining(page, {
|
||||
expectedText,
|
||||
beforeAssistantCount,
|
||||
timeoutMs: responseTimeoutMs,
|
||||
});
|
||||
await waitForDebugChatTextStable(page);
|
||||
const afterMessages = await visibleDebugChatMessages(page);
|
||||
const afterAssistantCount = countRole(afterMessages, "assistant");
|
||||
const afterUserCount = countRole(afterMessages, "user");
|
||||
const latestAssistantText = latestRoleText(afterMessages, "assistant");
|
||||
const failureSignal = findFailureSignal(latestAssistantText) || findFailureSignal(messagesText(afterMessages));
|
||||
const newAssistantCount = afterAssistantCount - beforeAssistantCount;
|
||||
const newUserCount = afterUserCount - beforeUserCount;
|
||||
|
||||
const base = {
|
||||
first_send: firstSend,
|
||||
followup_send: followupSend,
|
||||
first_to_followup_attempt_ms: followupStartedAt - firstStartedAt,
|
||||
followup_send_latency_ms: followupSentAt - followupStartedAt,
|
||||
before_assistant_count: beforeAssistantCount,
|
||||
pre_followup_assistant_count: preFollowupAssistantCount,
|
||||
after_assistant_count: afterAssistantCount,
|
||||
new_assistant_count: newAssistantCount,
|
||||
before_user_count: beforeUserCount,
|
||||
after_user_count: afterUserCount,
|
||||
new_user_count: newUserCount,
|
||||
latest_assistant_text: latestAssistantText,
|
||||
assistant_containing_expected_seen: waitResult.seen,
|
||||
failure_signal: failureSignal,
|
||||
};
|
||||
|
||||
if (failureSignal) {
|
||||
return {
|
||||
...base,
|
||||
status: "fail",
|
||||
reason: `Debug Chat displayed a known failure signal: ${failureSignal}`,
|
||||
};
|
||||
}
|
||||
if (!waitResult.seen) {
|
||||
return {
|
||||
...base,
|
||||
status: "fail",
|
||||
reason: `No new assistant message contained steering sentinel ${expectedText}.`,
|
||||
};
|
||||
}
|
||||
if (!latestAssistantText.includes(expectedText)) {
|
||||
return {
|
||||
...base,
|
||||
status: "fail",
|
||||
reason: `Latest assistant message did not contain steering sentinel ${expectedText}.`,
|
||||
};
|
||||
}
|
||||
if (newUserCount < 2) {
|
||||
return {
|
||||
...base,
|
||||
status: "fail",
|
||||
reason: `Expected two new user messages, saw ${newUserCount}.`,
|
||||
};
|
||||
}
|
||||
if (newAssistantCount !== 1) {
|
||||
return {
|
||||
...base,
|
||||
status: "fail",
|
||||
reason: `Expected one assistant response for one claimed steering run, saw ${newAssistantCount}. More than one usually means the follow-up became a separate run.`,
|
||||
};
|
||||
}
|
||||
if (latestAssistantText.includes("STEERING_NO_FOLLOWUP")) {
|
||||
return {
|
||||
...base,
|
||||
status: "fail",
|
||||
reason: "Runner answered the no-follow-up branch, so steering was not injected.",
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
...base,
|
||||
status: "pass",
|
||||
reason: `Follow-up sentinel ${expectedText} appeared in the only new assistant response after two user messages.`,
|
||||
};
|
||||
}
|
||||
|
||||
function debugChatInput(page) {
|
||||
return page
|
||||
.locator('input[placeholder*="message"], input[placeholder*="消息"], textarea[placeholder*="message"], textarea[placeholder*="消息"]')
|
||||
.last();
|
||||
}
|
||||
|
||||
async function sendPrompt(page, prompt, { enabledTimeoutMs }) {
|
||||
const input = debugChatInput(page);
|
||||
const inputVisible = await input.isVisible({ timeout: 5000 }).catch(() => false);
|
||||
if (!inputVisible) return { sent: false, reason: "Debug Chat input is not visible." };
|
||||
const inputEnabled = await input.isEnabled({ timeout: enabledTimeoutMs }).catch(() => false);
|
||||
if (!inputEnabled) return { sent: false, reason: `Debug Chat input was not enabled within ${enabledTimeoutMs}ms.` };
|
||||
|
||||
await input.fill(prompt).catch(async () => {
|
||||
await input.click();
|
||||
await input.pressSequentially(prompt);
|
||||
});
|
||||
await input.press("Enter");
|
||||
await page.getByText(prompt, { exact: false }).last().waitFor({ state: "visible", timeout: 10000 }).catch(() => {});
|
||||
return {
|
||||
sent: true,
|
||||
submitted_by: "keyboard_enter",
|
||||
};
|
||||
}
|
||||
|
||||
async function waitForLatestAssistantContaining(page, { expectedText, beforeAssistantCount, timeoutMs }) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
let lastMessages = [];
|
||||
let latestAssistantText = "";
|
||||
while (Date.now() < deadline) {
|
||||
const messages = await visibleDebugChatMessages(page);
|
||||
lastMessages = messages;
|
||||
latestAssistantText = latestRoleText(messages, "assistant");
|
||||
if (countRole(messages, "assistant") > beforeAssistantCount && latestAssistantText.includes(expectedText)) {
|
||||
return {
|
||||
seen: true,
|
||||
latest_assistant_text: latestAssistantText,
|
||||
messages,
|
||||
};
|
||||
}
|
||||
const failureSignal = findFailureSignal(latestAssistantText);
|
||||
if (failureSignal) {
|
||||
return {
|
||||
seen: false,
|
||||
latest_assistant_text: latestAssistantText,
|
||||
messages,
|
||||
failure_signal: failureSignal,
|
||||
};
|
||||
}
|
||||
await page.waitForTimeout(500);
|
||||
}
|
||||
return {
|
||||
seen: false,
|
||||
latest_assistant_text: latestAssistantText,
|
||||
messages: lastMessages,
|
||||
};
|
||||
}
|
||||
|
||||
async function inspectPipeline(page, { backendUrl, pipelineUrl, pipelineName, expectedRunnerId }) {
|
||||
const pipelineIdFromUrl = pipelineIdFromUrlValue(pipelineUrl);
|
||||
return await page.evaluate(async ({ backendUrl, pipelineIdFromUrl, pipelineName, expectedRunnerId }) => {
|
||||
const token = localStorage.getItem("token");
|
||||
if (!token) {
|
||||
return {
|
||||
status: "blocked",
|
||||
authenticated: false,
|
||||
reason: "Browser profile has no localStorage token.",
|
||||
};
|
||||
}
|
||||
const getJson = async (path) => {
|
||||
const response = await fetch(`${backendUrl}${path}`, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
return {
|
||||
status: response.status,
|
||||
json: await response.json().catch(() => ({})),
|
||||
};
|
||||
};
|
||||
|
||||
let pipelineId = pipelineIdFromUrl;
|
||||
let matchedBy = pipelineId ? "url" : "";
|
||||
if (!pipelineId) {
|
||||
if (!pipelineName) {
|
||||
return {
|
||||
status: "blocked",
|
||||
authenticated: true,
|
||||
pipeline_resolved: false,
|
||||
reason: "Set LANGBOT_LOCAL_AGENT_PIPELINE_URL or LANGBOT_LOCAL_AGENT_PIPELINE_NAME.",
|
||||
};
|
||||
}
|
||||
const list = await getJson("/api/v1/pipelines");
|
||||
const pipelines = list.json.data?.pipelines || [];
|
||||
const match = pipelines.find((pipeline) => pipeline.name === pipelineName);
|
||||
if (!match) {
|
||||
return {
|
||||
status: "blocked",
|
||||
authenticated: true,
|
||||
pipeline_resolved: false,
|
||||
list_status: list.status,
|
||||
reason: `Could not find pipeline named ${pipelineName}.`,
|
||||
};
|
||||
}
|
||||
pipelineId = match.uuid;
|
||||
matchedBy = "name";
|
||||
}
|
||||
|
||||
const loaded = await getJson(`/api/v1/pipelines/${encodeURIComponent(pipelineId)}`);
|
||||
const pipeline = loaded.json.data?.pipeline;
|
||||
if (loaded.status >= 400 || !pipeline) {
|
||||
return {
|
||||
status: "fail",
|
||||
authenticated: true,
|
||||
pipeline_resolved: false,
|
||||
pipeline_id: pipelineId,
|
||||
get_status: loaded.status,
|
||||
reason: loaded.json.msg || "Could not load pipeline.",
|
||||
};
|
||||
}
|
||||
const config = pipeline.config || {};
|
||||
const runner = config.ai?.runner || {};
|
||||
const runnerId = runner.id || runner.runner || "";
|
||||
if (!runnerId) {
|
||||
return {
|
||||
status: "blocked",
|
||||
authenticated: true,
|
||||
pipeline_resolved: true,
|
||||
pipeline_id: pipelineId,
|
||||
pipeline_name: pipeline.name,
|
||||
matched_by: matchedBy,
|
||||
reason: "Pipeline has no ai.runner.id or legacy ai.runner.runner.",
|
||||
};
|
||||
}
|
||||
if (expectedRunnerId && runnerId !== expectedRunnerId) {
|
||||
return {
|
||||
status: "blocked",
|
||||
authenticated: true,
|
||||
pipeline_resolved: true,
|
||||
pipeline_id: pipelineId,
|
||||
pipeline_name: pipeline.name,
|
||||
matched_by: matchedBy,
|
||||
runner_id: runnerId,
|
||||
expected_runner_id: expectedRunnerId,
|
||||
reason: `Pipeline runner mismatch: expected ${expectedRunnerId}, got ${runnerId}.`,
|
||||
};
|
||||
}
|
||||
return {
|
||||
status: "ready",
|
||||
authenticated: true,
|
||||
pipeline_resolved: true,
|
||||
pipeline_id: pipelineId,
|
||||
pipeline_name: pipeline.name,
|
||||
matched_by: matchedBy,
|
||||
runner_id: runnerId,
|
||||
expected_runner_id: expectedRunnerId || "",
|
||||
};
|
||||
}, { backendUrl, pipelineIdFromUrl, pipelineName, expectedRunnerId });
|
||||
}
|
||||
|
||||
async function inspectToolNames(page, { backendUrl }) {
|
||||
return await page.evaluate(async ({ backendUrl }) => {
|
||||
const token = localStorage.getItem("token");
|
||||
if (!token) {
|
||||
return {
|
||||
status: "blocked",
|
||||
authenticated: false,
|
||||
tool_names: [],
|
||||
reason: "Browser profile has no localStorage token.",
|
||||
};
|
||||
}
|
||||
const response = await fetch(`${backendUrl}/api/v1/tools`, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
const json = await response.json().catch(() => ({}));
|
||||
const toolNames = (json.data?.tools || [])
|
||||
.map((tool) => tool.name || tool.tool_name || tool.function?.name || "")
|
||||
.filter(Boolean)
|
||||
.sort();
|
||||
return {
|
||||
status: response.status >= 400 ? "fail" : "ready",
|
||||
authenticated: true,
|
||||
http_status: response.status,
|
||||
code: json.code ?? null,
|
||||
tool_names: toolNames,
|
||||
reason: response.status >= 400 ? json.msg || "Could not list tools." : "Tool list loaded.",
|
||||
};
|
||||
}, { backendUrl });
|
||||
}
|
||||
|
||||
async function resetPipelineDebugChat(page, { backendUrl, pipelineId, sessionType }) {
|
||||
return await page.evaluate(async ({ backendUrl, pipelineId, sessionType }) => {
|
||||
const token = localStorage.getItem("token");
|
||||
if (!token) {
|
||||
return {
|
||||
status: "blocked",
|
||||
authenticated: false,
|
||||
pipeline_id: pipelineId,
|
||||
session_type: sessionType,
|
||||
reason: "Browser profile has no localStorage token.",
|
||||
};
|
||||
}
|
||||
const response = await fetch(
|
||||
`${backendUrl}/api/v1/pipelines/${encodeURIComponent(pipelineId)}/ws/reset/${encodeURIComponent(sessionType)}`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
},
|
||||
);
|
||||
const json = await response.json().catch(() => ({}));
|
||||
return {
|
||||
status: response.status >= 400 ? "fail" : "ready",
|
||||
authenticated: true,
|
||||
pipeline_id: pipelineId,
|
||||
session_type: sessionType,
|
||||
reset_status: response.status,
|
||||
reset_code: json.code ?? null,
|
||||
reason: response.status >= 400 ? json.msg || "Debug Chat reset failed." : "Debug Chat session reset.",
|
||||
};
|
||||
}, { backendUrl, pipelineId, sessionType });
|
||||
}
|
||||
|
||||
function pipelineIdFromUrlValue(value) {
|
||||
const match = String(value || "").match(/\/pipelines?\/([^/?#]+)/i);
|
||||
return match ? decodeURIComponent(match[1]) : "";
|
||||
}
|
||||
|
||||
function countRole(messages, role) {
|
||||
return messages.filter((message) => message.role === role).length;
|
||||
}
|
||||
|
||||
function latestRoleText(messages, role) {
|
||||
return messages.filter((message) => message.role === role).at(-1)?.text || "";
|
||||
}
|
||||
|
||||
function messagesText(messages) {
|
||||
return messages.map((message) => message.text).join("\n");
|
||||
}
|
||||
|
||||
function findFailureSignal(text) {
|
||||
return DEBUG_CHAT_FAILURE_SIGNALS.find((signal) => String(text || "").includes(signal)) || "";
|
||||
}
|
||||
|
||||
function positiveInt(value, fallback) {
|
||||
const parsed = Number.parseInt(String(value || ""), 10);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
||||
}
|
||||
Executable
+185
@@ -0,0 +1,185 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { spawn } from "node:child_process";
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import { env } from "node:process";
|
||||
import {
|
||||
ensureEvidence,
|
||||
evidencePaths,
|
||||
exitCode,
|
||||
localIsoWithOffset,
|
||||
writeResult,
|
||||
} from "./lib/langbot-e2e.mjs";
|
||||
|
||||
function loadEnvDefaults(path) {
|
||||
if (!existsSync(path)) return;
|
||||
for (const rawLine of readFileSync(path, "utf8").split(/\r?\n/)) {
|
||||
const line = rawLine.trim();
|
||||
if (!line || line.startsWith("#")) continue;
|
||||
const sep = line.indexOf("=");
|
||||
if (sep === -1) continue;
|
||||
const key = line.slice(0, sep).trim();
|
||||
if (env[key]) continue;
|
||||
env[key] = line.slice(sep + 1).trim().replace(/^["']|["']$/g, "");
|
||||
}
|
||||
}
|
||||
|
||||
loadEnvDefaults("skills/.env");
|
||||
loadEnvDefaults("skills/.env.local");
|
||||
|
||||
const caseId = env.LBS_CASE_ID || "mcp-stdio-fixture-direct";
|
||||
const paths = evidencePaths(caseId);
|
||||
await ensureEvidence(paths);
|
||||
|
||||
const startedAt = new Date();
|
||||
const fixturePath = resolve(env.LANGBOT_MCP_FIXTURE_PATH || "skills/langbot-testing/fixtures/mcp/qa_mcp_echo_server.py");
|
||||
const langbotRepo = env.LANGBOT_REPO ? resolve(env.LANGBOT_REPO) : "";
|
||||
const uvCandidates = [
|
||||
env.LANGBOT_MCP_FIXTURE_UV,
|
||||
"uv",
|
||||
].filter(Boolean);
|
||||
const uv = uvCandidates.find((candidate) => candidate === "uv" || existsSync(candidate));
|
||||
const pythonCandidates = [
|
||||
env.LANGBOT_MCP_FIXTURE_PYTHON,
|
||||
langbotRepo ? `${langbotRepo}/.venv/bin/python` : "",
|
||||
"python3",
|
||||
].filter(Boolean);
|
||||
const python = pythonCandidates.find((candidate) => candidate === "python3" || existsSync(candidate));
|
||||
const command = langbotRepo && uv
|
||||
? { executable: uv, args: ["run", "python", fixturePath], cwd: langbotRepo, mode: "uv" }
|
||||
: python
|
||||
? { executable: python, args: [fixturePath], cwd: resolve("."), mode: "python" }
|
||||
: null;
|
||||
const expectedText = "qa_mcp_echo:mcp-stdio-fixture-ok";
|
||||
|
||||
const result = {
|
||||
source: "automation",
|
||||
case_id: caseId,
|
||||
run_id: paths.runId,
|
||||
started_at: startedAt.toISOString(),
|
||||
started_at_local: localIsoWithOffset(startedAt),
|
||||
finished_at: "",
|
||||
finished_at_local: "",
|
||||
status: "fail",
|
||||
reason: "",
|
||||
fixture_path: fixturePath,
|
||||
command,
|
||||
expected_text: expectedText,
|
||||
evidence: {
|
||||
automation_result_json: paths.automationResultJson,
|
||||
result_json: paths.resultJson,
|
||||
},
|
||||
};
|
||||
|
||||
function parseJsonLines(buffer) {
|
||||
return buffer
|
||||
.split(/\r?\n/)
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.map((line) => {
|
||||
try {
|
||||
return JSON.parse(line);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
})
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
async function request(child, id, method, params) {
|
||||
child.stdin.write(`${JSON.stringify({ jsonrpc: "2.0", id, method, params })}\n`);
|
||||
}
|
||||
|
||||
async function run() {
|
||||
if (!command) {
|
||||
result.status = "env_issue";
|
||||
result.reason = "No uv or Python interpreter found. Set LANGBOT_REPO, LANGBOT_MCP_FIXTURE_UV, or LANGBOT_MCP_FIXTURE_PYTHON.";
|
||||
return;
|
||||
}
|
||||
if (!existsSync(fixturePath)) {
|
||||
result.status = "env_issue";
|
||||
result.reason = `MCP fixture not found: ${fixturePath}`;
|
||||
return;
|
||||
}
|
||||
|
||||
const child = spawn(command.executable, command.args, {
|
||||
cwd: command.cwd,
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
});
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
child.stdout.setEncoding("utf8");
|
||||
child.stderr.setEncoding("utf8");
|
||||
child.stdout.on("data", (chunk) => {
|
||||
stdout += chunk;
|
||||
});
|
||||
child.stderr.on("data", (chunk) => {
|
||||
stderr += chunk;
|
||||
});
|
||||
|
||||
const timeout = setTimeout(() => child.kill("SIGTERM"), 10_000);
|
||||
try {
|
||||
await new Promise((resolveReady) => setTimeout(resolveReady, 100));
|
||||
await request(child, 1, "initialize", {
|
||||
protocolVersion: "2024-11-05",
|
||||
capabilities: {},
|
||||
clientInfo: { name: "langbot-skills", version: "0" },
|
||||
});
|
||||
await new Promise((resolveReady) => setTimeout(resolveReady, 200));
|
||||
child.stdin.write(`${JSON.stringify({ jsonrpc: "2.0", method: "notifications/initialized", params: {} })}\n`);
|
||||
await request(child, 2, "tools/list", {});
|
||||
await request(child, 3, "tools/call", {
|
||||
name: "qa_mcp_echo",
|
||||
arguments: { text: "mcp-stdio-fixture-ok" },
|
||||
});
|
||||
await new Promise((resolveDone) => setTimeout(resolveDone, 1500));
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
child.kill("SIGTERM");
|
||||
}
|
||||
|
||||
const messages = parseJsonLines(stdout);
|
||||
if (/No module named ['"]mcp['"]|ModuleNotFoundError/i.test(stderr)) {
|
||||
result.status = "env_issue";
|
||||
result.reason = `Python environment cannot import mcp. Set LANGBOT_MCP_FIXTURE_PYTHON to a LangBot venv Python. stderr=${stderr.trim()}`;
|
||||
return;
|
||||
}
|
||||
const listResult = messages.find((message) => message.id === 2)?.result;
|
||||
const callResult = messages.find((message) => message.id === 3)?.result;
|
||||
const toolNames = Array.isArray(listResult?.tools)
|
||||
? listResult.tools.map((tool) => tool.name)
|
||||
: [];
|
||||
const callText = Array.isArray(callResult?.content)
|
||||
? callResult.content.map((item) => item.text || "").join("\n")
|
||||
: "";
|
||||
|
||||
if (!toolNames.includes("qa_mcp_echo")) {
|
||||
result.status = "fail";
|
||||
result.reason = `MCP fixture did not list qa_mcp_echo. stderr=${stderr.trim()}`;
|
||||
return;
|
||||
}
|
||||
if (!callText.includes(expectedText)) {
|
||||
result.status = "fail";
|
||||
result.reason = `MCP fixture call did not return ${expectedText}. stderr=${stderr.trim()}`;
|
||||
return;
|
||||
}
|
||||
|
||||
result.status = "pass";
|
||||
result.reason = "MCP stdio fixture listed qa_mcp_echo and returned the deterministic tool result without a model provider.";
|
||||
}
|
||||
|
||||
try {
|
||||
await run();
|
||||
} catch (error) {
|
||||
result.status = "fail";
|
||||
result.reason = error instanceof Error ? error.message : String(error);
|
||||
} finally {
|
||||
const finishedAt = new Date();
|
||||
result.finished_at = finishedAt.toISOString();
|
||||
result.finished_at_local = localIsoWithOffset(finishedAt);
|
||||
await writeResult(paths, result);
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
}
|
||||
|
||||
process.exit(exitCode(result.status));
|
||||
@@ -0,0 +1,234 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { resolve } from "node:path";
|
||||
import { env } from "node:process";
|
||||
import {
|
||||
createBrowser,
|
||||
ensureEvidence,
|
||||
evidencePaths,
|
||||
exitCode,
|
||||
localIsoWithOffset,
|
||||
safeScreenshot,
|
||||
writeResult,
|
||||
} from "./lib/langbot-e2e.mjs";
|
||||
|
||||
function loadEnvDefaults(path) {
|
||||
if (!existsSync(path)) return;
|
||||
for (const rawLine of readFileSync(path, "utf8").split(/\r?\n/)) {
|
||||
const line = rawLine.trim();
|
||||
if (!line || line.startsWith("#")) continue;
|
||||
const sep = line.indexOf("=");
|
||||
if (sep === -1) continue;
|
||||
const key = line.slice(0, sep).trim();
|
||||
if (env[key]) continue;
|
||||
env[key] = line.slice(sep + 1).trim().replace(/^["']|["']$/g, "");
|
||||
}
|
||||
}
|
||||
|
||||
loadEnvDefaults("skills/.env");
|
||||
loadEnvDefaults("skills/.env.local");
|
||||
|
||||
const caseId = env.LBS_CASE_ID || "mcp-stdio-register";
|
||||
const paths = evidencePaths(caseId);
|
||||
await ensureEvidence(paths);
|
||||
|
||||
const startedAt = new Date();
|
||||
const serverName = env.LANGBOT_MCP_SERVER_NAME || "qa-local-stdio";
|
||||
const expectedTool = env.LANGBOT_MCP_EXPECTED_TOOL || "qa_mcp_echo";
|
||||
const fixturePath = resolve(env.LANGBOT_MCP_FIXTURE_PATH || "skills/langbot-testing/fixtures/mcp/qa_mcp_echo_server.py");
|
||||
const fixtureCommand = env.LANGBOT_MCP_FIXTURE_COMMAND || "python";
|
||||
const fixtureArgs = env.LANGBOT_MCP_FIXTURE_ARGS
|
||||
? JSON.parse(env.LANGBOT_MCP_FIXTURE_ARGS)
|
||||
: [fixturePath];
|
||||
const startupTimeoutSec = Number(env.LANGBOT_MCP_STARTUP_TIMEOUT_SEC || "300");
|
||||
const readyTimeoutMs = Number(env.LANGBOT_MCP_READY_TIMEOUT_MS || "360000");
|
||||
const backendUrl = env.LANGBOT_BACKEND_URL || "";
|
||||
const apiDiagnosticPath = resolve(paths.evidenceDir, "api-diagnostic.json");
|
||||
|
||||
let browser;
|
||||
const result = {
|
||||
source: "automation",
|
||||
case_id: caseId,
|
||||
run_id: paths.runId,
|
||||
started_at: startedAt.toISOString(),
|
||||
started_at_local: localIsoWithOffset(startedAt),
|
||||
finished_at: "",
|
||||
finished_at_local: "",
|
||||
status: "fail",
|
||||
reason: "",
|
||||
server_name: serverName,
|
||||
fixture_path: fixturePath,
|
||||
expected_tool: expectedTool,
|
||||
evidence: {
|
||||
console_log: paths.consoleLog,
|
||||
network_log: paths.networkLog,
|
||||
screenshot: paths.screenshot,
|
||||
api_diagnostic_json: apiDiagnosticPath,
|
||||
automation_result_json: paths.automationResultJson,
|
||||
result_json: paths.resultJson,
|
||||
},
|
||||
evidence_collected: ["api_diagnostic"],
|
||||
};
|
||||
|
||||
async function run() {
|
||||
if (!backendUrl) {
|
||||
result.status = "env_issue";
|
||||
result.reason = "LANGBOT_BACKEND_URL is not configured.";
|
||||
return;
|
||||
}
|
||||
if (!existsSync(fixturePath)) {
|
||||
result.status = "env_issue";
|
||||
result.reason = `MCP fixture not found: ${fixturePath}`;
|
||||
return;
|
||||
}
|
||||
|
||||
browser = await createBrowser(paths);
|
||||
const { page } = browser;
|
||||
await page.goto(env.LANGBOT_FRONTEND_URL, { waitUntil: "domcontentloaded" });
|
||||
await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {});
|
||||
|
||||
const diagnostic = await page.evaluate(async ({
|
||||
backendUrl,
|
||||
serverName,
|
||||
expectedTool,
|
||||
fixturePath,
|
||||
fixtureCommand,
|
||||
fixtureArgs,
|
||||
startupTimeoutSec,
|
||||
readyTimeoutMs,
|
||||
}) => {
|
||||
const token = localStorage.getItem("token");
|
||||
if (!token) {
|
||||
return {
|
||||
authenticated: false,
|
||||
save_status: 0,
|
||||
save_code: null,
|
||||
save_msg: "Browser profile has no localStorage token.",
|
||||
tool_names: [],
|
||||
has_expected_tool: false,
|
||||
runtime_status: null,
|
||||
runtime_tool_names: [],
|
||||
runtime_error: "",
|
||||
};
|
||||
}
|
||||
|
||||
const headers = {
|
||||
Authorization: `Bearer ${token}`,
|
||||
"Content-Type": "application/json",
|
||||
};
|
||||
const serverConfig = {
|
||||
name: serverName,
|
||||
mode: "stdio",
|
||||
enable: true,
|
||||
extra_args: {
|
||||
command: fixtureCommand,
|
||||
args: fixtureArgs,
|
||||
env: {},
|
||||
box: {
|
||||
startup_timeout_sec: startupTimeoutSec,
|
||||
},
|
||||
},
|
||||
};
|
||||
const getJson = async (path) => {
|
||||
const response = await fetch(`${backendUrl}${path}`, { headers });
|
||||
return {
|
||||
status: response.status,
|
||||
json: await response.json().catch(() => ({})),
|
||||
};
|
||||
};
|
||||
const sendJson = async (method, path, body) => {
|
||||
const response = await fetch(`${backendUrl}${path}`, {
|
||||
method,
|
||||
headers,
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
return {
|
||||
status: response.status,
|
||||
json: await response.json().catch(() => ({})),
|
||||
};
|
||||
};
|
||||
|
||||
const serverPath = `/api/v1/mcp/servers/${encodeURIComponent(serverName)}`;
|
||||
const beforeServer = await getJson(serverPath);
|
||||
const save = beforeServer.status === 404
|
||||
? await sendJson("POST", "/api/v1/mcp/servers", serverConfig)
|
||||
: await sendJson("PUT", serverPath, serverConfig);
|
||||
|
||||
const deadline = Date.now() + readyTimeoutMs;
|
||||
let lastTools = [];
|
||||
let lastRuntime = null;
|
||||
while (Date.now() < deadline) {
|
||||
await new Promise((resolveReady) => setTimeout(resolveReady, 500));
|
||||
const tools = await getJson("/api/v1/tools");
|
||||
const server = await getJson(serverPath);
|
||||
lastTools = (tools.json.data?.tools || [])
|
||||
.map((tool) => tool.name || tool.tool_name || tool.function?.name || "")
|
||||
.filter(Boolean)
|
||||
.sort();
|
||||
lastRuntime = server.json.data?.server?.runtime_info || null;
|
||||
if (lastTools.includes(expectedTool)) break;
|
||||
}
|
||||
|
||||
return {
|
||||
authenticated: true,
|
||||
before_status: beforeServer.status,
|
||||
save_status: save.status,
|
||||
save_code: save.json.code ?? null,
|
||||
save_msg: save.json.msg || "",
|
||||
tool_names: lastTools,
|
||||
has_expected_tool: lastTools.includes(expectedTool),
|
||||
runtime_status: lastRuntime?.status || null,
|
||||
runtime_tool_names: (lastRuntime?.tools || [])
|
||||
.map((tool) => tool.name || tool.tool_name || "")
|
||||
.filter(Boolean)
|
||||
.sort(),
|
||||
runtime_tool_count: lastRuntime?.tool_count ?? null,
|
||||
runtime_error: lastRuntime?.error_message || "",
|
||||
};
|
||||
}, { backendUrl, serverName, expectedTool, fixturePath, fixtureCommand, fixtureArgs, startupTimeoutSec, readyTimeoutMs });
|
||||
|
||||
await writeFile(apiDiagnosticPath, `${JSON.stringify(diagnostic, null, 2)}\n`, "utf8");
|
||||
await safeScreenshot(page, paths.screenshot);
|
||||
|
||||
if (!diagnostic.authenticated) {
|
||||
result.status = "blocked";
|
||||
result.reason = "Browser profile is not authenticated for LangBot; cannot update MCP server.";
|
||||
return;
|
||||
}
|
||||
if (diagnostic.save_status >= 400 || diagnostic.save_code !== 0) {
|
||||
result.status = "fail";
|
||||
result.reason = `Failed to save MCP server ${serverName}: ${diagnostic.save_status} ${diagnostic.save_msg}`;
|
||||
return;
|
||||
}
|
||||
if (diagnostic.runtime_status !== "connected") {
|
||||
result.status = "fail";
|
||||
result.reason = `MCP server ${serverName} is not connected after save: ${diagnostic.runtime_status || "missing runtime"}. ${diagnostic.runtime_error}`;
|
||||
return;
|
||||
}
|
||||
if (!diagnostic.has_expected_tool || !diagnostic.runtime_tool_names.includes(expectedTool)) {
|
||||
result.status = "fail";
|
||||
result.reason = `MCP server ${serverName} did not expose ${expectedTool}. See ${apiDiagnosticPath}.`;
|
||||
return;
|
||||
}
|
||||
|
||||
result.status = "pass";
|
||||
result.reason = `MCP server ${serverName} is connected and exposes ${expectedTool} through LangBot /api/v1/tools.`;
|
||||
}
|
||||
|
||||
try {
|
||||
await run();
|
||||
} catch (error) {
|
||||
result.status = /Playwright is not installed|LANGBOT_FRONTEND_URL/.test(error.message) ? "env_issue" : "fail";
|
||||
result.reason = error instanceof Error ? error.message : String(error);
|
||||
} finally {
|
||||
if (browser) await browser.close().catch(() => {});
|
||||
const finishedAt = new Date();
|
||||
result.finished_at = finishedAt.toISOString();
|
||||
result.finished_at_local = localIsoWithOffset(finishedAt);
|
||||
await writeResult(paths, result);
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
}
|
||||
|
||||
process.exit(exitCode(result.status));
|
||||
Executable
+728
@@ -0,0 +1,728 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { spawn } from "node:child_process";
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import { resolve } from "node:path";
|
||||
import { env } from "node:process";
|
||||
import {
|
||||
openPipelineDebugChat,
|
||||
runDebugChatPrompt,
|
||||
setDebugChatStreamOutput,
|
||||
} from "./lib/debug-chat.mjs";
|
||||
import {
|
||||
createBrowser,
|
||||
ensureEvidence,
|
||||
evidencePaths,
|
||||
exitCode,
|
||||
localIsoWithOffset,
|
||||
pathExists,
|
||||
safeScreenshot,
|
||||
writeResult,
|
||||
} from "./lib/langbot-e2e.mjs";
|
||||
|
||||
const caseId = env.LBS_CASE_ID || "pipeline-debug-chat";
|
||||
const paths = evidencePaths(caseId);
|
||||
await ensureEvidence(paths);
|
||||
|
||||
const expectedText = env.LANGBOT_E2E_EXPECTED_TEXT || "OK";
|
||||
const prompt = env.LANGBOT_E2E_PROMPT || `请只回复 ${expectedText},用于前端调试测试。`;
|
||||
const responseTimeoutMs = Number.parseInt(env.LANGBOT_E2E_RESPONSE_TIMEOUT_MS || "120000", 10);
|
||||
const safeResponseTimeoutMs = Number.isFinite(responseTimeoutMs) && responseTimeoutMs > 0 ? responseTimeoutMs : 120000;
|
||||
const streamOutput = /^(0|false)$/i.test(env.LANGBOT_E2E_STREAM_OUTPUT || "")
|
||||
? false
|
||||
: /^(1|true)$/i.test(env.LANGBOT_E2E_STREAM_OUTPUT || "")
|
||||
? true
|
||||
: null;
|
||||
const failureSignals = (env.LANGBOT_E2E_FAILURE_SIGNALS || "")
|
||||
.split(/\r?\n/)
|
||||
.map((item) => item.trim())
|
||||
.filter(Boolean);
|
||||
const imageBase64Path = env.LANGBOT_E2E_IMAGE_BASE64_PATH || "";
|
||||
const imagePathEnv = env.LANGBOT_E2E_IMAGE_PATH || "";
|
||||
const backendUrl = env.LANGBOT_BACKEND_URL || "";
|
||||
const pipelineRequired = env.LANGBOT_E2E_PIPELINE_REQUIRED === "1";
|
||||
const pipelineUrl = pipelineRequired
|
||||
? env.LANGBOT_E2E_PIPELINE_URL
|
||||
: (env.LANGBOT_E2E_PIPELINE_URL || env.LANGBOT_PIPELINE_URL);
|
||||
const pipelineName = pipelineRequired
|
||||
? env.LANGBOT_E2E_PIPELINE_NAME
|
||||
: (env.LANGBOT_E2E_PIPELINE_NAME || env.LANGBOT_PIPELINE_NAME);
|
||||
const expectedRunnerId = env.LANGBOT_E2E_EXPECTED_RUNNER_ID || "";
|
||||
const resetDebugChat = boolFromEnv(env.LANGBOT_E2E_RESET_DEBUG_CHAT, false);
|
||||
const restoreRunnerConfig = boolFromEnv(env.LANGBOT_E2E_RESTORE_RUNNER_CONFIG, true);
|
||||
const debugChatSessionType = env.LANGBOT_E2E_DEBUG_CHAT_SESSION_TYPE || "person";
|
||||
const pipelineConfigDiagnosticPath = resolve(paths.evidenceDir, "pipeline-config-diagnostic.json");
|
||||
const debugChatResetDiagnosticPath = resolve(paths.evidenceDir, "debug-chat-reset-diagnostic.json");
|
||||
const pipelineConfigRestoreDiagnosticPath = resolve(paths.evidenceDir, "pipeline-config-restore-diagnostic.json");
|
||||
const startedAt = new Date();
|
||||
|
||||
let browser;
|
||||
let restorePlan = null;
|
||||
let result = {
|
||||
source: "automation",
|
||||
case_id: caseId,
|
||||
run_id: paths.runId,
|
||||
started_at: startedAt.toISOString(),
|
||||
started_at_local: localIsoWithOffset(startedAt),
|
||||
finished_at: "",
|
||||
finished_at_local: "",
|
||||
status: "fail",
|
||||
reason: "",
|
||||
url: "",
|
||||
prompt,
|
||||
expected_text: expectedText,
|
||||
response_timeout_ms: safeResponseTimeoutMs,
|
||||
stream_output: streamOutput,
|
||||
image_fixture: imageBase64Path || imagePathEnv,
|
||||
prompt_count: 1,
|
||||
chat_results: [],
|
||||
evidence: {
|
||||
console_log: paths.consoleLog,
|
||||
network_log: paths.networkLog,
|
||||
screenshot: paths.screenshot,
|
||||
automation_result_json: paths.automationResultJson,
|
||||
result_json: paths.resultJson,
|
||||
},
|
||||
evidence_collected: ["ui", "screenshot", "console", "network"],
|
||||
};
|
||||
|
||||
function boolFromEnv(value, defaultValue) {
|
||||
if (value === undefined || value === "") return defaultValue;
|
||||
if (/^(0|false|no|off)$/i.test(value)) return false;
|
||||
if (/^(1|true|yes|on)$/i.test(value)) return true;
|
||||
return defaultValue;
|
||||
}
|
||||
|
||||
function parseJsonEnv(key, fallback) {
|
||||
const raw = env[key];
|
||||
if (!raw) return fallback;
|
||||
try {
|
||||
return JSON.parse(raw);
|
||||
} catch (error) {
|
||||
throw new Error(`${key} must be valid JSON: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
function promptStepsFromEnv() {
|
||||
const rawSteps = parseJsonEnv("LANGBOT_E2E_PROMPTS_JSON", null);
|
||||
if (rawSteps === null) {
|
||||
return [{ prompt, expectedText, responseTimeoutMs: safeResponseTimeoutMs }];
|
||||
}
|
||||
if (!Array.isArray(rawSteps) || rawSteps.length === 0) {
|
||||
throw new Error("LANGBOT_E2E_PROMPTS_JSON must be a non-empty JSON array.");
|
||||
}
|
||||
return rawSteps.map((item, index) => {
|
||||
if (typeof item === "string") {
|
||||
return { prompt: item, expectedText, responseTimeoutMs: safeResponseTimeoutMs };
|
||||
}
|
||||
if (!item || typeof item !== "object" || typeof item.prompt !== "string" || !item.prompt) {
|
||||
throw new Error(`LANGBOT_E2E_PROMPTS_JSON[${index}] must be a string or an object with a prompt string.`);
|
||||
}
|
||||
const stepTimeout = Number.parseInt(String(item.response_timeout_ms || item.responseTimeoutMs || safeResponseTimeoutMs), 10);
|
||||
return {
|
||||
prompt: item.prompt,
|
||||
expectedText: String(item.expected_text || item.expectedText || expectedText),
|
||||
responseTimeoutMs: Number.isFinite(stepTimeout) && stepTimeout > 0 ? stepTimeout : safeResponseTimeoutMs,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function expandEnvRefs(value) {
|
||||
return String(value || "").replace(/\$\{([A-Z][A-Z0-9_]*)\}|\$([A-Z][A-Z0-9_]*)/g, (_match, braced, bare) => {
|
||||
return env[braced || bare] || "";
|
||||
});
|
||||
}
|
||||
|
||||
function textList(value) {
|
||||
if (value === undefined || value === null || value === "") return [];
|
||||
return Array.isArray(value) ? value.map(String) : [String(value)];
|
||||
}
|
||||
|
||||
function runArgv(argv, { cwd = "", timeoutMs = 30_000 } = {}) {
|
||||
return new Promise((resolveRun) => {
|
||||
if (!Array.isArray(argv) || argv.length === 0 || !argv.every((item) => typeof item === "string" && item)) {
|
||||
resolveRun({
|
||||
status: "fail",
|
||||
reason: "Filesystem command check requires a non-empty argv string array.",
|
||||
exit_code: null,
|
||||
stdout: "",
|
||||
stderr: "",
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const child = spawn(argv[0], argv.slice(1), {
|
||||
cwd: cwd ? resolve(cwd) : undefined,
|
||||
env,
|
||||
shell: false,
|
||||
});
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
let timedOut = false;
|
||||
const timer = setTimeout(() => {
|
||||
timedOut = true;
|
||||
child.kill("SIGTERM");
|
||||
}, timeoutMs);
|
||||
|
||||
child.stdout.on("data", (chunk) => {
|
||||
stdout += chunk.toString();
|
||||
});
|
||||
child.stderr.on("data", (chunk) => {
|
||||
stderr += chunk.toString();
|
||||
});
|
||||
child.on("error", (error) => {
|
||||
clearTimeout(timer);
|
||||
resolveRun({
|
||||
status: "fail",
|
||||
reason: error.message,
|
||||
exit_code: null,
|
||||
stdout,
|
||||
stderr,
|
||||
});
|
||||
});
|
||||
child.on("close", (code) => {
|
||||
clearTimeout(timer);
|
||||
resolveRun({
|
||||
status: timedOut ? "fail" : "pass",
|
||||
reason: timedOut ? `Command timed out after ${timeoutMs} ms.` : "",
|
||||
exit_code: code,
|
||||
stdout,
|
||||
stderr,
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function runFilesystemChecks(checks) {
|
||||
if (!Array.isArray(checks) || checks.length === 0) {
|
||||
return { status: "not_required", checks: [] };
|
||||
}
|
||||
const results = [];
|
||||
for (let index = 0; index < checks.length; index += 1) {
|
||||
const check = checks[index];
|
||||
if (!check || typeof check !== "object") {
|
||||
results.push({ index, status: "fail", reason: "Filesystem check must be an object." });
|
||||
continue;
|
||||
}
|
||||
const contains = textList(check.contains);
|
||||
const notContains = textList(check.not_contains || check.notContains);
|
||||
const expectedExitCode = Number.isInteger(check.exit_code)
|
||||
? check.exit_code
|
||||
: Number.isInteger(check.expected_exit_code)
|
||||
? check.expected_exit_code
|
||||
: 0;
|
||||
const expectedStdout = textList(check.stdout_contains || check.expected_stdout || check.expectedStdout);
|
||||
|
||||
if (check.path) {
|
||||
const path = resolve(expandEnvRefs(check.path));
|
||||
let text = "";
|
||||
try {
|
||||
text = await readFile(path, "utf8");
|
||||
} catch (error) {
|
||||
results.push({ index, status: "fail", type: "file", path, reason: error.message });
|
||||
continue;
|
||||
}
|
||||
const missing = contains.filter((needle) => !text.includes(needle));
|
||||
const forbidden = notContains.filter((needle) => text.includes(needle));
|
||||
results.push({
|
||||
index,
|
||||
status: missing.length || forbidden.length ? "fail" : "pass",
|
||||
type: "file",
|
||||
path,
|
||||
missing,
|
||||
forbidden,
|
||||
reason: missing.length
|
||||
? `Missing expected text: ${missing.join(", ")}`
|
||||
: forbidden.length
|
||||
? `Found forbidden text: ${forbidden.join(", ")}`
|
||||
: "",
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
if (check.argv) {
|
||||
const cwd = check.cwd ? expandEnvRefs(check.cwd) : "";
|
||||
const timeoutMs = Number.parseInt(String(check.timeout_ms || check.timeoutMs || "30000"), 10);
|
||||
const run = await runArgv(check.argv.map(expandEnvRefs), {
|
||||
cwd,
|
||||
timeoutMs: Number.isFinite(timeoutMs) && timeoutMs > 0 ? timeoutMs : 30_000,
|
||||
});
|
||||
const missingStdout = expectedStdout.filter((needle) => !run.stdout.includes(needle));
|
||||
const exitMatches = run.exit_code === expectedExitCode;
|
||||
results.push({
|
||||
index,
|
||||
status: run.status === "pass" && exitMatches && missingStdout.length === 0 ? "pass" : "fail",
|
||||
type: "command",
|
||||
argv: check.argv,
|
||||
cwd,
|
||||
exit_code: run.exit_code,
|
||||
expected_exit_code: expectedExitCode,
|
||||
missing_stdout: missingStdout,
|
||||
stdout_preview: run.stdout.slice(0, 2000),
|
||||
stderr_preview: run.stderr.slice(0, 2000),
|
||||
reason: run.reason
|
||||
|| (!exitMatches ? `Expected exit code ${expectedExitCode}, saw ${run.exit_code}.` : "")
|
||||
|| (missingStdout.length ? `Missing stdout text: ${missingStdout.join(", ")}` : ""),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
results.push({ index, status: "fail", reason: "Filesystem check requires either path or argv." });
|
||||
}
|
||||
const failed = results.filter((item) => item.status !== "pass");
|
||||
return {
|
||||
status: failed.length ? "fail" : "pass",
|
||||
checks: results,
|
||||
reason: failed.length ? `Filesystem checks failed: ${failed.map((item) => item.index).join(", ")}` : "",
|
||||
};
|
||||
}
|
||||
|
||||
function pipelineIdFromUrl(url) {
|
||||
if (!url) return "";
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
return parsed.searchParams.get("id") || "";
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
function sanitizePipelineDiagnostic(diagnostic) {
|
||||
const { restore_config: _restoreConfig, ...safe } = diagnostic || {};
|
||||
return safe;
|
||||
}
|
||||
|
||||
async function prepareImageFixture(paths) {
|
||||
if (imagePathEnv) return resolve(imagePathEnv);
|
||||
if (!imageBase64Path) return "";
|
||||
const source = resolve(imageBase64Path);
|
||||
const target = resolve(paths.evidenceDir, "image-fixture.png");
|
||||
const encoded = await readFile(source, "utf8");
|
||||
await writeFile(target, Buffer.from(encoded.replace(/\s+/g, ""), "base64"));
|
||||
return target;
|
||||
}
|
||||
|
||||
async function inspectAndPatchPipelineConfig(page, {
|
||||
backendUrl,
|
||||
pipelineUrl,
|
||||
pipelineName,
|
||||
runnerConfigPatch,
|
||||
expectedRunnerId,
|
||||
}) {
|
||||
const pipelineIdFromUrlValue = pipelineIdFromUrl(pipelineUrl) || pipelineIdFromUrl(page.url());
|
||||
return await page.evaluate(async ({
|
||||
backendUrl,
|
||||
pipelineIdFromUrlValue,
|
||||
pipelineName,
|
||||
runnerConfigPatch,
|
||||
expectedRunnerId,
|
||||
}) => {
|
||||
const token = localStorage.getItem("token");
|
||||
if (!token) {
|
||||
return {
|
||||
status: "blocked",
|
||||
authenticated: false,
|
||||
reason: "Browser profile has no localStorage token.",
|
||||
};
|
||||
}
|
||||
|
||||
const headers = {
|
||||
Authorization: `Bearer ${token}`,
|
||||
"Content-Type": "application/json",
|
||||
};
|
||||
const getJson = async (path) => {
|
||||
const response = await fetch(`${backendUrl}${path}`, { headers });
|
||||
return {
|
||||
status: response.status,
|
||||
json: await response.json().catch(() => ({})),
|
||||
};
|
||||
};
|
||||
const putJson = async (path, body) => {
|
||||
const response = await fetch(`${backendUrl}${path}`, {
|
||||
method: "PUT",
|
||||
headers,
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
return {
|
||||
status: response.status,
|
||||
json: await response.json().catch(() => ({})),
|
||||
};
|
||||
};
|
||||
|
||||
let pipelineId = pipelineIdFromUrlValue || "";
|
||||
let matchedBy = pipelineId ? "url" : "";
|
||||
if (!pipelineId && pipelineName) {
|
||||
const list = await getJson("/api/v1/pipelines");
|
||||
const pipelines = list.json.data?.pipelines || [];
|
||||
const match = pipelines.find((pipeline) => pipeline.name === pipelineName);
|
||||
if (!match) {
|
||||
return {
|
||||
status: "blocked",
|
||||
authenticated: true,
|
||||
pipeline_resolved: false,
|
||||
list_status: list.status,
|
||||
reason: `Could not find pipeline named ${pipelineName}.`,
|
||||
};
|
||||
}
|
||||
pipelineId = match.uuid;
|
||||
matchedBy = "name";
|
||||
}
|
||||
|
||||
if (!pipelineId) {
|
||||
return {
|
||||
status: "blocked",
|
||||
authenticated: true,
|
||||
pipeline_resolved: false,
|
||||
reason: "Could not resolve pipeline id from URL or pipeline name.",
|
||||
};
|
||||
}
|
||||
|
||||
const before = await getJson(`/api/v1/pipelines/${encodeURIComponent(pipelineId)}`);
|
||||
const pipeline = before.json.data?.pipeline;
|
||||
if (before.status >= 400 || !pipeline) {
|
||||
return {
|
||||
status: "fail",
|
||||
authenticated: true,
|
||||
pipeline_resolved: false,
|
||||
pipeline_id: pipelineId,
|
||||
get_status: before.status,
|
||||
reason: before.json.msg || "Could not load pipeline.",
|
||||
};
|
||||
}
|
||||
|
||||
const config = JSON.parse(JSON.stringify(pipeline.config || {}));
|
||||
const aiConfig = config.ai && typeof config.ai === "object" ? config.ai : {};
|
||||
const runner = aiConfig.runner && typeof aiConfig.runner === "object" ? aiConfig.runner : {};
|
||||
const runnerId = runner.id || runner.runner || "";
|
||||
if (!runnerId) {
|
||||
return {
|
||||
status: "blocked",
|
||||
authenticated: true,
|
||||
pipeline_resolved: true,
|
||||
pipeline_id: pipelineId,
|
||||
pipeline_name: pipeline.name,
|
||||
matched_by: matchedBy,
|
||||
reason: "Pipeline has no ai.runner.id or legacy ai.runner.runner.",
|
||||
};
|
||||
}
|
||||
if (expectedRunnerId && runnerId !== expectedRunnerId) {
|
||||
return {
|
||||
status: "blocked",
|
||||
authenticated: true,
|
||||
pipeline_resolved: true,
|
||||
pipeline_id: pipelineId,
|
||||
pipeline_name: pipeline.name,
|
||||
matched_by: matchedBy,
|
||||
runner_id: runnerId,
|
||||
expected_runner_id: expectedRunnerId,
|
||||
reason: `Pipeline runner mismatch: expected ${expectedRunnerId}, got ${runnerId}.`,
|
||||
};
|
||||
}
|
||||
|
||||
const runnerConfigs = aiConfig.runner_config && typeof aiConfig.runner_config === "object"
|
||||
? aiConfig.runner_config
|
||||
: {};
|
||||
const currentRunnerConfig = runnerConfigs[runnerId] && typeof runnerConfigs[runnerId] === "object"
|
||||
? runnerConfigs[runnerId]
|
||||
: {};
|
||||
const patchKeys = Object.keys(runnerConfigPatch || {});
|
||||
const baseDiagnostic = {
|
||||
status: "ready",
|
||||
authenticated: true,
|
||||
pipeline_resolved: true,
|
||||
pipeline_id: pipelineId,
|
||||
pipeline_name: pipeline.name,
|
||||
matched_by: matchedBy,
|
||||
runner_id: runnerId,
|
||||
expected_runner_id: expectedRunnerId || "",
|
||||
patch_keys: patchKeys,
|
||||
runner_config_before_keys: Object.keys(currentRunnerConfig),
|
||||
patched: patchKeys.length > 0,
|
||||
};
|
||||
|
||||
if (patchKeys.length === 0) {
|
||||
return baseDiagnostic;
|
||||
}
|
||||
|
||||
const updatedRunnerConfig = {
|
||||
...currentRunnerConfig,
|
||||
...runnerConfigPatch,
|
||||
};
|
||||
const updatedConfig = {
|
||||
...config,
|
||||
ai: {
|
||||
...aiConfig,
|
||||
runner: {
|
||||
...runner,
|
||||
id: runnerId,
|
||||
},
|
||||
runner_config: {
|
||||
...runnerConfigs,
|
||||
[runnerId]: updatedRunnerConfig,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const update = await putJson(`/api/v1/pipelines/${encodeURIComponent(pipelineId)}`, {
|
||||
config: updatedConfig,
|
||||
});
|
||||
if (update.status >= 400) {
|
||||
return {
|
||||
...baseDiagnostic,
|
||||
status: "fail",
|
||||
put_status: update.status,
|
||||
put_code: update.json.code ?? null,
|
||||
reason: update.json.msg || "Pipeline config update failed.",
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
...baseDiagnostic,
|
||||
put_status: update.status,
|
||||
put_code: update.json.code ?? null,
|
||||
runner_config_after_keys: Object.keys(updatedRunnerConfig),
|
||||
restore_config: config,
|
||||
};
|
||||
}, {
|
||||
backendUrl,
|
||||
pipelineIdFromUrlValue,
|
||||
pipelineName,
|
||||
runnerConfigPatch,
|
||||
expectedRunnerId,
|
||||
});
|
||||
}
|
||||
|
||||
async function restorePipelineConfig(page, { backendUrl, pipelineId, config }) {
|
||||
return await page.evaluate(async ({ backendUrl, pipelineId, config }) => {
|
||||
const token = localStorage.getItem("token");
|
||||
if (!token) {
|
||||
return {
|
||||
status: "blocked",
|
||||
authenticated: false,
|
||||
pipeline_id: pipelineId,
|
||||
reason: "Browser profile has no localStorage token.",
|
||||
};
|
||||
}
|
||||
const response = await fetch(`${backendUrl}/api/v1/pipelines/${encodeURIComponent(pipelineId)}`, {
|
||||
method: "PUT",
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({ config }),
|
||||
});
|
||||
const json = await response.json().catch(() => ({}));
|
||||
return {
|
||||
status: response.status >= 400 ? "fail" : "ready",
|
||||
authenticated: true,
|
||||
pipeline_id: pipelineId,
|
||||
put_status: response.status,
|
||||
put_code: json.code ?? null,
|
||||
reason: response.status >= 400 ? json.msg || "Pipeline config restore failed." : "Pipeline config restored.",
|
||||
};
|
||||
}, { backendUrl, pipelineId, config });
|
||||
}
|
||||
|
||||
async function resetPipelineDebugChat(page, { backendUrl, pipelineId, sessionType }) {
|
||||
return await page.evaluate(async ({ backendUrl, pipelineId, sessionType }) => {
|
||||
const token = localStorage.getItem("token");
|
||||
if (!token) {
|
||||
return {
|
||||
status: "blocked",
|
||||
authenticated: false,
|
||||
pipeline_id: pipelineId,
|
||||
session_type: sessionType,
|
||||
reason: "Browser profile has no localStorage token.",
|
||||
};
|
||||
}
|
||||
const response = await fetch(
|
||||
`${backendUrl}/api/v1/pipelines/${encodeURIComponent(pipelineId)}/ws/reset/${encodeURIComponent(sessionType)}`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
},
|
||||
);
|
||||
const json = await response.json().catch(() => ({}));
|
||||
return {
|
||||
status: response.status >= 400 ? "fail" : "ready",
|
||||
authenticated: true,
|
||||
pipeline_id: pipelineId,
|
||||
session_type: sessionType,
|
||||
reset_status: response.status,
|
||||
reset_code: json.code ?? null,
|
||||
reason: response.status >= 400 ? json.msg || "Debug Chat reset failed." : "Debug Chat session reset.",
|
||||
};
|
||||
}, { backendUrl, pipelineId, sessionType });
|
||||
}
|
||||
|
||||
try {
|
||||
browser = await createBrowser(paths);
|
||||
const { page } = browser;
|
||||
const imagePath = await prepareImageFixture(paths);
|
||||
const promptSteps = promptStepsFromEnv();
|
||||
const filesystemChecks = parseJsonEnv("LANGBOT_E2E_FILESYSTEM_CHECKS_JSON", []);
|
||||
const runnerConfigPatch = parseJsonEnv("LANGBOT_E2E_RUNNER_CONFIG_PATCH_JSON", {});
|
||||
const runnerPatchKeys = Object.keys(runnerConfigPatch);
|
||||
if (runnerPatchKeys.length > 0 || resetDebugChat || expectedRunnerId) {
|
||||
if (!backendUrl) {
|
||||
result.status = "env_issue";
|
||||
result.reason = "LANGBOT_BACKEND_URL is required for runner config patch, runner assertion, or Debug Chat reset.";
|
||||
throw new Error(result.reason);
|
||||
}
|
||||
}
|
||||
result.prompt_count = promptSteps.length;
|
||||
result.prompt = promptSteps.length === 1 ? promptSteps[0].prompt : `${promptSteps.length} prompts`;
|
||||
result.expected_text = promptSteps.at(-1)?.expectedText || expectedText;
|
||||
|
||||
const openResult = await openPipelineDebugChat(page, {
|
||||
pipelineUrl,
|
||||
pipelineName,
|
||||
envHint: pipelineRequired
|
||||
? "case-specific pipeline env mapped to LANGBOT_E2E_PIPELINE_URL or LANGBOT_E2E_PIPELINE_NAME"
|
||||
: "LANGBOT_PIPELINE_URL or LANGBOT_PIPELINE_NAME",
|
||||
});
|
||||
result.url = page.url();
|
||||
|
||||
if (!openResult.opened) {
|
||||
result.status = openResult.status;
|
||||
result.reason = openResult.reason;
|
||||
} else {
|
||||
result.status = "running";
|
||||
result.reason = "";
|
||||
if (runnerPatchKeys.length > 0 || resetDebugChat || expectedRunnerId) {
|
||||
const pipelineDiagnostic = await inspectAndPatchPipelineConfig(page, {
|
||||
backendUrl,
|
||||
pipelineUrl,
|
||||
pipelineName,
|
||||
runnerConfigPatch,
|
||||
expectedRunnerId,
|
||||
});
|
||||
const safeDiagnostic = sanitizePipelineDiagnostic(pipelineDiagnostic);
|
||||
await writeFile(pipelineConfigDiagnosticPath, `${JSON.stringify(safeDiagnostic, null, 2)}\n`, "utf8");
|
||||
result.evidence.pipeline_config_diagnostic_json = pipelineConfigDiagnosticPath;
|
||||
result.pipeline_config = safeDiagnostic;
|
||||
if (!result.evidence_collected.includes("api_diagnostic")) result.evidence_collected.push("api_diagnostic");
|
||||
|
||||
if (pipelineDiagnostic.status === "fail" || pipelineDiagnostic.status === "blocked") {
|
||||
result.status = pipelineDiagnostic.status;
|
||||
result.reason = pipelineDiagnostic.reason || "Pipeline config preparation failed.";
|
||||
} else {
|
||||
if (pipelineDiagnostic.restore_config && restoreRunnerConfig) {
|
||||
restorePlan = {
|
||||
backendUrl,
|
||||
pipelineId: pipelineDiagnostic.pipeline_id,
|
||||
config: pipelineDiagnostic.restore_config,
|
||||
};
|
||||
}
|
||||
if (resetDebugChat) {
|
||||
const resetDiagnostic = await resetPipelineDebugChat(page, {
|
||||
backendUrl,
|
||||
pipelineId: pipelineDiagnostic.pipeline_id,
|
||||
sessionType: debugChatSessionType,
|
||||
});
|
||||
await writeFile(debugChatResetDiagnosticPath, `${JSON.stringify(resetDiagnostic, null, 2)}\n`, "utf8");
|
||||
result.evidence.debug_chat_reset_diagnostic_json = debugChatResetDiagnosticPath;
|
||||
result.debug_chat_reset = resetDiagnostic;
|
||||
if (resetDiagnostic.status === "fail" || resetDiagnostic.status === "blocked") {
|
||||
result.status = resetDiagnostic.status;
|
||||
result.reason = resetDiagnostic.reason || "Debug Chat reset failed.";
|
||||
} else {
|
||||
await page.waitForTimeout(1000);
|
||||
const reopenResult = await openPipelineDebugChat(page, {
|
||||
pipelineUrl,
|
||||
pipelineName,
|
||||
envHint: pipelineRequired
|
||||
? "case-specific pipeline env mapped to LANGBOT_E2E_PIPELINE_URL or LANGBOT_E2E_PIPELINE_NAME"
|
||||
: "LANGBOT_PIPELINE_URL or LANGBOT_PIPELINE_NAME",
|
||||
});
|
||||
result.url = page.url();
|
||||
if (!reopenResult.opened) {
|
||||
result.status = reopenResult.status;
|
||||
result.reason = reopenResult.reason;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (result.status === "fail" || result.status === "blocked" || result.status === "env_issue") {
|
||||
// Preparation already determined the outcome.
|
||||
} else {
|
||||
const streamResult = await setDebugChatStreamOutput(page, streamOutput);
|
||||
if (streamResult.status === "blocked" || streamResult.status === "fail") {
|
||||
result.status = streamResult.status;
|
||||
result.reason = streamResult.reason;
|
||||
} else {
|
||||
for (let index = 0; index < promptSteps.length; index += 1) {
|
||||
const step = promptSteps[index];
|
||||
const chatResult = await runDebugChatPrompt(page, {
|
||||
prompt: step.prompt,
|
||||
expectedText: step.expectedText,
|
||||
responseTimeoutMs: step.responseTimeoutMs,
|
||||
imagePath: index === 0 ? imagePath : "",
|
||||
failureSignals: failureSignals.length > 0 ? failureSignals : undefined,
|
||||
});
|
||||
result.chat_results.push({
|
||||
index,
|
||||
expected_text: step.expectedText,
|
||||
status: chatResult.status,
|
||||
reason: chatResult.reason,
|
||||
min_expected_count: chatResult.min_expected_count,
|
||||
final_count: chatResult.final_count,
|
||||
before_assistant_expected_count: chatResult.before_assistant_expected_count,
|
||||
after_assistant_expected_count: chatResult.after_assistant_expected_count,
|
||||
failure_signal: chatResult.failure_signal || "",
|
||||
});
|
||||
result.status = chatResult.status;
|
||||
result.reason = `Prompt ${index + 1}/${promptSteps.length}: ${chatResult.reason}`;
|
||||
if (chatResult.status !== "pass") break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (result.status === "pass" && filesystemChecks.length > 0) {
|
||||
const filesystemResult = await runFilesystemChecks(filesystemChecks);
|
||||
result.filesystem_checks = filesystemResult;
|
||||
if (!result.evidence_collected.includes("filesystem")) result.evidence_collected.push("filesystem");
|
||||
if (filesystemResult.status === "fail") {
|
||||
result.status = "fail";
|
||||
result.reason = filesystemResult.reason || "Filesystem checks failed.";
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (!["env_issue", "blocked", "fail", "pass"].includes(result.status) || !result.reason) {
|
||||
result.status = /Playwright is not installed|LANGBOT_FRONTEND_URL/.test(error.message) ? "env_issue" : "fail";
|
||||
}
|
||||
result.reason = result.reason || error.message;
|
||||
} finally {
|
||||
if (browser?.page) await safeScreenshot(browser.page, paths.screenshot);
|
||||
if (browser?.page && restorePlan) {
|
||||
const restoreDiagnostic = await restorePipelineConfig(browser.page, restorePlan).catch((error) => ({
|
||||
status: "fail",
|
||||
pipeline_id: restorePlan.pipelineId,
|
||||
reason: error.message,
|
||||
}));
|
||||
await writeFile(pipelineConfigRestoreDiagnosticPath, `${JSON.stringify(restoreDiagnostic, null, 2)}\n`, "utf8");
|
||||
result.evidence.pipeline_config_restore_diagnostic_json = pipelineConfigRestoreDiagnosticPath;
|
||||
result.pipeline_config_restore = restoreDiagnostic;
|
||||
}
|
||||
if (browser) await browser.close().catch(() => {});
|
||||
const finishedAt = new Date();
|
||||
result.finished_at = finishedAt.toISOString();
|
||||
result.finished_at_local = localIsoWithOffset(finishedAt);
|
||||
const existingEvidence = {};
|
||||
for (const [key, value] of Object.entries(result.evidence)) {
|
||||
if (typeof value !== "string") continue;
|
||||
const isResultFile = value === paths.automationResultJson || value === paths.resultJson;
|
||||
if (isResultFile || await pathExists(value)) existingEvidence[key] = value;
|
||||
}
|
||||
result.evidence = existingEvidence;
|
||||
await writeResult(paths, result);
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
}
|
||||
|
||||
process.exit(exitCode(result.status));
|
||||
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { env } from "node:process";
|
||||
import {
|
||||
bodyText,
|
||||
createBrowser,
|
||||
ensureEvidence,
|
||||
evidencePaths,
|
||||
loadEnvFiles,
|
||||
resetAndAuthLocalUser,
|
||||
safeScreenshot,
|
||||
setBrowserToken,
|
||||
verifyBrowserToken,
|
||||
writeResult,
|
||||
} from "./lib/langbot-e2e.mjs";
|
||||
|
||||
const caseId = "refresh-local-login";
|
||||
const paths = evidencePaths(caseId);
|
||||
await loadEnvFiles();
|
||||
await ensureEvidence(paths);
|
||||
|
||||
const result = {
|
||||
source: "automation",
|
||||
case_id: caseId,
|
||||
status: "fail",
|
||||
reason: "",
|
||||
user: env.LANGBOT_E2E_LOGIN_USER || "",
|
||||
frontend_url: env.LANGBOT_FRONTEND_URL || "",
|
||||
backend_url: env.LANGBOT_BACKEND_URL || "",
|
||||
backend_token_check: null,
|
||||
browser_token_check: null,
|
||||
evidence: {
|
||||
console_log: paths.consoleLog,
|
||||
network_log: paths.networkLog,
|
||||
screenshot: paths.screenshot,
|
||||
automation_result_json: paths.automationResultJson,
|
||||
result_json: paths.resultJson,
|
||||
},
|
||||
evidence_collected: ["ui", "screenshot", "console", "api_diagnostic"],
|
||||
};
|
||||
|
||||
let browser;
|
||||
|
||||
try {
|
||||
const backendUrl = env.LANGBOT_BACKEND_URL;
|
||||
const frontendUrl = env.LANGBOT_FRONTEND_URL;
|
||||
const user = env.LANGBOT_E2E_LOGIN_USER;
|
||||
const password = env.LANGBOT_E2E_LOGIN_PASSWORD || "LangBotE2ELocalPass!2026";
|
||||
if (!backendUrl) throw new Error("LANGBOT_BACKEND_URL is not configured.");
|
||||
if (!frontendUrl) throw new Error("LANGBOT_FRONTEND_URL is not configured.");
|
||||
if (!user) throw new Error("LANGBOT_E2E_LOGIN_USER is required.");
|
||||
|
||||
const auth = await resetAndAuthLocalUser({ backendUrl, user, password });
|
||||
result.backend_token_check = auth.check;
|
||||
|
||||
browser = await createBrowser(paths);
|
||||
const { page } = browser;
|
||||
await setBrowserToken(page, frontendUrl, auth.token);
|
||||
const browserCheck = await verifyBrowserToken(page, backendUrl);
|
||||
result.browser_token_check = browserCheck;
|
||||
if (!browserCheck.authenticated) {
|
||||
throw new Error(browserCheck.reason || "Browser token check failed.");
|
||||
}
|
||||
|
||||
await page.goto(`${frontendUrl.replace(/\/$/, "")}/home/monitoring`, { waitUntil: "domcontentloaded" });
|
||||
await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {});
|
||||
const text = await bodyText(page);
|
||||
if (!text.includes("Dashboard") && !text.includes("Pipelines") && !text.includes("流水线")) {
|
||||
throw new Error("Token was written, but authenticated navigation was not visible.");
|
||||
}
|
||||
|
||||
result.status = "pass";
|
||||
result.reason = "Browser profile localStorage token refreshed.";
|
||||
} catch (error) {
|
||||
result.status = "fail";
|
||||
result.reason = error.message;
|
||||
} finally {
|
||||
if (browser?.page) await safeScreenshot(browser.page, paths.screenshot);
|
||||
if (browser) await browser.close().catch(() => {});
|
||||
await writeResult(paths, result);
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
}
|
||||
|
||||
process.exit(result.status === "pass" ? 0 : 1);
|
||||
Executable
+107
@@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import {
|
||||
bodyText,
|
||||
createBrowser,
|
||||
ensureEvidence,
|
||||
evidencePaths,
|
||||
exitCode,
|
||||
gotoFrontend,
|
||||
isLoginUrl,
|
||||
loadEnvFiles,
|
||||
localIsoWithOffset,
|
||||
safeScreenshot,
|
||||
verifyBrowserToken,
|
||||
writeResult,
|
||||
} from "./lib/langbot-e2e.mjs";
|
||||
|
||||
const caseId = "webui-login-state";
|
||||
await loadEnvFiles();
|
||||
const paths = evidencePaths(caseId);
|
||||
await ensureEvidence(paths);
|
||||
|
||||
const startedAt = new Date();
|
||||
let browser;
|
||||
let result = {
|
||||
source: "automation",
|
||||
case_id: caseId,
|
||||
run_id: paths.runId,
|
||||
started_at: startedAt.toISOString(),
|
||||
started_at_local: localIsoWithOffset(startedAt),
|
||||
finished_at: "",
|
||||
finished_at_local: "",
|
||||
status: "fail",
|
||||
reason: "",
|
||||
url: "",
|
||||
auth: null,
|
||||
evidence: {
|
||||
console_log: paths.consoleLog,
|
||||
network_log: paths.networkLog,
|
||||
screenshot: paths.screenshot,
|
||||
automation_result_json: paths.automationResultJson,
|
||||
result_json: paths.resultJson,
|
||||
},
|
||||
evidence_collected: ["ui", "screenshot", "console"],
|
||||
};
|
||||
|
||||
try {
|
||||
browser = await createBrowser(paths);
|
||||
const { page } = browser;
|
||||
await gotoFrontend(page);
|
||||
result.url = page.url();
|
||||
|
||||
const backendUrl = process.env.LANGBOT_BACKEND_URL || "";
|
||||
if (!backendUrl) {
|
||||
result.status = "env_issue";
|
||||
result.reason = "LANGBOT_BACKEND_URL is not configured.";
|
||||
await safeScreenshot(page, paths.screenshot);
|
||||
throw new Error(result.reason);
|
||||
}
|
||||
|
||||
const auth = await verifyBrowserToken(page, backendUrl);
|
||||
result.auth = auth;
|
||||
const text = await bodyText(page);
|
||||
const navigationSignals = [
|
||||
"Dashboard",
|
||||
"Bots",
|
||||
"Pipelines",
|
||||
"Knowledge",
|
||||
"Plugins",
|
||||
"首页",
|
||||
"机器人",
|
||||
"流水线",
|
||||
"知识库",
|
||||
"插件",
|
||||
];
|
||||
const matchedSignal = navigationSignals.find((signal) => text.includes(signal));
|
||||
|
||||
if (!auth.authenticated) {
|
||||
result.status = "blocked";
|
||||
result.reason = auth.reason || "Browser profile token was not accepted by backend.";
|
||||
} else if (isLoginUrl(page.url()) || /登录|Login|Sign in/i.test(text)) {
|
||||
result.status = "fail";
|
||||
result.reason = "Backend accepted the token, but the WebUI still showed the login page.";
|
||||
} else if (!matchedSignal) {
|
||||
result.status = "fail";
|
||||
result.reason = "Opened WebUI, but no known LangBot navigation signal was visible.";
|
||||
} else {
|
||||
result.status = "pass";
|
||||
result.reason = `Authenticated navigation signal visible: ${matchedSignal}`;
|
||||
}
|
||||
|
||||
await safeScreenshot(page, paths.screenshot);
|
||||
} catch (error) {
|
||||
if (!["env_issue", "blocked", "fail", "pass"].includes(result.status) || !result.reason) {
|
||||
result.status = /Playwright is not installed|LANGBOT_FRONTEND_URL/.test(error.message) ? "env_issue" : "fail";
|
||||
result.reason = error.message;
|
||||
}
|
||||
} finally {
|
||||
if (browser) await browser.close().catch(() => {});
|
||||
const finishedAt = new Date();
|
||||
result.finished_at = finishedAt.toISOString();
|
||||
result.finished_at_local = localIsoWithOffset(finishedAt);
|
||||
await writeResult(paths, result);
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
}
|
||||
|
||||
process.exit(exitCode(result.status));
|
||||
Reference in New Issue
Block a user