mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-09-16 14:57:15 +00:00
feat(runner): unify plugin execution across agents and event processors
This commit is contained in:
RockChinQ
@@ -24,7 +24,10 @@ function loadEnvDefaults(path) {
|
||||
if (sep === -1) continue;
|
||||
const key = line.slice(0, sep).trim();
|
||||
if (env[key]) continue;
|
||||
env[key] = line.slice(sep + 1).trim().replace(/^["']|["']$/g, "");
|
||||
env[key] = line
|
||||
.slice(sep + 1)
|
||||
.trim()
|
||||
.replace(/^["']|["']$/g, "");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,12 +49,16 @@ function redactMessage(text) {
|
||||
return String(text ?? "")
|
||||
.replace(/\bbearer\s+[A-Za-z0-9._~+/=-]{8,}/gi, "Bearer [redacted]")
|
||||
.replace(/\bsk-[A-Za-z0-9_-]{6,}\b/g, "[redacted]")
|
||||
.replace(/(api[_-]?key|authorization|credential|jwt|oauth|password|secret|token)\s*[:=]\s*["']?[^"',\s]+/gi, "$1=[redacted]");
|
||||
.replace(
|
||||
/(api[_-]?key|authorization|credential|jwt|oauth|password|secret|token)\s*[:=]\s*["']?[^"',\s]+/gi,
|
||||
"$1=[redacted]",
|
||||
);
|
||||
}
|
||||
|
||||
function isEnvironmentError(message) {
|
||||
return /Playwright is not installed|LANGBOT_FRONTEND_URL|LANGBOT_BACKEND_URL|ERR_CONNECTION_REFUSED|ECONNREFUSED|net::ERR_|fetch failed|timed out/i
|
||||
.test(message);
|
||||
return /Playwright is not installed|LANGBOT_FRONTEND_URL|LANGBOT_BACKEND_URL|ERR_CONNECTION_REFUSED|ECONNREFUSED|net::ERR_|fetch failed|timed out/i.test(
|
||||
message,
|
||||
);
|
||||
}
|
||||
|
||||
loadEnvDefaults("skills/.env");
|
||||
@@ -80,9 +87,15 @@ const targets = [
|
||||
},
|
||||
{
|
||||
id: "acp-agent-runner",
|
||||
expected_runner_id: "plugin:langbot-team/ACPAgentRunner/default",
|
||||
pipeline_url: firstEnv("LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL", "LANGBOT_AGENT_RUNNER_PIPELINE_URL"),
|
||||
pipeline_name: firstEnv("LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME", "LANGBOT_AGENT_RUNNER_PIPELINE_NAME"),
|
||||
expected_runner_id: "plugin:langbot-team/ACPRunner/default",
|
||||
pipeline_url: firstEnv(
|
||||
"LANGBOT_ACP_RUNNER_PIPELINE_URL",
|
||||
"LANGBOT_RUNNER_PIPELINE_URL",
|
||||
),
|
||||
pipeline_name: firstEnv(
|
||||
"LANGBOT_ACP_RUNNER_PIPELINE_NAME",
|
||||
"LANGBOT_RUNNER_PIPELINE_NAME",
|
||||
),
|
||||
require_func_call_model: false,
|
||||
require_vision_model: false,
|
||||
},
|
||||
@@ -111,20 +124,29 @@ const result = {
|
||||
automation_result_json: paths.automationResultJson,
|
||||
result_json: paths.resultJson,
|
||||
},
|
||||
evidence_collected: ["ui", "screenshot", "console", "network", "api_diagnostic"],
|
||||
evidence_collected: [
|
||||
"ui",
|
||||
"screenshot",
|
||||
"console",
|
||||
"network",
|
||||
"api_diagnostic",
|
||||
],
|
||||
};
|
||||
|
||||
async function run() {
|
||||
if (!backendUrl || !frontendUrl) {
|
||||
result.status = "env_issue";
|
||||
result.reason = "LANGBOT_FRONTEND_URL and LANGBOT_BACKEND_URL must be configured.";
|
||||
result.reason =
|
||||
"LANGBOT_FRONTEND_URL and LANGBOT_BACKEND_URL must be configured.";
|
||||
return;
|
||||
}
|
||||
|
||||
browser = await createBrowser(paths);
|
||||
const { page } = browser;
|
||||
await page.goto(frontendUrl, { waitUntil: "domcontentloaded" });
|
||||
await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {});
|
||||
await page
|
||||
.waitForLoadState("networkidle", { timeout: 10_000 })
|
||||
.catch(() => {});
|
||||
const workspace = await ensureBrowserWorkspace(page, backendUrl);
|
||||
if (workspace.status !== "pass") {
|
||||
result.status = workspace.status;
|
||||
@@ -132,309 +154,428 @@ async function run() {
|
||||
return;
|
||||
}
|
||||
|
||||
const diagnostic = await page.evaluate(async ({ backendUrl, targets, testModels }) => {
|
||||
const blockers = [];
|
||||
const envIssues = [];
|
||||
const warnings = [];
|
||||
const checks = [];
|
||||
const diagnostic = await page.evaluate(
|
||||
async ({ backendUrl, targets, testModels }) => {
|
||||
const blockers = [];
|
||||
const envIssues = [];
|
||||
const warnings = [];
|
||||
const checks = [];
|
||||
|
||||
const addCheck = (name, status, detail = {}) => {
|
||||
checks.push({ name, status, ...detail });
|
||||
if (status === "blocked") blockers.push({ name, ...detail });
|
||||
if (status === "env_issue") envIssues.push({ name, ...detail });
|
||||
};
|
||||
const safeMessage = (value) => String(value ?? "")
|
||||
.replace(/\bbearer\s+[A-Za-z0-9._~+/=-]{8,}/gi, "Bearer [redacted]")
|
||||
.replace(/\bsk-[A-Za-z0-9_-]{6,}\b/g, "[redacted]")
|
||||
.replace(/(api[_-]?key|authorization|credential|jwt|oauth|password|secret|token)\s*[:=]\s*["']?[^"',\s]+/gi, "$1=[redacted]");
|
||||
|
||||
const token = localStorage.getItem("token");
|
||||
if (!token) {
|
||||
addCheck("browser-auth", "blocked", { reason: "Browser profile has no localStorage token." });
|
||||
return { authenticated: false, blockers, env_issues: envIssues, warnings, checks };
|
||||
}
|
||||
|
||||
const headers = {
|
||||
Authorization: `Bearer ${token}`,
|
||||
"Content-Type": "application/json",
|
||||
"X-Workspace-Id": localStorage.getItem("langbot_active_workspace_uuid") || "",
|
||||
};
|
||||
const getJson = async (path) => {
|
||||
const response = await fetch(`${backendUrl}${path}`, { headers });
|
||||
return {
|
||||
status: response.status,
|
||||
json: await response.json().catch(() => ({})),
|
||||
const addCheck = (name, status, detail = {}) => {
|
||||
checks.push({ name, status, ...detail });
|
||||
if (status === "blocked") blockers.push({ name, ...detail });
|
||||
if (status === "env_issue") envIssues.push({ name, ...detail });
|
||||
};
|
||||
};
|
||||
const postJson = async (path, body) => {
|
||||
const response = await fetch(`${backendUrl}${path}`, {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
return {
|
||||
status: response.status,
|
||||
json: await response.json().catch(() => ({})),
|
||||
};
|
||||
};
|
||||
const safeMessage = (value) =>
|
||||
String(value ?? "")
|
||||
.replace(/\bbearer\s+[A-Za-z0-9._~+/=-]{8,}/gi, "Bearer [redacted]")
|
||||
.replace(/\bsk-[A-Za-z0-9_-]{6,}\b/g, "[redacted]")
|
||||
.replace(
|
||||
/(api[_-]?key|authorization|credential|jwt|oauth|password|secret|token)\s*[:=]\s*["']?[^"',\s]+/gi,
|
||||
"$1=[redacted]",
|
||||
);
|
||||
|
||||
const tokenCheck = await getJson("/api/v1/user/check-token");
|
||||
addCheck(
|
||||
"browser-auth",
|
||||
tokenCheck.status < 400 && (tokenCheck.json.code ?? 0) === 0 ? "pass" : "blocked",
|
||||
{ http_status: tokenCheck.status, code: tokenCheck.json.code ?? null, reason: safeMessage(tokenCheck.json.msg || "") },
|
||||
);
|
||||
|
||||
const systemInfo = await getJson("/api/v1/system/info");
|
||||
addCheck(
|
||||
"backend-system-info",
|
||||
systemInfo.status < 400 ? "pass" : "env_issue",
|
||||
{
|
||||
http_status: systemInfo.status,
|
||||
version: systemInfo.json.data?.version || systemInfo.json.data?.system?.version || "",
|
||||
},
|
||||
);
|
||||
|
||||
const pluginSystem = await getJson("/api/v1/system/status/plugin-system");
|
||||
addCheck(
|
||||
"plugin-system",
|
||||
pluginSystem.status < 400 && (pluginSystem.json.code ?? 0) === 0 ? "pass" : "env_issue",
|
||||
{
|
||||
http_status: pluginSystem.status,
|
||||
code: pluginSystem.json.code ?? null,
|
||||
status: pluginSystem.json.data?.status || pluginSystem.json.data?.state || "",
|
||||
reason: safeMessage(pluginSystem.json.msg || ""),
|
||||
},
|
||||
);
|
||||
|
||||
const boxStatus = await getJson("/api/v1/box/status");
|
||||
addCheck(
|
||||
"box-runtime",
|
||||
boxStatus.status < 400 && (boxStatus.json.code ?? 0) === 0 ? "pass" : "env_issue",
|
||||
{
|
||||
http_status: boxStatus.status,
|
||||
code: boxStatus.json.code ?? null,
|
||||
status: boxStatus.json.data?.status || "",
|
||||
backend: boxStatus.json.data?.backend || "",
|
||||
reason: safeMessage(boxStatus.json.msg || ""),
|
||||
},
|
||||
);
|
||||
|
||||
const plugins = await getJson("/api/v1/plugins");
|
||||
const installedPluginIds = (plugins.json.data?.plugins || [])
|
||||
.map((plugin) => {
|
||||
const metadata = plugin.manifest?.manifest?.metadata || plugin.manifest?.metadata || plugin.metadata || {};
|
||||
return metadata.author && metadata.name ? `${metadata.author}/${metadata.name}` : "";
|
||||
})
|
||||
.filter(Boolean);
|
||||
const requiredPlugins = ["langbot-team/LocalAgent", "langbot-team/ACPAgentRunner", "qa/plugin-smoke"];
|
||||
const pluginPresence = Object.fromEntries(requiredPlugins.map((id) => [id, installedPluginIds.includes(id)]));
|
||||
for (const [id, present] of Object.entries(pluginPresence)) {
|
||||
addCheck(`plugin:${id}`, present ? "pass" : "blocked", { plugin_id: id, reason: present ? "" : "Required plugin is not listed by /api/v1/plugins." });
|
||||
}
|
||||
|
||||
const tools = await getJson("/api/v1/tools");
|
||||
const toolNames = (tools.json.data?.tools || [])
|
||||
.map((tool) => tool.name || tool.tool_name || tool.function?.name || "")
|
||||
.filter(Boolean)
|
||||
.sort();
|
||||
addCheck(
|
||||
"tool:qa_plugin_echo",
|
||||
toolNames.includes("qa_plugin_echo") ? "pass" : "blocked",
|
||||
{ reason: toolNames.includes("qa_plugin_echo") ? "" : "qa-plugin-smoke tool qa_plugin_echo is not exposed through /api/v1/tools." },
|
||||
);
|
||||
if (!toolNames.includes("qa_mcp_echo")) {
|
||||
warnings.push({
|
||||
name: "tool:qa_mcp_echo",
|
||||
reason: "qa_mcp_echo is not currently exposed. This is acceptable before mcp-stdio-register, but mcp-stdio-tool-call must run after registration.",
|
||||
});
|
||||
}
|
||||
|
||||
const modelResponse = await getJson("/api/v1/provider/models/llm");
|
||||
const models = (modelResponse.json.data?.models || []).map((model) => ({
|
||||
uuid: model.uuid,
|
||||
name: model.name,
|
||||
abilities: Array.isArray(model.abilities) ? model.abilities : [],
|
||||
provider_uuid: model.provider_uuid || model.provider?.uuid || "",
|
||||
provider_name: model.provider_name || model.provider?.name || "",
|
||||
requester: model.requester || model.provider?.requester || "",
|
||||
}));
|
||||
addCheck(
|
||||
"llm-model-list",
|
||||
modelResponse.status < 400 && (modelResponse.json.code ?? 0) === 0 ? "pass" : "env_issue",
|
||||
{ http_status: modelResponse.status, model_count: models.length, reason: safeMessage(modelResponse.json.msg || "") },
|
||||
);
|
||||
const modelById = new Map(models.map((model) => [model.uuid, model]));
|
||||
|
||||
const pipelineList = await getJson("/api/v1/pipelines");
|
||||
const pipelines = pipelineList.json.data?.pipelines || [];
|
||||
addCheck(
|
||||
"pipeline-list",
|
||||
pipelineList.status < 400 && (pipelineList.json.code ?? 0) === 0 ? "pass" : "blocked",
|
||||
{ http_status: pipelineList.status, pipeline_count: pipelines.length, reason: safeMessage(pipelineList.json.msg || "") },
|
||||
);
|
||||
|
||||
const resolvedPipelines = [];
|
||||
const modelTested = new Set();
|
||||
for (const target of targets) {
|
||||
let pipelineId = "";
|
||||
let matchedBy = "";
|
||||
if (target.pipeline_url) {
|
||||
try {
|
||||
pipelineId = new URL(target.pipeline_url).searchParams.get("id") || "";
|
||||
matchedBy = pipelineId ? "url" : "";
|
||||
} catch {
|
||||
pipelineId = "";
|
||||
}
|
||||
}
|
||||
if (!pipelineId && target.pipeline_name) {
|
||||
const match = pipelines.find((pipeline) => pipeline.name === target.pipeline_name);
|
||||
if (match) {
|
||||
pipelineId = match.uuid;
|
||||
matchedBy = "name";
|
||||
}
|
||||
}
|
||||
if (!pipelineId) {
|
||||
addCheck(`pipeline:${target.id}`, "blocked", {
|
||||
target: target.id,
|
||||
reason: "Required pipeline env is missing or could not resolve to a pipeline id.",
|
||||
const token = localStorage.getItem("token");
|
||||
if (!token) {
|
||||
addCheck("browser-auth", "blocked", {
|
||||
reason: "Browser profile has no localStorage token.",
|
||||
});
|
||||
continue;
|
||||
return {
|
||||
authenticated: false,
|
||||
blockers,
|
||||
env_issues: envIssues,
|
||||
warnings,
|
||||
checks,
|
||||
};
|
||||
}
|
||||
|
||||
const response = await getJson(`/api/v1/pipelines/${encodeURIComponent(pipelineId)}`);
|
||||
const pipeline = response.json.data?.pipeline;
|
||||
if (response.status >= 400 || !pipeline) {
|
||||
addCheck(`pipeline:${target.id}`, "blocked", {
|
||||
target: target.id,
|
||||
pipeline_id: pipelineId,
|
||||
http_status: response.status,
|
||||
reason: safeMessage(response.json.msg || "Could not load pipeline."),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const config = pipeline.config || {};
|
||||
const aiConfig = config.ai && typeof config.ai === "object" ? config.ai : {};
|
||||
const runner = aiConfig.runner && typeof aiConfig.runner === "object" ? aiConfig.runner : {};
|
||||
const runnerId = runner.id || "";
|
||||
const runnerConfigs = aiConfig.runner_config && typeof aiConfig.runner_config === "object" ? aiConfig.runner_config : {};
|
||||
const runnerConfig = runnerConfigs[runnerId] && typeof runnerConfigs[runnerId] === "object" ? runnerConfigs[runnerId] : {};
|
||||
const pipelineSummary = {
|
||||
target: target.id,
|
||||
pipeline_id: pipelineId,
|
||||
pipeline_name: pipeline.name,
|
||||
matched_by: matchedBy,
|
||||
runner_id: runnerId,
|
||||
expected_runner_id: target.expected_runner_id,
|
||||
runner_config_keys: Object.keys(runnerConfig).sort(),
|
||||
const headers = {
|
||||
Authorization: `Bearer ${token}`,
|
||||
"Content-Type": "application/json",
|
||||
"X-Workspace-Id":
|
||||
localStorage.getItem("langbot_active_workspace_uuid") || "",
|
||||
};
|
||||
const getJson = async (path) => {
|
||||
const response = await fetch(`${backendUrl}${path}`, { headers });
|
||||
return {
|
||||
status: response.status,
|
||||
json: await response.json().catch(() => ({})),
|
||||
};
|
||||
};
|
||||
const postJson = async (path, body) => {
|
||||
const response = await fetch(`${backendUrl}${path}`, {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
return {
|
||||
status: response.status,
|
||||
json: await response.json().catch(() => ({})),
|
||||
};
|
||||
};
|
||||
resolvedPipelines.push(pipelineSummary);
|
||||
|
||||
const tokenCheck = await getJson("/api/v1/user/check-token");
|
||||
addCheck(
|
||||
`pipeline:${target.id}:runner`,
|
||||
runnerId === target.expected_runner_id ? "pass" : "blocked",
|
||||
"browser-auth",
|
||||
tokenCheck.status < 400 && (tokenCheck.json.code ?? 0) === 0
|
||||
? "pass"
|
||||
: "blocked",
|
||||
{
|
||||
...pipelineSummary,
|
||||
reason: runnerId === target.expected_runner_id ? "" : `Expected ${target.expected_runner_id}, got ${runnerId || "<missing>"}.`,
|
||||
http_status: tokenCheck.status,
|
||||
code: tokenCheck.json.code ?? null,
|
||||
reason: safeMessage(tokenCheck.json.msg || ""),
|
||||
},
|
||||
);
|
||||
|
||||
if (target.require_func_call_model || target.require_vision_model || (testModels && target.id === "local-agent")) {
|
||||
const modelConfig = runnerConfig.model;
|
||||
const primaryModelId = typeof modelConfig === "string"
|
||||
? modelConfig
|
||||
: modelConfig && typeof modelConfig === "object"
|
||||
? modelConfig.primary || ""
|
||||
const systemInfo = await getJson("/api/v1/system/info");
|
||||
addCheck(
|
||||
"backend-system-info",
|
||||
systemInfo.status < 400 ? "pass" : "env_issue",
|
||||
{
|
||||
http_status: systemInfo.status,
|
||||
version:
|
||||
systemInfo.json.data?.version ||
|
||||
systemInfo.json.data?.system?.version ||
|
||||
"",
|
||||
},
|
||||
);
|
||||
|
||||
const pluginSystem = await getJson("/api/v1/system/status/plugin-system");
|
||||
addCheck(
|
||||
"plugin-system",
|
||||
pluginSystem.status < 400 && (pluginSystem.json.code ?? 0) === 0
|
||||
? "pass"
|
||||
: "env_issue",
|
||||
{
|
||||
http_status: pluginSystem.status,
|
||||
code: pluginSystem.json.code ?? null,
|
||||
status:
|
||||
pluginSystem.json.data?.status ||
|
||||
pluginSystem.json.data?.state ||
|
||||
"",
|
||||
reason: safeMessage(pluginSystem.json.msg || ""),
|
||||
},
|
||||
);
|
||||
|
||||
const boxStatus = await getJson("/api/v1/box/status");
|
||||
addCheck(
|
||||
"box-runtime",
|
||||
boxStatus.status < 400 && (boxStatus.json.code ?? 0) === 0
|
||||
? "pass"
|
||||
: "env_issue",
|
||||
{
|
||||
http_status: boxStatus.status,
|
||||
code: boxStatus.json.code ?? null,
|
||||
status: boxStatus.json.data?.status || "",
|
||||
backend: boxStatus.json.data?.backend || "",
|
||||
reason: safeMessage(boxStatus.json.msg || ""),
|
||||
},
|
||||
);
|
||||
|
||||
const plugins = await getJson("/api/v1/plugins");
|
||||
const installedPluginIds = (plugins.json.data?.plugins || [])
|
||||
.map((plugin) => {
|
||||
const metadata =
|
||||
plugin.manifest?.manifest?.metadata ||
|
||||
plugin.manifest?.metadata ||
|
||||
plugin.metadata ||
|
||||
{};
|
||||
return metadata.author && metadata.name
|
||||
? `${metadata.author}/${metadata.name}`
|
||||
: "";
|
||||
if (!primaryModelId) {
|
||||
addCheck(`pipeline:${target.id}:primary-model`, "blocked", {
|
||||
...pipelineSummary,
|
||||
reason: "Local-agent runner config has no primary model.",
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const model = modelById.get(primaryModelId);
|
||||
if (!model) {
|
||||
addCheck(`pipeline:${target.id}:primary-model`, "blocked", {
|
||||
...pipelineSummary,
|
||||
model_uuid: primaryModelId,
|
||||
reason: "Primary model is not listed by /api/v1/provider/models/llm.",
|
||||
});
|
||||
continue;
|
||||
}
|
||||
addCheck(`pipeline:${target.id}:primary-model`, "pass", {
|
||||
...pipelineSummary,
|
||||
model: {
|
||||
uuid: model.uuid,
|
||||
name: model.name,
|
||||
abilities: model.abilities,
|
||||
provider_name: model.provider_name,
|
||||
requester: model.requester,
|
||||
},
|
||||
})
|
||||
.filter(Boolean);
|
||||
const requiredPlugins = [
|
||||
"langbot-team/LocalAgent",
|
||||
"langbot-team/ACPRunner",
|
||||
"qa/plugin-smoke",
|
||||
];
|
||||
const pluginPresence = Object.fromEntries(
|
||||
requiredPlugins.map((id) => [id, installedPluginIds.includes(id)]),
|
||||
);
|
||||
for (const [id, present] of Object.entries(pluginPresence)) {
|
||||
addCheck(`plugin:${id}`, present ? "pass" : "blocked", {
|
||||
plugin_id: id,
|
||||
reason: present
|
||||
? ""
|
||||
: "Required plugin is not listed by /api/v1/plugins.",
|
||||
});
|
||||
if (target.require_func_call_model) {
|
||||
addCheck(
|
||||
`pipeline:${target.id}:func-call-model`,
|
||||
model.abilities.includes("func_call") ? "pass" : "env_issue",
|
||||
{
|
||||
model_uuid: model.uuid,
|
||||
model_name: model.name,
|
||||
abilities: model.abilities,
|
||||
reason: model.abilities.includes("func_call") ? "" : "Release gate includes tool-call cases; the local-agent primary model must advertise func_call.",
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
const tools = await getJson("/api/v1/tools");
|
||||
const toolNames = (tools.json.data?.tools || [])
|
||||
.map((tool) => tool.name || tool.tool_name || tool.function?.name || "")
|
||||
.filter(Boolean)
|
||||
.sort();
|
||||
addCheck(
|
||||
"tool:qa_plugin_echo",
|
||||
toolNames.includes("qa_plugin_echo") ? "pass" : "blocked",
|
||||
{
|
||||
reason: toolNames.includes("qa_plugin_echo")
|
||||
? ""
|
||||
: "qa-plugin-smoke tool qa_plugin_echo is not exposed through /api/v1/tools.",
|
||||
},
|
||||
);
|
||||
if (!toolNames.includes("qa_mcp_echo")) {
|
||||
warnings.push({
|
||||
name: "tool:qa_mcp_echo",
|
||||
reason:
|
||||
"qa_mcp_echo is not currently exposed. This is acceptable before mcp-stdio-register, but mcp-stdio-tool-call must run after registration.",
|
||||
});
|
||||
}
|
||||
|
||||
const modelResponse = await getJson("/api/v1/provider/models/llm");
|
||||
const models = (modelResponse.json.data?.models || []).map((model) => ({
|
||||
uuid: model.uuid,
|
||||
name: model.name,
|
||||
abilities: Array.isArray(model.abilities) ? model.abilities : [],
|
||||
provider_uuid: model.provider_uuid || model.provider?.uuid || "",
|
||||
provider_name: model.provider_name || model.provider?.name || "",
|
||||
requester: model.requester || model.provider?.requester || "",
|
||||
}));
|
||||
addCheck(
|
||||
"llm-model-list",
|
||||
modelResponse.status < 400 && (modelResponse.json.code ?? 0) === 0
|
||||
? "pass"
|
||||
: "env_issue",
|
||||
{
|
||||
http_status: modelResponse.status,
|
||||
model_count: models.length,
|
||||
reason: safeMessage(modelResponse.json.msg || ""),
|
||||
},
|
||||
);
|
||||
const modelById = new Map(models.map((model) => [model.uuid, model]));
|
||||
|
||||
const pipelineList = await getJson("/api/v1/pipelines");
|
||||
const pipelines = pipelineList.json.data?.pipelines || [];
|
||||
addCheck(
|
||||
"pipeline-list",
|
||||
pipelineList.status < 400 && (pipelineList.json.code ?? 0) === 0
|
||||
? "pass"
|
||||
: "blocked",
|
||||
{
|
||||
http_status: pipelineList.status,
|
||||
pipeline_count: pipelines.length,
|
||||
reason: safeMessage(pipelineList.json.msg || ""),
|
||||
},
|
||||
);
|
||||
|
||||
const resolvedPipelines = [];
|
||||
const modelTested = new Set();
|
||||
for (const target of targets) {
|
||||
let pipelineId = "";
|
||||
let matchedBy = "";
|
||||
if (target.pipeline_url) {
|
||||
try {
|
||||
pipelineId =
|
||||
new URL(target.pipeline_url).searchParams.get("id") || "";
|
||||
matchedBy = pipelineId ? "url" : "";
|
||||
} catch {
|
||||
pipelineId = "";
|
||||
}
|
||||
}
|
||||
if (target.require_vision_model) {
|
||||
addCheck(
|
||||
`pipeline:${target.id}:vision-model`,
|
||||
model.abilities.includes("vision") ? "pass" : "env_issue",
|
||||
{
|
||||
model_uuid: model.uuid,
|
||||
model_name: model.name,
|
||||
abilities: model.abilities,
|
||||
reason: model.abilities.includes("vision") ? "" : "Release gate includes multimodal cases; the local-agent primary model must advertise vision.",
|
||||
},
|
||||
if (!pipelineId && target.pipeline_name) {
|
||||
const match = pipelines.find(
|
||||
(pipeline) => pipeline.name === target.pipeline_name,
|
||||
);
|
||||
if (match) {
|
||||
pipelineId = match.uuid;
|
||||
matchedBy = "name";
|
||||
}
|
||||
}
|
||||
if (testModels && !modelTested.has(model.uuid)) {
|
||||
modelTested.add(model.uuid);
|
||||
const modelTest = await postJson(`/api/v1/provider/models/llm/${encodeURIComponent(model.uuid)}/test`, { extra_args: {} });
|
||||
const passed = modelTest.status < 400 && (modelTest.json.code ?? 0) === 0;
|
||||
addCheck(
|
||||
`model-test:${model.name}`,
|
||||
passed ? "pass" : "env_issue",
|
||||
{
|
||||
model_uuid: model.uuid,
|
||||
model_name: model.name,
|
||||
http_status: modelTest.status,
|
||||
code: modelTest.json.code ?? null,
|
||||
reason: passed ? "" : safeMessage(modelTest.json.msg || modelTest.json.message || "Model test failed."),
|
||||
if (!pipelineId) {
|
||||
addCheck(`pipeline:${target.id}`, "blocked", {
|
||||
target: target.id,
|
||||
reason:
|
||||
"Required pipeline env is missing or could not resolve to a pipeline id.",
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const response = await getJson(
|
||||
`/api/v1/pipelines/${encodeURIComponent(pipelineId)}`,
|
||||
);
|
||||
const pipeline = response.json.data?.pipeline;
|
||||
if (response.status >= 400 || !pipeline) {
|
||||
addCheck(`pipeline:${target.id}`, "blocked", {
|
||||
target: target.id,
|
||||
pipeline_id: pipelineId,
|
||||
http_status: response.status,
|
||||
reason: safeMessage(
|
||||
response.json.msg || "Could not load pipeline.",
|
||||
),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const config = pipeline.config || {};
|
||||
const aiConfig =
|
||||
config.ai && typeof config.ai === "object" ? config.ai : {};
|
||||
const runner =
|
||||
aiConfig.runner && typeof aiConfig.runner === "object"
|
||||
? aiConfig.runner
|
||||
: {};
|
||||
const runnerId = runner.id || "";
|
||||
const runnerConfigs =
|
||||
aiConfig.runner_config && typeof aiConfig.runner_config === "object"
|
||||
? aiConfig.runner_config
|
||||
: {};
|
||||
const runnerConfig =
|
||||
runnerConfigs[runnerId] && typeof runnerConfigs[runnerId] === "object"
|
||||
? runnerConfigs[runnerId]
|
||||
: {};
|
||||
const pipelineSummary = {
|
||||
target: target.id,
|
||||
pipeline_id: pipelineId,
|
||||
pipeline_name: pipeline.name,
|
||||
matched_by: matchedBy,
|
||||
runner_id: runnerId,
|
||||
expected_runner_id: target.expected_runner_id,
|
||||
runner_config_keys: Object.keys(runnerConfig).sort(),
|
||||
};
|
||||
resolvedPipelines.push(pipelineSummary);
|
||||
|
||||
addCheck(
|
||||
`pipeline:${target.id}:runner`,
|
||||
runnerId === target.expected_runner_id ? "pass" : "blocked",
|
||||
{
|
||||
...pipelineSummary,
|
||||
reason:
|
||||
runnerId === target.expected_runner_id
|
||||
? ""
|
||||
: `Expected ${target.expected_runner_id}, got ${runnerId || "<missing>"}.`,
|
||||
},
|
||||
);
|
||||
|
||||
if (
|
||||
target.require_func_call_model ||
|
||||
target.require_vision_model ||
|
||||
(testModels && target.id === "local-agent")
|
||||
) {
|
||||
const modelConfig = runnerConfig.model;
|
||||
const primaryModelId =
|
||||
typeof modelConfig === "string"
|
||||
? modelConfig
|
||||
: modelConfig && typeof modelConfig === "object"
|
||||
? modelConfig.primary || ""
|
||||
: "";
|
||||
if (!primaryModelId) {
|
||||
addCheck(`pipeline:${target.id}:primary-model`, "blocked", {
|
||||
...pipelineSummary,
|
||||
reason: "Local-agent runner config has no primary model.",
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const model = modelById.get(primaryModelId);
|
||||
if (!model) {
|
||||
addCheck(`pipeline:${target.id}:primary-model`, "blocked", {
|
||||
...pipelineSummary,
|
||||
model_uuid: primaryModelId,
|
||||
reason:
|
||||
"Primary model is not listed by /api/v1/provider/models/llm.",
|
||||
});
|
||||
continue;
|
||||
}
|
||||
addCheck(`pipeline:${target.id}:primary-model`, "pass", {
|
||||
...pipelineSummary,
|
||||
model: {
|
||||
uuid: model.uuid,
|
||||
name: model.name,
|
||||
abilities: model.abilities,
|
||||
provider_name: model.provider_name,
|
||||
requester: model.requester,
|
||||
},
|
||||
);
|
||||
});
|
||||
if (target.require_func_call_model) {
|
||||
addCheck(
|
||||
`pipeline:${target.id}:func-call-model`,
|
||||
model.abilities.includes("func_call") ? "pass" : "env_issue",
|
||||
{
|
||||
model_uuid: model.uuid,
|
||||
model_name: model.name,
|
||||
abilities: model.abilities,
|
||||
reason: model.abilities.includes("func_call")
|
||||
? ""
|
||||
: "Release gate includes tool-call cases; the local-agent primary model must advertise func_call.",
|
||||
},
|
||||
);
|
||||
}
|
||||
if (target.require_vision_model) {
|
||||
addCheck(
|
||||
`pipeline:${target.id}:vision-model`,
|
||||
model.abilities.includes("vision") ? "pass" : "env_issue",
|
||||
{
|
||||
model_uuid: model.uuid,
|
||||
model_name: model.name,
|
||||
abilities: model.abilities,
|
||||
reason: model.abilities.includes("vision")
|
||||
? ""
|
||||
: "Release gate includes multimodal cases; the local-agent primary model must advertise vision.",
|
||||
},
|
||||
);
|
||||
}
|
||||
if (testModels && !modelTested.has(model.uuid)) {
|
||||
modelTested.add(model.uuid);
|
||||
const modelTest = await postJson(
|
||||
`/api/v1/provider/models/llm/${encodeURIComponent(model.uuid)}/test`,
|
||||
{ extra_args: {} },
|
||||
);
|
||||
const passed =
|
||||
modelTest.status < 400 && (modelTest.json.code ?? 0) === 0;
|
||||
addCheck(
|
||||
`model-test:${model.name}`,
|
||||
passed ? "pass" : "env_issue",
|
||||
{
|
||||
model_uuid: model.uuid,
|
||||
model_name: model.name,
|
||||
http_status: modelTest.status,
|
||||
code: modelTest.json.code ?? null,
|
||||
reason: passed
|
||||
? ""
|
||||
: safeMessage(
|
||||
modelTest.json.msg ||
|
||||
modelTest.json.message ||
|
||||
"Model test failed.",
|
||||
),
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
authenticated: true,
|
||||
blockers,
|
||||
env_issues: envIssues,
|
||||
warnings,
|
||||
checks,
|
||||
resolved_pipelines: resolvedPipelines,
|
||||
tools: {
|
||||
required: ["qa_plugin_echo"],
|
||||
optional_before_register: ["qa_mcp_echo"],
|
||||
present: toolNames.filter((name) => ["qa_plugin_echo", "qa_mcp_echo"].includes(name)),
|
||||
},
|
||||
models,
|
||||
};
|
||||
}, { backendUrl, targets, testModels });
|
||||
return {
|
||||
authenticated: true,
|
||||
blockers,
|
||||
env_issues: envIssues,
|
||||
warnings,
|
||||
checks,
|
||||
resolved_pipelines: resolvedPipelines,
|
||||
tools: {
|
||||
required: ["qa_plugin_echo"],
|
||||
optional_before_register: ["qa_mcp_echo"],
|
||||
present: toolNames.filter((name) =>
|
||||
["qa_plugin_echo", "qa_mcp_echo"].includes(name),
|
||||
),
|
||||
},
|
||||
models,
|
||||
};
|
||||
},
|
||||
{ backendUrl, targets, testModels },
|
||||
);
|
||||
|
||||
diagnostic.blockers = (diagnostic.blockers || []).map((item) => ({ ...item, reason: redactMessage(item.reason || "") }));
|
||||
diagnostic.env_issues = (diagnostic.env_issues || []).map((item) => ({ ...item, reason: redactMessage(item.reason || "") }));
|
||||
await writeFile(diagnosticPath, `${JSON.stringify(diagnostic, null, 2)}\n`, "utf8");
|
||||
diagnostic.blockers = (diagnostic.blockers || []).map((item) => ({
|
||||
...item,
|
||||
reason: redactMessage(item.reason || ""),
|
||||
}));
|
||||
diagnostic.env_issues = (diagnostic.env_issues || []).map((item) => ({
|
||||
...item,
|
||||
reason: redactMessage(item.reason || ""),
|
||||
}));
|
||||
await writeFile(
|
||||
diagnosticPath,
|
||||
`${JSON.stringify(diagnostic, null, 2)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
await safeScreenshot(page, paths.screenshot);
|
||||
|
||||
const blockers = diagnostic.blockers || [];
|
||||
@@ -447,31 +588,49 @@ async function run() {
|
||||
result.reason = `Preflight environment issue: ${envIssues.map((item) => item.name).join(", ")}`;
|
||||
} else {
|
||||
result.status = "pass";
|
||||
result.reason = "Release gate preflight passed: auth, plugin runtime, required pipelines, runner ids, tools, and local-agent model checks are ready.";
|
||||
result.reason =
|
||||
"Release gate preflight passed: auth, plugin runtime, required pipelines, runner ids, tools, and local-agent model checks are ready.";
|
||||
}
|
||||
result.check_count = Array.isArray(diagnostic.checks) ? diagnostic.checks.length : 0;
|
||||
result.warning_count = Array.isArray(diagnostic.warnings) ? diagnostic.warnings.length : 0;
|
||||
result.check_count = Array.isArray(diagnostic.checks)
|
||||
? diagnostic.checks.length
|
||||
: 0;
|
||||
result.warning_count = Array.isArray(diagnostic.warnings)
|
||||
? diagnostic.warnings.length
|
||||
: 0;
|
||||
}
|
||||
|
||||
try {
|
||||
await run();
|
||||
} catch (error) {
|
||||
const message = redactMessage(error instanceof Error ? error.message : String(error));
|
||||
const message = redactMessage(
|
||||
error instanceof Error ? error.message : String(error),
|
||||
);
|
||||
result.status = isEnvironmentError(message) ? "env_issue" : "fail";
|
||||
result.reason = message;
|
||||
await writeFile(diagnosticPath, `${JSON.stringify({
|
||||
authenticated: false,
|
||||
blockers: [],
|
||||
env_issues: result.status === "env_issue" ? [{ name: "preflight-runtime", reason: message }] : [],
|
||||
warnings: [],
|
||||
checks: [
|
||||
await writeFile(
|
||||
diagnosticPath,
|
||||
`${JSON.stringify(
|
||||
{
|
||||
name: "preflight-runtime",
|
||||
status: result.status,
|
||||
reason: message,
|
||||
authenticated: false,
|
||||
blockers: [],
|
||||
env_issues:
|
||||
result.status === "env_issue"
|
||||
? [{ name: "preflight-runtime", reason: message }]
|
||||
: [],
|
||||
warnings: [],
|
||||
checks: [
|
||||
{
|
||||
name: "preflight-runtime",
|
||||
status: result.status,
|
||||
reason: message,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}, null, 2)}\n`, "utf8").catch(() => {});
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
"utf8",
|
||||
).catch(() => {});
|
||||
} finally {
|
||||
if (browser) await browser.close().catch(() => {});
|
||||
const finishedAt = new Date();
|
||||
|
||||
Reference in New Issue
Block a user