mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-08-13 14:10:59 +00:00
test(agent-runner): strengthen local agent e2e gate
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import { resolve } from "node:path";
|
||||
import { env } from "node:process";
|
||||
import {
|
||||
@@ -46,6 +46,8 @@ const startupTimeoutSec = Number(env.LANGBOT_MCP_STARTUP_TIMEOUT_SEC || "300");
|
||||
const readyTimeoutMs = Number(env.LANGBOT_MCP_READY_TIMEOUT_MS || "360000");
|
||||
const backendUrl = env.LANGBOT_BACKEND_URL || "";
|
||||
const apiDiagnosticPath = resolve(paths.evidenceDir, "api-diagnostic.json");
|
||||
const envLocalPath = resolve("skills/.env.local");
|
||||
const serverUuidEnvKey = env.LANGBOT_MCP_SERVER_UUID_ENV_KEY || "LANGBOT_MCP_QA_STDIO_SERVER_UUID";
|
||||
|
||||
let browser;
|
||||
const result = {
|
||||
@@ -70,6 +72,7 @@ const result = {
|
||||
result_json: paths.resultJson,
|
||||
},
|
||||
evidence_collected: ["api_diagnostic"],
|
||||
wrote_env: false,
|
||||
};
|
||||
|
||||
async function run() {
|
||||
@@ -159,6 +162,7 @@ async function run() {
|
||||
const deadline = Date.now() + readyTimeoutMs;
|
||||
let lastTools = [];
|
||||
let lastRuntime = null;
|
||||
let lastServer = null;
|
||||
while (Date.now() < deadline) {
|
||||
await new Promise((resolveReady) => setTimeout(resolveReady, 500));
|
||||
const tools = await getJson("/api/v1/tools");
|
||||
@@ -167,7 +171,8 @@ async function run() {
|
||||
.map((tool) => tool.name || tool.tool_name || tool.function?.name || "")
|
||||
.filter(Boolean)
|
||||
.sort();
|
||||
lastRuntime = server.json.data?.server?.runtime_info || null;
|
||||
lastServer = server.json.data?.server || null;
|
||||
lastRuntime = lastServer?.runtime_info || null;
|
||||
if (lastTools.includes(expectedTool)) break;
|
||||
}
|
||||
|
||||
@@ -177,6 +182,7 @@ async function run() {
|
||||
save_status: save.status,
|
||||
save_code: save.json.code ?? null,
|
||||
save_msg: save.json.msg || "",
|
||||
server_uuid: lastServer?.uuid || save.json.data?.uuid || "",
|
||||
tool_names: lastTools,
|
||||
has_expected_tool: lastTools.includes(expectedTool),
|
||||
runtime_status: lastRuntime?.status || null,
|
||||
@@ -212,11 +218,47 @@ async function run() {
|
||||
result.reason = `MCP server ${serverName} did not expose ${expectedTool}. See ${apiDiagnosticPath}.`;
|
||||
return;
|
||||
}
|
||||
if (!diagnostic.server_uuid) {
|
||||
result.status = "fail";
|
||||
result.reason = `MCP server ${serverName} exposed ${expectedTool}, but the server UUID was not returned. See ${apiDiagnosticPath}.`;
|
||||
return;
|
||||
}
|
||||
|
||||
await upsertEnvLocal(envLocalPath, {
|
||||
[serverUuidEnvKey]: diagnostic.server_uuid,
|
||||
});
|
||||
result.wrote_env = true;
|
||||
result.server_uuid = diagnostic.server_uuid;
|
||||
result.server_uuid_env_key = serverUuidEnvKey;
|
||||
|
||||
result.status = "pass";
|
||||
result.reason = `MCP server ${serverName} is connected and exposes ${expectedTool} through LangBot /api/v1/tools.`;
|
||||
}
|
||||
|
||||
async function upsertEnvLocal(path, updates) {
|
||||
let text = "";
|
||||
try {
|
||||
text = await readFile(path, "utf8");
|
||||
} catch {
|
||||
text = "";
|
||||
}
|
||||
|
||||
const lines = text ? text.split(/\r?\n/) : [];
|
||||
const seen = new Set();
|
||||
const updated = lines.map((line) => {
|
||||
const match = line.match(/^([A-Z][A-Z0-9_]*)=/);
|
||||
if (!match || !Object.prototype.hasOwnProperty.call(updates, match[1])) return line;
|
||||
seen.add(match[1]);
|
||||
return `${match[1]}=${updates[match[1]]}`;
|
||||
});
|
||||
|
||||
for (const [key, value] of Object.entries(updates)) {
|
||||
if (!seen.has(key)) updated.push(`${key}=${value}`);
|
||||
}
|
||||
|
||||
await writeFile(path, `${updated.filter((line, index) => line || index < updated.length - 1).join("\n")}\n`, "utf8");
|
||||
}
|
||||
|
||||
try {
|
||||
await run();
|
||||
} catch (error) {
|
||||
|
||||
Reference in New Issue
Block a user