test(agent-runner): strengthen local agent e2e gate

This commit is contained in:
huanghuoguoguo
2026-07-01 20:20:43 +08:00
parent d0f8f080e9
commit 995888f6b2
44 changed files with 5678 additions and 1040 deletions
+44 -2
View File
@@ -1,7 +1,7 @@
#!/usr/bin/env node
import { existsSync, readFileSync } from "node:fs";
import { writeFile } from "node:fs/promises";
import { readFile, writeFile } from "node:fs/promises";
import { resolve } from "node:path";
import { env } from "node:process";
import {
@@ -46,6 +46,8 @@ const startupTimeoutSec = Number(env.LANGBOT_MCP_STARTUP_TIMEOUT_SEC || "300");
const readyTimeoutMs = Number(env.LANGBOT_MCP_READY_TIMEOUT_MS || "360000");
const backendUrl = env.LANGBOT_BACKEND_URL || "";
const apiDiagnosticPath = resolve(paths.evidenceDir, "api-diagnostic.json");
const envLocalPath = resolve("skills/.env.local");
const serverUuidEnvKey = env.LANGBOT_MCP_SERVER_UUID_ENV_KEY || "LANGBOT_MCP_QA_STDIO_SERVER_UUID";
let browser;
const result = {
@@ -70,6 +72,7 @@ const result = {
result_json: paths.resultJson,
},
evidence_collected: ["api_diagnostic"],
wrote_env: false,
};
async function run() {
@@ -159,6 +162,7 @@ async function run() {
const deadline = Date.now() + readyTimeoutMs;
let lastTools = [];
let lastRuntime = null;
let lastServer = null;
while (Date.now() < deadline) {
await new Promise((resolveReady) => setTimeout(resolveReady, 500));
const tools = await getJson("/api/v1/tools");
@@ -167,7 +171,8 @@ async function run() {
.map((tool) => tool.name || tool.tool_name || tool.function?.name || "")
.filter(Boolean)
.sort();
lastRuntime = server.json.data?.server?.runtime_info || null;
lastServer = server.json.data?.server || null;
lastRuntime = lastServer?.runtime_info || null;
if (lastTools.includes(expectedTool)) break;
}
@@ -177,6 +182,7 @@ async function run() {
save_status: save.status,
save_code: save.json.code ?? null,
save_msg: save.json.msg || "",
server_uuid: lastServer?.uuid || save.json.data?.uuid || "",
tool_names: lastTools,
has_expected_tool: lastTools.includes(expectedTool),
runtime_status: lastRuntime?.status || null,
@@ -212,11 +218,47 @@ async function run() {
result.reason = `MCP server ${serverName} did not expose ${expectedTool}. See ${apiDiagnosticPath}.`;
return;
}
if (!diagnostic.server_uuid) {
result.status = "fail";
result.reason = `MCP server ${serverName} exposed ${expectedTool}, but the server UUID was not returned. See ${apiDiagnosticPath}.`;
return;
}
await upsertEnvLocal(envLocalPath, {
[serverUuidEnvKey]: diagnostic.server_uuid,
});
result.wrote_env = true;
result.server_uuid = diagnostic.server_uuid;
result.server_uuid_env_key = serverUuidEnvKey;
result.status = "pass";
result.reason = `MCP server ${serverName} is connected and exposes ${expectedTool} through LangBot /api/v1/tools.`;
}
async function upsertEnvLocal(path, updates) {
let text = "";
try {
text = await readFile(path, "utf8");
} catch {
text = "";
}
const lines = text ? text.split(/\r?\n/) : [];
const seen = new Set();
const updated = lines.map((line) => {
const match = line.match(/^([A-Z][A-Z0-9_]*)=/);
if (!match || !Object.prototype.hasOwnProperty.call(updates, match[1])) return line;
seen.add(match[1]);
return `${match[1]}=${updates[match[1]]}`;
});
for (const [key, value] of Object.entries(updates)) {
if (!seen.has(key)) updated.push(`${key}=${value}`);
}
await writeFile(path, `${updated.filter((line, index) => line || index < updated.length - 1).join("\n")}\n`, "utf8");
}
try {
await run();
} catch (error) {