xinyin025/LangBot
1
0
Fork 0
mirror of https://github.com/langbot-app/LangBot.git synced 2026-07-17 01:46:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(agent-runner): strengthen local agent e2e gate

Browse Source
This commit is contained in:
huanghuoguoguo
2026-07-01 20:20:43 +08:00
parent d0f8f080e9
commit 995888f6b2
44 changed files with 5678 additions and 1040 deletions
Download Patch File Download Diff File Expand all files Collapse all files
skills/scripts/e2e/ensure-langrag-sentinel-kb.mjs
+1 -1
View File
@@ -21,7 +21,7 @@ await ensureEvidence(paths);
const backendUrl = env.LANGBOT_BACKEND_URL || "";
const user = env.LANGBOT_E2E_LOGIN_USER || "";
const password = env.LANGBOT_E2E_LOGIN_PASSWORD || "LangBotE2ELocalPass!2026";
const expectedText = env.LANGBOT_E2E_EXPECTED_TEXT || "azalea-cobalt-7421";
const expectedText = env.LANGBOT_E2E_RAG_EXPECTED_TEXT || "azalea-cobalt-7421";
const query = env.LANGBOT_E2E_RETRIEVE_QUERY || "What is the local agent runner retrieval sentinel?";
const writeEnv = process.argv.includes("--write-env");
const checkOnly = process.argv.includes("--check-only");
skills/scripts/e2e/ensure-local-agent-pipeline.mjs
+423 -95
View File
@@ -18,12 +18,14 @@ import {
writeResult,
} from "./lib/langbot-e2e.mjs";
const RUNNER_ID = "local-agent";
const RUNNER_ID = "plugin:langbot/local-agent/default";
const SPACE_PROVIDER_UUID = "00000000-0000-0000-0000-000000000000";
const DEFAULT_PIPELINE_NAME = "Agent QA Local Agent Debug Chat";
const DEFAULT_LOCAL_PASSWORD = "LangBotE2ELocalPass!2026";
const DEFAULT_MODEL_TEST_LIMIT = 8;
const DEFAULT_MODEL_FALLBACK_COUNT = 3;
const DEFAULT_FAKE_MODEL_UUID = "langbot-e2e-fake-local-agent-model";
const DEFAULT_FAKE_PROVIDER_NAME = "LangBot E2E Fake OpenAI Provider";
const caseId = "ensure-local-agent-pipeline";
await loadEnvFiles();
@@ -31,7 +33,10 @@ const paths = evidencePaths(caseId);
await ensureEvidence(paths);
const writeEnv = process.argv.includes("--write-env");
const pipelineName = env.LANGBOT_E2E_CREATE_PIPELINE_NAME || env.LANGBOT_LOCAL_AGENT_PIPELINE_NAME || DEFAULT_PIPELINE_NAME;
const pipelineName =
env.LANGBOT_E2E_CREATE_PIPELINE_NAME ||
env.LANGBOT_LOCAL_AGENT_PIPELINE_NAME ||
DEFAULT_PIPELINE_NAME;
const frontendUrl = env.LANGBOT_FRONTEND_URL || "";
const backendUrl = env.LANGBOT_BACKEND_URL || "";
const envLocalPath = resolve("skills/.env.local");
@@ -51,6 +56,7 @@ const result = {
selected_model_id: "",
selected_model_name: "",
fallback_model_ids: [],
fake_provider: null,
model_count: 0,
space_model_count: 0,
scanned_space_model_count: 0,
@@ -83,7 +89,9 @@ try {
const password = env.LANGBOT_E2E_LOGIN_PASSWORD || DEFAULT_LOCAL_PASSWORD;
if (!user) {
result.status = "env_issue";
throw new Error("LANGBOT_E2E_LOGIN_USER is required so this setup can create/update the pipeline via backend API.");
throw new Error(
"LANGBOT_E2E_LOGIN_USER is required so this setup can create/update the pipeline via backend API.",
);
}
const auth = await resetAndAuthLocalUser({ backendUrl, user, password });
@@ -97,7 +105,9 @@ try {
result.wizard = wizard;
if (wizard.status !== "pass") {
result.status = "fail";
throw new Error(wizard.reason || "Failed to mark the local QA wizard as skipped.");
throw new Error(
wizard.reason || "Failed to mark the local QA wizard as skipped.",
);
}
const prepared = await ensureLocalAgentPipeline({
@@ -118,10 +128,12 @@ try {
LANGBOT_PIPELINE_NAME: result.pipeline_name || pipelineName,
LANGBOT_LOCAL_AGENT_PIPELINE_URL: result.pipeline_url,
LANGBOT_LOCAL_AGENT_PIPELINE_NAME: result.pipeline_name || pipelineName,
...(result.selected_model_id ? {
LANGBOT_LOCAL_AGENT_MODEL_UUID: result.selected_model_id,
LANGBOT_E2E_MODEL_UUID: result.selected_model_id,
} : {}),
...(result.selected_model_id
? {
LANGBOT_LOCAL_AGENT_MODEL_UUID: result.selected_model_id,
LANGBOT_E2E_MODEL_UUID: result.selected_model_id,
}
: {}),
});
result.wrote_env = true;
}
@@ -132,12 +144,21 @@ try {
const browserCheck = await verifyBrowserToken(page, backendUrl);
result.browser_token_check = browserCheck;
if (!browserCheck.authenticated) {
throw new Error(browserCheck.reason || "Browser token check failed after setup.");
throw new Error(
browserCheck.reason || "Browser token check failed after setup.",
);
}
await page.goto(result.pipeline_url || frontendUrl, { waitUntil: "domcontentloaded" });
await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {});
await page.goto(result.pipeline_url || frontendUrl, {
waitUntil: "domcontentloaded",
});
await page
.waitForLoadState("networkidle", { timeout: 10_000 })
.catch(() => {});
const text = await bodyText(page);
result.page_signal = ["Pipelines", "流水线", pipelineName].find((signal) => text.includes(signal)) || "";
result.page_signal =
["Pipelines", "流水线", pipelineName].find((signal) =>
text.includes(signal),
) || "";
} catch (error) {
result.status = result.status === "env_issue" ? "env_issue" : "fail";
result.reason = result.reason || error.message;
@@ -148,24 +169,51 @@ try {
console.log(JSON.stringify(result, null, 2));
}
process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1);
process.exit(
result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1,
);
async function skipWizard({ backendUrl, token }) {
const response = await apiJson(backendUrl, "/api/v1/system/wizard/completed", {
method: "POST",
token,
body: { status: "skipped" },
});
const response = await apiJson(
backendUrl,
"/api/v1/system/wizard/completed",
{
method: "POST",
token,
body: { status: "skipped" },
},
);
const ok = response.status < 400 && response.json.code === 0;
return {
status: ok ? "pass" : "fail",
http_status: response.status,
code: response.json.code ?? null,
reason: ok ? "Wizard marked skipped for local QA." : response.json.msg || "Wizard status update failed.",
reason: ok
? "Wizard marked skipped for local QA."
: response.json.msg || "Wizard status update failed.",
};
}
async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runnerId }) {
async function ensureLocalAgentPipeline({
backendUrl,
token,
pipelineName,
runnerId,
}) {
const fakeProviderBaseUrl =
env.LANGBOT_E2E_FAKE_PROVIDER_BASE_URL ||
env.LANGBOT_FAKE_PROVIDER_BASE_URL ||
"";
let fakeModel = null;
if (fakeProviderBaseUrl) {
fakeModel = await ensureFakeProviderModel({
backendUrl,
token,
baseUrl: fakeProviderBaseUrl,
});
if (fakeModel.status !== "pass") return fakeModel;
}
const [pipelineList, modelList] = await Promise.all([
apiJson(backendUrl, "/api/v1/pipelines", { token }),
apiJson(backendUrl, "/api/v1/provider/models/llm", { token }),
@@ -199,7 +247,9 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne
.map((item) => item.trim())
.filter(Boolean),
);
const spaceModels = models.filter((model) => isSpaceModel(model) && !skippedModelIds.has(model.uuid));
const spaceModels = models.filter(
(model) => isSpaceModel(model) && !skippedModelIds.has(model.uuid),
);
const pipelines = pipelineList.json.data?.pipelines || [];
let pipeline = pipelines.find((item) => item.name === pipelineName) || null;
let created = false;
@@ -210,7 +260,8 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne
token,
body: {
name: pipelineName,
description: "Local QA pipeline for AgentRunner Debug Chat smoke tests.",
description:
"Local QA pipeline for AgentRunner Debug Chat smoke tests.",
emoji: "QA",
},
});
@@ -224,7 +275,11 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne
};
}
const pipelineId = createdResponse.json.data?.uuid || "";
const loaded = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipelineId)}`, { token });
const loaded = await apiJson(
backendUrl,
`/api/v1/pipelines/${encodeURIComponent(pipelineId)}`,
{ token },
);
pipeline = loaded.json.data?.pipeline || null;
created = true;
}
@@ -238,7 +293,11 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne
};
}
const loaded = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, { token });
const loaded = await apiJson(
backendUrl,
`/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`,
{ token },
);
if (isApiFailure(loaded) || !loaded.json.data?.pipeline) {
return {
status: "fail",
@@ -251,32 +310,53 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne
}
pipeline = loaded.json.data.pipeline;
const config = pipeline.config && typeof pipeline.config === "object" ? pipeline.config : {};
const config =
pipeline.config && typeof pipeline.config === "object"
? pipeline.config
: {};
const ai = config.ai && typeof config.ai === "object" ? config.ai : {};
const rawExistingLocalAgentConfig = ai["local-agent"] && typeof ai["local-agent"] === "object"
? ai["local-agent"]
: {};
const runnerConfigs =
ai.runner_config && typeof ai.runner_config === "object"
? ai.runner_config
: {};
const rawExistingLocalAgentConfig =
runnerConfigs[runnerId] && typeof runnerConfigs[runnerId] === "object"
? runnerConfigs[runnerId]
: {};
const existingLocalAgentConfig = rawExistingLocalAgentConfig;
const existingModel = existingLocalAgentConfig.model && typeof existingLocalAgentConfig.model === "object"
? existingLocalAgentConfig.model
: {};
const requestedModelId = env.LANGBOT_LOCAL_AGENT_MODEL_UUID || env.LANGBOT_E2E_MODEL_UUID || "";
const selected = await selectWorkingSpaceModel({
backendUrl,
token,
models,
skippedModelIds,
skippedModelNames,
requestedModelId,
existingModelId: existingModel.primary || "",
});
const existingModel =
existingLocalAgentConfig.model &&
typeof existingLocalAgentConfig.model === "object"
? existingLocalAgentConfig.model
: {};
const requestedModelId =
env.LANGBOT_LOCAL_AGENT_MODEL_UUID || env.LANGBOT_E2E_MODEL_UUID || "";
const selected = fakeModel
? {
status: "pass",
reason: "",
selected_model_id: fakeModel.model_uuid,
selected_model_name: fakeModel.model_name,
fallback_model_ids: [],
scanned_space_model_count: 0,
tested_model_count: 0,
model_tests: [],
}
: await selectWorkingSpaceModel({
backendUrl,
token,
models,
skippedModelIds,
skippedModelNames,
requestedModelId,
existingModelId: existingModel.primary || "",
});
const selectedModelId = selected.selected_model_id || "";
const localAgentConfig = {
timeout: 300,
prompt: [{ role: "system", content: "You are a helpful assistant." }],
"remove-think": false,
"knowledge-bases": [],
"box-session-id-template": "{launcher_type}_{launcher_id}",
"retrieval-top-k": 5,
"rerank-model": "",
"rerank-top-k": 5,
@@ -288,7 +368,6 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne
"context-reserve-tokens": 16384,
"context-keep-recent-tokens": 20000,
"context-summary-tokens": 8000,
...existingLocalAgentConfig,
// Current backend truncation still reads this field directly.
"max-round": positiveInteger(existingLocalAgentConfig["max-round"], 10),
model: {
@@ -303,23 +382,30 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne
runner: {
...(ai.runner && typeof ai.runner === "object" ? ai.runner : {}),
id: runnerId,
runner: runnerId,
"expire-time": 0,
},
"local-agent": localAgentConfig,
runner_config: {
...runnerConfigs,
[runnerId]: localAgentConfig,
},
},
};
const updateResponse = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, {
method: "PUT",
token,
body: {
name: pipelineName,
description: "Local QA pipeline for AgentRunner Debug Chat smoke tests.",
emoji: "QA",
config: updatedConfig,
const updateResponse = await apiJson(
backendUrl,
`/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`,
{
method: "PUT",
token,
body: {
name: pipelineName,
description:
"Local QA pipeline for AgentRunner Debug Chat smoke tests.",
emoji: "QA",
config: updatedConfig,
},
},
});
);
if (isApiFailure(updateResponse)) {
return {
status: "fail",
@@ -341,7 +427,8 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne
status: selectedModelId ? "pass" : "env_issue",
reason: selectedModelId
? `Local-agent pipeline is configured for Debug Chat with Space model ${selected.selected_model_name || selectedModelId} and ${selected.fallback_model_ids.length} fallback(s).`
: selected.reason || "No working Space LLM model is configured in this LangBot instance.",
: selected.reason ||
"No working Space LLM model is configured in this LangBot instance.",
pipeline_id: pipeline.uuid,
pipeline_name: pipelineName,
model_count: models.length,
@@ -352,21 +439,214 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne
selected_model_id: selectedModelId,
selected_model_name: selected.selected_model_name,
fallback_model_ids: selected.fallback_model_ids,
fake_provider: fakeModel,
created,
updated: true,
};
}
async function ensureFakeProviderModel({ backendUrl, token, baseUrl }) {
const modelUuid = env.LANGBOT_E2E_FAKE_MODEL_UUID || DEFAULT_FAKE_MODEL_UUID;
const modelName =
env.LANGBOT_E2E_FAKE_MODEL_NAME ||
env.LANGBOT_FAKE_PROVIDER_MODEL ||
env.LANGBOT_FAKE_PROVIDER_MODEL_NAME ||
"langbot-e2e-fake-model";
const providerName =
env.LANGBOT_E2E_FAKE_PROVIDER_NAME || DEFAULT_FAKE_PROVIDER_NAME;
const providerRequester =
env.LANGBOT_E2E_FAKE_PROVIDER_REQUESTER || "openai-chat-completions";
const apiKey =
env.LANGBOT_E2E_FAKE_PROVIDER_API_KEY ||
env.LANGBOT_FAKE_PROVIDER_API_KEY ||
"fake-key";
const providersResponse = await apiJson(
backendUrl,
"/api/v1/provider/providers",
{ token },
);
if (isApiFailure(providersResponse)) {
return {
status: "fail",
reason:
providersResponse.json.msg ||
"Failed to list providers before creating fake provider.",
provider_status: providersResponse.status,
};
}
const normalizedBaseUrl = String(baseUrl || "").replace(/\/$/, "");
const providers = providersResponse.json.data?.providers || [];
let provider = providers.find(
(item) =>
item.name === providerName ||
(item.requester === providerRequester &&
String(item.base_url || "").replace(/\/$/, "") === normalizedBaseUrl),
);
const providerBody = {
name: providerName,
requester: providerRequester,
base_url: normalizedBaseUrl,
api_keys: [apiKey],
};
if (provider?.uuid) {
const updateResponse = await apiJson(
backendUrl,
`/api/v1/provider/providers/${encodeURIComponent(provider.uuid)}`,
{
method: "PUT",
token,
body: providerBody,
},
);
if (isApiFailure(updateResponse)) {
return {
status: "fail",
reason: updateResponse.json.msg || "Failed to update fake provider.",
provider_status: updateResponse.status,
};
}
} else {
const createProviderResponse = await apiJson(
backendUrl,
"/api/v1/provider/providers",
{
method: "POST",
token,
body: providerBody,
},
);
if (isApiFailure(createProviderResponse)) {
return {
status: "fail",
reason:
createProviderResponse.json.msg || "Failed to create fake provider.",
provider_status: createProviderResponse.status,
};
}
provider = { uuid: createProviderResponse.json.data?.uuid || "" };
}
if (!provider?.uuid) {
return {
status: "fail",
reason: "Fake provider did not return a provider uuid.",
};
}
let resolvedModelUuid = modelUuid;
let modelResponse = await apiJson(
backendUrl,
`/api/v1/provider/models/llm/${encodeURIComponent(modelUuid)}`,
{
token,
},
);
if (modelResponse.status === 404) {
const providerModelsResponse = await apiJson(
backendUrl,
`/api/v1/provider/models/llm?provider_uuid=${encodeURIComponent(provider.uuid)}`,
{ token },
);
if (!isApiFailure(providerModelsResponse)) {
const existingModel = (
providerModelsResponse.json.data?.models || []
).find((item) => item.name === modelName);
if (existingModel?.uuid) {
resolvedModelUuid = existingModel.uuid;
modelResponse = await apiJson(
backendUrl,
`/api/v1/provider/models/llm/${encodeURIComponent(resolvedModelUuid)}`,
{
token,
},
);
}
}
}
const modelBody = {
name: modelName,
provider_uuid: provider.uuid,
abilities: ["func_call", "vision"],
context_length: 8192,
extra_args: {},
prefered_ranking: 0,
};
if (modelResponse.status === 404) {
const createModelResponse = await apiJson(
backendUrl,
"/api/v1/provider/models/llm",
{
method: "POST",
token,
body: {
uuid: modelUuid,
...modelBody,
},
},
);
if (isApiFailure(createModelResponse)) {
return {
status: "fail",
reason: createModelResponse.json.msg || "Failed to create fake model.",
model_status: createModelResponse.status,
};
}
resolvedModelUuid = createModelResponse.json.data?.uuid || modelUuid;
} else if (isApiFailure(modelResponse)) {
return {
status: "fail",
reason: modelResponse.json.msg || "Failed to load fake model.",
model_status: modelResponse.status,
};
} else {
const updateModelResponse = await apiJson(
backendUrl,
`/api/v1/provider/models/llm/${encodeURIComponent(resolvedModelUuid)}`,
{
method: "PUT",
token,
body: modelBody,
},
);
if (isApiFailure(updateModelResponse)) {
return {
status: "fail",
reason: updateModelResponse.json.msg || "Failed to update fake model.",
model_status: updateModelResponse.status,
};
}
}
return {
status: "pass",
provider_uuid: provider.uuid,
provider_requester: providerRequester,
base_url: normalizedBaseUrl,
model_uuid: resolvedModelUuid,
model_name: modelName,
};
}
function isApiFailure(response) {
return response.status >= 400 || (response.json.code !== undefined && response.json.code !== 0);
return (
response.status >= 400 ||
(response.json.code !== undefined && response.json.code !== 0)
);
}
function isSpaceModel(model) {
const provider = model?.provider && typeof model.provider === "object" ? model.provider : {};
return model?.provider_uuid === SPACE_PROVIDER_UUID
|| provider.uuid === SPACE_PROVIDER_UUID
|| provider.requester === "space-chat-completions"
|| provider.name === "LangBot Models";
const provider =
model?.provider && typeof model.provider === "object" ? model.provider : {};
return (
model?.provider_uuid === SPACE_PROVIDER_UUID ||
provider.uuid === SPACE_PROVIDER_UUID ||
provider.requester === "space-chat-completions" ||
provider.name === "LangBot Models"
);
}
async function selectWorkingSpaceModel({
@@ -379,15 +659,24 @@ async function selectWorkingSpaceModel({
existingModelId,
}) {
const modelTests = [];
const testLimit = positiveInteger(env.LANGBOT_E2E_MODEL_TEST_LIMIT, DEFAULT_MODEL_TEST_LIMIT);
const fallbackCount = positiveInteger(env.LANGBOT_E2E_MODEL_FALLBACK_COUNT, DEFAULT_MODEL_FALLBACK_COUNT);
const testLimit = positiveInteger(
env.LANGBOT_E2E_MODEL_TEST_LIMIT,
DEFAULT_MODEL_TEST_LIMIT,
);
const fallbackCount = positiveInteger(
env.LANGBOT_E2E_MODEL_FALLBACK_COUNT,
DEFAULT_MODEL_FALLBACK_COUNT,
);
const workingModels = [];
const spaceModels = rankModels(models.filter((model) => (
model.uuid
&& isSpaceModel(model)
&& !skippedModelIds.has(model.uuid)
&& !skippedModelNames.has(model.name)
)));
const spaceModels = rankModels(
models.filter(
(model) =>
model.uuid &&
isSpaceModel(model) &&
!skippedModelIds.has(model.uuid) &&
!skippedModelNames.has(model.name),
),
);
const requestedModel = requestedModelId
? spaceModels.find((model) => model.uuid === requestedModelId) || null
: null;
@@ -396,7 +685,9 @@ async function selectWorkingSpaceModel({
: null;
const candidates = uniqueCandidates([
...(requestedModel ? [existingCandidate(requestedModel, "requested")] : []),
...(existingModel ? [existingCandidate(existingModel, "existing-pipeline")] : []),
...(existingModel
? [existingCandidate(existingModel, "existing-pipeline")]
: []),
...spaceModels.map((model) => existingCandidate(model, "configured-space")),
]);
@@ -405,9 +696,16 @@ async function selectWorkingSpaceModel({
scanResult = await scanSpaceModels({ backendUrl, token });
if (scanResult.status === "pass") {
const knownNames = new Set(spaceModels.map((model) => model.name));
candidates.push(...scanResult.models
.filter((model) => model.name && !knownNames.has(model.name) && !skippedModelNames.has(model.name))
.map((model) => scannedCandidate(model)));
candidates.push(
...scanResult.models
.filter(
(model) =>
model.name &&
!knownNames.has(model.name) &&
!skippedModelNames.has(model.name),
)
.map((model) => scannedCandidate(model)),
);
}
}
@@ -435,14 +733,16 @@ async function selectWorkingSpaceModel({
};
}
const baseReason = unique.length === 0
? scanResult.reason || "No Space LLM model candidates are available."
: `No working Space LLM model found after testing ${modelTests.length} candidate(s).`;
const baseReason =
unique.length === 0
? scanResult.reason || "No Space LLM model candidates are available."
: `No working Space LLM model found after testing ${modelTests.length} candidate(s).`;
return {
status: "env_issue",
reason: requestedModelId && !requestedModel
? `Requested Space LLM model ${requestedModelId} is missing or skipped; ${baseReason}`
: baseReason,
reason:
requestedModelId && !requestedModel
? `Requested Space LLM model ${requestedModelId} is missing or skipped; ${baseReason}`
: baseReason,
selected_model_id: "",
selected_model_name: "",
fallback_model_ids: [],
@@ -462,7 +762,11 @@ async function scanSpaceModels({ backendUrl, token }) {
return {
status: "env_issue",
models: [],
reason: safeReason(response.json.msg || response.json.message || "Failed to scan Space LLM models."),
reason: safeReason(
response.json.msg ||
response.json.message ||
"Failed to scan Space LLM models.",
),
};
}
return {
@@ -492,28 +796,42 @@ async function ensureAndTestModel({ backendUrl, token, candidate }) {
if (isApiFailure(create) || !modelUuid) {
return modelTestResult(candidate, {
status: "fail",
reason: safeReason(create.json.msg || "Failed to create scanned Space model."),
reason: safeReason(
create.json.msg || "Failed to create scanned Space model.",
),
http_status: create.status,
});
}
created = true;
}
const test = await apiJson(backendUrl, `/api/v1/provider/models/llm/${encodeURIComponent(modelUuid)}/test`, {
method: "POST",
token,
body: { extra_args: {} },
});
const test = await apiJson(
backendUrl,
`/api/v1/provider/models/llm/${encodeURIComponent(modelUuid)}/test`,
{
method: "POST",
token,
body: { extra_args: {} },
},
);
const passed = !isApiFailure(test);
if (!passed && created) {
await apiJson(backendUrl, `/api/v1/provider/models/llm/${encodeURIComponent(modelUuid)}`, {
method: "DELETE",
token,
}).catch(() => {});
await apiJson(
backendUrl,
`/api/v1/provider/models/llm/${encodeURIComponent(modelUuid)}`,
{
method: "DELETE",
token,
},
).catch(() => {});
}
return modelTestResult(candidate, {
status: passed ? "pass" : "fail",
reason: passed ? "" : safeReason(test.json.msg || test.json.message || "Space model test failed."),
reason: passed
? ""
: safeReason(
test.json.msg || test.json.message || "Space model test failed.",
),
http_status: test.status,
model_uuid: modelUuid,
created,
@@ -558,7 +876,9 @@ function uniqueCandidates(candidates) {
const seen = new Set();
const result = [];
for (const candidate of candidates) {
const key = candidate.uuid ? `uuid:${candidate.uuid}` : `name:${candidate.name}`;
const key = candidate.uuid
? `uuid:${candidate.uuid}`
: `name:${candidate.name}`;
if (!candidate.name || seen.has(key)) continue;
seen.add(key);
result.push(candidate);
@@ -568,8 +888,12 @@ function uniqueCandidates(candidates) {
function rankModels(models) {
return [...models].sort((left, right) => {
const leftRank = Number.isFinite(Number(left.prefered_ranking)) ? Number(left.prefered_ranking) : 9999;
const rightRank = Number.isFinite(Number(right.prefered_ranking)) ? Number(right.prefered_ranking) : 9999;
const leftRank = Number.isFinite(Number(left.prefered_ranking))
? Number(left.prefered_ranking)
: 9999;
const rightRank = Number.isFinite(Number(right.prefered_ranking))
? Number(right.prefered_ranking)
: 9999;
if (leftRank !== rightRank) return leftRank - rightRank;
return String(left.name || "").localeCompare(String(right.name || ""));
});
@@ -605,5 +929,9 @@ async function upsertEnvLocal(path, updates) {
for (const [key, value] of Object.entries(updates)) {
if (!seen.has(key)) next.push(`${key}=${value}`);
}
await writeFile(path, `${next.filter((line, index) => line !== "" || index < next.length - 1).join("\n")}\n`, "utf8");
await writeFile(
path,
`${next.filter((line, index) => line !== "" || index < next.length - 1).join("\n")}\n`,
"utf8",
);
}
skills/scripts/e2e/fake-openai-provider.mjs
+350 -14
View File
@@ -9,7 +9,43 @@ const args = parseArgs(process.argv.slice(2));
const host = args.host || env.LANGBOT_FAKE_PROVIDER_HOST || "127.0.0.1";
const port = integer(args.port ?? env.LANGBOT_FAKE_PROVIDER_PORT, 0);
const stateFile = args["state-file"] || env.LANGBOT_FAKE_PROVIDER_STATE_FILE || "";
const modelName = env.LANGBOT_FAKE_PROVIDER_MODEL_NAME || "gpt-4o-mini";
const modelName = args.model || env.LANGBOT_FAKE_PROVIDER_MODEL_NAME || env.LANGBOT_FAKE_PROVIDER_MODEL || "gpt-4o-mini";
const COMPACTION_SENTINEL = "qa_compaction_sentinel_7391";
const COMBO_SENTINEL = "qa_combo_compaction_sentinel_2406";
const RAG_SENTINEL = "azalea-cobalt-7421";
const COMBO_TOOL_INPUT = "combo-tool-ok-local-agent";
const COMBO_TOOL_RESULT = `qa-plugin-smoke:${COMBO_TOOL_INPUT}`;
const COMBO_FINAL = `COMBO_FINAL ${COMBO_SENTINEL} ${RAG_SENTINEL} ${COMBO_TOOL_RESULT}`;
const MULTITOOL_SENTINEL = "qa_multitool_compaction_sentinel_6718";
const MULTITOOL_TOOL_A_INPUT = "multi-tool-a-local-agent";
const MULTITOOL_TOOL_B_INPUT = "multi-tool-b-local-agent";
const MULTITOOL_TOOL_A_RESULT = `qa-plugin-smoke:${MULTITOOL_TOOL_A_INPUT}`;
const MULTITOOL_TOOL_B_RESULT = `qa-plugin-smoke:${MULTITOOL_TOOL_B_INPUT}`;
const MULTITOOL_FINAL = [
"MULTITOOL_COMBO_FINAL",
MULTITOOL_SENTINEL,
RAG_SENTINEL,
MULTITOOL_TOOL_A_RESULT,
MULTITOOL_TOOL_B_RESULT,
].join(" ");
const PARALLEL_SENTINEL = "qa_parallel_compaction_sentinel_8142";
const PARALLEL_TOOL_A_INPUT = "parallel-tool-a-local-agent";
const PARALLEL_TOOL_B_INPUT = "parallel-tool-b-local-agent";
const PARALLEL_TOOL_A_RESULT = `qa-plugin-smoke:${PARALLEL_TOOL_A_INPUT}`;
const PARALLEL_TOOL_B_RESULT = `qa-plugin-smoke:${PARALLEL_TOOL_B_INPUT}`;
const PARALLEL_FINAL = [
"PARALLEL_COMBO_FINAL",
PARALLEL_SENTINEL,
RAG_SENTINEL,
PARALLEL_TOOL_A_RESULT,
PARALLEL_TOOL_B_RESULT,
].join(" ");
const LOOP_LIMIT_INPUT = "loop-limit-repeat-local-agent";
const TOOL_ERROR_INPUT = "tool-error-recovery-local-agent";
const TOOL_ERROR_FINAL = "TOOL_ERROR_RECOVERY_FINAL qa-plugin-smoke forced failure observed";
const STEERING_FOLLOWUP_SENTINEL = "qa_steering_sentinel_6194";
const STEERING_SLEEP_INPUT = "steering-e2e-anchor";
const STEERING_SLEEP_RESULT = `qa-plugin-smoke:sleep:8:${STEERING_SLEEP_INPUT}`;
const config = {
response_text: env.LANGBOT_FAKE_PROVIDER_RESPONSE_TEXT || "OK",
first_token_delay_ms: integer(env.LANGBOT_FAKE_PROVIDER_FIRST_TOKEN_DELAY_MS, 25),
@@ -88,6 +124,7 @@ const server = createServer(async (request, response) => {
created: 1,
owned_by: "langbot-qa",
type: "llm",
context_length: 8192,
},
],
});
@@ -100,7 +137,8 @@ const server = createServer(async (request, response) => {
const requestId = `chatcmpl-langbot-fake-${requestCount}`;
const shouldFail = requestCount <= config.fail_first_n
|| (config.fail_every_n > 0 && requestCount % config.fail_every_n === 0);
const replyText = responseTextForBody(body);
const replyMessage = buildResponse(body);
const replyText = replyMessage.content || "";
requestRecord = recordRequest({
id: requestId,
request_number: requestCount,
@@ -140,7 +178,7 @@ const server = createServer(async (request, response) => {
await streamCompletion(response, {
requestId,
model: body.model || modelName,
content: replyText,
message: replyMessage,
failAfterFirstChunk: config.fail_after_first_chunk,
requestRecord,
startedPerf,
@@ -150,7 +188,7 @@ const server = createServer(async (request, response) => {
sendJson(response, 200, completionPayload({
requestId,
model: body.model || modelName,
content: replyText,
message: replyMessage,
}));
markRequestTiming(requestRecord, "first_chunk", startedPerf);
markRequestTiming(requestRecord, "first_content_chunk", startedPerf);
@@ -270,8 +308,8 @@ function sendJson(response, status, payload) {
response.end(text);
}
function completionPayload({ requestId, model, content }) {
const completionTokens = tokenEstimate(content);
function completionPayload({ requestId, model, message }) {
const completionTokens = tokenEstimate(message.content || JSON.stringify(message.tool_calls || []));
return {
id: requestId,
object: "chat.completion",
@@ -280,11 +318,8 @@ function completionPayload({ requestId, model, content }) {
choices: [
{
index: 0,
message: {
role: "assistant",
content,
},
finish_reason: "stop",
message,
finish_reason: message.tool_calls?.length ? "tool_calls" : "stop",
},
],
usage: {
@@ -298,7 +333,7 @@ function completionPayload({ requestId, model, content }) {
async function streamCompletion(response, {
requestId,
model,
content,
message,
failAfterFirstChunk: failMidStream,
requestRecord,
startedPerf,
@@ -319,7 +354,56 @@ async function streamCompletion(response, {
choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }],
});
const chunks = splitContent(content);
const chunks = message.tool_calls?.length ? [] : splitContent(message.content || "");
if (message.tool_calls?.length) {
await sleep(config.chunk_delay_ms);
markRequestTiming(requestRecord, "first_content_chunk", startedPerf);
requestRecord.content_chunk_count = (requestRecord.content_chunk_count || 0) + 1;
writeSse(response, {
id: requestId,
object: "chat.completion.chunk",
created: Math.floor(Date.now() / 1000),
model,
choices: [
{
index: 0,
delta: {
tool_calls: message.tool_calls.map((call, index) => ({
index,
id: call.id,
type: call.type,
function: {
name: call.function.name,
arguments: call.function.arguments,
},
})),
},
finish_reason: null,
},
],
});
await sleep(config.chunk_delay_ms);
writeSse(response, {
id: requestId,
object: "chat.completion.chunk",
created: Math.floor(Date.now() / 1000),
model,
choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }],
usage: {
prompt_tokens: 8,
completion_tokens: tokenEstimate(JSON.stringify(message.tool_calls)),
total_tokens: 8 + tokenEstimate(JSON.stringify(message.tool_calls)),
},
});
response.write("data: [DONE]\n\n");
response.end();
finishRequestRecord(requestRecord, startedPerf, {
status: "ok",
http_status: 200,
});
return;
}
for (let index = 0; index < chunks.length; index += 1) {
await sleep(config.chunk_delay_ms);
if (index === 0) markRequestTiming(requestRecord, "first_content_chunk", startedPerf);
@@ -342,7 +426,7 @@ async function streamCompletion(response, {
}
await sleep(config.chunk_delay_ms);
const completionTokens = tokenEstimate(content);
const completionTokens = tokenEstimate(message.content || "");
writeSse(response, {
id: requestId,
object: "chat.completion.chunk",
@@ -399,6 +483,258 @@ function responseTextForBody(body) {
return config.response_text;
}
function messageText(messages = []) {
return messages
.map((message) => {
const parts = [contentText(message?.content)];
if (Array.isArray(message?.tool_calls)) {
parts.push(
message.tool_calls
.map((call) => `${call?.function?.name || call?.name || ""} ${call?.function?.arguments || ""}`)
.join("\n"),
);
}
return parts.filter(Boolean).join("\n");
})
.join("\n");
}
function contentText(content) {
if (typeof content === "string") return content;
if (Array.isArray(content)) return content.map((part) => contentText(part)).join("");
if (content && typeof content === "object") {
for (const key of ["text", "content", "message", "error", "value"]) {
if (content[key] !== undefined && content[key] !== null) {
return contentText(content[key]);
}
}
return JSON.stringify(content);
}
return "";
}
function latestUserText(messages = []) {
const latest = [...messages].reverse().find((message) => message?.role === "user");
return contentText(latest?.content);
}
function toolNames(tools = []) {
return tools
.map((tool) => tool?.function?.name || tool?.name || "")
.filter(Boolean);
}
function firstToolName(tools = [], candidates = []) {
const names = toolNames(tools);
return candidates.find((candidate) => names.includes(candidate)) || "";
}
function isSummarizationRequest(messages = []) {
const text = messageText(messages);
return /context summarization assistant|conversation to summarize|<previous-summary>/i.test(text);
}
function buildSummaryResponse(text) {
const references = [];
if (text.includes(COMPACTION_SENTINEL)) references.push(COMPACTION_SENTINEL);
if (text.includes(COMBO_SENTINEL)) references.push(COMBO_SENTINEL);
if (text.includes(MULTITOOL_SENTINEL)) references.push(MULTITOOL_SENTINEL);
if (text.includes(PARALLEL_SENTINEL)) references.push(PARALLEL_SENTINEL);
if (/RAG_TOOL_COMBO_GOAL/i.test(text)) references.push("RAG_TOOL_COMBO_GOAL");
if (/MULTITOOL_RAG_GOAL/i.test(text)) references.push("MULTITOOL_RAG_GOAL");
if (/PARALLEL_RAG_GOAL/i.test(text)) references.push("PARALLEL_RAG_GOAL");
const critical = references.length
? references.map((reference) => `- ${reference}`).join("\n")
: "- (none)";
return {
role: "assistant",
content: [
"## Goal",
"Preserve deterministic LangBot E2E context across compaction.",
"",
"## Constraints & Preferences",
"- Keep exact sentinel values verbatim.",
"",
"## Progress",
"### Done",
"- [x] Earlier conversation was compacted by the fake provider.",
"",
"### In Progress",
"- [ ] Continue the current Debug Chat run.",
"",
"### Blocked",
"- (none)",
"",
"## Key Decisions",
"- **Deterministic QA**: Return only known sentinel values from compacted input.",
"",
"## Next Steps",
"1. Answer the current user request using preserved context.",
"",
"## Critical Context",
critical,
].join("\n"),
};
}
function buildResponse(payload) {
const text = messageText(payload.messages || []);
const current = latestUserText(payload.messages || []);
const tools = payload.tools || [];
if (isSummarizationRequest(payload.messages || [])) {
return buildSummaryResponse(text);
}
const pluginTool = firstToolName(tools, ["qa_plugin_echo"]);
const pluginFailTool = firstToolName(tools, ["qa_plugin_fail"]);
const pluginSleepTool = firstToolName(tools, ["qa_plugin_sleep"]);
const mcpTool = firstToolName(tools, ["qa_mcp_echo"]);
if (/STEERING_NO_FOLLOWUP|qa_plugin_sleep|steering-e2e-anchor|qa_steering_sentinel_6194/i.test(current || text)) {
if (text.includes(STEERING_FOLLOWUP_SENTINEL) && text.includes(STEERING_SLEEP_RESULT)) {
return { role: "assistant", content: STEERING_FOLLOWUP_SENTINEL };
}
if (text.includes(STEERING_SLEEP_RESULT) && !text.includes(STEERING_FOLLOWUP_SENTINEL)) {
return { role: "assistant", content: "STEERING_NO_FOLLOWUP" };
}
if (pluginSleepTool && !text.includes(STEERING_SLEEP_RESULT)) {
return toolCall("call_qa_plugin_sleep_steering", pluginSleepTool, { seconds: 8, text: STEERING_SLEEP_INPUT });
}
}
if (/测试暗号是什么|original sentinel|first.*sentinel/i.test(current)) {
return {
role: "assistant",
content: text.includes(COMPACTION_SENTINEL) ? COMPACTION_SENTINEL : "COMPACTION_SENTINEL_MISSING",
};
}
if ((current.includes(COMPACTION_SENTINEL)
|| current.includes(COMBO_SENTINEL)
|| current.includes(MULTITOOL_SENTINEL)
|| current.includes(PARALLEL_SENTINEL))
&& /请只回复 MEMORY_SET|only reply MEMORY_SET/i.test(current)) {
return { role: "assistant", content: "MEMORY_SET" };
}
if (/PARALLEL_CONTEXT_PRESSURE_READY/.test(current)) return { role: "assistant", content: "PARALLEL_CONTEXT_PRESSURE_READY" };
if (/MULTITOOL_CONTEXT_PRESSURE_READY/.test(current)) return { role: "assistant", content: "MULTITOOL_CONTEXT_PRESSURE_READY" };
if (/COMBO_CONTEXT_PRESSURE_READY/.test(current)) return { role: "assistant", content: "COMBO_CONTEXT_PRESSURE_READY" };
if (/CONTEXT_PRESSURE_READY/.test(current)) return { role: "assistant", content: "CONTEXT_PRESSURE_READY" };
if (/qa_mcp_echo:mcp-ok-local-agent/.test(text)) return { role: "assistant", content: "qa_mcp_echo:mcp-ok-local-agent" };
if (/qa_mcp_echo|mcp-ok-local-agent/i.test(current || text) && mcpTool && !/qa_mcp_echo:mcp-ok-local-agent/.test(text)) {
return toolCall("call_qa_mcp_echo", mcpTool, { text: "mcp-ok-local-agent" });
}
if (/LOOP_LIMIT|loop-limit-repeat-local-agent|iteration limit/i.test(current || text) && pluginTool) {
return toolCall(`call_qa_plugin_echo_loop_limit_${Date.now()}`, pluginTool, { text: LOOP_LIMIT_INPUT });
}
if (/TOOL_ERROR_RECOVERY|tool-error-recovery-local-agent|qa_plugin_fail/i.test(current || text)) {
if (/(?:Error:|Tool execution failed:|ActionCallError:|RuntimeError:)/i.test(text)
&& /(?:qa-plugin-smoke forced failure|qa_plugin_fail|tool-error-recovery-local-agent)/i.test(text)) {
return { role: "assistant", content: TOOL_ERROR_FINAL };
}
if (pluginFailTool) {
return toolCall("call_qa_plugin_fail_recovery", pluginFailTool, { text: TOOL_ERROR_INPUT });
}
}
const isParallelComboRequest = /PARALLEL_COMBO|parallel-tool-a-local-agent|parallel-tool-b-local-agent|PARALLEL_RAG_GOAL/i.test(current || text);
if (isParallelComboRequest && pluginTool && !text.includes(PARALLEL_TOOL_A_RESULT) && !text.includes(PARALLEL_TOOL_B_RESULT)) {
return toolCalls([
["call_qa_plugin_echo_parallel_a", pluginTool, { text: PARALLEL_TOOL_A_INPUT }],
["call_qa_plugin_echo_parallel_b", pluginTool, { text: PARALLEL_TOOL_B_INPUT }],
]);
}
if (isParallelComboRequest && (text.includes(PARALLEL_TOOL_A_RESULT) || text.includes(PARALLEL_TOOL_B_RESULT))) {
return missingOrFinal(text, [
[PARALLEL_SENTINEL, "parallel-memory"],
[RAG_SENTINEL, "rag"],
[PARALLEL_TOOL_A_RESULT, "tool-a"],
[PARALLEL_TOOL_B_RESULT, "tool-b"],
], "PARALLEL_COMBO_MISSING", PARALLEL_FINAL);
}
const isMultiToolComboRequest = /MULTITOOL_COMBO|multi-tool-a-local-agent|multi-tool-b-local-agent|MULTITOOL_RAG_GOAL/i.test(current || text);
if (isMultiToolComboRequest && pluginTool && !text.includes(MULTITOOL_TOOL_A_RESULT)) {
return toolCall("call_qa_plugin_echo_multi_a", pluginTool, { text: MULTITOOL_TOOL_A_INPUT });
}
if (isMultiToolComboRequest && pluginTool && text.includes(MULTITOOL_TOOL_A_RESULT) && !text.includes(MULTITOOL_TOOL_B_RESULT)) {
return toolCall("call_qa_plugin_echo_multi_b", pluginTool, { text: MULTITOOL_TOOL_B_INPUT });
}
if (isMultiToolComboRequest && text.includes(MULTITOOL_TOOL_A_RESULT) && text.includes(MULTITOOL_TOOL_B_RESULT)) {
return missingOrFinal(text, [
[MULTITOOL_SENTINEL, "multi-memory"],
[RAG_SENTINEL, "rag"],
[MULTITOOL_TOOL_A_RESULT, "tool-a"],
[MULTITOOL_TOOL_B_RESULT, "tool-b"],
], "MULTITOOL_COMBO_MISSING", MULTITOOL_FINAL);
}
const isComboRequest = /qa_combo|\bCOMBO_FINAL\b|combo-tool-ok-local-agent|RAG_TOOL_COMBO_GOAL/i.test(current || text);
if (isComboRequest && pluginTool && !text.includes(COMBO_TOOL_RESULT)) {
return toolCall("call_qa_plugin_echo_combo", pluginTool, { text: COMBO_TOOL_INPUT });
}
if (isComboRequest && text.includes(COMBO_TOOL_RESULT)) {
return missingOrFinal(text, [
[COMBO_SENTINEL, "combo-memory"],
[RAG_SENTINEL, "rag"],
[COMBO_TOOL_RESULT, "tool-result"],
], "COMBO_MISSING", COMBO_FINAL);
}
if (/qa-plugin-smoke:plugin-tool-ok-local-agent/.test(text)) {
return { role: "assistant", content: "qa-plugin-smoke:plugin-tool-ok-local-agent" };
}
if (/qa_plugin_echo|plugin-tool-ok-local-agent/i.test(current || text) && pluginTool && !/qa-plugin-smoke:plugin-tool-ok-local-agent/.test(text)) {
return toolCall("call_qa_plugin_echo", pluginTool, { text: "plugin-tool-ok-local-agent" });
}
const e2eTool = firstToolName(tools, ["e2e_lookup"]);
if (/Use the e2e lookup tool|e2e lookup tool|tool loop/i.test(current || text) && e2eTool && !/tool-result:alpha/.test(text)) {
return toolCall("call_e2e_lookup", e2eTool, { query: "alpha" });
}
if (/tool-result:alpha/.test(text)) return { role: "assistant", content: "Tool loop final answer after tool-result:alpha" };
if (/RAG_SENTINEL/.test(text)) return { role: "assistant", content: "RAG final answer with RAG_SENTINEL" };
if (/NONSTREAM_OK/i.test(current || text)) return { role: "assistant", content: "NONSTREAM_OK" };
if (/IMAGE_OK/i.test(current || text)
&& /(?:\[Image\]|langbot_input_attachments|data:image\/|\"type\":\s*\"image\")/i.test(current || text)) {
return { role: "assistant", content: "IMAGE_OK" };
}
if (/azalea-cobalt-7421/.test(text)) return { role: "assistant", content: "azalea-cobalt-7421" };
return { role: "assistant", content: responseTextForBody(payload) };
}
function toolCall(id, name, args) {
return toolCalls([[id, name, args]]);
}
function toolCalls(calls) {
return {
role: "assistant",
content: "",
tool_calls: calls.map(([id, name, args]) => ({
id,
type: "function",
function: {
name,
arguments: JSON.stringify(args),
},
})),
};
}
function missingOrFinal(text, requirements, prefix, finalText) {
const missing = requirements
.filter(([needle]) => !text.includes(needle))
.map(([, label]) => label);
return {
role: "assistant",
content: missing.length ? `${prefix}_${missing.join("_")}` : finalText,
};
}
function flattenContent(content) {
if (typeof content === "string") return content;
if (Array.isArray(content)) {
skills/scripts/e2e/install-qa-plugin-smoke.mjs
+62 -18
View File
@@ -26,7 +26,12 @@ const packagePath = resolve(
|| "skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/dist/qa-plugin-smoke-0.1.0.lbpkg",
);
const expectedPluginId = env.LANGBOT_E2E_EXPECTED_PLUGIN_ID || "qa/plugin-smoke";
const expectedTool = env.LANGBOT_E2E_EXPECTED_TOOL || (expectedPluginId === "qa/plugin-smoke" ? "qa_plugin_echo" : "");
const expectedTools = (env.LANGBOT_E2E_EXPECTED_TOOLS || env.LANGBOT_E2E_EXPECTED_TOOL || (
expectedPluginId === "qa/plugin-smoke" ? "qa_plugin_echo" : ""
))
.split(",")
.map((item) => item.trim())
.filter(Boolean);
const expectedRunnerId = env.LANGBOT_E2E_EXPECTED_RUNNER_ID || "";
const result = {
@@ -40,6 +45,7 @@ const result = {
package_preview: null,
task_id: null,
task: null,
reinstall_reason: "",
plugin_present_before: false,
plugin_present_after: false,
tool_names: [],
@@ -64,21 +70,23 @@ try {
}
result.plugin_present_before = await hasPlugin(backendUrl, auth.token);
if (!result.plugin_present_before) {
const form = new FormData();
form.set("file", new Blob([bytes]), packagePath.split("/").pop());
const response = await fetch(`${backendUrl.replace(/\/$/, "")}/api/v1/plugins/install/local`, {
method: "POST",
headers: { Authorization: `Bearer ${auth.token}` },
body: form,
});
const json = await response.json().catch(() => ({}));
if (response.status >= 400 || json.code !== 0) {
throw new Error(json.msg || `Plugin install request failed with HTTP ${response.status}.`);
if (expectedTools.length > 0) {
result.tool_names = await listToolNames(backendUrl, auth.token);
}
const missingToolsBefore = expectedTools.filter((tool) => !result.tool_names.includes(tool));
if (result.plugin_present_before && missingToolsBefore.length > 0) {
result.reinstall_reason = `Installed plugin is missing expected tools: ${missingToolsBefore.join(", ")}`;
const removeTask = await removePlugin(backendUrl, auth.token);
if (!isTaskComplete(removeTask)) {
throw new Error(`Plugin reinstall cleanup did not complete successfully: ${JSON.stringify(removeTask)}`);
}
result.task_id = json.data?.task_id ?? null;
if (!result.task_id) throw new Error("Plugin install response did not include task_id.");
result.task = await waitForTask(backendUrl, auth.token, result.task_id);
result.plugin_present_before = false;
await sleep(1000);
}
if (!result.plugin_present_before) {
result.task = await installPlugin(backendUrl, auth.token, bytes, packagePath);
result.task_id = result.task?.id || result.task_id;
if (!isTaskComplete(result.task)) {
throw new Error(`Plugin install task did not complete successfully: ${JSON.stringify(result.task)}`);
}
@@ -87,10 +95,11 @@ try {
await sleep(1000);
result.plugin_present_after = await hasPlugin(backendUrl, auth.token);
if (!result.plugin_present_after) throw new Error(`${expectedPluginId} is not listed by /api/v1/plugins after install.`);
if (expectedTool) {
if (expectedTools.length > 0) {
result.tool_names = await listToolNames(backendUrl, auth.token);
if (!result.tool_names.includes(expectedTool)) {
throw new Error(`${expectedTool} is not listed by /api/v1/tools after install.`);
const missingTools = expectedTools.filter((tool) => !result.tool_names.includes(tool));
if (missingTools.length > 0) {
throw new Error(`${missingTools.join(", ")} is not listed by /api/v1/tools after install.`);
}
}
if (expectedRunnerId) {
@@ -121,6 +130,41 @@ async function hasPlugin(backendUrl, token) {
});
}
async function installPlugin(backendUrl, token, bytes, packagePath) {
const form = new FormData();
form.set("file", new Blob([bytes]), packagePath.split("/").pop());
const response = await fetch(`${backendUrl.replace(/\/$/, "")}/api/v1/plugins/install/local`, {
method: "POST",
headers: { Authorization: `Bearer ${token}` },
body: form,
});
const json = await response.json().catch(() => ({}));
if (response.status >= 400 || json.code !== 0) {
throw new Error(json.msg || `Plugin install request failed with HTTP ${response.status}.`);
}
const taskId = json.data?.task_id ?? null;
if (!taskId) throw new Error("Plugin install response did not include task_id.");
const task = await waitForTask(backendUrl, token, taskId);
return { id: taskId, ...task };
}
async function removePlugin(backendUrl, token) {
const [author, pluginName] = expectedPluginId.split("/");
if (!author || !pluginName) throw new Error(`Invalid expected plugin id: ${expectedPluginId}`);
const response = await apiJson(
backendUrl,
`/api/v1/plugins/${encodeURIComponent(author)}/${encodeURIComponent(pluginName)}?delete_data=false`,
{ method: "DELETE", token },
);
if (response.status >= 400 || response.json.code !== 0) {
throw new Error(response.json.msg || `Plugin delete request failed with HTTP ${response.status}.`);
}
const taskId = response.json.data?.task_id ?? null;
if (!taskId) throw new Error("Plugin delete response did not include task_id.");
const task = await waitForTask(backendUrl, token, taskId);
return { id: taskId, ...task };
}
async function previewPackage(backendUrl, token, bytes, packagePath) {
const form = new FormData();
form.set("file", new Blob([bytes]), packagePath.split("/").pop());
skills/scripts/e2e/lib/langbot-e2e.mjs
+159 -3
View File
@@ -1,5 +1,5 @@
import { appendFile, mkdir, readFile, stat, writeFile } from "node:fs/promises";
import { join, resolve } from "node:path";
import { basename, join, resolve } from "node:path";
import { env } from "node:process";
const secretRe = /(?:authorization|bearer|token|secret|password|api[_-]?key|jwt|oauth)\s*[:=]\s*["']?[^"',\s]+/gi;
@@ -71,6 +71,86 @@ export async function writeResult(paths, result) {
}
}
function browserDiagnosticFindings(source, text) {
const findings = [];
const lines = String(text || "").split(/\r?\n/);
for (let index = 0; index < lines.length; index += 1) {
const line = lines[index];
if (!line) continue;
const lineNumber = index + 1;
if (source === "console") {
const checks = [
["pageerror", /\[pageerror\]/i],
["frontend_uncaught_error", /\[error\].*(?:\bUncaught\b|Unhandled(?: promise rejection|Rejection)|TypeError|ReferenceError)/i],
["http_5xx", /Failed to load resource: the server responded with a status of 5\d\d/i],
["api_server_error", /\[error\].*Server error:/i],
["plugin_runtime_timeout", /\[error\].*Action [A-Za-z0-9_]+ call timed out/i],
];
for (const [kind, regex] of checks) {
if (!regex.test(line)) continue;
findings.push({
source,
severity: "fail",
kind,
line: lineNumber,
excerpt: redact(line.trim()),
});
break;
}
continue;
}
if (source === "network") {
if (/\[response\]\s+5\d\d\b/i.test(line)) {
findings.push({
source,
severity: "fail",
kind: "http_5xx",
line: lineNumber,
excerpt: redact(line.trim()),
});
continue;
}
if (/\[requestfailed\]/i.test(line) && !/net::ERR_ABORTED/i.test(line)) {
findings.push({
source,
severity: "warning",
kind: "request_failed",
line: lineNumber,
excerpt: redact(line.trim()),
});
}
}
}
return findings;
}
export async function scanBrowserDiagnostics(paths) {
const sources = [
["console", paths.consoleLog],
["network", paths.networkLog],
];
const findings = [];
for (const [source, path] of sources) {
let text = "";
try {
text = await readFile(path, "utf8");
} catch {
continue;
}
findings.push(...browserDiagnosticFindings(source, text));
}
const hasFailure = findings.some((finding) => finding.severity === "fail");
return {
status: hasFailure ? "fail" : "pass",
findings,
reason: hasFailure
? `Browser diagnostics found ${findings.filter((finding) => finding.severity === "fail").length} failing signal(s).`
: "No failing browser diagnostics found.",
};
}
export async function loadEnvFiles(paths = ["skills/.env", "skills/.env.local"]) {
const processEnvKeys = new Set(Object.keys(env));
for (const path of paths) {
@@ -92,8 +172,28 @@ export async function loadEnvFiles(paths = ["skills/.env", "skills/.env.local"])
}
}
export async function readRecoveryKey(repo = env.LANGBOT_REPO || "../LangBot") {
const configPath = resolve(repo, "data/config.yaml");
export async function resolveLangBotRepo(repo = env.LANGBOT_REPO || "", cwd = process.cwd()) {
if (repo) return resolve(repo);
const candidates = [
resolve(cwd),
basename(cwd) === "skills" ? resolve(cwd, "..") : "",
resolve(cwd, "../LangBot"),
resolve(cwd, "LangBot"),
].filter(Boolean);
const seen = new Set();
for (const candidate of candidates) {
if (seen.has(candidate)) continue;
seen.add(candidate);
if (await pathExists(resolve(candidate, "data/config.yaml"))) return candidate;
}
return resolve(cwd, "../LangBot");
}
export async function readRecoveryKey(repo = env.LANGBOT_REPO || "") {
const configPath = resolve(await resolveLangBotRepo(repo), "data/config.yaml");
const config = await readFile(configPath, "utf8");
const match = config.match(/^\s*recovery_key:\s*['"]?([^'"\s#]+)['"]?\s*$/m);
return match?.[1] || "";
@@ -202,6 +302,62 @@ export async function verifyBrowserToken(page, backendUrl) {
}, backendUrl);
}
export async function ensureAuthenticatedBrowser(page, {
frontendUrl = env.LANGBOT_FRONTEND_URL || "",
backendUrl = env.LANGBOT_BACKEND_URL || "",
user = env.LANGBOT_E2E_LOGIN_USER || "",
password = env.LANGBOT_E2E_LOGIN_PASSWORD || "LangBotE2ELocalPass!2026",
recoveryKey = "",
} = {}) {
if (!frontendUrl) return { status: "env_issue", reason: "LANGBOT_FRONTEND_URL is not configured." };
if (!backendUrl) return { status: "env_issue", reason: "LANGBOT_BACKEND_URL is not configured." };
const current = await verifyBrowserToken(page, backendUrl).catch((error) => ({
authenticated: false,
reason: error.message,
}));
if (current.authenticated) {
return {
status: "pass",
reason: "Existing browser token is valid.",
backend_token_check: null,
browser_token_check: current,
injected: false,
};
}
if (!user) {
return {
status: "blocked",
reason: "Browser profile is not authenticated for LANGBOT_FRONTEND_URL, and LANGBOT_E2E_LOGIN_USER is not configured for automatic local login.",
backend_token_check: null,
browser_token_check: current,
injected: false,
};
}
const auth = await resetAndAuthLocalUser({ backendUrl, user, password, recoveryKey });
await setBrowserToken(page, frontendUrl, auth.token);
const browserCheck = await verifyBrowserToken(page, backendUrl);
if (!browserCheck.authenticated) {
return {
status: "blocked",
reason: browserCheck.reason || "Browser token check failed after automatic local login.",
backend_token_check: auth.check,
browser_token_check: browserCheck,
injected: true,
};
}
return {
status: "pass",
reason: "Browser token injected from local recovery login.",
backend_token_check: auth.check,
browser_token_check: browserCheck,
injected: true,
};
}
export function exitCode(status) {
if (status === "pass") return 0;
if (status === "blocked" || status === "env_issue") return 2;
skills/scripts/e2e/local-agent-steering-debug-chat.mjs
+14
View File
@@ -11,6 +11,7 @@ import {
} from "./lib/debug-chat.mjs";
import {
createBrowser,
ensureAuthenticatedBrowser,
ensureEvidence,
evidencePaths,
exitCode,
@@ -74,6 +75,7 @@ const result = {
pipeline_config: null,
debug_chat_reset: null,
tool_diagnostic: null,
browser_auth: null,
steering: null,
evidence: {
console_log: paths.consoleLog,
@@ -95,6 +97,18 @@ try {
browser = await createBrowser(paths);
const { page } = browser;
const authDiagnostic = await ensureAuthenticatedBrowser(page, {
frontendUrl: env.LANGBOT_FRONTEND_URL || "",
backendUrl,
});
result.browser_auth = authDiagnostic;
if (!result.evidence_collected.includes("api_diagnostic")) result.evidence_collected.push("api_diagnostic");
if (authDiagnostic.status === "env_issue" || authDiagnostic.status === "blocked" || authDiagnostic.status === "fail") {
result.status = authDiagnostic.status;
result.reason = authDiagnostic.reason || "Browser authentication failed.";
throw new Error(result.reason);
}
const openResult = await openPipelineDebugChat(page, {
pipelineUrl,
pipelineName,
skills/scripts/e2e/mcp-stdio-register.mjs
+44 -2
View File
@@ -1,7 +1,7 @@
#!/usr/bin/env node
import { existsSync, readFileSync } from "node:fs";
import { writeFile } from "node:fs/promises";
import { readFile, writeFile } from "node:fs/promises";
import { resolve } from "node:path";
import { env } from "node:process";
import {
@@ -46,6 +46,8 @@ const startupTimeoutSec = Number(env.LANGBOT_MCP_STARTUP_TIMEOUT_SEC || "300");
const readyTimeoutMs = Number(env.LANGBOT_MCP_READY_TIMEOUT_MS || "360000");
const backendUrl = env.LANGBOT_BACKEND_URL || "";
const apiDiagnosticPath = resolve(paths.evidenceDir, "api-diagnostic.json");
const envLocalPath = resolve("skills/.env.local");
const serverUuidEnvKey = env.LANGBOT_MCP_SERVER_UUID_ENV_KEY || "LANGBOT_MCP_QA_STDIO_SERVER_UUID";
let browser;
const result = {
@@ -70,6 +72,7 @@ const result = {
result_json: paths.resultJson,
},
evidence_collected: ["api_diagnostic"],
wrote_env: false,
};
async function run() {
@@ -159,6 +162,7 @@ async function run() {
const deadline = Date.now() + readyTimeoutMs;
let lastTools = [];
let lastRuntime = null;
let lastServer = null;
while (Date.now() < deadline) {
await new Promise((resolveReady) => setTimeout(resolveReady, 500));
const tools = await getJson("/api/v1/tools");
@@ -167,7 +171,8 @@ async function run() {
.map((tool) => tool.name || tool.tool_name || tool.function?.name || "")
.filter(Boolean)
.sort();
lastRuntime = server.json.data?.server?.runtime_info || null;
lastServer = server.json.data?.server || null;
lastRuntime = lastServer?.runtime_info || null;
if (lastTools.includes(expectedTool)) break;
}
@@ -177,6 +182,7 @@ async function run() {
save_status: save.status,
save_code: save.json.code ?? null,
save_msg: save.json.msg || "",
server_uuid: lastServer?.uuid || save.json.data?.uuid || "",
tool_names: lastTools,
has_expected_tool: lastTools.includes(expectedTool),
runtime_status: lastRuntime?.status || null,
@@ -212,11 +218,47 @@ async function run() {
result.reason = `MCP server ${serverName} did not expose ${expectedTool}. See ${apiDiagnosticPath}.`;
return;
}
if (!diagnostic.server_uuid) {
result.status = "fail";
result.reason = `MCP server ${serverName} exposed ${expectedTool}, but the server UUID was not returned. See ${apiDiagnosticPath}.`;
return;
}
await upsertEnvLocal(envLocalPath, {
[serverUuidEnvKey]: diagnostic.server_uuid,
});
result.wrote_env = true;
result.server_uuid = diagnostic.server_uuid;
result.server_uuid_env_key = serverUuidEnvKey;
result.status = "pass";
result.reason = `MCP server ${serverName} is connected and exposes ${expectedTool} through LangBot /api/v1/tools.`;
}
async function upsertEnvLocal(path, updates) {
let text = "";
try {
text = await readFile(path, "utf8");
} catch {
text = "";
}
const lines = text ? text.split(/\r?\n/) : [];
const seen = new Set();
const updated = lines.map((line) => {
const match = line.match(/^([A-Z][A-Z0-9_]*)=/);
if (!match || !Object.prototype.hasOwnProperty.call(updates, match[1])) return line;
seen.add(match[1]);
return `${match[1]}=${updates[match[1]]}`;
});
for (const [key, value] of Object.entries(updates)) {
if (!seen.has(key)) updated.push(`${key}=${value}`);
}
await writeFile(path, `${updated.filter((line, index) => line || index < updated.length - 1).join("\n")}\n`, "utf8");
}
try {
await run();
} catch (error) {
skills/scripts/e2e/pipeline-debug-chat.mjs
+308 -15
View File
@@ -11,12 +11,14 @@ import {
} from "./lib/debug-chat.mjs";
import {
createBrowser,
ensureAuthenticatedBrowser,
ensureEvidence,
evidencePaths,
exitCode,
localIsoWithOffset,
pathExists,
safeScreenshot,
scanBrowserDiagnostics,
writeResult,
} from "./lib/langbot-e2e.mjs";
@@ -50,15 +52,19 @@ const pipelineName = pipelineRequired
const expectedRunnerId = env.LANGBOT_E2E_EXPECTED_RUNNER_ID || "";
const resetDebugChat = boolFromEnv(env.LANGBOT_E2E_RESET_DEBUG_CHAT, false);
const restoreRunnerConfig = boolFromEnv(env.LANGBOT_E2E_RESTORE_RUNNER_CONFIG, true);
const restoreExtensions = boolFromEnv(env.LANGBOT_E2E_RESTORE_EXTENSIONS, true);
const debugChatSessionType = env.LANGBOT_E2E_DEBUG_CHAT_SESSION_TYPE || "person";
const pipelineConfigDiagnosticPath = resolve(paths.evidenceDir, "pipeline-config-diagnostic.json");
const pipelineExtensionsDiagnosticPath = resolve(paths.evidenceDir, "pipeline-extensions-diagnostic.json");
const debugChatResetDiagnosticPath = resolve(paths.evidenceDir, "debug-chat-reset-diagnostic.json");
const pipelineConfigRestoreDiagnosticPath = resolve(paths.evidenceDir, "pipeline-config-restore-diagnostic.json");
const pipelineExtensionsRestoreDiagnosticPath = resolve(paths.evidenceDir, "pipeline-extensions-restore-diagnostic.json");
const metricsPath = resolve(paths.evidenceDir, "metrics.json");
const startedAt = new Date();
let browser;
let restorePlan = null;
let restoreConfigPlan = null;
let restoreExtensionsPlan = null;
let result = {
source: "automation",
case_id: caseId,
@@ -317,6 +323,11 @@ function sanitizePipelineDiagnostic(diagnostic) {
return safe;
}
function sanitizePipelineExtensionsDiagnostic(diagnostic) {
const { restore_extensions: _restoreExtensions, ...safe } = diagnostic || {};
return safe;
}
async function prepareImageFixture(paths) {
if (imagePathEnv) return resolve(imagePathEnv);
if (!imageBase64Path) return "";
@@ -451,6 +462,31 @@ async function inspectAndPatchPipelineConfig(page, {
? runnerConfigs[runnerId]
: {};
const patchKeys = Object.keys(runnerConfigPatch || {});
const sensitiveConfigKeyRe = /(?:api[_-]?key|authorization|bearer|credential|jwt|oauth|password|secret)/i;
const sanitizeConfigValue = (key, value, depth = 0) => {
if (sensitiveConfigKeyRe.test(String(key))) return "[redacted]";
if (typeof value === "string") {
return value.length > 1200 ? `${value.slice(0, 1200)}...[truncated ${value.length - 1200} chars]` : value;
}
if (Array.isArray(value)) {
const items = value.slice(0, 50).map((item) => sanitizeConfigValue(key, item, depth + 1));
if (value.length > 50) items.push(`[truncated ${value.length - 50} items]`);
return items;
}
if (value && typeof value === "object") {
if (depth >= 3) return "[object truncated]";
return Object.fromEntries(
Object.entries(value).map(([childKey, childValue]) => [
childKey,
sanitizeConfigValue(childKey, childValue, depth + 1),
]),
);
}
return value;
};
const pickPatchValues = (config) => Object.fromEntries(
patchKeys.map((key) => [key, sanitizeConfigValue(key, config?.[key])]),
);
const baseDiagnostic = {
status: "ready",
authenticated: true,
@@ -462,6 +498,7 @@ async function inspectAndPatchPipelineConfig(page, {
expected_runner_id: expectedRunnerId || "",
patch_keys: patchKeys,
runner_config_before_keys: Object.keys(currentRunnerConfig),
runner_config_patch_before: pickPatchValues(currentRunnerConfig),
patched: patchKeys.length > 0,
};
@@ -506,6 +543,7 @@ async function inspectAndPatchPipelineConfig(page, {
put_status: update.status,
put_code: update.json.code ?? null,
runner_config_after_keys: Object.keys(updatedRunnerConfig),
runner_config_patch_after: pickPatchValues(updatedRunnerConfig),
restore_config: config,
};
}, {
@@ -548,6 +586,201 @@ async function restorePipelineConfig(page, { backendUrl, pipelineId, config }) {
}, { backendUrl, pipelineId, config });
}
async function inspectAndPatchPipelineExtensions(page, {
backendUrl,
pipelineId,
extensionsPatch,
}) {
return await page.evaluate(async ({
backendUrl,
pipelineId,
extensionsPatch,
}) => {
const token = localStorage.getItem("token");
if (!token) {
return {
status: "blocked",
authenticated: false,
pipeline_id: pipelineId,
reason: "Browser profile has no localStorage token.",
};
}
const headers = {
Authorization: `Bearer ${token}`,
"Content-Type": "application/json",
};
const getJson = async (path) => {
const response = await fetch(`${backendUrl}${path}`, { headers });
return {
status: response.status,
json: await response.json().catch(() => ({})),
};
};
const putJson = async (path, body) => {
const response = await fetch(`${backendUrl}${path}`, {
method: "PUT",
headers,
body: JSON.stringify(body),
});
return {
status: response.status,
json: await response.json().catch(() => ({})),
};
};
if (!pipelineId) {
return {
status: "blocked",
authenticated: true,
pipeline_resolved: false,
reason: "Pipeline id is required before patching extensions.",
};
}
const before = await getJson(`/api/v1/pipelines/${encodeURIComponent(pipelineId)}/extensions`);
let extensions = before.json.data || {};
let getStatus = before.status;
let getCode = before.json.code ?? null;
let fallbackReason = "";
if (before.status >= 400 || before.json.code !== 0) {
const pipelineResponse = await getJson(`/api/v1/pipelines/${encodeURIComponent(pipelineId)}`);
const pipeline = pipelineResponse.json.data?.pipeline || {};
const prefs = pipeline.extensions_preferences || {};
if (pipelineResponse.status >= 400 || pipelineResponse.json.code !== 0 || !pipeline.uuid) {
return {
status: "fail",
authenticated: true,
pipeline_id: pipelineId,
get_status: before.status,
get_code: before.json.code ?? null,
fallback_pipeline_status: pipelineResponse.status,
fallback_pipeline_code: pipelineResponse.json.code ?? null,
reason: before.json.msg || "Could not load pipeline extensions.",
};
}
fallbackReason = before.json.msg || "Could not load pipeline extensions; restored from pipeline preferences.";
extensions = {
enable_all_plugins: prefs.enable_all_plugins ?? true,
enable_all_mcp_servers: prefs.enable_all_mcp_servers ?? true,
enable_all_skills: prefs.enable_all_skills ?? true,
bound_plugins: prefs.plugins || [],
bound_mcp_servers: prefs.mcp_servers || [],
bound_skills: prefs.skills || [],
};
}
const patchKeys = Object.keys(extensionsPatch || {});
const restoreExtensions = {
enable_all_plugins: extensions.enable_all_plugins ?? true,
enable_all_mcp_servers: extensions.enable_all_mcp_servers ?? true,
enable_all_skills: extensions.enable_all_skills ?? true,
bound_plugins: extensions.bound_plugins || [],
bound_mcp_servers: extensions.bound_mcp_servers || [],
bound_skills: extensions.bound_skills || [],
};
const baseDiagnostic = {
status: "ready",
authenticated: true,
pipeline_id: pipelineId,
patch_keys: patchKeys,
patched: patchKeys.length > 0,
get_status: getStatus,
get_code: getCode,
fallback_reason: fallbackReason,
extensions_before: {
enable_all_plugins: restoreExtensions.enable_all_plugins,
enable_all_mcp_servers: restoreExtensions.enable_all_mcp_servers,
enable_all_skills: restoreExtensions.enable_all_skills,
bound_plugins: restoreExtensions.bound_plugins,
bound_mcp_servers: restoreExtensions.bound_mcp_servers,
bound_skills: restoreExtensions.bound_skills,
},
};
if (patchKeys.length === 0) {
return baseDiagnostic;
}
const updateBody = {
enable_all_plugins: Object.prototype.hasOwnProperty.call(extensionsPatch, "enable_all_plugins")
? Boolean(extensionsPatch.enable_all_plugins)
: restoreExtensions.enable_all_plugins,
enable_all_mcp_servers: Object.prototype.hasOwnProperty.call(extensionsPatch, "enable_all_mcp_servers")
? Boolean(extensionsPatch.enable_all_mcp_servers)
: restoreExtensions.enable_all_mcp_servers,
enable_all_skills: Object.prototype.hasOwnProperty.call(extensionsPatch, "enable_all_skills")
? Boolean(extensionsPatch.enable_all_skills)
: restoreExtensions.enable_all_skills,
bound_plugins: Object.prototype.hasOwnProperty.call(extensionsPatch, "bound_plugins")
? extensionsPatch.bound_plugins
: restoreExtensions.bound_plugins,
bound_mcp_servers: Object.prototype.hasOwnProperty.call(extensionsPatch, "bound_mcp_servers")
? extensionsPatch.bound_mcp_servers
: restoreExtensions.bound_mcp_servers,
bound_skills: Object.prototype.hasOwnProperty.call(extensionsPatch, "bound_skills")
? extensionsPatch.bound_skills
: restoreExtensions.bound_skills,
};
const update = await putJson(`/api/v1/pipelines/${encodeURIComponent(pipelineId)}/extensions`, updateBody);
if (update.status >= 400 || update.json.code !== 0) {
return {
...baseDiagnostic,
status: "fail",
put_status: update.status,
put_code: update.json.code ?? null,
reason: update.json.msg || "Pipeline extensions update failed.",
};
}
return {
...baseDiagnostic,
put_status: update.status,
put_code: update.json.code ?? null,
extensions_after: updateBody,
restore_extensions: restoreExtensions,
};
}, {
backendUrl,
pipelineId,
extensionsPatch,
});
}
async function restorePipelineExtensions(page, { backendUrl, pipelineId, extensions }) {
return await page.evaluate(async ({ backendUrl, pipelineId, extensions }) => {
const token = localStorage.getItem("token");
if (!token) {
return {
status: "blocked",
authenticated: false,
pipeline_id: pipelineId,
reason: "Browser profile has no localStorage token.",
};
}
const response = await fetch(`${backendUrl}/api/v1/pipelines/${encodeURIComponent(pipelineId)}/extensions`, {
method: "PUT",
headers: {
Authorization: `Bearer ${token}`,
"Content-Type": "application/json",
},
body: JSON.stringify(extensions),
});
const json = await response.json().catch(() => ({}));
return {
status: response.status >= 400 || json.code !== 0 ? "fail" : "ready",
authenticated: true,
pipeline_id: pipelineId,
put_status: response.status,
put_code: json.code ?? null,
reason: response.status >= 400 || json.code !== 0
? json.msg || "Pipeline extensions restore failed."
: "Pipeline extensions restored.",
};
}, { backendUrl, pipelineId, extensions });
}
async function resetPipelineDebugChat(page, { backendUrl, pipelineId, sessionType }) {
return await page.evaluate(async ({ backendUrl, pipelineId, sessionType }) => {
const token = localStorage.getItem("token");
@@ -590,11 +823,13 @@ try {
const promptSteps = promptStepsFromEnv();
const filesystemChecks = parseJsonEnv("LANGBOT_E2E_FILESYSTEM_CHECKS_JSON", []);
const runnerConfigPatch = parseJsonEnv("LANGBOT_E2E_RUNNER_CONFIG_PATCH_JSON", {});
const extensionsPatch = parseJsonEnv("LANGBOT_E2E_EXTENSIONS_PATCH_JSON", {});
const runnerPatchKeys = Object.keys(runnerConfigPatch);
if (runnerPatchKeys.length > 0 || resetDebugChat || expectedRunnerId) {
const extensionsPatchKeys = Object.keys(extensionsPatch);
if (runnerPatchKeys.length > 0 || extensionsPatchKeys.length > 0 || resetDebugChat || expectedRunnerId) {
if (!backendUrl) {
result.status = "env_issue";
result.reason = "LANGBOT_BACKEND_URL is required for runner config patch, runner assertion, or Debug Chat reset.";
result.reason = "LANGBOT_BACKEND_URL is required for runner config patch, extensions patch, runner assertion, or Debug Chat reset.";
throw new Error(result.reason);
}
}
@@ -602,13 +837,30 @@ try {
result.prompt = promptSteps.length === 1 ? promptSteps[0].prompt : `${promptSteps.length} prompts`;
result.expected_text = promptSteps.at(-1)?.expectedText || expectedText;
const openResult = await openPipelineDebugChat(page, {
pipelineUrl,
pipelineName,
envHint: pipelineRequired
? "case-specific pipeline env mapped to LANGBOT_E2E_PIPELINE_URL or LANGBOT_E2E_PIPELINE_NAME"
: "LANGBOT_PIPELINE_URL or LANGBOT_PIPELINE_NAME",
const authDiagnostic = await ensureAuthenticatedBrowser(page, {
frontendUrl: env.LANGBOT_FRONTEND_URL || "",
backendUrl,
});
result.browser_auth = authDiagnostic;
if (!result.evidence_collected.includes("api_diagnostic")) result.evidence_collected.push("api_diagnostic");
const authFailed = authDiagnostic.status === "env_issue" || authDiagnostic.status === "blocked" || authDiagnostic.status === "fail";
if (authFailed) {
result.status = authDiagnostic.status;
result.reason = authDiagnostic.reason || "Browser authentication failed.";
} else {
result.status = "running";
result.reason = "";
}
const openResult = authFailed
? { opened: false, status: result.status, reason: result.reason }
: await openPipelineDebugChat(page, {
pipelineUrl,
pipelineName,
envHint: pipelineRequired
? "case-specific pipeline env mapped to LANGBOT_E2E_PIPELINE_URL or LANGBOT_E2E_PIPELINE_NAME"
: "LANGBOT_PIPELINE_URL or LANGBOT_PIPELINE_NAME",
});
result.url = page.url();
if (!openResult.opened) {
@@ -617,7 +869,7 @@ try {
} else {
result.status = "running";
result.reason = "";
if (runnerPatchKeys.length > 0 || resetDebugChat || expectedRunnerId) {
if (runnerPatchKeys.length > 0 || extensionsPatchKeys.length > 0 || resetDebugChat || expectedRunnerId) {
const pipelineDiagnostic = await inspectAndPatchPipelineConfig(page, {
backendUrl,
pipelineUrl,
@@ -636,13 +888,35 @@ try {
result.reason = pipelineDiagnostic.reason || "Pipeline config preparation failed.";
} else {
if (pipelineDiagnostic.restore_config && restoreRunnerConfig) {
restorePlan = {
restoreConfigPlan = {
backendUrl,
pipelineId: pipelineDiagnostic.pipeline_id,
config: pipelineDiagnostic.restore_config,
};
}
if (resetDebugChat) {
if (extensionsPatchKeys.length > 0) {
const extensionsDiagnostic = await inspectAndPatchPipelineExtensions(page, {
backendUrl,
pipelineId: pipelineDiagnostic.pipeline_id,
extensionsPatch,
});
const safeExtensionsDiagnostic = sanitizePipelineExtensionsDiagnostic(extensionsDiagnostic);
await writeFile(pipelineExtensionsDiagnosticPath, `${JSON.stringify(safeExtensionsDiagnostic, null, 2)}\n`, "utf8");
result.evidence.pipeline_extensions_diagnostic_json = pipelineExtensionsDiagnosticPath;
result.pipeline_extensions = safeExtensionsDiagnostic;
if (!result.evidence_collected.includes("api_diagnostic")) result.evidence_collected.push("api_diagnostic");
if (extensionsDiagnostic.status === "fail" || extensionsDiagnostic.status === "blocked") {
result.status = extensionsDiagnostic.status;
result.reason = extensionsDiagnostic.reason || "Pipeline extensions preparation failed.";
} else if (extensionsDiagnostic.restore_extensions && restoreExtensions) {
restoreExtensionsPlan = {
backendUrl,
pipelineId: extensionsDiagnostic.pipeline_id,
extensions: extensionsDiagnostic.restore_extensions,
};
}
}
if (!["fail", "blocked", "env_issue"].includes(result.status) && resetDebugChat) {
const resetDiagnostic = await resetPipelineDebugChat(page, {
backendUrl,
pipelineId: pipelineDiagnostic.pipeline_id,
@@ -720,6 +994,15 @@ try {
result.reason = filesystemResult.reason || "Filesystem checks failed.";
}
}
if (result.status === "pass" && !boolFromEnv(env.LANGBOT_E2E_ALLOW_BROWSER_ERRORS, false)) {
const browserDiagnostics = await scanBrowserDiagnostics(paths);
result.browser_diagnostics = browserDiagnostics;
if (browserDiagnostics.status === "fail") {
result.status = "fail";
result.reason = browserDiagnostics.reason;
}
}
}
} catch (error) {
if (!["env_issue", "blocked", "fail", "pass"].includes(result.status) || !result.reason) {
@@ -728,10 +1011,20 @@ try {
result.reason = result.reason || error.message;
} finally {
if (browser?.page) await safeScreenshot(browser.page, paths.screenshot);
if (browser?.page && restorePlan) {
const restoreDiagnostic = await restorePipelineConfig(browser.page, restorePlan).catch((error) => ({
if (browser?.page && restoreExtensionsPlan) {
const restoreDiagnostic = await restorePipelineExtensions(browser.page, restoreExtensionsPlan).catch((error) => ({
status: "fail",
pipeline_id: restorePlan.pipelineId,
pipeline_id: restoreExtensionsPlan.pipelineId,
reason: error.message,
}));
await writeFile(pipelineExtensionsRestoreDiagnosticPath, `${JSON.stringify(restoreDiagnostic, null, 2)}\n`, "utf8");
result.evidence.pipeline_extensions_restore_diagnostic_json = pipelineExtensionsRestoreDiagnosticPath;
result.pipeline_extensions_restore = restoreDiagnostic;
}
if (browser?.page && restoreConfigPlan) {
const restoreDiagnostic = await restorePipelineConfig(browser.page, restoreConfigPlan).catch((error) => ({
status: "fail",
pipeline_id: restoreConfigPlan.pipelineId,
reason: error.message,
}));
await writeFile(pipelineConfigRestoreDiagnosticPath, `${JSON.stringify(restoreDiagnostic, null, 2)}\n`, "utf8");