mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-09-01 23:27:14 +00:00
fix(agent): harden runner integration and QA
This commit is contained in:
@@ -6,6 +6,7 @@ import { resolve } from "node:path";
|
||||
import { env } from "node:process";
|
||||
import {
|
||||
createBrowser,
|
||||
ensureBrowserWorkspace,
|
||||
ensureEvidence,
|
||||
evidencePaths,
|
||||
exitCode,
|
||||
@@ -124,6 +125,12 @@ async function run() {
|
||||
const { page } = browser;
|
||||
await page.goto(frontendUrl, { waitUntil: "domcontentloaded" });
|
||||
await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {});
|
||||
const workspace = await ensureBrowserWorkspace(page, backendUrl);
|
||||
if (workspace.status !== "pass") {
|
||||
result.status = workspace.status;
|
||||
result.reason = workspace.reason;
|
||||
return;
|
||||
}
|
||||
|
||||
const diagnostic = await page.evaluate(async ({ backendUrl, targets, testModels }) => {
|
||||
const blockers = [];
|
||||
@@ -150,6 +157,7 @@ async function run() {
|
||||
const headers = {
|
||||
Authorization: `Bearer ${token}`,
|
||||
"Content-Type": "application/json",
|
||||
"X-Workspace-Id": localStorage.getItem("langbot_active_workspace_uuid") || "",
|
||||
};
|
||||
const getJson = async (path) => {
|
||||
const response = await fetch(`${backendUrl}${path}`, { headers });
|
||||
|
||||
@@ -78,6 +78,14 @@ try {
|
||||
provider: "claude-code",
|
||||
location: "remote-ssh",
|
||||
workspace: remoteWorkspace,
|
||||
"env-json": JSON.stringify({
|
||||
ALL_PROXY: "",
|
||||
all_proxy: "",
|
||||
HTTP_PROXY: "",
|
||||
http_proxy: "",
|
||||
HTTPS_PROXY: "",
|
||||
https_proxy: "",
|
||||
}),
|
||||
"ssh-target": sshTarget,
|
||||
"ssh-port": Number.parseInt(sshPort, 10),
|
||||
"ssh-identity-file": sshIdentityFile,
|
||||
|
||||
@@ -4,6 +4,7 @@ import { readFile, writeFile } from "node:fs/promises";
|
||||
import { resolve } from "node:path";
|
||||
import { env } from "node:process";
|
||||
import {
|
||||
authenticatedApiHeaders,
|
||||
apiJson,
|
||||
ensureEvidence,
|
||||
evidencePaths,
|
||||
@@ -99,7 +100,15 @@ try {
|
||||
}
|
||||
result.store_task_id = store.json.data?.task_id || "";
|
||||
|
||||
const ready = await waitForSentinel(backendUrl, auth.token, result.kb_uuid, query, expectedText, waitMs);
|
||||
const ready = await waitForSentinel(
|
||||
backendUrl,
|
||||
auth.token,
|
||||
result.kb_uuid,
|
||||
query,
|
||||
expectedText,
|
||||
waitMs,
|
||||
upload.fileId,
|
||||
);
|
||||
result.file_statuses = ready.fileStatuses;
|
||||
if (ready.matched) {
|
||||
result.checked_bases.push(ready.checked);
|
||||
@@ -209,9 +218,7 @@ async function uploadDocument(backendUrl, token, path) {
|
||||
form.append("file", new Blob([bytes], { type: "text/plain" }), "sentinel-doc.txt");
|
||||
const response = await fetch(`${backendUrl.replace(/\/$/, "")}/api/v1/files/documents`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
},
|
||||
headers: await authenticatedApiHeaders(backendUrl, token, { contentType: "" }),
|
||||
body: form,
|
||||
});
|
||||
const json = await response.json().catch(() => ({}));
|
||||
@@ -222,13 +229,14 @@ async function uploadDocument(backendUrl, token, path) {
|
||||
return { fileId };
|
||||
}
|
||||
|
||||
async function waitForSentinel(backendUrl, token, kbUuid, query, expectedText, timeoutMs) {
|
||||
async function waitForSentinel(backendUrl, token, kbUuid, query, expectedText, timeoutMs, uploadedFileId) {
|
||||
const started = Date.now();
|
||||
let fileStatuses = [];
|
||||
let lastChecked = null;
|
||||
while (Date.now() - started < timeoutMs) {
|
||||
const files = await apiJson(backendUrl, `/api/v1/knowledge/bases/${encodeURIComponent(kbUuid)}/files`, { token });
|
||||
fileStatuses = files.json.data?.files || fileStatuses;
|
||||
const listedFiles = files.json.data?.files || [];
|
||||
fileStatuses = listedFiles.filter((item) => item.file_name === uploadedFileId);
|
||||
lastChecked = await retrieveSentinel(backendUrl, token, kbUuid, kbName, query, expectedText);
|
||||
if (lastChecked.matched) {
|
||||
return { matched: true, fileStatuses, checked: lastChecked };
|
||||
|
||||
@@ -4,9 +4,12 @@ import { readFile } from "node:fs/promises";
|
||||
import { resolve } from "node:path";
|
||||
import { env } from "node:process";
|
||||
import {
|
||||
authenticatedApiHeaders,
|
||||
apiJson,
|
||||
ensureEvidence,
|
||||
evidencePaths,
|
||||
isTaskComplete,
|
||||
isTaskFailed,
|
||||
loadEnvFiles,
|
||||
resetAndAuthLocalUser,
|
||||
writeResult,
|
||||
@@ -138,7 +141,7 @@ async function installPlugin(backendUrl, token, bytes, packagePath) {
|
||||
form.set("file", new Blob([bytes]), packagePath.split("/").pop());
|
||||
const response = await fetch(`${backendUrl.replace(/\/$/, "")}/api/v1/plugins/install/local`, {
|
||||
method: "POST",
|
||||
headers: { Authorization: `Bearer ${token}` },
|
||||
headers: await authenticatedApiHeaders(backendUrl, token, { contentType: "" }),
|
||||
body: form,
|
||||
});
|
||||
const json = await response.json().catch(() => ({}));
|
||||
@@ -173,7 +176,7 @@ async function previewPackage(backendUrl, token, bytes, packagePath) {
|
||||
form.set("file", new Blob([bytes]), packagePath.split("/").pop());
|
||||
const response = await fetch(`${backendUrl.replace(/\/$/, "")}/api/v1/plugins/install/local/preview`, {
|
||||
method: "POST",
|
||||
headers: { Authorization: `Bearer ${token}` },
|
||||
headers: await authenticatedApiHeaders(backendUrl, token, { contentType: "" }),
|
||||
body: form,
|
||||
});
|
||||
const json = await response.json().catch(() => ({}));
|
||||
@@ -214,32 +217,12 @@ async function waitForTask(backendUrl, token, taskId) {
|
||||
while (Date.now() < deadline) {
|
||||
const response = await apiJson(backendUrl, `/api/v1/system/tasks/${encodeURIComponent(taskId)}`, { token });
|
||||
last = response.json.data || response.json;
|
||||
if (isTaskComplete(last) || isTaskFailed(last)) return last;
|
||||
if (isTaskFailed(last) || isTaskComplete(last)) return last;
|
||||
await sleep(1000);
|
||||
}
|
||||
return last;
|
||||
}
|
||||
|
||||
function isTaskComplete(task) {
|
||||
const status = String(task?.status || task?.state || "").toLowerCase();
|
||||
const runtimeStatus = String(task?.runtime?.status || task?.runtime?.state || "").toLowerCase();
|
||||
return ["done", "completed", "success", "succeeded", "finished"].includes(status)
|
||||
|| ["done", "completed", "success", "succeeded", "finished"].includes(runtimeStatus)
|
||||
|| task?.done === true
|
||||
|| task?.completed === true
|
||||
|| (task?.runtime?.done === true && !task?.runtime?.exception);
|
||||
}
|
||||
|
||||
function isTaskFailed(task) {
|
||||
const status = String(task?.status || task?.state || "").toLowerCase();
|
||||
const runtimeStatus = String(task?.runtime?.status || task?.runtime?.state || "").toLowerCase();
|
||||
return ["failed", "error", "cancelled", "canceled"].includes(status)
|
||||
|| ["failed", "error", "cancelled", "canceled"].includes(runtimeStatus)
|
||||
|| task?.failed === true
|
||||
|| Boolean(task?.error)
|
||||
|| Boolean(task?.runtime?.exception);
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import {
|
||||
bodyText,
|
||||
createBrowser,
|
||||
ensureBrowserWorkspace,
|
||||
ensureEvidence,
|
||||
evidencePaths,
|
||||
exitCode,
|
||||
@@ -57,6 +58,12 @@ try {
|
||||
const { page } = browser;
|
||||
await page.goto(`${frontendUrl.replace(/\/$/, "")}/home/knowledge`, { waitUntil: "domcontentloaded" });
|
||||
await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {});
|
||||
const workspace = await ensureBrowserWorkspace(page, backendUrl);
|
||||
if (workspace.status !== "pass") {
|
||||
result.status = workspace.status;
|
||||
result.reason = workspace.reason;
|
||||
throw new Error(result.reason);
|
||||
}
|
||||
result.url = page.url();
|
||||
|
||||
const text = await bodyText(page);
|
||||
@@ -77,6 +84,7 @@ try {
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
"Content-Type": "application/json",
|
||||
"X-Workspace-Id": localStorage.getItem("langbot_active_workspace_uuid") || "",
|
||||
},
|
||||
body: JSON.stringify({ query }),
|
||||
});
|
||||
|
||||
@@ -424,18 +424,33 @@ async function fetchDebugChatHistory(page, { backendUrl, pipelineId, sessionType
|
||||
}
|
||||
return await page.evaluate(async ({ backendUrl, pipelineId, sessionType }) => {
|
||||
const token = localStorage.getItem("token") || "";
|
||||
const response = await fetch(
|
||||
`${backendUrl.replace(/\/$/, "")}/api/v1/pipelines/${encodeURIComponent(pipelineId)}/ws/messages/${encodeURIComponent(sessionType)}`,
|
||||
{ headers: token ? { Authorization: `Bearer ${token}` } : {} },
|
||||
);
|
||||
const json = await response.json().catch(() => ({}));
|
||||
return {
|
||||
status: response.ok && json.code === 0 ? "ready" : "fail",
|
||||
http_status: response.status,
|
||||
code: json.code ?? null,
|
||||
messages: json.data?.messages || [],
|
||||
reason: response.ok && json.code === 0 ? "" : json.msg || `Debug Chat history returned HTTP ${response.status}.`,
|
||||
};
|
||||
try {
|
||||
const response = await fetch(
|
||||
`${backendUrl.replace(/\/$/, "")}/api/v1/pipelines/${encodeURIComponent(pipelineId)}/ws/messages/${encodeURIComponent(sessionType)}`,
|
||||
{
|
||||
headers: token
|
||||
? {
|
||||
Authorization: `Bearer ${token}`,
|
||||
"X-Workspace-Id": localStorage.getItem("langbot_active_workspace_uuid") || "",
|
||||
}
|
||||
: {},
|
||||
},
|
||||
);
|
||||
const json = await response.json().catch(() => ({}));
|
||||
return {
|
||||
status: response.ok && json.code === 0 ? "ready" : "fail",
|
||||
http_status: response.status,
|
||||
code: json.code ?? null,
|
||||
messages: json.data?.messages || [],
|
||||
reason: response.ok && json.code === 0 ? "" : json.msg || `Debug Chat history returned HTTP ${response.status}.`,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
status: "retryable",
|
||||
messages: [],
|
||||
reason: `Debug Chat history request failed: ${error instanceof Error ? error.message : String(error)}`,
|
||||
};
|
||||
}
|
||||
}, { backendUrl, pipelineId, sessionType });
|
||||
}
|
||||
|
||||
@@ -454,6 +469,10 @@ async function waitForFinalDebugChatAssistant(page, {
|
||||
while (Date.now() < deadline) {
|
||||
lastHistory = await fetchDebugChatHistory(page, { backendUrl, pipelineId, sessionType });
|
||||
if (lastHistory.status === "fail") return lastHistory;
|
||||
if (lastHistory.status === "retryable") {
|
||||
await page.waitForTimeout(Math.min(250, Math.max(1, deadline - Date.now())));
|
||||
continue;
|
||||
}
|
||||
const assistants = lastHistory.messages.filter((message) => message.role === "assistant");
|
||||
const latest = assistants.at(-1);
|
||||
if (assistants.length > beforeAssistantCount && latest?.is_final === true) {
|
||||
@@ -480,12 +499,23 @@ export async function attachDebugChatImage(page, imagePath) {
|
||||
if (!await input.count()) {
|
||||
return { status: "fail", reason: "Could not find a Debug Chat image upload input." };
|
||||
}
|
||||
const previews = page.locator('[data-debug-chat-attachment-preview="true"]');
|
||||
const beforePreviewCount = await previews.count();
|
||||
await input.setInputFiles(imagePath);
|
||||
await page.locator("img").last().waitFor({ state: "visible", timeout: 10_000 }).catch(() => {});
|
||||
const previewVisible = await previews
|
||||
.nth(beforePreviewCount)
|
||||
.waitFor({ state: "visible", timeout: 10_000 })
|
||||
.then(() => true)
|
||||
.catch(() => false);
|
||||
if (!previewVisible) {
|
||||
return { status: "fail", reason: "Debug Chat did not show the selected image preview." };
|
||||
}
|
||||
return { status: "ready", reason: `Attached image fixture: ${imagePath}` };
|
||||
}
|
||||
|
||||
export async function sendDebugChatPrompt(page, prompt, imagePath = "") {
|
||||
const sentImages = page.locator('[data-debug-chat-message-image="true"]');
|
||||
const beforeSentImageCount = imagePath ? await sentImages.count() : 0;
|
||||
const imageResult = await attachDebugChatImage(page, imagePath);
|
||||
if (imageResult.status === "fail") return imageResult;
|
||||
|
||||
@@ -500,6 +530,16 @@ export async function sendDebugChatPrompt(page, prompt, imagePath = "") {
|
||||
const clickedSend = await clickFirstVisible(page, ["Send", "发送", "提交"], 1_500);
|
||||
if (!clickedSend) await page.keyboard.press("Enter");
|
||||
await page.getByText(prompt, { exact: false }).last().waitFor({ state: "visible", timeout: 10_000 }).catch(() => {});
|
||||
if (imagePath) {
|
||||
const sentImageVisible = await sentImages
|
||||
.nth(beforeSentImageCount)
|
||||
.waitFor({ state: "visible", timeout: 15_000 })
|
||||
.then(() => true)
|
||||
.catch(() => false);
|
||||
if (!sentImageVisible) {
|
||||
return { status: "fail", reason: "The sent Debug Chat message did not render its image attachment." };
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -535,7 +575,7 @@ export async function runDebugChatPrompt(page, {
|
||||
}
|
||||
|
||||
const responseStartedAt = Date.now();
|
||||
await waitForExpectedDebugChatText(page, {
|
||||
const expectedTextPromise = waitForExpectedDebugChatText(page, {
|
||||
expectedText,
|
||||
expectedTexts: requiredExpectedTexts,
|
||||
minExpectedCount,
|
||||
@@ -545,13 +585,19 @@ export async function runDebugChatPrompt(page, {
|
||||
beforeText,
|
||||
failureSignals,
|
||||
});
|
||||
const finalAssistant = await waitForFinalDebugChatAssistant(page, {
|
||||
const finalAssistantPromise = waitForFinalDebugChatAssistant(page, {
|
||||
backendUrl,
|
||||
pipelineId,
|
||||
sessionType,
|
||||
beforeAssistantCount: beforeHistoryAssistantCount,
|
||||
timeoutMs: Math.max(1, responseTimeoutMs - (Date.now() - responseStartedAt)),
|
||||
});
|
||||
if (backendUrl && pipelineId && sessionType) {
|
||||
await Promise.race([expectedTextPromise, finalAssistantPromise]);
|
||||
} else {
|
||||
await expectedTextPromise;
|
||||
}
|
||||
const finalAssistant = await finalAssistantPromise;
|
||||
await waitForDebugChatTextStable(page);
|
||||
|
||||
const afterText = await bodyText(page);
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
import { appendFile, mkdir, readFile, stat, writeFile } from "node:fs/promises";
|
||||
import { createHash } from "node:crypto";
|
||||
import { basename, join, resolve } from "node:path";
|
||||
import { env } from "node:process";
|
||||
|
||||
const secretRe = /(?:authorization|bearer|token|secret|password|api[_-]?key|jwt|oauth)\s*[:=]\s*["']?[^"',\s]+/gi;
|
||||
const ACTIVE_WORKSPACE_STORAGE_KEY = "langbot_active_workspace_uuid";
|
||||
const workspaceUuidCache = new Map();
|
||||
|
||||
export function redact(text) {
|
||||
return String(text ?? "")
|
||||
@@ -30,6 +33,19 @@ export function localIsoWithOffset(date = new Date()) {
|
||||
return `${yyyy}-${mm}-${dd}T${hh}:${mi}:${ss}.${ms}${sign}${pad(Math.floor(absolute / 60))}:${pad(absolute % 60)}`;
|
||||
}
|
||||
|
||||
export function boxWorkspaceNamespace(instanceUuid, workspaceUuid) {
|
||||
const instance = String(instanceUuid || "").trim();
|
||||
const workspace = String(workspaceUuid || "").trim();
|
||||
if (!instance || !workspace) {
|
||||
throw new Error("Box Workspace namespace requires instance and Workspace UUIDs.");
|
||||
}
|
||||
const digest = createHash("sha256")
|
||||
.update(`${instance}\0${workspace}`, "utf8")
|
||||
.digest("hex")
|
||||
.slice(0, 24);
|
||||
return `ws-${digest}`;
|
||||
}
|
||||
|
||||
export function evidencePaths(caseId) {
|
||||
const runId = env.LBS_RUN_ID || `${timestampSlug()}-${caseId}`;
|
||||
const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join("reports", "evidence", runId));
|
||||
@@ -103,6 +119,27 @@ export async function writeResult(paths, result) {
|
||||
}
|
||||
}
|
||||
|
||||
export function isTaskFailed(task) {
|
||||
const status = String(task?.status || task?.state || "").toLowerCase();
|
||||
const runtimeStatus = String(task?.runtime?.status || task?.runtime?.state || "").toLowerCase();
|
||||
return ["failed", "error", "cancelled", "canceled"].includes(status)
|
||||
|| ["failed", "error", "cancelled", "canceled"].includes(runtimeStatus)
|
||||
|| task?.failed === true
|
||||
|| Boolean(task?.error)
|
||||
|| Boolean(task?.runtime?.exception);
|
||||
}
|
||||
|
||||
export function isTaskComplete(task) {
|
||||
if (isTaskFailed(task)) return false;
|
||||
const status = String(task?.status || task?.state || "").toLowerCase();
|
||||
const runtimeStatus = String(task?.runtime?.status || task?.runtime?.state || "").toLowerCase();
|
||||
return ["done", "completed", "success", "succeeded", "finished"].includes(status)
|
||||
|| ["done", "completed", "success", "succeeded", "finished"].includes(runtimeStatus)
|
||||
|| task?.done === true
|
||||
|| task?.completed === true
|
||||
|| task?.runtime?.done === true;
|
||||
}
|
||||
|
||||
function browserDiagnosticFindings(source, text) {
|
||||
const findings = [];
|
||||
const lines = String(text || "").split(/\r?\n/);
|
||||
@@ -231,11 +268,98 @@ export async function readRecoveryKey(repo = env.LANGBOT_REPO || "") {
|
||||
return match?.[1] || "";
|
||||
}
|
||||
|
||||
export async function apiJson(backendUrl, path, { method = "GET", token = "", body } = {}) {
|
||||
function workspaceCacheKey(backendUrl, token) {
|
||||
return `${backendUrl.replace(/\/$/, "")}\0${token}`;
|
||||
}
|
||||
|
||||
function isAccountScopedApi(path, method) {
|
||||
const pathname = path.split("?", 1)[0];
|
||||
if (pathname === "/api/v1/workspaces/bootstrap") return true;
|
||||
if (pathname === "/api/v1/workspaces" && method === "GET") return true;
|
||||
return [
|
||||
"/api/v1/user/check-token",
|
||||
"/api/v1/user/info",
|
||||
"/api/v1/user/account-info",
|
||||
"/api/v1/user/change-password",
|
||||
].includes(pathname);
|
||||
}
|
||||
|
||||
export async function resolveWorkspaceUuid(
|
||||
backendUrl,
|
||||
token,
|
||||
preferredWorkspaceUuid = env.LANGBOT_WORKSPACE_UUID || "",
|
||||
) {
|
||||
if (!token) throw new Error("A user token is required to resolve the active Workspace.");
|
||||
|
||||
const normalizedBackendUrl = backendUrl.replace(/\/$/, "");
|
||||
const normalizedPreferred = preferredWorkspaceUuid.trim();
|
||||
const cacheKey = workspaceCacheKey(normalizedBackendUrl, token);
|
||||
const cached = workspaceUuidCache.get(cacheKey);
|
||||
if (cached && (!normalizedPreferred || cached === normalizedPreferred)) return cached;
|
||||
|
||||
const response = await fetch(`${normalizedBackendUrl}/api/v1/workspaces/bootstrap`, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
const json = await response.json().catch(() => ({}));
|
||||
if (response.status >= 400 || json.code !== 0) {
|
||||
throw new Error(json.msg || `Workspace bootstrap failed with HTTP ${response.status}.`);
|
||||
}
|
||||
|
||||
const workspaces = json.data?.workspaces || [];
|
||||
const workspaceUuids = workspaces
|
||||
.map((entry) => entry.workspace?.uuid || entry.uuid || "")
|
||||
.filter(Boolean);
|
||||
let workspaceUuid = normalizedPreferred;
|
||||
if (workspaceUuid && !workspaceUuids.includes(workspaceUuid)) {
|
||||
throw new Error(`Configured Workspace ${workspaceUuid} is not available to the authenticated Account.`);
|
||||
}
|
||||
if (!workspaceUuid && workspaceUuids.length === 1) {
|
||||
[workspaceUuid] = workspaceUuids;
|
||||
}
|
||||
if (!workspaceUuid && workspaceUuids.length === 0) {
|
||||
throw new Error("The authenticated Account has no available Workspace.");
|
||||
}
|
||||
if (!workspaceUuid) {
|
||||
throw new Error(
|
||||
"The authenticated Account has multiple Workspaces; set LANGBOT_WORKSPACE_UUID for this QA run.",
|
||||
);
|
||||
}
|
||||
|
||||
workspaceUuidCache.set(cacheKey, workspaceUuid);
|
||||
return workspaceUuid;
|
||||
}
|
||||
|
||||
export async function authenticatedApiHeaders(
|
||||
backendUrl,
|
||||
token,
|
||||
{ contentType = "application/json", workspaceUuid = "" } = {},
|
||||
) {
|
||||
const resolvedWorkspaceUuid = await resolveWorkspaceUuid(backendUrl, token, workspaceUuid);
|
||||
return {
|
||||
Authorization: `Bearer ${token}`,
|
||||
...(contentType ? { "Content-Type": contentType } : {}),
|
||||
"X-Workspace-Id": resolvedWorkspaceUuid,
|
||||
};
|
||||
}
|
||||
|
||||
export async function apiJson(
|
||||
backendUrl,
|
||||
path,
|
||||
{ method = "GET", token = "", body, skipWorkspace = false, workspaceUuid = "" } = {},
|
||||
) {
|
||||
const normalizedMethod = method.toUpperCase();
|
||||
const headers = { "Content-Type": "application/json" };
|
||||
if (token) headers.Authorization = `Bearer ${token}`;
|
||||
if (token) {
|
||||
headers.Authorization = `Bearer ${token}`;
|
||||
if (!skipWorkspace && !isAccountScopedApi(path, normalizedMethod)) {
|
||||
headers["X-Workspace-Id"] = await resolveWorkspaceUuid(backendUrl, token, workspaceUuid);
|
||||
}
|
||||
}
|
||||
const response = await fetch(`${backendUrl.replace(/\/$/, "")}${path}`, {
|
||||
method,
|
||||
method: normalizedMethod,
|
||||
headers,
|
||||
body: body === undefined ? undefined : JSON.stringify(body),
|
||||
});
|
||||
@@ -301,6 +425,38 @@ export async function setBrowserToken(page, frontendUrl, token) {
|
||||
await page.evaluate((value) => localStorage.setItem("token", value), token);
|
||||
}
|
||||
|
||||
export async function ensureBrowserWorkspace(
|
||||
page,
|
||||
backendUrl,
|
||||
preferredWorkspaceUuid = env.LANGBOT_WORKSPACE_UUID || "",
|
||||
) {
|
||||
const browserContext = await page.evaluate((storageKey) => ({
|
||||
token: localStorage.getItem("token") || "",
|
||||
workspaceUuid: localStorage.getItem(storageKey) || "",
|
||||
}), ACTIVE_WORKSPACE_STORAGE_KEY);
|
||||
if (!browserContext.token) {
|
||||
return { status: "blocked", reason: "Browser profile has no localStorage token.", workspace_uuid: "" };
|
||||
}
|
||||
|
||||
try {
|
||||
const workspaceUuid = await resolveWorkspaceUuid(
|
||||
backendUrl,
|
||||
browserContext.token,
|
||||
preferredWorkspaceUuid || browserContext.workspaceUuid,
|
||||
);
|
||||
await page.evaluate(({ storageKey, workspaceUuid: value }) => {
|
||||
localStorage.setItem(storageKey, value);
|
||||
}, { storageKey: ACTIVE_WORKSPACE_STORAGE_KEY, workspaceUuid });
|
||||
return {
|
||||
status: "pass",
|
||||
reason: "Browser Workspace selection is initialized.",
|
||||
workspace_uuid: workspaceUuid,
|
||||
};
|
||||
} catch (error) {
|
||||
return { status: "blocked", reason: error.message, workspace_uuid: "" };
|
||||
}
|
||||
}
|
||||
|
||||
export async function verifyBrowserToken(page, backendUrl) {
|
||||
return await page.evaluate(async (baseUrl) => {
|
||||
const token = localStorage.getItem("token");
|
||||
@@ -349,11 +505,23 @@ export async function ensureAuthenticatedBrowser(page, {
|
||||
reason: error.message,
|
||||
}));
|
||||
if (current.authenticated) {
|
||||
const workspace = await ensureBrowserWorkspace(page, backendUrl);
|
||||
if (workspace.status !== "pass") {
|
||||
return {
|
||||
status: workspace.status,
|
||||
reason: workspace.reason,
|
||||
backend_token_check: null,
|
||||
browser_token_check: current,
|
||||
workspace,
|
||||
injected: false,
|
||||
};
|
||||
}
|
||||
return {
|
||||
status: "pass",
|
||||
reason: "Existing browser token is valid.",
|
||||
backend_token_check: null,
|
||||
browser_token_check: current,
|
||||
workspace,
|
||||
injected: false,
|
||||
};
|
||||
}
|
||||
@@ -381,11 +549,24 @@ export async function ensureAuthenticatedBrowser(page, {
|
||||
};
|
||||
}
|
||||
|
||||
const workspace = await ensureBrowserWorkspace(page, backendUrl);
|
||||
if (workspace.status !== "pass") {
|
||||
return {
|
||||
status: workspace.status,
|
||||
reason: workspace.reason,
|
||||
backend_token_check: auth.check,
|
||||
browser_token_check: browserCheck,
|
||||
workspace,
|
||||
injected: true,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
status: "pass",
|
||||
reason: "Browser token injected from local recovery login.",
|
||||
backend_token_check: auth.check,
|
||||
browser_token_check: browserCheck,
|
||||
workspace,
|
||||
injected: true,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -161,7 +161,8 @@ try {
|
||||
result.status = resetDiagnostic.status;
|
||||
result.reason = resetDiagnostic.reason || "Debug Chat reset failed.";
|
||||
} else {
|
||||
await page.waitForTimeout(1000);
|
||||
await page.reload({ waitUntil: "commit" });
|
||||
await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {});
|
||||
const reopenResult = await openPipelineDebugChat(page, {
|
||||
pipelineUrl,
|
||||
pipelineName,
|
||||
@@ -406,6 +407,7 @@ async function inspectPipeline(page, { backendUrl, pipelineUrl, pipelineName, ex
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
"Content-Type": "application/json",
|
||||
"X-Workspace-Id": localStorage.getItem("langbot_active_workspace_uuid") || "",
|
||||
},
|
||||
});
|
||||
return {
|
||||
@@ -508,6 +510,7 @@ async function inspectToolNames(page, { backendUrl }) {
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
"Content-Type": "application/json",
|
||||
"X-Workspace-Id": localStorage.getItem("langbot_active_workspace_uuid") || "",
|
||||
},
|
||||
});
|
||||
const json = await response.json().catch(() => ({}));
|
||||
@@ -545,6 +548,7 @@ async function resetPipelineDebugChat(page, { backendUrl, pipelineId, sessionTyp
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
"Content-Type": "application/json",
|
||||
"X-Workspace-Id": localStorage.getItem("langbot_active_workspace_uuid") || "",
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
@@ -35,22 +35,15 @@ await ensureEvidence(paths);
|
||||
const startedAt = new Date();
|
||||
const fixturePath = resolve(env.LANGBOT_MCP_FIXTURE_PATH || "skills/langbot-testing/fixtures/mcp/qa_mcp_echo_server.py");
|
||||
const langbotRepo = env.LANGBOT_REPO ? resolve(env.LANGBOT_REPO) : "";
|
||||
const uvCandidates = [
|
||||
env.LANGBOT_MCP_FIXTURE_UV,
|
||||
"uv",
|
||||
].filter(Boolean);
|
||||
const uv = uvCandidates.find((candidate) => candidate === "uv" || existsSync(candidate));
|
||||
const pythonCandidates = [
|
||||
env.LANGBOT_MCP_FIXTURE_PYTHON,
|
||||
langbotRepo ? `${langbotRepo}/.venv/bin/python` : "",
|
||||
"python3",
|
||||
].filter(Boolean);
|
||||
const python = pythonCandidates.find((candidate) => candidate === "python3" || existsSync(candidate));
|
||||
const command = langbotRepo && uv
|
||||
? { executable: uv, args: ["run", "python", fixturePath], cwd: langbotRepo, mode: "uv" }
|
||||
: python
|
||||
? { executable: python, args: [fixturePath], cwd: resolve("."), mode: "python" }
|
||||
: null;
|
||||
const command = python
|
||||
? { executable: python, args: [fixturePath], cwd: resolve("."), mode: "python" }
|
||||
: null;
|
||||
const expectedText = "qa_mcp_echo:mcp-stdio-fixture-ok";
|
||||
|
||||
const result = {
|
||||
@@ -94,7 +87,7 @@ async function request(child, id, method, params) {
|
||||
async function run() {
|
||||
if (!command) {
|
||||
result.status = "env_issue";
|
||||
result.reason = "No uv or Python interpreter found. Set LANGBOT_REPO, LANGBOT_MCP_FIXTURE_UV, or LANGBOT_MCP_FIXTURE_PYTHON.";
|
||||
result.reason = "No Python interpreter found. Set LANGBOT_REPO or LANGBOT_MCP_FIXTURE_PYTHON.";
|
||||
return;
|
||||
}
|
||||
if (!existsSync(fixturePath)) {
|
||||
|
||||
@@ -6,6 +6,7 @@ import { resolve } from "node:path";
|
||||
import { env } from "node:process";
|
||||
import {
|
||||
createBrowser,
|
||||
ensureBrowserWorkspace,
|
||||
ensureEvidence,
|
||||
evidencePaths,
|
||||
exitCode,
|
||||
@@ -91,6 +92,12 @@ async function run() {
|
||||
const { page } = browser;
|
||||
await page.goto(env.LANGBOT_FRONTEND_URL, { waitUntil: "domcontentloaded" });
|
||||
await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {});
|
||||
const workspace = await ensureBrowserWorkspace(page, backendUrl);
|
||||
if (workspace.status !== "pass") {
|
||||
result.status = workspace.status;
|
||||
result.reason = workspace.reason;
|
||||
return;
|
||||
}
|
||||
|
||||
const diagnostic = await page.evaluate(async ({
|
||||
backendUrl,
|
||||
@@ -101,6 +108,7 @@ async function run() {
|
||||
fixtureArgs,
|
||||
startupTimeoutSec,
|
||||
readyTimeoutMs,
|
||||
workspaceUuid,
|
||||
}) => {
|
||||
const token = localStorage.getItem("token");
|
||||
if (!token) {
|
||||
@@ -120,6 +128,7 @@ async function run() {
|
||||
const headers = {
|
||||
Authorization: `Bearer ${token}`,
|
||||
"Content-Type": "application/json",
|
||||
"X-Workspace-Id": workspaceUuid,
|
||||
};
|
||||
const serverConfig = {
|
||||
name: serverName,
|
||||
@@ -193,7 +202,17 @@ async function run() {
|
||||
runtime_tool_count: lastRuntime?.tool_count ?? null,
|
||||
runtime_error: lastRuntime?.error_message || "",
|
||||
};
|
||||
}, { backendUrl, serverName, expectedTool, fixturePath, fixtureCommand, fixtureArgs, startupTimeoutSec, readyTimeoutMs });
|
||||
}, {
|
||||
backendUrl,
|
||||
serverName,
|
||||
expectedTool,
|
||||
fixturePath,
|
||||
fixtureCommand,
|
||||
fixtureArgs,
|
||||
startupTimeoutSec,
|
||||
readyTimeoutMs,
|
||||
workspaceUuid: workspace.workspace_uuid,
|
||||
});
|
||||
|
||||
await writeFile(apiDiagnosticPath, `${JSON.stringify(diagnostic, null, 2)}\n`, "utf8");
|
||||
await safeScreenshot(page, paths.screenshot);
|
||||
|
||||
@@ -401,6 +401,7 @@ async function inspectAndPatchPipelineConfig(page, {
|
||||
const headers = {
|
||||
Authorization: `Bearer ${token}`,
|
||||
"Content-Type": "application/json",
|
||||
"X-Workspace-Id": localStorage.getItem("langbot_active_workspace_uuid") || "",
|
||||
};
|
||||
const getJson = async (path) => {
|
||||
const response = await fetch(`${backendUrl}${path}`, { headers });
|
||||
@@ -607,6 +608,7 @@ async function restorePipelineConfig(page, { backendUrl, pipelineId, config }) {
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
"Content-Type": "application/json",
|
||||
"X-Workspace-Id": localStorage.getItem("langbot_active_workspace_uuid") || "",
|
||||
},
|
||||
body: JSON.stringify({ config }),
|
||||
});
|
||||
@@ -645,6 +647,7 @@ async function inspectAndPatchPipelineExtensions(page, {
|
||||
const headers = {
|
||||
Authorization: `Bearer ${token}`,
|
||||
"Content-Type": "application/json",
|
||||
"X-Workspace-Id": localStorage.getItem("langbot_active_workspace_uuid") || "",
|
||||
};
|
||||
const getJson = async (path) => {
|
||||
const response = await fetch(`${backendUrl}${path}`, { headers });
|
||||
@@ -800,6 +803,7 @@ async function restorePipelineExtensions(page, { backendUrl, pipelineId, extensi
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
"Content-Type": "application/json",
|
||||
"X-Workspace-Id": localStorage.getItem("langbot_active_workspace_uuid") || "",
|
||||
},
|
||||
body: JSON.stringify(extensions),
|
||||
});
|
||||
@@ -836,6 +840,7 @@ async function resetPipelineDebugChat(page, { backendUrl, pipelineId, sessionTyp
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
"Content-Type": "application/json",
|
||||
"X-Workspace-Id": localStorage.getItem("langbot_active_workspace_uuid") || "",
|
||||
},
|
||||
},
|
||||
);
|
||||
@@ -965,7 +970,8 @@ try {
|
||||
result.status = resetDiagnostic.status;
|
||||
result.reason = resetDiagnostic.reason || "Debug Chat reset failed.";
|
||||
} else {
|
||||
await page.waitForTimeout(1000);
|
||||
await page.reload({ waitUntil: "commit" });
|
||||
await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {});
|
||||
const reopenResult = await openPipelineDebugChat(page, {
|
||||
pipelineUrl,
|
||||
pipelineName,
|
||||
|
||||
@@ -1,17 +1,23 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { cp, mkdir, rm } from "node:fs/promises";
|
||||
import { cp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { env } from "node:process";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import {
|
||||
apiJson,
|
||||
boxWorkspaceNamespace,
|
||||
ensureEvidence,
|
||||
evidencePaths,
|
||||
loadEnvFiles,
|
||||
resetAndAuthLocalUser,
|
||||
resolveWorkspaceUuid,
|
||||
writeResult,
|
||||
} from "./lib/langbot-e2e.mjs";
|
||||
|
||||
const DEFAULT_LOCAL_PASSWORD = "LangBotE2ELocalPass!2026";
|
||||
|
||||
await loadEnvFiles();
|
||||
const paths = evidencePaths("reset-complex-agent-task");
|
||||
await ensureEvidence(paths);
|
||||
@@ -22,7 +28,7 @@ const source = resolve(
|
||||
"../../skills/langbot-testing/fixtures/complex-agent-task/workspace",
|
||||
);
|
||||
const repo = env.LANGBOT_REPO || "";
|
||||
const target = repo ? resolve(repo, "data/box/default/order-orchestrator") : "";
|
||||
let target = "";
|
||||
const result = {
|
||||
source: "setup_automation",
|
||||
case_id: "reset-complex-agent-task",
|
||||
@@ -36,6 +42,32 @@ const result = {
|
||||
|
||||
try {
|
||||
if (!repo) throw new Error("LANGBOT_REPO is required.");
|
||||
const backendUrl = env.LANGBOT_BACKEND_URL || "";
|
||||
const user = env.LANGBOT_E2E_LOGIN_USER || "";
|
||||
const password = env.LANGBOT_E2E_LOGIN_PASSWORD || DEFAULT_LOCAL_PASSWORD;
|
||||
if (!backendUrl) throw new Error("LANGBOT_BACKEND_URL is required.");
|
||||
if (!user) throw new Error("LANGBOT_E2E_LOGIN_USER is required.");
|
||||
|
||||
const auth = await resetAndAuthLocalUser({ backendUrl, user, password });
|
||||
const workspaceUuid = await resolveWorkspaceUuid(backendUrl, auth.token);
|
||||
const bootstrap = await apiJson(backendUrl, "/api/v1/workspaces/bootstrap", {
|
||||
token: auth.token,
|
||||
skipWorkspace: true,
|
||||
});
|
||||
const workspaceAccess = (bootstrap.json.data?.workspaces || []).find(
|
||||
(entry) => (entry.workspace?.uuid || entry.uuid) === workspaceUuid,
|
||||
);
|
||||
const instanceUuid = workspaceAccess?.workspace?.instance_uuid || workspaceAccess?.instance_uuid || "";
|
||||
if (!instanceUuid) throw new Error("Workspace bootstrap did not include instance_uuid.");
|
||||
|
||||
const boxWorkspace = env.LANGBOT_BOX_WORKSPACE_HOST_PATH || resolve(
|
||||
repo,
|
||||
"data/box/default/tenants",
|
||||
boxWorkspaceNamespace(instanceUuid, workspaceUuid),
|
||||
);
|
||||
target = resolve(boxWorkspace, "order-orchestrator");
|
||||
result.target = target;
|
||||
result.workspace_uuid = workspaceUuid;
|
||||
await rm(target, { recursive: true, force: true });
|
||||
await mkdir(dirname(target), { recursive: true });
|
||||
await cp(source, target, { recursive: true });
|
||||
@@ -50,6 +82,10 @@ try {
|
||||
if (baseline.error) throw baseline.error;
|
||||
if (baseline.status === 0) throw new Error("Complex task baseline unexpectedly passes; the fixture must start failing.");
|
||||
|
||||
await upsertEnvLocal(resolve("skills/.env.local"), {
|
||||
LANGBOT_COMPLEX_AGENT_WORKSPACE: target,
|
||||
});
|
||||
|
||||
result.status = "pass";
|
||||
result.reason = "Complex task workspace reset and failing baseline confirmed.";
|
||||
} catch (error) {
|
||||
@@ -60,3 +96,20 @@ try {
|
||||
await writeResult(paths, result);
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1);
|
||||
|
||||
async function upsertEnvLocal(path, updates) {
|
||||
let content = "";
|
||||
try {
|
||||
content = await readFile(path, "utf8");
|
||||
} catch (error) {
|
||||
if (error.code !== "ENOENT") throw error;
|
||||
}
|
||||
const lines = content ? content.split(/\r?\n/) : [];
|
||||
for (const [key, value] of Object.entries(updates)) {
|
||||
const replacement = `${key}=${value}`;
|
||||
const index = lines.findIndex((line) => line.startsWith(`${key}=`));
|
||||
if (index >= 0) lines[index] = replacement;
|
||||
else lines.push(replacement);
|
||||
}
|
||||
await writeFile(path, `${lines.filter(Boolean).join("\n")}\n`, "utf8");
|
||||
}
|
||||
|
||||
@@ -81,6 +81,7 @@ async function api(page, path, options = {}) {
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
"Content-Type": "application/json",
|
||||
"X-Workspace-Id": localStorage.getItem("langbot_active_workspace_uuid") || "",
|
||||
},
|
||||
body:
|
||||
options.body === undefined ? undefined : JSON.stringify(options.body),
|
||||
|
||||
@@ -156,6 +156,8 @@
|
||||
"agent-runner-runtime-chaos",
|
||||
"bot-event-routing-product-flow",
|
||||
"box-mcp-heartbeat-recovery",
|
||||
"claude-code-agent-debug-chat",
|
||||
"codex-agent-debug-chat",
|
||||
"dify-agent-debug-chat",
|
||||
"langbot-fake-provider-debug-chat-cross-pipeline-isolation",
|
||||
"langbot-fake-provider-debug-chat-fault-recovery",
|
||||
@@ -601,6 +603,58 @@
|
||||
"filesystem"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "claude-code-agent-debug-chat",
|
||||
"title": "Claude Code Agent can retrieve LangBot history through MCP",
|
||||
"mode": "agent-browser",
|
||||
"area": "pipeline",
|
||||
"type": "regression",
|
||||
"priority": "p2",
|
||||
"risk": "high",
|
||||
"ci_eligible": false,
|
||||
"tags": [
|
||||
"agent-runner",
|
||||
"claude-code",
|
||||
"external-runner",
|
||||
"mcp",
|
||||
"pipeline"
|
||||
],
|
||||
"automation": "scripts/e2e/pipeline-debug-chat.mjs",
|
||||
"setup_automation": [],
|
||||
"setup_provides_env": [],
|
||||
"evidence_required": [
|
||||
"ui",
|
||||
"screenshot",
|
||||
"console",
|
||||
"backend_log"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "codex-agent-debug-chat",
|
||||
"title": "Codex Agent can retrieve LangBot history through MCP",
|
||||
"mode": "agent-browser",
|
||||
"area": "pipeline",
|
||||
"type": "regression",
|
||||
"priority": "p2",
|
||||
"risk": "high",
|
||||
"ci_eligible": false,
|
||||
"tags": [
|
||||
"agent-runner",
|
||||
"codex",
|
||||
"external-runner",
|
||||
"mcp",
|
||||
"pipeline"
|
||||
],
|
||||
"automation": "scripts/e2e/pipeline-debug-chat.mjs",
|
||||
"setup_automation": [],
|
||||
"setup_provides_env": [],
|
||||
"evidence_required": [
|
||||
"ui",
|
||||
"screenshot",
|
||||
"console",
|
||||
"backend_log"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "dify-agent-debug-chat",
|
||||
"title": "Dify AgentRunner returns a response through Pipeline Debug Chat",
|
||||
@@ -1097,7 +1151,8 @@
|
||||
],
|
||||
"setup_provides_env": [
|
||||
"LANGBOT_LOCAL_AGENT_PIPELINE_URL",
|
||||
"LANGBOT_LOCAL_AGENT_PIPELINE_NAME"
|
||||
"LANGBOT_LOCAL_AGENT_PIPELINE_NAME",
|
||||
"LANGBOT_COMPLEX_AGENT_WORKSPACE"
|
||||
],
|
||||
"evidence_required": [
|
||||
"ui",
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
id: claude-code-agent-debug-chat
|
||||
title: "Claude Code Agent can retrieve LangBot history through MCP"
|
||||
mode: agent-browser
|
||||
area: pipeline
|
||||
type: regression
|
||||
priority: p2
|
||||
risk: high
|
||||
ci_eligible: false
|
||||
tags:
|
||||
- agent-runner
|
||||
- claude-code
|
||||
- external-runner
|
||||
- mcp
|
||||
- pipeline
|
||||
skills:
|
||||
- langbot-env-setup
|
||||
- langbot-testing
|
||||
env:
|
||||
- LANGBOT_FRONTEND_URL
|
||||
- LANGBOT_BACKEND_URL
|
||||
env_any:
|
||||
- LANGBOT_CLAUDE_CODE_AGENT_PIPELINE_URL|LANGBOT_CLAUDE_CODE_AGENT_PIPELINE_NAME
|
||||
automation: scripts/e2e/pipeline-debug-chat.mjs
|
||||
automation_env:
|
||||
- LANGBOT_FRONTEND_URL
|
||||
- LANGBOT_BACKEND_URL
|
||||
- LANGBOT_BROWSER_PROFILE
|
||||
- LANGBOT_CHROMIUM_EXECUTABLE
|
||||
- LANGBOT_CLAUDE_CODE_AGENT_PIPELINE_URL
|
||||
- LANGBOT_CLAUDE_CODE_AGENT_PIPELINE_NAME
|
||||
automation_pipeline_url_env: LANGBOT_CLAUDE_CODE_AGENT_PIPELINE_URL
|
||||
automation_pipeline_name_env: LANGBOT_CLAUDE_CODE_AGENT_PIPELINE_NAME
|
||||
automation_expected_runner_id: "plugin:langbot-team/ClaudeCodeAgent/default"
|
||||
automation_reset_debug_chat: "1"
|
||||
automation_prompts_json: '[{"prompt":"Remember this exact passcode for the LangBot conversation: claude-history-proof-6384. Do not repeat the passcode. Reply exactly CLAUDE_HISTORY_SEEDED with no other text.","expected_text":"CLAUDE_HISTORY_SEEDED","response_timeout_ms":"600000"},{"prompt":"This CLI turn has no reused Claude session. Recover the passcode from LangBot history by calling mcp__langbot_agent__langbot_history_page exactly once. If ToolSearch is available here, first load the tool with select:mcp__langbot_agent__langbot_history_page and call it directly. If this is a coordinator session that exposes only delegation tools, delegate to exactly one worker and require that worker to perform the same ToolSearch-then-call flow, then wait for its result. Use only the returned LangBot history. If the MCP call fails, do not guess or claim success. Reply with only the recovered passcode.","expected_text":"claude-history-proof-6384","response_timeout_ms":"600000"}]'
|
||||
automation_expected_text: "claude-history-proof-6384"
|
||||
automation_response_timeout_ms: "600000"
|
||||
preconditions:
|
||||
- "Claude Code CLI is installed, authenticated, and runnable non-interactively by the LangBot plugin process."
|
||||
- "The configured workspace exists and is writable by the Claude Code CLI."
|
||||
- "The pipeline has reuse-session=false so the second turn cannot use Claude CLI session memory."
|
||||
steps:
|
||||
- "Open LANGBOT_FRONTEND_URL."
|
||||
- "Open the Claude Code Agent QA pipeline."
|
||||
- "Confirm the pipeline runner is plugin:langbot-team/ClaudeCodeAgent/default."
|
||||
- "Open Debug Chat, seed the passcode, then ask a fresh Claude CLI turn to recover it through mcp__langbot_agent__langbot_history_page."
|
||||
checks:
|
||||
- "UI: The first Bot message contains CLAUDE_HISTORY_SEEDED and the second contains claude-history-proof-6384."
|
||||
- "Runner evidence: The second run records a real LangBot history MCP tool call in the main session or one delegated worker, not only model text."
|
||||
- "Logs: The request completes without claude_code process, MCP, approval, or runtime errors."
|
||||
- "Console: No unexpected frontend errors appear during Debug Chat."
|
||||
evidence_required:
|
||||
- ui
|
||||
- screenshot
|
||||
- console
|
||||
- backend_log
|
||||
success_patterns:
|
||||
- "CLAUDE_HISTORY_SEEDED"
|
||||
- "claude-history-proof-6384"
|
||||
- "Streaming completed"
|
||||
failure_patterns:
|
||||
- "claude_code.command_not_found"
|
||||
- "claude_code.process_exited"
|
||||
- "not available among the connected MCP tools"
|
||||
- "not accessible in this session"
|
||||
- "No such tool available"
|
||||
- "Agent runner not found"
|
||||
- "Agent runner plugin:langbot-team/ClaudeCodeAgent/default execution failed"
|
||||
troubleshooting:
|
||||
- backend-not-listening
|
||||
- plugin-runtime-timeout
|
||||
- proxy-env-mismatch
|
||||
@@ -0,0 +1,71 @@
|
||||
id: codex-agent-debug-chat
|
||||
title: "Codex Agent can retrieve LangBot history through MCP"
|
||||
mode: agent-browser
|
||||
area: pipeline
|
||||
type: regression
|
||||
priority: p2
|
||||
risk: high
|
||||
ci_eligible: false
|
||||
tags:
|
||||
- agent-runner
|
||||
- codex
|
||||
- external-runner
|
||||
- mcp
|
||||
- pipeline
|
||||
skills:
|
||||
- langbot-env-setup
|
||||
- langbot-testing
|
||||
env:
|
||||
- LANGBOT_FRONTEND_URL
|
||||
- LANGBOT_BACKEND_URL
|
||||
env_any:
|
||||
- LANGBOT_CODEX_AGENT_PIPELINE_URL|LANGBOT_CODEX_AGENT_PIPELINE_NAME
|
||||
automation: scripts/e2e/pipeline-debug-chat.mjs
|
||||
automation_env:
|
||||
- LANGBOT_FRONTEND_URL
|
||||
- LANGBOT_BACKEND_URL
|
||||
- LANGBOT_BROWSER_PROFILE
|
||||
- LANGBOT_CHROMIUM_EXECUTABLE
|
||||
- LANGBOT_CODEX_AGENT_PIPELINE_URL
|
||||
- LANGBOT_CODEX_AGENT_PIPELINE_NAME
|
||||
automation_pipeline_url_env: LANGBOT_CODEX_AGENT_PIPELINE_URL
|
||||
automation_pipeline_name_env: LANGBOT_CODEX_AGENT_PIPELINE_NAME
|
||||
automation_expected_runner_id: "plugin:langbot-team/CodexAgent/default"
|
||||
automation_reset_debug_chat: "1"
|
||||
automation_prompts_json: '[{"prompt":"Do not launch a subagent. Remember this exact passcode for the LangBot conversation: codex-history-proof-9157. Do not repeat the passcode. Reply exactly CODEX_HISTORY_SEEDED with no other text.","expected_text":"CODEX_HISTORY_SEEDED","response_timeout_ms":"600000"},{"prompt":"Do not launch a subagent. This CLI turn has no reused Codex session. Directly call the langbot_history_page tool exposed by the langbot_agent MCP server exactly once and wait for its result. Use only the returned LangBot history to recover the passcode from the previous user turn. If the tool is unavailable, do not guess or claim success. Reply with only the recovered passcode.","expected_text":"codex-history-proof-9157","response_timeout_ms":"600000"}]'
|
||||
automation_expected_text: "codex-history-proof-9157"
|
||||
automation_response_timeout_ms: "600000"
|
||||
preconditions:
|
||||
- "Codex CLI is installed, authenticated, and runnable non-interactively by the LangBot plugin process."
|
||||
- "The configured workspace exists and is writable, with approval policy never and sandbox mode appropriate for the QA host."
|
||||
- "The pipeline has reuse-session=false so the second turn cannot use Codex CLI thread memory."
|
||||
steps:
|
||||
- "Open LANGBOT_FRONTEND_URL."
|
||||
- "Open the Codex Agent QA pipeline."
|
||||
- "Confirm the pipeline runner is plugin:langbot-team/CodexAgent/default."
|
||||
- "Open Debug Chat, seed the passcode, then ask a fresh Codex CLI turn to recover it through langbot_agent.langbot_history_page."
|
||||
checks:
|
||||
- "UI: The first Bot message contains CODEX_HISTORY_SEEDED and the second contains codex-history-proof-9157."
|
||||
- "Runner evidence: The second run records a real LangBot history MCP tool call, not only model text."
|
||||
- "Logs: The request completes without codex process, MCP, approval, sandbox, or runtime errors."
|
||||
- "Console: No unexpected frontend errors appear during Debug Chat."
|
||||
evidence_required:
|
||||
- ui
|
||||
- screenshot
|
||||
- console
|
||||
- backend_log
|
||||
success_patterns:
|
||||
- "CODEX_HISTORY_SEEDED"
|
||||
- "codex-history-proof-9157"
|
||||
- "Streaming completed"
|
||||
failure_patterns:
|
||||
- "codex.command_not_found"
|
||||
- "codex.process_exited"
|
||||
- "not available among the connected MCP tools"
|
||||
- "not accessible in this session"
|
||||
- "Agent runner not found"
|
||||
- "Agent runner plugin:langbot-team/CodexAgent/default execution failed"
|
||||
troubleshooting:
|
||||
- backend-not-listening
|
||||
- plugin-runtime-timeout
|
||||
- proxy-env-mismatch
|
||||
@@ -44,15 +44,16 @@ automation_prompt: "Work autonomously on the project at /workspace/order-orchest
|
||||
automation_expected_text: "COMPLEX_AGENT_TASK_OK tests=12 acceptance=PASS"
|
||||
automation_response_timeout_ms: "900000"
|
||||
automation_stream_output: "1"
|
||||
automation_filesystem_checks_json: '[{"argv":["python3","skills/skills/langbot-testing/fixtures/complex-agent-task/verify.py","--workspace","${LANGBOT_REPO}/data/box/default/order-orchestrator"],"cwd":"${LANGBOT_REPO}","stdout_contains":"HOST_VERIFY_PASS tests=12 acceptance=PASS protected=PASS","exit_code":0,"timeout_ms":120000}]'
|
||||
automation_filesystem_checks_json: '[{"argv":["python3","skills/skills/langbot-testing/fixtures/complex-agent-task/verify.py","--workspace","${LANGBOT_COMPLEX_AGENT_WORKSPACE}"],"cwd":"${LANGBOT_REPO}","stdout_contains":"HOST_VERIFY_PASS tests=12 acceptance=PASS protected=PASS","exit_code":0,"timeout_ms":120000}]'
|
||||
setup_automation:
|
||||
- "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env"
|
||||
- "node:scripts/e2e/reset-complex-agent-task.mjs"
|
||||
setup_provides_env:
|
||||
- LANGBOT_LOCAL_AGENT_PIPELINE_URL
|
||||
- LANGBOT_LOCAL_AGENT_PIPELINE_NAME
|
||||
- LANGBOT_COMPLEX_AGENT_WORKSPACE
|
||||
preconditions:
|
||||
- "The local Box workspace is mounted at LANGBOT_REPO/data/box/default and exposed to native tools as /workspace."
|
||||
- "The active tenant Box workspace is available on the host and exposed to native tools as /workspace."
|
||||
- "The selected real model supports sustained function calling across file inspection, edits, and test reruns."
|
||||
steps:
|
||||
- "Reset the isolated order-orchestrator workspace and confirm its baseline test suite fails."
|
||||
|
||||
Vendored
BIN
Binary file not shown.
@@ -59,16 +59,56 @@ import {
|
||||
waitForDebugChatReady,
|
||||
} from "../scripts/e2e/lib/debug-chat.mjs";
|
||||
import {
|
||||
apiJson,
|
||||
beginBackendLogCapture,
|
||||
boxWorkspaceNamespace,
|
||||
clickFirstVisible,
|
||||
ensureAuthenticatedBrowser,
|
||||
finishBackendLogCapture,
|
||||
isTaskComplete,
|
||||
isTaskFailed,
|
||||
resolveLangBotRepo,
|
||||
scanBrowserDiagnostics,
|
||||
} from "../scripts/e2e/lib/langbot-e2e.mjs";
|
||||
|
||||
const root = process.cwd();
|
||||
|
||||
test("Box workspace namespace matches the SDK tenancy contract", () => {
|
||||
assert.equal(
|
||||
boxWorkspaceNamespace(
|
||||
"instance_8e83c14e-db31-4746-97f4-2dc165136a18",
|
||||
"29907f84-54a8-4da9-903f-04d84ff8793a",
|
||||
),
|
||||
"ws-8410680088facf21327b7542",
|
||||
);
|
||||
});
|
||||
|
||||
test("e2e task helpers reject finished tasks with runtime exceptions", () => {
|
||||
const task = {
|
||||
runtime: {
|
||||
done: true,
|
||||
state: "FINISHED",
|
||||
exception: "ValueError: plugin not found",
|
||||
},
|
||||
};
|
||||
|
||||
assert.equal(isTaskFailed(task), true);
|
||||
assert.equal(isTaskComplete(task), false);
|
||||
});
|
||||
|
||||
test("e2e task helpers accept successful finished tasks", () => {
|
||||
const task = {
|
||||
runtime: {
|
||||
done: true,
|
||||
state: "FINISHED",
|
||||
exception: null,
|
||||
},
|
||||
};
|
||||
|
||||
assert.equal(isTaskFailed(task), false);
|
||||
assert.equal(isTaskComplete(task), true);
|
||||
});
|
||||
|
||||
test("clickFirstVisible waits for a later visible DOM match", async () => {
|
||||
let pollCount = 0;
|
||||
let clickedIndex = -1;
|
||||
@@ -218,19 +258,36 @@ test("e2e helper can inject a local login token into a fresh browser context", a
|
||||
{ status: auth === "Bearer fresh-token" ? 200 : 401 },
|
||||
);
|
||||
}
|
||||
if (url.endsWith("/api/v1/workspaces/bootstrap")) {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
code: 0,
|
||||
data: { workspaces: [{ workspace: { uuid: "workspace-test" } }] },
|
||||
}),
|
||||
{ status: 200 },
|
||||
);
|
||||
}
|
||||
return new Response(JSON.stringify({ code: -1, msg: "unexpected" }), {
|
||||
status: 404,
|
||||
});
|
||||
}) as typeof fetch;
|
||||
|
||||
let token = "";
|
||||
let workspaceUuid = "";
|
||||
const page = {
|
||||
addInitScript: async () => {},
|
||||
goto: async () => {},
|
||||
evaluate: async (fn: unknown, arg: string) => {
|
||||
evaluate: async (fn: unknown, arg: unknown) => {
|
||||
const text = String(fn);
|
||||
if (text.includes("workspaceUuid: localStorage.getItem")) {
|
||||
return { token, workspaceUuid };
|
||||
}
|
||||
if (text.includes("workspaceUuid: value")) {
|
||||
workspaceUuid = (arg as { workspaceUuid: string }).workspaceUuid;
|
||||
return undefined;
|
||||
}
|
||||
if (text.includes("localStorage.setItem")) {
|
||||
token = arg;
|
||||
token = String(arg);
|
||||
return undefined;
|
||||
}
|
||||
if (text.includes("localStorage.getItem")) {
|
||||
@@ -263,6 +320,7 @@ test("e2e helper can inject a local login token into a fresh browser context", a
|
||||
assert.equal(result.status, "pass");
|
||||
assert.equal(result.injected, true);
|
||||
assert.equal(token, "fresh-token");
|
||||
assert.equal(workspaceUuid, "workspace-test");
|
||||
} finally {
|
||||
globalThis.fetch = previousFetch;
|
||||
if (previousRepo === undefined) delete process.env.LANGBOT_REPO;
|
||||
@@ -271,6 +329,43 @@ test("e2e helper can inject a local login token into a fresh browser context", a
|
||||
}
|
||||
});
|
||||
|
||||
test("apiJson bootstraps and sends the selected Workspace for scoped APIs", async () => {
|
||||
const previousFetch = globalThis.fetch;
|
||||
const requests: Array<{ url: string; headers: Record<string, string> }> = [];
|
||||
try {
|
||||
globalThis.fetch = (async (url: string, init?: RequestInit) => {
|
||||
const headers = (init?.headers || {}) as Record<string, string>;
|
||||
requests.push({ url, headers });
|
||||
if (url.endsWith("/api/v1/workspaces/bootstrap")) {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
code: 0,
|
||||
data: { workspaces: [{ workspace: { uuid: "workspace-api-test" } }] },
|
||||
}),
|
||||
{ status: 200 },
|
||||
);
|
||||
}
|
||||
return new Response(JSON.stringify({ code: 0, data: { tools: [] } }), {
|
||||
status: 200,
|
||||
});
|
||||
}) as typeof fetch;
|
||||
|
||||
const response = await apiJson(
|
||||
"http://127.0.0.1:5300",
|
||||
"/api/v1/tools",
|
||||
{ token: "workspace-api-token" },
|
||||
);
|
||||
|
||||
assert.equal(response.status, 200);
|
||||
assert.equal(requests.length, 2);
|
||||
assert.equal(requests[0].url, "http://127.0.0.1:5300/api/v1/workspaces/bootstrap");
|
||||
assert.equal(requests[0].headers["X-Workspace-Id"], undefined);
|
||||
assert.equal(requests[1].headers["X-Workspace-Id"], "workspace-api-test");
|
||||
} finally {
|
||||
globalThis.fetch = previousFetch;
|
||||
}
|
||||
});
|
||||
|
||||
function ctx(args: string[]): CommandContext {
|
||||
return { root, args };
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user