Files
LangBot/skills/scripts/e2e/lib/debug-chat.mjs
T
2026-08-01 09:23:41 +08:00

663 lines
25 KiB
JavaScript
Executable File

import {
bodyText,
clickFirstVisible,
clickFirstVisibleLocator,
countOccurrences,
gotoFrontend,
isLoginUrl,
} from "./langbot-e2e.mjs";
export const DEBUG_CHAT_FAILURE_SIGNALS = [
"Agent runner temporarily unavailable",
"All models failed during streaming setup",
"调用超时",
"超时",
];
export function minExpectedOccurrences(beforeText, expectedText, prompt) {
const beforeCount = countOccurrences(beforeText, expectedText);
return beforeCount + (String(prompt).includes(expectedText) ? 2 : 1);
}
export function latestExpectedLeafMatches(latestExpectedLeaf, prompt) {
return Boolean(latestExpectedLeaf)
&& latestExpectedLeaf !== prompt
&& !String(latestExpectedLeaf).includes(prompt);
}
export function findNewFailureSignal(beforeText, afterText, failureSignals = DEBUG_CHAT_FAILURE_SIGNALS) {
return failureSignals.find((signal) => countOccurrences(afterText, signal) > countOccurrences(beforeText, signal)) || "";
}
export function hasDebugChatOutcome(text, expectedText, minExpectedCount, failureBaselines = []) {
if (countOccurrences(text, expectedText) >= minExpectedCount) return true;
return failureBaselines.some(({ signal, count }) => countOccurrences(text, signal) > count);
}
function findFailureSignalInText(text, failureSignals = DEBUG_CHAT_FAILURE_SIGNALS) {
return failureSignals.find((signal) => String(text || "").includes(signal)) || "";
}
function countExpectedInMessages(messages, expectedText) {
return messages
.filter((message) => message.role === "assistant")
.reduce((count, message) => count + countOccurrences(message.text, expectedText), 0);
}
function debugChatInput(page) {
return page
.locator('input[placeholder*="message"], input[placeholder*="消息"], textarea[placeholder*="message"], textarea[placeholder*="消息"]')
.last();
}
async function clickDebugChatTab(page) {
const label = /^(?:Debug Chat|调试聊天|调试对话|对话调试)$/i;
const configuredTimeout = Number.parseInt(
process.env.LANGBOT_E2E_UI_READY_TIMEOUT_MS
|| process.env.LANGBOT_E2E_NAVIGATION_TIMEOUT_MS
|| "30000",
10,
);
const timeout = Number.isFinite(configuredTimeout) && configuredTimeout > 0
? configuredTimeout
: 30_000;
return await clickFirstVisibleLocator(page, [
page.getByRole("tab", { name: label }),
page.locator('[data-slot="tabs-trigger"]').filter({ hasText: label }),
page.getByText(label, { exact: true }),
], timeout);
}
export async function waitForDebugChatReady(page, timeout = 20_000) {
const input = debugChatInput(page);
const visible = await input.isVisible({ timeout }).catch(() => false);
if (!visible) {
return {
ready: false,
reason: "Debug Chat tab was clicked, but the Debug Chat input did not become visible.",
};
}
const deadline = Date.now() + timeout;
let enabled = false;
while (Date.now() < deadline) {
enabled = await input.isEnabled().catch(() => false);
if (enabled) break;
await page.waitForTimeout(Math.min(250, Math.max(1, deadline - Date.now())));
}
if (!enabled) {
return {
ready: false,
reason: "Debug Chat input is visible but disabled; WebSocket may not be connected.",
};
}
return { ready: true, reason: "" };
}
export function classifyDebugChatResult({
beforeText,
afterText,
expectedText,
expectedTexts = null,
prompt,
latestExpectedLeaf,
latestFailureLeaf,
beforeMessages = null,
afterMessages = null,
latestAssistantText = "",
latestAssistantIsFinal = null,
maxNewAssistantMessages = null,
failureSignals = DEBUG_CHAT_FAILURE_SIGNALS,
}) {
const requiredExpectedTexts = [...new Set(
(Array.isArray(expectedTexts) && expectedTexts.length > 0 ? expectedTexts : [expectedText])
.map(String)
.filter(Boolean),
)];
const minExpectedCount = minExpectedOccurrences(beforeText, expectedText, prompt);
const finalCount = countOccurrences(afterText, expectedText);
const failureText = findNewFailureSignal(beforeText, afterText, failureSignals);
const promptContainsExpected = String(prompt).includes(expectedText);
const hasMessageEvidence = Array.isArray(beforeMessages) && Array.isArray(afterMessages);
const beforeAssistantExpectedCount = hasMessageEvidence
? countExpectedInMessages(beforeMessages, expectedText)
: null;
const afterAssistantExpectedCount = hasMessageEvidence
? countExpectedInMessages(afterMessages, expectedText)
: null;
const beforeAssistantMessageCount = hasMessageEvidence
? beforeMessages.filter((message) => message.role === "assistant").length
: null;
const afterAssistantMessageCount = hasMessageEvidence
? afterMessages.filter((message) => message.role === "assistant").length
: null;
const newAssistantMessageCount = hasMessageEvidence
? afterAssistantMessageCount - beforeAssistantMessageCount
: null;
const assistantMessageEvidence = {
before_assistant_message_count: beforeAssistantMessageCount,
after_assistant_message_count: afterAssistantMessageCount,
new_assistant_message_count: newAssistantMessageCount,
};
if (hasMessageEvidence) {
const missingExpectedTexts = requiredExpectedTexts.filter(
(text) => !String(latestAssistantText).includes(text),
);
const latestAssistantFailure = findFailureSignalInText(latestAssistantText, failureSignals);
if (latestAssistantFailure) {
return {
status: "fail",
reason: `Debug Chat displayed a known failure signal in the latest assistant message: ${latestAssistantFailure}`,
min_expected_count: minExpectedCount,
final_count: finalCount,
failure_signal: latestAssistantFailure,
before_assistant_expected_count: beforeAssistantExpectedCount,
after_assistant_expected_count: afterAssistantExpectedCount,
...assistantMessageEvidence,
};
}
if (latestAssistantIsFinal === false) {
return {
status: "fail",
reason: "The latest assistant message contained the expected text but was not final.",
min_expected_count: minExpectedCount,
final_count: finalCount,
before_assistant_expected_count: beforeAssistantExpectedCount,
after_assistant_expected_count: afterAssistantExpectedCount,
...assistantMessageEvidence,
latest_assistant_is_final: false,
};
}
if (maxNewAssistantMessages !== null && newAssistantMessageCount > maxNewAssistantMessages) {
return {
status: "fail",
reason: `Debug Chat created ${newAssistantMessageCount} assistant messages; expected at most ${maxNewAssistantMessages}.`,
min_expected_count: minExpectedCount,
final_count: finalCount,
before_assistant_expected_count: beforeAssistantExpectedCount,
after_assistant_expected_count: afterAssistantExpectedCount,
...assistantMessageEvidence,
};
}
if (newAssistantMessageCount > 0 && missingExpectedTexts.length === 0) {
return {
status: "pass",
reason: requiredExpectedTexts.length === 1
? `Expected text appeared in a new assistant message: ${expectedText}`
: `All ${requiredExpectedTexts.length} expected text fragments appeared in a new assistant message.`,
min_expected_count: minExpectedCount,
final_count: finalCount,
before_assistant_expected_count: beforeAssistantExpectedCount,
after_assistant_expected_count: afterAssistantExpectedCount,
...assistantMessageEvidence,
missing_expected_texts: [],
};
}
if (failureText) {
return {
status: "fail",
reason: `Debug Chat displayed a known failure signal: ${failureText}`,
min_expected_count: minExpectedCount,
final_count: finalCount,
failure_signal: failureText,
before_assistant_expected_count: beforeAssistantExpectedCount,
after_assistant_expected_count: afterAssistantExpectedCount,
...assistantMessageEvidence,
};
}
return {
status: "fail",
reason: missingExpectedTexts.length > 0
? `A new assistant message was missing expected text: ${missingExpectedTexts.join(", ")}`
: `Expected text did not appear in a new assistant message. Expected assistant occurrences to increase above ${beforeAssistantExpectedCount}, saw ${afterAssistantExpectedCount}.`,
min_expected_count: minExpectedCount,
final_count: finalCount,
before_assistant_expected_count: beforeAssistantExpectedCount,
after_assistant_expected_count: afterAssistantExpectedCount,
...assistantMessageEvidence,
missing_expected_texts: missingExpectedTexts,
};
}
if (failureText) {
return {
status: "fail",
reason: `Debug Chat displayed a known failure signal: ${failureText}`,
min_expected_count: minExpectedCount,
final_count: finalCount,
failure_signal: failureText,
before_assistant_expected_count: beforeAssistantExpectedCount,
after_assistant_expected_count: afterAssistantExpectedCount,
};
}
if (latestExpectedLeafMatches(latestExpectedLeaf, prompt) && finalCount >= minExpectedCount) {
return {
status: "pass",
reason: `Expected text appeared in the latest visible response leaf: ${expectedText}`,
min_expected_count: minExpectedCount,
final_count: finalCount,
};
}
if (!promptContainsExpected && finalCount >= minExpectedCount) {
return {
status: "pass",
reason: `Expected text appeared enough times for user prompt plus bot response: ${expectedText}`,
min_expected_count: minExpectedCount,
final_count: finalCount,
};
}
return {
status: "fail",
reason: `Bot response did not appear. Expected ${minExpectedCount} occurrences of ${expectedText}, saw ${finalCount}.`,
min_expected_count: minExpectedCount,
final_count: finalCount,
};
}
export async function openPipelineDebugChat(page, { pipelineUrl, pipelineName, envHint = "LANGBOT_PIPELINE_URL or LANGBOT_PIPELINE_NAME" }) {
if (pipelineUrl) {
let alreadyAtPipeline = false;
try {
alreadyAtPipeline = new URL(page.url()).href === new URL(pipelineUrl).href;
} catch {
// Invalid URLs are handled by page.goto below.
}
if (!alreadyAtPipeline) {
await page.goto(pipelineUrl, { waitUntil: "commit" });
await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {});
}
} else {
if (!pipelineName) {
return {
opened: false,
status: "blocked",
reason: `Set ${envHint} before running pipeline-debug-chat automation.`,
};
}
await gotoFrontend(page);
if (isLoginUrl(page.url())) {
return {
opened: false,
status: "blocked",
reason: "Browser profile is not authenticated for LANGBOT_FRONTEND_URL.",
};
}
const clickedPipelines = await clickFirstVisible(page, ["Pipelines", "流水线"], 4_000);
if (!clickedPipelines) {
return { opened: false, status: "fail", reason: "Could not find Pipelines navigation." };
}
await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {});
const clickedPipeline = await clickFirstVisible(page, [pipelineName], 5_000);
if (!clickedPipeline) {
return { opened: false, status: "blocked", reason: `Could not find pipeline named ${pipelineName}.` };
}
}
if (isLoginUrl(page.url())) {
return {
opened: false,
status: "blocked",
reason: "Browser profile is not authenticated for LANGBOT_FRONTEND_URL.",
};
}
const clickedDebug = await clickDebugChatTab(page);
if (!clickedDebug) {
return { opened: false, status: "fail", reason: "Could not find the Debug Chat tab." };
}
await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {});
const ready = await waitForDebugChatReady(page);
if (!ready.ready) {
return { opened: false, status: "fail", reason: ready.reason };
}
return { opened: true };
}
export async function latestVisibleLeafText(page, needles) {
return await page.evaluate((items) => {
const isVisible = (element) => {
const style = window.getComputedStyle(element);
const rect = element.getBoundingClientRect();
return style.visibility !== "hidden"
&& style.display !== "none"
&& rect.width > 0
&& rect.height > 0;
};
const leaves = [];
for (const element of document.body.querySelectorAll("*")) {
if (!isVisible(element)) continue;
const text = element.innerText?.trim();
if (!text || text.length > 4000) continue;
const visibleChildHasText = Array.from(element.children).some((child) => (
isVisible(child) && child.innerText?.trim()
));
if (visibleChildHasText) continue;
if (!items.some((needle) => text.includes(needle))) continue;
leaves.push(text);
}
return leaves.at(-1) || "";
}, needles);
}
export async function visibleDebugChatMessages(page) {
return await page.evaluate(() => {
const isVisible = (element) => {
const style = window.getComputedStyle(element);
const rect = element.getBoundingClientRect();
return style.visibility !== "hidden"
&& style.display !== "none"
&& rect.width > 0
&& rect.height > 0;
};
const classText = (element) => String(element.getAttribute("class") || "");
return Array.from(document.querySelectorAll("div.max-w-3xl"))
.filter((element) => isVisible(element))
.map((element) => {
const row = element.parentElement;
const text = element.innerText?.trim() || "";
const isUser = classText(element).includes("user-message-bubble")
|| classText(row).includes("justify-end");
return {
role: isUser ? "user" : "assistant",
text,
};
})
.filter((message) => message.text);
});
}
export async function waitForExpectedDebugChatText(page, {
expectedText,
expectedTexts = null,
minExpectedCount,
minExpectedCounts = null,
timeoutMs,
beforeText = "",
failureSignals = DEBUG_CHAT_FAILURE_SIGNALS,
}) {
const requiredExpectedTexts = [...new Set(
(Array.isArray(expectedTexts) && expectedTexts.length > 0 ? expectedTexts : [expectedText])
.map(String)
.filter(Boolean),
)];
const expectedRequirements = requiredExpectedTexts.map((text, index) => ({
text,
min: Array.isArray(minExpectedCounts) && Number.isFinite(minExpectedCounts[index])
? minExpectedCounts[index]
: (text === expectedText ? minExpectedCount : minExpectedOccurrences(beforeText, text, "")),
}));
const failureBaselines = failureSignals.map((signal) => ({
signal,
count: countOccurrences(beforeText, signal),
}));
await page.waitForFunction(
({ requirements, failures }) => {
const text = document.body.innerText;
if (requirements.every((item) => text.split(item.text).length - 1 >= item.min)) return true;
return failures.some(({ signal, count }) => text.split(signal).length - 1 > count);
},
{ requirements: expectedRequirements, failures: failureBaselines },
{ timeout: timeoutMs },
).catch(() => {});
}
export async function waitForDebugChatTextStable(page, { timeoutMs = 5_000, quietMs = 750 } = {}) {
const startedAt = Date.now();
let lastText = await bodyText(page);
let stableSince = Date.now();
while (Date.now() - startedAt < timeoutMs) {
await page.waitForTimeout(250);
const currentText = await bodyText(page);
if (currentText !== lastText) {
lastText = currentText;
stableSince = Date.now();
continue;
}
if (Date.now() - stableSince >= quietMs) return;
}
}
async function fetchDebugChatHistory(page, { backendUrl, pipelineId, sessionType }) {
if (!backendUrl || !pipelineId || !sessionType) {
return { status: "not_required", messages: [] };
}
return await page.evaluate(async ({ backendUrl, pipelineId, sessionType }) => {
const token = localStorage.getItem("token") || "";
try {
const response = await fetch(
`${backendUrl.replace(/\/$/, "")}/api/v1/pipelines/${encodeURIComponent(pipelineId)}/ws/messages/${encodeURIComponent(sessionType)}`,
{
headers: token
? {
Authorization: `Bearer ${token}`,
"X-Workspace-Id": localStorage.getItem("langbot_active_workspace_uuid") || "",
}
: {},
},
);
const json = await response.json().catch(() => ({}));
return {
status: response.ok && json.code === 0 ? "ready" : "fail",
http_status: response.status,
code: json.code ?? null,
messages: json.data?.messages || [],
reason: response.ok && json.code === 0 ? "" : json.msg || `Debug Chat history returned HTTP ${response.status}.`,
};
} catch (error) {
return {
status: "retryable",
messages: [],
reason: `Debug Chat history request failed: ${error instanceof Error ? error.message : String(error)}`,
};
}
}, { backendUrl, pipelineId, sessionType });
}
async function waitForFinalDebugChatAssistant(page, {
backendUrl,
pipelineId,
sessionType,
beforeAssistantCount,
timeoutMs,
}) {
if (!backendUrl || !pipelineId || !sessionType) {
return { status: "not_required", latest_assistant_is_final: null };
}
const deadline = Date.now() + Math.max(1, timeoutMs);
let lastHistory = null;
while (Date.now() < deadline) {
lastHistory = await fetchDebugChatHistory(page, { backendUrl, pipelineId, sessionType });
if (lastHistory.status === "fail") return lastHistory;
if (lastHistory.status === "retryable") {
await page.waitForTimeout(Math.min(250, Math.max(1, deadline - Date.now())));
continue;
}
const assistants = lastHistory.messages.filter((message) => message.role === "assistant");
const latest = assistants.at(-1);
if (assistants.length > beforeAssistantCount && latest?.is_final === true) {
return {
status: "pass",
latest_assistant_is_final: true,
assistant_message_count: assistants.length,
};
}
await page.waitForTimeout(Math.min(250, Math.max(1, deadline - Date.now())));
}
const assistants = (lastHistory?.messages || []).filter((message) => message.role === "assistant");
return {
status: "fail",
reason: "Timed out waiting for the new assistant message to become final.",
latest_assistant_is_final: assistants.at(-1)?.is_final === true,
assistant_message_count: assistants.length,
};
}
export async function attachDebugChatImage(page, imagePath) {
if (!imagePath) return { status: "not_required", reason: "" };
const input = page.locator('input[type="file"][accept*="image"], input[type="file"]').first();
if (!await input.count()) {
return { status: "fail", reason: "Could not find a Debug Chat image upload input." };
}
const previews = page.locator('[data-debug-chat-attachment-preview="true"]');
const beforePreviewCount = await previews.count();
await input.setInputFiles(imagePath);
const previewVisible = await previews
.nth(beforePreviewCount)
.waitFor({ state: "visible", timeout: 10_000 })
.then(() => true)
.catch(() => false);
if (!previewVisible) {
return { status: "fail", reason: "Debug Chat did not show the selected image preview." };
}
return { status: "ready", reason: `Attached image fixture: ${imagePath}` };
}
export async function sendDebugChatPrompt(page, prompt, imagePath = "") {
const sentImages = page.locator('[data-debug-chat-message-image="true"]');
const beforeSentImageCount = imagePath ? await sentImages.count() : 0;
const imageResult = await attachDebugChatImage(page, imagePath);
if (imageResult.status === "fail") return imageResult;
const input = debugChatInput(page);
const inputVisible = await input.isVisible({ timeout: 5_000 }).catch(() => false);
const inputEnabled = inputVisible && await input.isEnabled({ timeout: 10_000 }).catch(() => false);
if (!inputVisible || !inputEnabled) return false;
await input.fill(prompt).catch(async () => {
await input.click();
await input.pressSequentially(prompt);
});
const clickedSend = await clickFirstVisible(page, ["Send", "发送", "提交"], 1_500);
if (!clickedSend) await page.keyboard.press("Enter");
await page.getByText(prompt, { exact: false }).last().waitFor({ state: "visible", timeout: 10_000 }).catch(() => {});
if (imagePath) {
const sentImageVisible = await sentImages
.nth(beforeSentImageCount)
.waitFor({ state: "visible", timeout: 15_000 })
.then(() => true)
.catch(() => false);
if (!sentImageVisible) {
return { status: "fail", reason: "The sent Debug Chat message did not render its image attachment." };
}
}
return true;
}
export async function runDebugChatPrompt(page, {
prompt,
expectedText,
expectedTexts = null,
responseTimeoutMs,
imagePath = "",
backendUrl = "",
pipelineId = "",
sessionType = "person",
maxNewAssistantMessages = null,
failureSignals = DEBUG_CHAT_FAILURE_SIGNALS,
}) {
const beforeText = await bodyText(page);
const beforeMessages = await visibleDebugChatMessages(page);
const beforeHistory = await fetchDebugChatHistory(page, { backendUrl, pipelineId, sessionType });
const beforeHistoryAssistantCount = beforeHistory.messages.filter((message) => message.role === "assistant").length;
const requiredExpectedTexts = [...new Set(
(Array.isArray(expectedTexts) && expectedTexts.length > 0 ? expectedTexts : [expectedText])
.map(String)
.filter(Boolean),
)];
const minExpectedCount = minExpectedOccurrences(beforeText, expectedText, prompt);
const minExpectedCounts = requiredExpectedTexts.map(
(text) => minExpectedOccurrences(beforeText, text, prompt),
);
const sent = await sendDebugChatPrompt(page, prompt, imagePath);
if (sent !== true) {
if (sent && typeof sent === "object" && typeof sent.reason === "string") return sent;
return { status: "fail", reason: "Could not find a Debug Chat text input." };
}
const responseStartedAt = Date.now();
const expectedTextPromise = waitForExpectedDebugChatText(page, {
expectedText,
expectedTexts: requiredExpectedTexts,
minExpectedCount,
minExpectedCounts,
prompt,
timeoutMs: responseTimeoutMs,
beforeText,
failureSignals,
});
const finalAssistantPromise = waitForFinalDebugChatAssistant(page, {
backendUrl,
pipelineId,
sessionType,
beforeAssistantCount: beforeHistoryAssistantCount,
timeoutMs: Math.max(1, responseTimeoutMs - (Date.now() - responseStartedAt)),
});
if (backendUrl && pipelineId && sessionType) {
await Promise.race([expectedTextPromise, finalAssistantPromise]);
} else {
await expectedTextPromise;
}
const finalAssistant = await finalAssistantPromise;
await waitForDebugChatTextStable(page);
const afterText = await bodyText(page);
const afterMessages = await visibleDebugChatMessages(page);
const latestAssistantText = afterMessages.filter((message) => message.role === "assistant").at(-1)?.text || "";
const latestExpectedLeaf = await latestVisibleLeafText(page, [expectedText]);
const failureText = findNewFailureSignal(beforeText, afterText, failureSignals);
const latestFailureLeaf = failureText ? await latestVisibleLeafText(page, [failureText]) : "";
const classified = classifyDebugChatResult({
beforeText,
afterText,
expectedText,
expectedTexts: requiredExpectedTexts,
prompt,
latestExpectedLeaf,
latestFailureLeaf,
beforeMessages,
afterMessages,
latestAssistantText,
latestAssistantIsFinal: finalAssistant.latest_assistant_is_final,
maxNewAssistantMessages,
failureSignals,
});
return {
...classified,
latest_assistant_is_final: finalAssistant.latest_assistant_is_final,
final_assistant_wait_status: finalAssistant.status,
final_assistant_wait_reason: finalAssistant.reason || "",
};
}
export async function setDebugChatStreamOutput(page, desired) {
if (desired === null || desired === undefined) return { status: "not_required", reason: "" };
const streamSwitch = page.locator('[role="switch"]').first();
if (!await streamSwitch.isVisible({ timeout: 5_000 }).catch(() => false)) {
return { status: "blocked", reason: "Debug Chat stream switch was not visible." };
}
if (!await streamSwitch.isEnabled({ timeout: 10_000 }).catch(() => false)) {
return { status: "blocked", reason: "Debug Chat stream switch was visible but disabled." };
}
const checked = (await streamSwitch.getAttribute("aria-checked").catch(() => null)) === "true";
if (checked !== desired) {
await streamSwitch.click();
await page.waitForFunction(
({ selector, expected }) => document.querySelector(selector)?.getAttribute("aria-checked") === String(expected),
{ selector: '[role="switch"]', expected: desired },
{ timeout: 5_000 },
).catch(() => {});
}
const finalChecked = (await streamSwitch.getAttribute("aria-checked").catch(() => null)) === "true";
if (finalChecked !== desired) {
return {
status: "fail",
reason: `Debug Chat stream switch did not reach requested state: ${desired ? "on" : "off"}.`,
};
}
return { status: "ready", reason: `Debug Chat stream switch is ${desired ? "on" : "off"}.` };
}