xinyin025/LangBot
1
0
Fork 0
mirror of https://github.com/langbot-app/LangBot.git synced 2026-07-26 14:26:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(skills): harden local agent QA automation

Browse Source
This commit is contained in:
huanghuoguoguo
2026-07-26 21:29:30 +08:00
parent e32210301a
commit 4bae8728a6
7 changed files with 176 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
skills/scripts/e2e/lib/debug-chat.mjs
+54 -11
View File
@@ -65,7 +65,7 @@ async function clickDebugChatTab(page) {
return Boolean(await clickFirstVisible(page, ["Debug Chat", "调试聊天", "调试对话"], 2_000));
}
async function waitForDebugChatReady(page, timeout = 20_000) {
export async function waitForDebugChatReady(page, timeout = 20_000) {
const input = debugChatInput(page);
const visible = await input.isVisible({ timeout }).catch(() => false);
if (!visible) {
@@ -75,7 +75,13 @@ async function waitForDebugChatReady(page, timeout = 20_000) {
};
}
const enabled = await input.isEnabled({ timeout }).catch(() => false);
const deadline = Date.now() + timeout;
let enabled = false;
while (Date.now() < deadline) {
enabled = await input.isEnabled().catch(() => false);
if (enabled) break;
await page.waitForTimeout(Math.min(250, Math.max(1, deadline - Date.now())));
}
if (!enabled) {
return {
ready: false,
@@ -90,6 +96,7 @@ export function classifyDebugChatResult({
beforeText,
afterText,
expectedText,
expectedTexts = null,
prompt,
latestExpectedLeaf,
latestFailureLeaf,
@@ -99,6 +106,11 @@ export function classifyDebugChatResult({
maxNewAssistantMessages = null,
failureSignals = DEBUG_CHAT_FAILURE_SIGNALS,
}) {
const requiredExpectedTexts = [...new Set(
(Array.isArray(expectedTexts) && expectedTexts.length > 0 ? expectedTexts : [expectedText])
.map(String)
.filter(Boolean),
)];
const minExpectedCount = minExpectedOccurrences(beforeText, expectedText, prompt);
const finalCount = countOccurrences(afterText, expectedText);
const failureText = findNewFailureSignal(beforeText, afterText, failureSignals);
@@ -110,9 +122,6 @@ export function classifyDebugChatResult({
const afterAssistantExpectedCount = hasMessageEvidence
? countExpectedInMessages(afterMessages, expectedText)
: null;
const assistantExpectedIncreased = hasMessageEvidence
? afterAssistantExpectedCount > beforeAssistantExpectedCount
: false;
const beforeAssistantMessageCount = hasMessageEvidence
? beforeMessages.filter((message) => message.role === "assistant").length
: null;
@@ -129,6 +138,9 @@ export function classifyDebugChatResult({
};
if (hasMessageEvidence) {
const missingExpectedTexts = requiredExpectedTexts.filter(
(text) => !String(latestAssistantText).includes(text),
);
const latestAssistantFailure = findFailureSignalInText(latestAssistantText, failureSignals);
if (latestAssistantFailure) {
return {
@@ -153,15 +165,18 @@ export function classifyDebugChatResult({
...assistantMessageEvidence,
};
}
if (assistantExpectedIncreased && String(latestAssistantText).includes(expectedText)) {
if (newAssistantMessageCount > 0 && missingExpectedTexts.length === 0) {
return {
status: "pass",
reason: `Expected text appeared in a new assistant message: ${expectedText}`,
reason: requiredExpectedTexts.length === 1
? `Expected text appeared in a new assistant message: ${expectedText}`
: `All ${requiredExpectedTexts.length} expected text fragments appeared in a new assistant message.`,
min_expected_count: minExpectedCount,
final_count: finalCount,
before_assistant_expected_count: beforeAssistantExpectedCount,
after_assistant_expected_count: afterAssistantExpectedCount,
...assistantMessageEvidence,
missing_expected_texts: [],
};
}
if (failureText) {
@@ -178,12 +193,15 @@ export function classifyDebugChatResult({
}
return {
status: "fail",
reason: `Expected text did not appear in a new assistant message. Expected assistant occurrences to increase above ${beforeAssistantExpectedCount}, saw ${afterAssistantExpectedCount}.`,
reason: missingExpectedTexts.length > 0
? `A new assistant message was missing expected text: ${missingExpectedTexts.join(", ")}`
: `Expected text did not appear in a new assistant message. Expected assistant occurrences to increase above ${beforeAssistantExpectedCount}, saw ${afterAssistantExpectedCount}.`,
min_expected_count: minExpectedCount,
final_count: finalCount,
before_assistant_expected_count: beforeAssistantExpectedCount,
after_assistant_expected_count: afterAssistantExpectedCount,
...assistantMessageEvidence,
missing_expected_texts: missingExpectedTexts,
};
}
if (failureText) {
@@ -327,22 +345,35 @@ export async function visibleDebugChatMessages(page) {
export async function waitForExpectedDebugChatText(page, {
expectedText,
expectedTexts = null,
minExpectedCount,
minExpectedCounts = null,
timeoutMs,
beforeText = "",
failureSignals = DEBUG_CHAT_FAILURE_SIGNALS,
}) {
const requiredExpectedTexts = [...new Set(
(Array.isArray(expectedTexts) && expectedTexts.length > 0 ? expectedTexts : [expectedText])
.map(String)
.filter(Boolean),
)];
const expectedRequirements = requiredExpectedTexts.map((text, index) => ({
text,
min: Array.isArray(minExpectedCounts) && Number.isFinite(minExpectedCounts[index])
? minExpectedCounts[index]
: (text === expectedText ? minExpectedCount : minExpectedOccurrences(beforeText, text, "")),
}));
const failureBaselines = failureSignals.map((signal) => ({
signal,
count: countOccurrences(beforeText, signal),
}));
await page.waitForFunction(
({ expected, min, failures }) => {
({ requirements, failures }) => {
const text = document.body.innerText;
if (text.split(expected).length - 1 >= min) return true;
if (requirements.every((item) => text.split(item.text).length - 1 >= item.min)) return true;
return failures.some(({ signal, count }) => text.split(signal).length - 1 > count);
},
{ expected: expectedText, min: minExpectedCount, failures: failureBaselines },
{ requirements: expectedRequirements, failures: failureBaselines },
{ timeout: timeoutMs },
).catch(() => {});
}
@@ -395,6 +426,7 @@ export async function sendDebugChatPrompt(page, prompt, imagePath = "") {
export async function runDebugChatPrompt(page, {
prompt,
expectedText,
expectedTexts = null,
responseTimeoutMs,
imagePath = "",
maxNewAssistantMessages = null,
@@ -402,7 +434,15 @@ export async function runDebugChatPrompt(page, {
}) {
const beforeText = await bodyText(page);
const beforeMessages = await visibleDebugChatMessages(page);
const requiredExpectedTexts = [...new Set(
(Array.isArray(expectedTexts) && expectedTexts.length > 0 ? expectedTexts : [expectedText])
.map(String)
.filter(Boolean),
)];
const minExpectedCount = minExpectedOccurrences(beforeText, expectedText, prompt);
const minExpectedCounts = requiredExpectedTexts.map(
(text) => minExpectedOccurrences(beforeText, text, prompt),
);
const sent = await sendDebugChatPrompt(page, prompt, imagePath);
if (sent !== true) {
if (sent && typeof sent === "object" && typeof sent.reason === "string") return sent;
@@ -411,7 +451,9 @@ export async function runDebugChatPrompt(page, {
await waitForExpectedDebugChatText(page, {
expectedText,
expectedTexts: requiredExpectedTexts,
minExpectedCount,
minExpectedCounts,
prompt,
timeoutMs: responseTimeoutMs,
beforeText,
@@ -430,6 +472,7 @@ export async function runDebugChatPrompt(page, {
beforeText,
afterText,
expectedText,
expectedTexts: requiredExpectedTexts,
prompt,
latestExpectedLeaf,
latestFailureLeaf,
skills/scripts/e2e/pipeline-debug-chat.mjs
+12 -3
View File
@@ -142,22 +142,28 @@ function stats(values) {
function promptStepsFromEnv() {
const rawSteps = parseJsonEnv("LANGBOT_E2E_PROMPTS_JSON", null);
if (rawSteps === null) {
return [{ prompt, expectedText, responseTimeoutMs: safeResponseTimeoutMs }];
return [{ prompt, expectedText, expectedTexts: [expectedText], responseTimeoutMs: safeResponseTimeoutMs }];
}
if (!Array.isArray(rawSteps) || rawSteps.length === 0) {
throw new Error("LANGBOT_E2E_PROMPTS_JSON must be a non-empty JSON array.");
}
return rawSteps.map((item, index) => {
if (typeof item === "string") {
return { prompt: item, expectedText, responseTimeoutMs: safeResponseTimeoutMs };
return { prompt: item, expectedText, expectedTexts: [expectedText], responseTimeoutMs: safeResponseTimeoutMs };
}
if (!item || typeof item !== "object" || typeof item.prompt !== "string" || !item.prompt) {
throw new Error(`LANGBOT_E2E_PROMPTS_JSON[${index}] must be a string or an object with a prompt string.`);
}
const stepTimeout = Number.parseInt(String(item.response_timeout_ms || item.responseTimeoutMs || safeResponseTimeoutMs), 10);
const stepExpectedText = String(item.expected_text || item.expectedText || expectedText);
const additionalExpectedTexts = item.expected_texts || item.expectedTexts || [];
if (!Array.isArray(additionalExpectedTexts)) {
throw new Error(`LANGBOT_E2E_PROMPTS_JSON[${index}].expected_texts must be an array.`);
}
return {
prompt: item.prompt,
expectedText: String(item.expected_text || item.expectedText || expectedText),
expectedText: stepExpectedText,
expectedTexts: [...new Set([stepExpectedText, ...additionalExpectedTexts.map(String)].filter(Boolean))],
responseTimeoutMs: Number.isFinite(stepTimeout) && stepTimeout > 0 ? stepTimeout : safeResponseTimeoutMs,
};
});
@@ -990,6 +996,7 @@ try {
const chatResult = await runDebugChatPrompt(page, {
prompt: step.prompt,
expectedText: step.expectedText,
expectedTexts: step.expectedTexts,
responseTimeoutMs: step.responseTimeoutMs,
imagePath: index === 0 ? imagePath : "",
maxNewAssistantMessages: streamOutput === false ? 1 : null,
@@ -999,6 +1006,7 @@ try {
result.chat_results.push({
index,
expected_text: step.expectedText,
expected_texts: step.expectedTexts,
status: chatResult.status,
reason: chatResult.reason,
response_duration_ms: promptDurationMs,
@@ -1010,6 +1018,7 @@ try {
after_assistant_message_count: chatResult.after_assistant_message_count,
new_assistant_message_count: chatResult.new_assistant_message_count,
failure_signal: chatResult.failure_signal || "",
missing_expected_texts: chatResult.missing_expected_texts || [],
});
result.status = chatResult.status;
result.reason = `Prompt ${index + 1}/${promptSteps.length}: ${chatResult.reason}`;