LangBot/skills/scripts/e2e/lib/debug-chat.mjs

import {
  bodyText,
  clickFirstVisible,
  countOccurrences,
  gotoFrontend,
  isLoginUrl,
} from "./langbot-e2e.mjs";

export const DEBUG_CHAT_FAILURE_SIGNALS = [
  "Agent runner temporarily unavailable",
  "All models failed during streaming setup",
  "调用超时",
  "超时",
];

export function minExpectedOccurrences(beforeText, expectedText, prompt) {
  const beforeCount = countOccurrences(beforeText, expectedText);
  return beforeCount + (String(prompt).includes(expectedText) ? 2 : 1);
}

export function latestExpectedLeafMatches(latestExpectedLeaf, prompt) {
  return Boolean(latestExpectedLeaf)
    && latestExpectedLeaf !== prompt
    && !String(latestExpectedLeaf).includes(prompt);
}

export function findNewFailureSignal(beforeText, afterText, failureSignals = DEBUG_CHAT_FAILURE_SIGNALS) {
  return failureSignals.find((signal) => countOccurrences(afterText, signal) > countOccurrences(beforeText, signal)) || "";
}

function findFailureSignalInText(text, failureSignals = DEBUG_CHAT_FAILURE_SIGNALS) {
  return failureSignals.find((signal) => String(text || "").includes(signal)) || "";
}

function countExpectedInMessages(messages, expectedText) {
  return messages
    .filter((message) => message.role === "assistant")
    .reduce((count, message) => count + countOccurrences(message.text, expectedText), 0);
}

function debugChatInput(page) {
  return page
    .locator('input[placeholder*="message"], input[placeholder*="消息"], textarea[placeholder*="message"], textarea[placeholder*="消息"]')
    .last();
}

async function clickDebugChatTab(page) {
  const tabByRole = page.getByRole("tab", { name: /Debug Chat|调试聊天|调试对话|Debug|调试/i }).first();
  if (await tabByRole.isVisible({ timeout: 3_000 }).catch(() => false)) {
    await tabByRole.click();
    return true;
  }

  const tabBySelector = page.locator('[role="tab"]').filter({ hasText: /Debug Chat|调试聊天|调试对话|Debug|调试/i }).first();
  if (await tabBySelector.isVisible({ timeout: 2_000 }).catch(() => false)) {
    await tabBySelector.click();
    return true;
  }

  return Boolean(await clickFirstVisible(page, ["Debug Chat", "调试聊天", "调试对话"], 2_000));
}

async function waitForDebugChatReady(page, timeout = 20_000) {
  const input = debugChatInput(page);
  const visible = await input.isVisible({ timeout }).catch(() => false);
  if (!visible) {
    return {
      ready: false,
      reason: "Debug Chat tab was clicked, but the Debug Chat input did not become visible.",
    };
  }

  const enabled = await input.isEnabled({ timeout }).catch(() => false);
  if (!enabled) {
    return {
      ready: false,
      reason: "Debug Chat input is visible but disabled; WebSocket may not be connected.",
    };
  }

  return { ready: true, reason: "" };
}

export function classifyDebugChatResult({
  beforeText,
  afterText,
  expectedText,
  prompt,
  latestExpectedLeaf,
  latestFailureLeaf,
  beforeMessages = null,
  afterMessages = null,
  latestAssistantText = "",
  failureSignals = DEBUG_CHAT_FAILURE_SIGNALS,
}) {
  const minExpectedCount = minExpectedOccurrences(beforeText, expectedText, prompt);
  const finalCount = countOccurrences(afterText, expectedText);
  const failureText = findNewFailureSignal(beforeText, afterText, failureSignals);
  const promptContainsExpected = String(prompt).includes(expectedText);
  const hasMessageEvidence = Array.isArray(beforeMessages) && Array.isArray(afterMessages);
  const beforeAssistantExpectedCount = hasMessageEvidence
    ? countExpectedInMessages(beforeMessages, expectedText)
    : null;
  const afterAssistantExpectedCount = hasMessageEvidence
    ? countExpectedInMessages(afterMessages, expectedText)
    : null;
  const assistantExpectedIncreased = hasMessageEvidence
    ? afterAssistantExpectedCount > beforeAssistantExpectedCount
    : false;

  if (hasMessageEvidence) {
    const latestAssistantFailure = findFailureSignalInText(latestAssistantText, failureSignals);
    if (latestAssistantFailure) {
      return {
        status: "fail",
        reason: `Debug Chat displayed a known failure signal in the latest assistant message: ${latestAssistantFailure}`,
        min_expected_count: minExpectedCount,
        final_count: finalCount,
        failure_signal: latestAssistantFailure,
        before_assistant_expected_count: beforeAssistantExpectedCount,
        after_assistant_expected_count: afterAssistantExpectedCount,
      };
    }
    if (assistantExpectedIncreased && String(latestAssistantText).includes(expectedText)) {
      return {
        status: "pass",
        reason: `Expected text appeared in a new assistant message: ${expectedText}`,
        min_expected_count: minExpectedCount,
        final_count: finalCount,
        before_assistant_expected_count: beforeAssistantExpectedCount,
        after_assistant_expected_count: afterAssistantExpectedCount,
      };
    }
    if (failureText) {
      return {
        status: "fail",
        reason: `Debug Chat displayed a known failure signal: ${failureText}`,
        min_expected_count: minExpectedCount,
        final_count: finalCount,
        failure_signal: failureText,
        before_assistant_expected_count: beforeAssistantExpectedCount,
        after_assistant_expected_count: afterAssistantExpectedCount,
      };
    }
    return {
      status: "fail",
      reason: `Expected text did not appear in a new assistant message. Expected assistant occurrences to increase above ${beforeAssistantExpectedCount}, saw ${afterAssistantExpectedCount}.`,
      min_expected_count: minExpectedCount,
      final_count: finalCount,
      before_assistant_expected_count: beforeAssistantExpectedCount,
      after_assistant_expected_count: afterAssistantExpectedCount,
    };
  }
  if (failureText) {
    return {
      status: "fail",
      reason: `Debug Chat displayed a known failure signal: ${failureText}`,
      min_expected_count: minExpectedCount,
      final_count: finalCount,
      failure_signal: failureText,
      before_assistant_expected_count: beforeAssistantExpectedCount,
      after_assistant_expected_count: afterAssistantExpectedCount,
    };
  }
  if (latestExpectedLeafMatches(latestExpectedLeaf, prompt) && finalCount >= minExpectedCount) {
    return {
      status: "pass",
      reason: `Expected text appeared in the latest visible response leaf: ${expectedText}`,
      min_expected_count: minExpectedCount,
      final_count: finalCount,
    };
  }
  if (!promptContainsExpected && finalCount >= minExpectedCount) {
    return {
      status: "pass",
      reason: `Expected text appeared enough times for user prompt plus bot response: ${expectedText}`,
      min_expected_count: minExpectedCount,
      final_count: finalCount,
    };
  }
  return {
    status: "fail",
    reason: `Bot response did not appear. Expected ${minExpectedCount} occurrences of ${expectedText}, saw ${finalCount}.`,
    min_expected_count: minExpectedCount,
    final_count: finalCount,
  };
}

export async function openPipelineDebugChat(page, { pipelineUrl, pipelineName, envHint = "LANGBOT_PIPELINE_URL or LANGBOT_PIPELINE_NAME" }) {
  if (pipelineUrl) {
    await page.goto(pipelineUrl, { waitUntil: "domcontentloaded" });
    await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {});
  } else {
    if (!pipelineName) {
      return {
        opened: false,
        status: "blocked",
        reason: `Set ${envHint} before running pipeline-debug-chat automation.`,
      };
    }
    await gotoFrontend(page);
    if (isLoginUrl(page.url())) {
      return {
        opened: false,
        status: "blocked",
        reason: "Browser profile is not authenticated for LANGBOT_FRONTEND_URL.",
      };
    }
    const clickedPipelines = await clickFirstVisible(page, ["Pipelines", "流水线"], 4_000);
    if (!clickedPipelines) {
      return { opened: false, status: "fail", reason: "Could not find Pipelines navigation." };
    }
    await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {});
    const clickedPipeline = await clickFirstVisible(page, [pipelineName], 5_000);
    if (!clickedPipeline) {
      return { opened: false, status: "blocked", reason: `Could not find pipeline named ${pipelineName}.` };
    }
  }

  if (isLoginUrl(page.url())) {
    return {
      opened: false,
      status: "blocked",
      reason: "Browser profile is not authenticated for LANGBOT_FRONTEND_URL.",
    };
  }

  const clickedDebug = await clickDebugChatTab(page);
  if (!clickedDebug) {
    return { opened: false, status: "fail", reason: "Could not find the Debug Chat tab." };
  }
  await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {});
  const ready = await waitForDebugChatReady(page);
  if (!ready.ready) {
    return { opened: false, status: "fail", reason: ready.reason };
  }
  return { opened: true };
}

export async function latestVisibleLeafText(page, needles) {
  return await page.evaluate((items) => {
    const isVisible = (element) => {
      const style = window.getComputedStyle(element);
      const rect = element.getBoundingClientRect();
      return style.visibility !== "hidden"
        && style.display !== "none"
        && rect.width > 0
        && rect.height > 0;
    };
    const leaves = [];
    for (const element of document.body.querySelectorAll("*")) {
      if (!isVisible(element)) continue;
      const text = element.innerText?.trim();
      if (!text || text.length > 4000) continue;
      const visibleChildHasText = Array.from(element.children).some((child) => (
        isVisible(child) && child.innerText?.trim()
      ));
      if (visibleChildHasText) continue;
      if (!items.some((needle) => text.includes(needle))) continue;
      leaves.push(text);
    }
    return leaves.at(-1) || "";
  }, needles);
}

export async function visibleDebugChatMessages(page) {
  return await page.evaluate(() => {
    const isVisible = (element) => {
      const style = window.getComputedStyle(element);
      const rect = element.getBoundingClientRect();
      return style.visibility !== "hidden"
        && style.display !== "none"
        && rect.width > 0
        && rect.height > 0;
    };
    const classText = (element) => String(element.getAttribute("class") || "");
    return Array.from(document.querySelectorAll("div.max-w-3xl"))
      .filter((element) => isVisible(element))
      .map((element) => {
        const row = element.parentElement;
        const text = element.innerText?.trim() || "";
        const isUser = classText(element).includes("user-message-bubble")
          || classText(row).includes("justify-end");
        return {
          role: isUser ? "user" : "assistant",
          text,
        };
      })
      .filter((message) => message.text);
  });
}

export async function waitForExpectedDebugChatText(page, { expectedText, minExpectedCount, timeoutMs }) {
  await page.waitForFunction(
    ({ expected, min }) => {
      return document.body.innerText.split(expected).length - 1 >= min;
    },
    { expected: expectedText, min: minExpectedCount },
    { timeout: timeoutMs },
  ).catch(() => {});
}

export async function waitForDebugChatTextStable(page, { timeoutMs = 5_000, quietMs = 750 } = {}) {
  const startedAt = Date.now();
  let lastText = await bodyText(page);
  let stableSince = Date.now();
  while (Date.now() - startedAt < timeoutMs) {
    await page.waitForTimeout(250);
    const currentText = await bodyText(page);
    if (currentText !== lastText) {
      lastText = currentText;
      stableSince = Date.now();
      continue;
    }
    if (Date.now() - stableSince >= quietMs) return;
  }
}

export async function attachDebugChatImage(page, imagePath) {
  if (!imagePath) return { status: "not_required", reason: "" };
  const input = page.locator('input[type="file"][accept*="image"], input[type="file"]').first();
  if (!await input.count()) {
    return { status: "fail", reason: "Could not find a Debug Chat image upload input." };
  }
  await input.setInputFiles(imagePath);
  await page.locator("img").last().waitFor({ state: "visible", timeout: 10_000 }).catch(() => {});
  return { status: "ready", reason: `Attached image fixture: ${imagePath}` };
}

export async function sendDebugChatPrompt(page, prompt, imagePath = "") {
  const imageResult = await attachDebugChatImage(page, imagePath);
  if (imageResult.status === "fail") return imageResult;

  const input = debugChatInput(page);
  const inputVisible = await input.isVisible({ timeout: 5_000 }).catch(() => false);
  const inputEnabled = inputVisible && await input.isEnabled({ timeout: 10_000 }).catch(() => false);
  if (!inputVisible || !inputEnabled) return false;
  await input.fill(prompt).catch(async () => {
    await input.click();
    await input.pressSequentially(prompt);
  });
  const clickedSend = await clickFirstVisible(page, ["Send", "发送", "提交"], 1_500);
  if (!clickedSend) await page.keyboard.press("Enter");
  await page.getByText(prompt, { exact: false }).last().waitFor({ state: "visible", timeout: 10_000 }).catch(() => {});
  return true;
}

export async function runDebugChatPrompt(page, { prompt, expectedText, responseTimeoutMs, imagePath = "", failureSignals = DEBUG_CHAT_FAILURE_SIGNALS }) {
  const beforeText = await bodyText(page);
  const beforeMessages = await visibleDebugChatMessages(page);
  const minExpectedCount = minExpectedOccurrences(beforeText, expectedText, prompt);
  const sent = await sendDebugChatPrompt(page, prompt, imagePath);
  if (sent !== true) {
    if (sent && typeof sent === "object" && typeof sent.reason === "string") return sent;
    return { status: "fail", reason: "Could not find a Debug Chat text input." };
  }

  await waitForExpectedDebugChatText(page, {
    expectedText,
    minExpectedCount,
    prompt,
    timeoutMs: responseTimeoutMs,
  });
  await waitForDebugChatTextStable(page);

  const afterText = await bodyText(page);
  const afterMessages = await visibleDebugChatMessages(page);
  const latestAssistantText = afterMessages.filter((message) => message.role === "assistant").at(-1)?.text || "";
  const latestExpectedLeaf = await latestVisibleLeafText(page, [expectedText]);
  const failureText = findNewFailureSignal(beforeText, afterText, failureSignals);
  const latestFailureLeaf = failureText ? await latestVisibleLeafText(page, [failureText]) : "";

  return classifyDebugChatResult({
    beforeText,
    afterText,
    expectedText,
    prompt,
    latestExpectedLeaf,
    latestFailureLeaf,
    beforeMessages,
    afterMessages,
    latestAssistantText,
    failureSignals,
  });
}

export async function setDebugChatStreamOutput(page, desired) {
  if (desired === null || desired === undefined) return { status: "not_required", reason: "" };

  const streamSwitch = page.locator('[role="switch"]').first();
  if (!await streamSwitch.isVisible({ timeout: 5_000 }).catch(() => false)) {
    return { status: "blocked", reason: "Debug Chat stream switch was not visible." };
  }
  if (!await streamSwitch.isEnabled({ timeout: 10_000 }).catch(() => false)) {
    return { status: "blocked", reason: "Debug Chat stream switch was visible but disabled." };
  }

  const checked = (await streamSwitch.getAttribute("aria-checked").catch(() => null)) === "true";
  if (checked !== desired) {
    await streamSwitch.click();
    await page.waitForFunction(
      ({ selector, expected }) => document.querySelector(selector)?.getAttribute("aria-checked") === String(expected),
      { selector: '[role="switch"]', expected: desired },
      { timeout: 5_000 },
    ).catch(() => {});
  }

  const finalChecked = (await streamSwitch.getAttribute("aria-checked").catch(() => null)) === "true";
  if (finalChecked !== desired) {
    return {
      status: "fail",
      reason: `Debug Chat stream switch did not reach requested state: ${desired ? "on" : "off"}.`,
    };
  }
  return { status: "ready", reason: `Debug Chat stream switch is ${desired ? "on" : "off"}.` };
}