fix(agent): harden runner delivery validation

2026-08-09 20:50:58 +00:00 · 2026-07-27 16:01:57 +08:00
parent 7d3b53bbce
commit 9c32ea4063
11 changed files with 403 additions and 47 deletions
@@ -57,6 +57,9 @@ try {
  const python = join(repo, ".venv", "bin", "python");
  const args = ["scripts/e2e/agent-run-ledger-audit.py", "--repo", repo, "--output", auditPath];
  if (env.LANGBOT_AGENT_RUN_ID) args.push("--run-id", env.LANGBOT_AGENT_RUN_ID);
+  if (env.LANGBOT_AGENT_TOOL_AUTHORIZATION_MODE) {
+    args.push("--tool-authorization-mode", env.LANGBOT_AGENT_TOOL_AUTHORIZATION_MODE);
+  }
  const execution = await run(python, args, process.cwd());
  const report = JSON.parse(await readFile(auditPath, "utf8"));
  result.status = report.status;
@@ -16,7 +16,12 @@ import sqlalchemy
 import yaml
 from sqlalchemy.ext.asyncio import create_async_engine

-from agent_run_ledger_policy import classify_invalid_tool_argument_errors, load_ledger_json
+from agent_run_ledger_policy import (
+    classify_invalid_tool_argument_errors,
+    classify_tool_authorization,
+    invalid_tool_argument_error_signal,
+    load_ledger_json,
+)


 def database_url(repo: pathlib.Path) -> str:
@@ -80,6 +85,7 @@ async def audit(
    expected_tool_name: str | None = None,
    expected_parameters: dict | None = None,
    expected_result_text: str | None = None,
+    tool_authorization_mode: str = "strict",
 ) -> dict:
    engine = create_async_engine(database_url(repo))
    failures: list[dict] = []
@@ -175,7 +181,6 @@ async def audit(
        r"invalid json(?! arguments)|unauthori[sz]ed|permission denied|forbidden|timed?\s*out|timeout",
        re.I,
    )
-    invalid_tool_arguments_pattern = re.compile(r"invalid json arguments", re.I)

    def error_surface(value: object) -> list[str]:
        """Collect diagnostic fields without treating normal tool parameters as errors."""
@@ -221,10 +226,10 @@ async def audit(
        if match:
            suspicious_errors.append({"sequence": row["sequence"], "type": event_type, "signal": match.group(0)})
        elif event_type == "tool.call.completed":
-            match = invalid_tool_arguments_pattern.search(diagnostic_text)
-            if match:
+            signal = invalid_tool_argument_error_signal(diagnostic_text)
+            if signal:
                invalid_tool_argument_errors.append(
-                    {"sequence": row["sequence"], "type": event_type, "signal": match.group(0)}
+                    {"sequence": row["sequence"], "type": event_type, "signal": signal}
                )

    if run_row["status"] != "completed":
@@ -248,8 +253,12 @@ async def audit(
            failures.append({"kind": "tool_call_order", "tool_call_id": call_id})
        if started[0]["tool_name"] not in allowed_tools:
            unauthorized_calls.append({"tool_call_id": call_id, "tool_name": started[0]["tool_name"]})
-    if unauthorized_calls:
-        failures.append({"kind": "unauthorized_tool_calls", "calls": unauthorized_calls})
+    authorization_failures, authorization_warnings = classify_tool_authorization(
+        unauthorized_calls,
+        authorization_mode=tool_authorization_mode,
+    )
+    failures.extend(authorization_failures)
+    warnings.extend(authorization_warnings)
    unrecovered_argument_errors, recovered_argument_warnings = classify_invalid_tool_argument_errors(
        invalid_tool_argument_errors,
        successful_tool_completion_sequences=successful_tool_completion_sequences,
@@ -305,6 +314,8 @@ async def audit(
        "tool_call_completed": sum(len(items) for items in completions.values()),
        "tool_call_ids": len(all_call_ids),
        "authorized_tool_count": len(allowed_tools),
+        "tool_authorization_mode": tool_authorization_mode,
+        "runner_native_tool_call_count": len(unauthorized_calls) if tool_authorization_mode == "runner-native" else 0,
        "invalid_event_json": invalid_event_json,
        "suspicious_error_count": len(suspicious_errors),
        "recovered_tool_argument_error_count": len(recovered_argument_warnings),
@@ -334,6 +345,11 @@ def main() -> int:
    parser.add_argument("--expected-tool-name")
    parser.add_argument("--expected-parameters-json")
    parser.add_argument("--expected-result-text")
+    parser.add_argument(
+        "--tool-authorization-mode",
+        choices=("strict", "runner-native"),
+        default="strict",
+    )
    parser.add_argument("--output", required=True)
    args = parser.parse_args()
    try:
@@ -351,6 +367,7 @@ def main() -> int:
            expected_tool_name=args.expected_tool_name,
            expected_parameters=expected_parameters,
            expected_result_text=args.expected_result_text,
+            tool_authorization_mode=args.tool_authorization_mode,
        ))
    except Exception as exc:  # noqa: BLE001 - probe must classify environment failures
        report = {"status": "env_issue", "reason": str(exc), "failures": [], "warnings": []}
@@ -3,6 +3,13 @@
 from __future__ import annotations

 import json
+import re
+
+
+_INVALID_TOOL_ARGUMENT_PATTERN = re.compile(
+    r"invalid json arguments|\b\d+\s+validation errors?\s+for\s+[A-Za-z_][A-Za-z0-9_]*Args\b",
+    re.IGNORECASE,
+)


 def load_ledger_json(value: str | None, *, field: str, failures: list[dict]) -> object:
@@ -16,6 +23,12 @@ def load_ledger_json(value: str | None, *, field: str, failures: list[dict]) ->
        return {}


+def invalid_tool_argument_error_signal(value: str) -> str:
+    """Return the persisted signal for malformed model-supplied tool arguments."""
+    match = _INVALID_TOOL_ARGUMENT_PATTERN.search(value)
+    return match.group(0) if match else ""
+
+
 def classify_invalid_tool_argument_errors(
    events: list[dict],
    *,
@@ -41,3 +54,25 @@ def classify_invalid_tool_argument_errors(
        else:
            failures.append(event)
    return failures, warnings
+
+
+def classify_tool_authorization(
+    calls: list[dict],
+    *,
+    authorization_mode: str,
+) -> tuple[list[dict], list[dict]]:
+    """Classify tool names absent from the Host authorization snapshot."""
+    if not calls:
+        return [], []
+    if authorization_mode == "runner-native":
+        return [], [
+            {
+                "kind": "runner_native_tool_calls",
+                "calls": calls,
+                "reason": (
+                    "External runner tool telemetry is not a LangBot Host tool call; "
+                    "the runner's own permission system governs it."
+                ),
+            }
+        ]
+    return [{"kind": "unauthorized_tool_calls", "calls": calls}], []
@@ -1,6 +1,7 @@
 import {
  bodyText,
  clickFirstVisible,
+  clickFirstVisibleLocator,
  countOccurrences,
  gotoFrontend,
  isLoginUrl,
@@ -50,19 +51,21 @@ function debugChatInput(page) {
 }

 async function clickDebugChatTab(page) {
-  const tabByRole = page.getByRole("tab", { name: /Debug Chat|调试聊天|调试对话|Debug|调试/i }).first();
-  if (await tabByRole.isVisible({ timeout: 3_000 }).catch(() => false)) {
-    await tabByRole.click();
-    return true;
-  }
-
-  const tabBySelector = page.locator('[role="tab"]').filter({ hasText: /Debug Chat|调试聊天|调试对话|Debug|调试/i }).first();
-  if (await tabBySelector.isVisible({ timeout: 2_000 }).catch(() => false)) {
-    await tabBySelector.click();
-    return true;
-  }
-
-  return Boolean(await clickFirstVisible(page, ["Debug Chat", "调试聊天", "调试对话"], 2_000));
+  const label = /^(?:Debug Chat|调试聊天|调试对话|对话调试)$/i;
+  const configuredTimeout = Number.parseInt(
+    process.env.LANGBOT_E2E_UI_READY_TIMEOUT_MS
+      || process.env.LANGBOT_E2E_NAVIGATION_TIMEOUT_MS
+      || "30000",
+    10,
+  );
+  const timeout = Number.isFinite(configuredTimeout) && configuredTimeout > 0
+    ? configuredTimeout
+    : 30_000;
+  return await clickFirstVisibleLocator(page, [
+    page.getByRole("tab", { name: label }),
+    page.locator('[data-slot="tabs-trigger"]').filter({ hasText: label }),
+    page.getByText(label, { exact: true }),
+  ], timeout);
 }

 export async function waitForDebugChatReady(page, timeout = 20_000) {
@@ -103,6 +106,7 @@ export function classifyDebugChatResult({
  beforeMessages = null,
  afterMessages = null,
  latestAssistantText = "",
+  latestAssistantIsFinal = null,
  maxNewAssistantMessages = null,
  failureSignals = DEBUG_CHAT_FAILURE_SIGNALS,
 }) {
@@ -154,6 +158,18 @@ export function classifyDebugChatResult({
        ...assistantMessageEvidence,
      };
    }
+    if (latestAssistantIsFinal === false) {
+      return {
+        status: "fail",
+        reason: "The latest assistant message contained the expected text but was not final.",
+        min_expected_count: minExpectedCount,
+        final_count: finalCount,
+        before_assistant_expected_count: beforeAssistantExpectedCount,
+        after_assistant_expected_count: afterAssistantExpectedCount,
+        ...assistantMessageEvidence,
+        latest_assistant_is_final: false,
+      };
+    }
    if (maxNewAssistantMessages !== null && newAssistantMessageCount > maxNewAssistantMessages) {
      return {
        status: "fail",
@@ -241,8 +257,16 @@ export function classifyDebugChatResult({

 export async function openPipelineDebugChat(page, { pipelineUrl, pipelineName, envHint = "LANGBOT_PIPELINE_URL or LANGBOT_PIPELINE_NAME" }) {
  if (pipelineUrl) {
-    await page.goto(pipelineUrl, { waitUntil: "domcontentloaded" });
-    await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {});
+    let alreadyAtPipeline = false;
+    try {
+      alreadyAtPipeline = new URL(page.url()).href === new URL(pipelineUrl).href;
+    } catch {
+      // Invalid URLs are handled by page.goto below.
+    }
+    if (!alreadyAtPipeline) {
+      await page.goto(pipelineUrl, { waitUntil: "commit" });
+      await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {});
+    }
  } else {
    if (!pipelineName) {
      return {
@@ -394,6 +418,62 @@ export async function waitForDebugChatTextStable(page, { timeoutMs = 5_000, quie
  }
 }

+async function fetchDebugChatHistory(page, { backendUrl, pipelineId, sessionType }) {
+  if (!backendUrl || !pipelineId || !sessionType) {
+    return { status: "not_required", messages: [] };
+  }
+  return await page.evaluate(async ({ backendUrl, pipelineId, sessionType }) => {
+    const token = localStorage.getItem("token") || "";
+    const response = await fetch(
+      `${backendUrl.replace(/\/$/, "")}/api/v1/pipelines/${encodeURIComponent(pipelineId)}/ws/messages/${encodeURIComponent(sessionType)}`,
+      { headers: token ? { Authorization: `Bearer ${token}` } : {} },
+    );
+    const json = await response.json().catch(() => ({}));
+    return {
+      status: response.ok && json.code === 0 ? "ready" : "fail",
+      http_status: response.status,
+      code: json.code ?? null,
+      messages: json.data?.messages || [],
+      reason: response.ok && json.code === 0 ? "" : json.msg || `Debug Chat history returned HTTP ${response.status}.`,
+    };
+  }, { backendUrl, pipelineId, sessionType });
+}
+
+async function waitForFinalDebugChatAssistant(page, {
+  backendUrl,
+  pipelineId,
+  sessionType,
+  beforeAssistantCount,
+  timeoutMs,
+}) {
+  if (!backendUrl || !pipelineId || !sessionType) {
+    return { status: "not_required", latest_assistant_is_final: null };
+  }
+  const deadline = Date.now() + Math.max(1, timeoutMs);
+  let lastHistory = null;
+  while (Date.now() < deadline) {
+    lastHistory = await fetchDebugChatHistory(page, { backendUrl, pipelineId, sessionType });
+    if (lastHistory.status === "fail") return lastHistory;
+    const assistants = lastHistory.messages.filter((message) => message.role === "assistant");
+    const latest = assistants.at(-1);
+    if (assistants.length > beforeAssistantCount && latest?.is_final === true) {
+      return {
+        status: "pass",
+        latest_assistant_is_final: true,
+        assistant_message_count: assistants.length,
+      };
+    }
+    await page.waitForTimeout(Math.min(250, Math.max(1, deadline - Date.now())));
+  }
+  const assistants = (lastHistory?.messages || []).filter((message) => message.role === "assistant");
+  return {
+    status: "fail",
+    reason: "Timed out waiting for the new assistant message to become final.",
+    latest_assistant_is_final: assistants.at(-1)?.is_final === true,
+    assistant_message_count: assistants.length,
+  };
+}
+
 export async function attachDebugChatImage(page, imagePath) {
  if (!imagePath) return { status: "not_required", reason: "" };
  const input = page.locator('input[type="file"][accept*="image"], input[type="file"]').first();
@@ -429,11 +509,16 @@ export async function runDebugChatPrompt(page, {
  expectedTexts = null,
  responseTimeoutMs,
  imagePath = "",
+  backendUrl = "",
+  pipelineId = "",
+  sessionType = "person",
  maxNewAssistantMessages = null,
  failureSignals = DEBUG_CHAT_FAILURE_SIGNALS,
 }) {
  const beforeText = await bodyText(page);
  const beforeMessages = await visibleDebugChatMessages(page);
+  const beforeHistory = await fetchDebugChatHistory(page, { backendUrl, pipelineId, sessionType });
+  const beforeHistoryAssistantCount = beforeHistory.messages.filter((message) => message.role === "assistant").length;
  const requiredExpectedTexts = [...new Set(
    (Array.isArray(expectedTexts) && expectedTexts.length > 0 ? expectedTexts : [expectedText])
      .map(String)
@@ -449,6 +534,7 @@ export async function runDebugChatPrompt(page, {
    return { status: "fail", reason: "Could not find a Debug Chat text input." };
  }

+  const responseStartedAt = Date.now();
  await waitForExpectedDebugChatText(page, {
    expectedText,
    expectedTexts: requiredExpectedTexts,
@@ -459,6 +545,13 @@ export async function runDebugChatPrompt(page, {
    beforeText,
    failureSignals,
  });
+  const finalAssistant = await waitForFinalDebugChatAssistant(page, {
+    backendUrl,
+    pipelineId,
+    sessionType,
+    beforeAssistantCount: beforeHistoryAssistantCount,
+    timeoutMs: Math.max(1, responseTimeoutMs - (Date.now() - responseStartedAt)),
+  });
  await waitForDebugChatTextStable(page);

  const afterText = await bodyText(page);
@@ -468,7 +561,7 @@ export async function runDebugChatPrompt(page, {
  const failureText = findNewFailureSignal(beforeText, afterText, failureSignals);
  const latestFailureLeaf = failureText ? await latestVisibleLeafText(page, [failureText]) : "";

-  return classifyDebugChatResult({
+  const classified = classifyDebugChatResult({
    beforeText,
    afterText,
    expectedText,
@@ -479,9 +572,16 @@ export async function runDebugChatPrompt(page, {
    beforeMessages,
    afterMessages,
    latestAssistantText,
+    latestAssistantIsFinal: finalAssistant.latest_assistant_is_final,
    maxNewAssistantMessages,
    failureSignals,
  });
+  return {
+    ...classified,
+    latest_assistant_is_final: finalAssistant.latest_assistant_is_final,
+    final_assistant_wait_status: finalAssistant.status,
+    final_assistant_wait_reason: finalAssistant.reason || "",
+  };
 }

 export async function setDebugChatStreamOutput(page, desired) {
@@ -428,6 +428,10 @@ export async function createBrowser(paths) {
    context = await browser.newContext({ viewport: { width: 1440, height: 960 } });
  }
  const page = context.pages()[0] || await context.newPage();
+  const navigationTimeoutMs = Number.parseInt(env.LANGBOT_E2E_NAVIGATION_TIMEOUT_MS || "30000", 10);
+  if (Number.isFinite(navigationTimeoutMs) && navigationTimeoutMs > 0) {
+    page.setDefaultNavigationTimeout(navigationTimeoutMs);
+  }

  page.on("console", (message) => {
    appendLine(paths.consoleLog, `[${message.type()}] ${message.text()}`).catch(() => {});
@@ -483,29 +487,42 @@ export function countOccurrences(haystack, needle) {
  return String(haystack).split(needle).length - 1;
 }

-export async function clickFirstVisible(page, labels, timeout = 2_000) {
-  for (const label of labels) {
-    const roleButton = page.getByRole("button", { name: label }).first();
-    if (await roleButton.isVisible({ timeout }).catch(() => false)) {
-      await roleButton.click();
-      return label;
+async function clickVisibleCandidate(page, candidates, timeout) {
+  const deadline = Date.now() + Math.max(1, timeout);
+  do {
+    for (const candidate of candidates) {
+      const count = await candidate.locator.count().catch(() => 0);
+      for (let index = 0; index < count; index += 1) {
+        const element = candidate.locator.nth(index);
+        if (!await element.isVisible().catch(() => false)) continue;
+        const remaining = Math.max(1, deadline - Date.now());
+        const clicked = await element.click({ timeout: Math.min(1_000, remaining) })
+          .then(() => true)
+          .catch(() => false);
+        if (clicked) return candidate.value;
+      }
    }
-
-    const roleLink = page.getByRole("link", { name: label }).first();
-    if (await roleLink.isVisible({ timeout }).catch(() => false)) {
-      await roleLink.click();
-      return label;
-    }
-
-    const text = page.getByText(label, { exact: false }).first();
-    if (await text.isVisible({ timeout }).catch(() => false)) {
-      await text.click();
-      return label;
-    }
-  }
+    const remaining = deadline - Date.now();
+    if (remaining <= 0) break;
+    await page.waitForTimeout(Math.min(100, remaining));
+  } while (Date.now() < deadline);
  return null;
 }

+export async function clickFirstVisibleLocator(page, locators, timeout = 2_000) {
+  const candidates = locators.map((locator) => ({ locator, value: true }));
+  return Boolean(await clickVisibleCandidate(page, candidates, timeout));
+}
+
+export async function clickFirstVisible(page, labels, timeout = 2_000) {
+  const candidates = labels.flatMap((label) => [
+    { locator: page.getByRole("button", { name: label }), value: label },
+    { locator: page.getByRole("link", { name: label }), value: label },
+    { locator: page.getByText(label, { exact: false }), value: label },
+  ]);
+  return await clickVisibleCandidate(page, candidates, timeout);
+}
+
 export async function fillFirstTextInput(page, value) {
  const candidates = [
    page.getByRole("textbox").last(),
@@ -57,6 +57,7 @@ const resetDebugChat = boolFromEnv(env.LANGBOT_E2E_RESET_DEBUG_CHAT, false);
 const restoreRunnerConfig = boolFromEnv(env.LANGBOT_E2E_RESTORE_RUNNER_CONFIG, true);
 const restoreExtensions = boolFromEnv(env.LANGBOT_E2E_RESTORE_EXTENSIONS, true);
 const debugChatSessionType = env.LANGBOT_E2E_DEBUG_CHAT_SESSION_TYPE || "person";
+const maxNewAssistantMessages = Number.parseInt(env.LANGBOT_E2E_MAX_NEW_ASSISTANT_MESSAGES || "1", 10);
 const pipelineConfigDiagnosticPath = resolve(paths.evidenceDir, "pipeline-config-diagnostic.json");
 const pipelineExtensionsDiagnosticPath = resolve(paths.evidenceDir, "pipeline-extensions-diagnostic.json");
 const debugChatResetDiagnosticPath = resolve(paths.evidenceDir, "debug-chat-reset-diagnostic.json");
@@ -873,7 +874,7 @@ try {
  result.expected_text = promptSteps.at(-1)?.expectedText || expectedText;

  const authDiagnostic = await ensureAuthenticatedBrowser(page, {
-    frontendUrl: env.LANGBOT_FRONTEND_URL || "",
+    frontendUrl: pipelineUrl || env.LANGBOT_FRONTEND_URL || "",
    backendUrl,
  });
  result.browser_auth = authDiagnostic;
@@ -999,7 +1000,12 @@ try {
            expectedTexts: step.expectedTexts,
            responseTimeoutMs: step.responseTimeoutMs,
            imagePath: index === 0 ? imagePath : "",
-            maxNewAssistantMessages: streamOutput === false ? 1 : null,
+            backendUrl,
+            pipelineId: result.pipeline_config?.pipeline_id || pipelineIdFromUrl(pipelineUrl),
+            sessionType: debugChatSessionType,
+            maxNewAssistantMessages: Number.isFinite(maxNewAssistantMessages) && maxNewAssistantMessages >= 0
+              ? maxNewAssistantMessages
+              : 1,
            failureSignals: failureSignals.length > 0 ? failureSignals : undefined,
          });
          const promptDurationMs = Date.now() - promptStartedAt;
@@ -1017,6 +1023,9 @@ try {
            before_assistant_message_count: chatResult.before_assistant_message_count,
            after_assistant_message_count: chatResult.after_assistant_message_count,
            new_assistant_message_count: chatResult.new_assistant_message_count,
+            latest_assistant_is_final: chatResult.latest_assistant_is_final,
+            final_assistant_wait_status: chatResult.final_assistant_wait_status,
+            final_assistant_wait_reason: chatResult.final_assistant_wait_reason,
            failure_signal: chatResult.failure_signal || "",
            missing_expected_texts: chatResult.missing_expected_texts || [],
          });