test(skills): harden local agent QA automation

skills/scripts/e2e/lib/debug-chat.mjs

@@ -65,7 +65,7 @@ async function clickDebugChatTab(page) {
   return Boolean(await clickFirstVisible(page, ["Debug Chat", "调试聊天", "调试对话"], 2_000));
 }
 
-async function waitForDebugChatReady(page, timeout = 20_000) {
+export async function waitForDebugChatReady(page, timeout = 20_000) {
   const input = debugChatInput(page);
   const visible = await input.isVisible({ timeout }).catch(() => false);
   if (!visible) {
@@ -75,7 +75,13 @@ async function waitForDebugChatReady(page, timeout = 20_000) {
     };
   }
 
-  const enabled = await input.isEnabled({ timeout }).catch(() => false);
+  const deadline = Date.now() + timeout;
+  let enabled = false;
+  while (Date.now() < deadline) {
+    enabled = await input.isEnabled().catch(() => false);
+    if (enabled) break;
+    await page.waitForTimeout(Math.min(250, Math.max(1, deadline - Date.now())));
+  }
   if (!enabled) {
     return {
       ready: false,
@@ -90,6 +96,7 @@ export function classifyDebugChatResult({
   beforeText,
   afterText,
   expectedText,
+  expectedTexts = null,
   prompt,
   latestExpectedLeaf,
   latestFailureLeaf,
@@ -99,6 +106,11 @@ export function classifyDebugChatResult({
   maxNewAssistantMessages = null,
   failureSignals = DEBUG_CHAT_FAILURE_SIGNALS,
 }) {
+  const requiredExpectedTexts = [...new Set(
+    (Array.isArray(expectedTexts) && expectedTexts.length > 0 ? expectedTexts : [expectedText])
+      .map(String)
+      .filter(Boolean),
+  )];
   const minExpectedCount = minExpectedOccurrences(beforeText, expectedText, prompt);
   const finalCount = countOccurrences(afterText, expectedText);
   const failureText = findNewFailureSignal(beforeText, afterText, failureSignals);
@@ -110,9 +122,6 @@ export function classifyDebugChatResult({
   const afterAssistantExpectedCount = hasMessageEvidence
     ? countExpectedInMessages(afterMessages, expectedText)
     : null;
-  const assistantExpectedIncreased = hasMessageEvidence
-    ? afterAssistantExpectedCount > beforeAssistantExpectedCount
-    : false;
   const beforeAssistantMessageCount = hasMessageEvidence
     ? beforeMessages.filter((message) => message.role === "assistant").length
     : null;
@@ -129,6 +138,9 @@ export function classifyDebugChatResult({
   };
 
   if (hasMessageEvidence) {
+    const missingExpectedTexts = requiredExpectedTexts.filter(
+      (text) => !String(latestAssistantText).includes(text),
+    );
     const latestAssistantFailure = findFailureSignalInText(latestAssistantText, failureSignals);
     if (latestAssistantFailure) {
       return {
@@ -153,15 +165,18 @@ export function classifyDebugChatResult({
         ...assistantMessageEvidence,
       };
     }
-    if (assistantExpectedIncreased && String(latestAssistantText).includes(expectedText)) {
+    if (newAssistantMessageCount > 0 && missingExpectedTexts.length === 0) {
       return {
         status: "pass",
-        reason: `Expected text appeared in a new assistant message: ${expectedText}`,
+        reason: requiredExpectedTexts.length === 1
+          ? `Expected text appeared in a new assistant message: ${expectedText}`
+          : `All ${requiredExpectedTexts.length} expected text fragments appeared in a new assistant message.`,
         min_expected_count: minExpectedCount,
         final_count: finalCount,
         before_assistant_expected_count: beforeAssistantExpectedCount,
         after_assistant_expected_count: afterAssistantExpectedCount,
         ...assistantMessageEvidence,
+        missing_expected_texts: [],
       };
     }
     if (failureText) {
@@ -178,12 +193,15 @@ export function classifyDebugChatResult({
     }
     return {
       status: "fail",
-      reason: `Expected text did not appear in a new assistant message. Expected assistant occurrences to increase above ${beforeAssistantExpectedCount}, saw ${afterAssistantExpectedCount}.`,
+      reason: missingExpectedTexts.length > 0
+        ? `A new assistant message was missing expected text: ${missingExpectedTexts.join(", ")}`
+        : `Expected text did not appear in a new assistant message. Expected assistant occurrences to increase above ${beforeAssistantExpectedCount}, saw ${afterAssistantExpectedCount}.`,
       min_expected_count: minExpectedCount,
       final_count: finalCount,
       before_assistant_expected_count: beforeAssistantExpectedCount,
       after_assistant_expected_count: afterAssistantExpectedCount,
       ...assistantMessageEvidence,
+      missing_expected_texts: missingExpectedTexts,
     };
   }
   if (failureText) {
@@ -327,22 +345,35 @@ export async function visibleDebugChatMessages(page) {
 
 export async function waitForExpectedDebugChatText(page, {
   expectedText,
+  expectedTexts = null,
   minExpectedCount,
+  minExpectedCounts = null,
   timeoutMs,
   beforeText = "",
   failureSignals = DEBUG_CHAT_FAILURE_SIGNALS,
 }) {
+  const requiredExpectedTexts = [...new Set(
+    (Array.isArray(expectedTexts) && expectedTexts.length > 0 ? expectedTexts : [expectedText])
+      .map(String)
+      .filter(Boolean),
+  )];
+  const expectedRequirements = requiredExpectedTexts.map((text, index) => ({
+    text,
+    min: Array.isArray(minExpectedCounts) && Number.isFinite(minExpectedCounts[index])
+      ? minExpectedCounts[index]
+      : (text === expectedText ? minExpectedCount : minExpectedOccurrences(beforeText, text, "")),
+  }));
   const failureBaselines = failureSignals.map((signal) => ({
     signal,
     count: countOccurrences(beforeText, signal),
   }));
   await page.waitForFunction(
-    ({ expected, min, failures }) => {
+    ({ requirements, failures }) => {
       const text = document.body.innerText;
-      if (text.split(expected).length - 1 >= min) return true;
+      if (requirements.every((item) => text.split(item.text).length - 1 >= item.min)) return true;
       return failures.some(({ signal, count }) => text.split(signal).length - 1 > count);
     },
-    { expected: expectedText, min: minExpectedCount, failures: failureBaselines },
+    { requirements: expectedRequirements, failures: failureBaselines },
     { timeout: timeoutMs },
   ).catch(() => {});
 }
@@ -395,6 +426,7 @@ export async function sendDebugChatPrompt(page, prompt, imagePath = "") {
 export async function runDebugChatPrompt(page, {
   prompt,
   expectedText,
+  expectedTexts = null,
   responseTimeoutMs,
   imagePath = "",
   maxNewAssistantMessages = null,
@@ -402,7 +434,15 @@ export async function runDebugChatPrompt(page, {
 }) {
   const beforeText = await bodyText(page);
   const beforeMessages = await visibleDebugChatMessages(page);
+  const requiredExpectedTexts = [...new Set(
+    (Array.isArray(expectedTexts) && expectedTexts.length > 0 ? expectedTexts : [expectedText])
+      .map(String)
+      .filter(Boolean),
+  )];
   const minExpectedCount = minExpectedOccurrences(beforeText, expectedText, prompt);
+  const minExpectedCounts = requiredExpectedTexts.map(
+    (text) => minExpectedOccurrences(beforeText, text, prompt),
+  );
   const sent = await sendDebugChatPrompt(page, prompt, imagePath);
   if (sent !== true) {
     if (sent && typeof sent === "object" && typeof sent.reason === "string") return sent;
@@ -411,7 +451,9 @@ export async function runDebugChatPrompt(page, {
 
   await waitForExpectedDebugChatText(page, {
     expectedText,
+    expectedTexts: requiredExpectedTexts,
     minExpectedCount,
+    minExpectedCounts,
     prompt,
     timeoutMs: responseTimeoutMs,
     beforeText,
@@ -430,6 +472,7 @@ export async function runDebugChatPrompt(page, {
     beforeText,
     afterText,
     expectedText,
+    expectedTexts: requiredExpectedTexts,
     prompt,
     latestExpectedLeaf,
     latestFailureLeaf,

skills/scripts/e2e/pipeline-debug-chat.mjs

@@ -142,22 +142,28 @@ function stats(values) {
 function promptStepsFromEnv() {
   const rawSteps = parseJsonEnv("LANGBOT_E2E_PROMPTS_JSON", null);
   if (rawSteps === null) {
-    return [{ prompt, expectedText, responseTimeoutMs: safeResponseTimeoutMs }];
+    return [{ prompt, expectedText, expectedTexts: [expectedText], responseTimeoutMs: safeResponseTimeoutMs }];
   }
   if (!Array.isArray(rawSteps) || rawSteps.length === 0) {
     throw new Error("LANGBOT_E2E_PROMPTS_JSON must be a non-empty JSON array.");
   }
   return rawSteps.map((item, index) => {
     if (typeof item === "string") {
-      return { prompt: item, expectedText, responseTimeoutMs: safeResponseTimeoutMs };
+      return { prompt: item, expectedText, expectedTexts: [expectedText], responseTimeoutMs: safeResponseTimeoutMs };
     }
     if (!item || typeof item !== "object" || typeof item.prompt !== "string" || !item.prompt) {
       throw new Error(`LANGBOT_E2E_PROMPTS_JSON[${index}] must be a string or an object with a prompt string.`);
     }
     const stepTimeout = Number.parseInt(String(item.response_timeout_ms || item.responseTimeoutMs || safeResponseTimeoutMs), 10);
+    const stepExpectedText = String(item.expected_text || item.expectedText || expectedText);
+    const additionalExpectedTexts = item.expected_texts || item.expectedTexts || [];
+    if (!Array.isArray(additionalExpectedTexts)) {
+      throw new Error(`LANGBOT_E2E_PROMPTS_JSON[${index}].expected_texts must be an array.`);
+    }
     return {
       prompt: item.prompt,
-      expectedText: String(item.expected_text || item.expectedText || expectedText),
+      expectedText: stepExpectedText,
+      expectedTexts: [...new Set([stepExpectedText, ...additionalExpectedTexts.map(String)].filter(Boolean))],
       responseTimeoutMs: Number.isFinite(stepTimeout) && stepTimeout > 0 ? stepTimeout : safeResponseTimeoutMs,
     };
   });
@@ -990,6 +996,7 @@ try {
           const chatResult = await runDebugChatPrompt(page, {
             prompt: step.prompt,
             expectedText: step.expectedText,
+            expectedTexts: step.expectedTexts,
             responseTimeoutMs: step.responseTimeoutMs,
             imagePath: index === 0 ? imagePath : "",
             maxNewAssistantMessages: streamOutput === false ? 1 : null,
@@ -999,6 +1006,7 @@ try {
           result.chat_results.push({
             index,
             expected_text: step.expectedText,
+            expected_texts: step.expectedTexts,
             status: chatResult.status,
             reason: chatResult.reason,
             response_duration_ms: promptDurationMs,
@@ -1010,6 +1018,7 @@ try {
             after_assistant_message_count: chatResult.after_assistant_message_count,
             new_assistant_message_count: chatResult.new_assistant_message_count,
             failure_signal: chatResult.failure_signal || "",
+            missing_expected_texts: chatResult.missing_expected_texts || [],
           });
           result.status = chatResult.status;
           result.reason = `Prompt ${index + 1}/${promptSteps.length}: ${chatResult.reason}`;

skills/skills/langbot-testing/cases/local-agent-multitool-rag-compaction-debug-chat.yaml

@@ -45,7 +45,7 @@ automation_reset_debug_chat: "1"
 automation_debug_chat_session_type: "person"
 automation_expected_text: "MULTITOOL_COMBO_FINAL qa_multitool_compaction_sentinel_6718 azalea-cobalt-7421 qa-plugin-smoke:multi-tool-a-local-agent qa-plugin-smoke:multi-tool-b-local-agent"
 automation_response_timeout_ms: "180000"
-automation_prompts_json: '[{"prompt":"请记住这个用于 local-agent 多工具组合回归测试的暗号：qa_multitool_compaction_sentinel_6718。请只回复 MEMORY_SET。","expected_text":"MEMORY_SET","response_timeout_ms":"180000"},{"prompt":"下面这轮只用于制造长历史压力，目标标记 MULTITOOL_RAG_GOAL 必须被压缩保留。请忽略填充内容，最后只回复 MULTITOOL_CONTEXT_PRESSURE_READY。填充片段 M001 multitool context padding. M002 multitool context padding. M003 multitool context padding. M004 multitool context padding. M005 multitool context padding. M006 multitool context padding. M007 multitool context padding. M008 multitool context padding. M009 multitool context padding. M010 multitool context padding. M011 multitool context padding. M012 multitool context padding. M013 multitool context padding. M014 multitool context padding. M015 multitool context padding. M016 multitool context padding. M017 multitool context padding. M018 multitool context padding. M019 multitool context padding. M020 multitool context padding. M021 multitool context padding. M022 multitool context padding. M023 multitool context padding. M024 multitool context padding. M025 multitool context padding. M026 multitool context padding. M027 multitool context padding. M028 multitool context padding. M029 multitool context padding. M030 multitool context padding. M031 multitool context padding. M032 multitool context padding.","expected_text":"MULTITOOL_CONTEXT_PRESSURE_READY","response_timeout_ms":"180000"},{"prompt":"MULTITOOL_COMBO final check: using the knowledge base, the compacted earlier passcode, and qa_plugin_echo twice with exactly multi-tool-a-local-agent then multi-tool-b-local-agent, return only MULTITOOL_COMBO_FINAL plus the passcode, the RAG sentinel, and both plugin tool results.","expected_text":"MULTITOOL_COMBO_FINAL qa_multitool_compaction_sentinel_6718 azalea-cobalt-7421 qa-plugin-smoke:multi-tool-a-local-agent qa-plugin-smoke:multi-tool-b-local-agent","response_timeout_ms":"180000"}]'
+automation_prompts_json: '[{"prompt":"请记住这个用于 local-agent 多工具组合回归测试的暗号：qa_multitool_compaction_sentinel_6718。请只回复 MEMORY_SET。","expected_text":"MEMORY_SET","response_timeout_ms":"180000"},{"prompt":"下面这轮只用于制造长历史压力，目标标记 MULTITOOL_RAG_GOAL 必须被压缩保留。请忽略填充内容，最后只回复 MULTITOOL_CONTEXT_PRESSURE_READY。填充片段 M001 multitool context padding. M002 multitool context padding. M003 multitool context padding. M004 multitool context padding. M005 multitool context padding. M006 multitool context padding. M007 multitool context padding. M008 multitool context padding. M009 multitool context padding. M010 multitool context padding. M011 multitool context padding. M012 multitool context padding. M013 multitool context padding. M014 multitool context padding. M015 multitool context padding. M016 multitool context padding. M017 multitool context padding. M018 multitool context padding. M019 multitool context padding. M020 multitool context padding. M021 multitool context padding. M022 multitool context padding. M023 multitool context padding. M024 multitool context padding. M025 multitool context padding. M026 multitool context padding. M027 multitool context padding. M028 multitool context padding. M029 multitool context padding. M030 multitool context padding. M031 multitool context padding. M032 multitool context padding.","expected_text":"MULTITOOL_CONTEXT_PRESSURE_READY","response_timeout_ms":"180000"},{"prompt":"MULTITOOL_COMBO final check: using the knowledge base, the compacted earlier passcode, and qa_plugin_echo twice with exactly multi-tool-a-local-agent then multi-tool-b-local-agent, return only MULTITOOL_COMBO_FINAL plus the passcode, the RAG sentinel, and both plugin tool results.","expected_text":"MULTITOOL_COMBO_FINAL","expected_texts":["qa_multitool_compaction_sentinel_6718","azalea-cobalt-7421","qa-plugin-smoke:multi-tool-a-local-agent","qa-plugin-smoke:multi-tool-b-local-agent"],"response_timeout_ms":"180000"}]'
 setup_automation:
   - "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env"
   - "node:scripts/e2e/ensure-langrag-sentinel-kb.mjs --write-env"

skills/skills/langbot-testing/cases/local-agent-parallel-tools-rag-compaction-debug-chat.yaml

@@ -45,7 +45,7 @@ automation_reset_debug_chat: "1"
 automation_debug_chat_session_type: "person"
 automation_expected_text: "PARALLEL_COMBO_FINAL qa_parallel_compaction_sentinel_8142 azalea-cobalt-7421 qa-plugin-smoke:parallel-tool-a-local-agent qa-plugin-smoke:parallel-tool-b-local-agent"
 automation_response_timeout_ms: "180000"
-automation_prompts_json: '[{"prompt":"请记住这个用于 local-agent 并行工具组合回归测试的暗号：qa_parallel_compaction_sentinel_8142。请只回复 MEMORY_SET。","expected_text":"MEMORY_SET","response_timeout_ms":"180000"},{"prompt":"下面这轮只用于制造长历史压力，目标标记 PARALLEL_RAG_GOAL 必须被压缩保留。请忽略填充内容，最后只回复 PARALLEL_CONTEXT_PRESSURE_READY。填充片段 P001 parallel context padding. P002 parallel context padding. P003 parallel context padding. P004 parallel context padding. P005 parallel context padding. P006 parallel context padding. P007 parallel context padding. P008 parallel context padding. P009 parallel context padding. P010 parallel context padding. P011 parallel context padding. P012 parallel context padding. P013 parallel context padding. P014 parallel context padding. P015 parallel context padding. P016 parallel context padding. P017 parallel context padding. P018 parallel context padding. P019 parallel context padding. P020 parallel context padding. P021 parallel context padding. P022 parallel context padding. P023 parallel context padding. P024 parallel context padding. P025 parallel context padding. P026 parallel context padding. P027 parallel context padding. P028 parallel context padding. P029 parallel context padding. P030 parallel context padding. P031 parallel context padding. P032 parallel context padding.","expected_text":"PARALLEL_CONTEXT_PRESSURE_READY","response_timeout_ms":"180000"},{"prompt":"PARALLEL_COMBO final check: using the knowledge base, the compacted earlier passcode, and one model turn that calls qa_plugin_echo for both parallel-tool-a-local-agent and parallel-tool-b-local-agent, return only PARALLEL_COMBO_FINAL plus the passcode, the RAG sentinel, and both plugin tool results.","expected_text":"PARALLEL_COMBO_FINAL qa_parallel_compaction_sentinel_8142 azalea-cobalt-7421 qa-plugin-smoke:parallel-tool-a-local-agent qa-plugin-smoke:parallel-tool-b-local-agent","response_timeout_ms":"180000"}]'
+automation_prompts_json: '[{"prompt":"请记住这个用于 local-agent 并行工具组合回归测试的暗号：qa_parallel_compaction_sentinel_8142。请只回复 MEMORY_SET。","expected_text":"MEMORY_SET","response_timeout_ms":"180000"},{"prompt":"下面这轮只用于制造长历史压力，目标标记 PARALLEL_RAG_GOAL 必须被压缩保留。请忽略填充内容，最后只回复 PARALLEL_CONTEXT_PRESSURE_READY。填充片段 P001 parallel context padding. P002 parallel context padding. P003 parallel context padding. P004 parallel context padding. P005 parallel context padding. P006 parallel context padding. P007 parallel context padding. P008 parallel context padding. P009 parallel context padding. P010 parallel context padding. P011 parallel context padding. P012 parallel context padding. P013 parallel context padding. P014 parallel context padding. P015 parallel context padding. P016 parallel context padding. P017 parallel context padding. P018 parallel context padding. P019 parallel context padding. P020 parallel context padding. P021 parallel context padding. P022 parallel context padding. P023 parallel context padding. P024 parallel context padding. P025 parallel context padding. P026 parallel context padding. P027 parallel context padding. P028 parallel context padding. P029 parallel context padding. P030 parallel context padding. P031 parallel context padding. P032 parallel context padding.","expected_text":"PARALLEL_CONTEXT_PRESSURE_READY","response_timeout_ms":"180000"},{"prompt":"PARALLEL_COMBO final check: using the knowledge base, the compacted earlier passcode, and one model turn that calls qa_plugin_echo for both parallel-tool-a-local-agent and parallel-tool-b-local-agent, return only PARALLEL_COMBO_FINAL plus the passcode, the RAG sentinel, and both plugin tool results.","expected_text":"PARALLEL_COMBO_FINAL","expected_texts":["qa_parallel_compaction_sentinel_8142","azalea-cobalt-7421","qa-plugin-smoke:parallel-tool-a-local-agent","qa-plugin-smoke:parallel-tool-b-local-agent"],"response_timeout_ms":"180000"}]'
 setup_automation:
   - "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env"
   - "node:scripts/e2e/ensure-langrag-sentinel-kb.mjs --write-env"

skills/skills/langbot-testing/fixtures/complex-agent-task/verify.py

@@ -6,6 +6,18 @@ import sys
 from pathlib import Path
 
 
+def has_initial_failure_section(report: str) -> bool:
+    folded = report.casefold()
+    return any(
+        marker in folded
+        for marker in (
+            "initial fail",
+            "initially fail",
+            "baseline fail",
+        )
+    )
+
+
 def main() -> int:
     parser = argparse.ArgumentParser()
     parser.add_argument("--workspace", required=True)
@@ -43,7 +55,7 @@ def main() -> int:
     assert result["acceptance"] == "PASS"
     report = (workspace / "AGENT_REPORT.md").read_text(encoding="utf-8")
     folded_report = report.casefold()
-    assert "initial" in folded_report and "fail" in folded_report, "report missing initial failure section"
+    assert has_initial_failure_section(report), "report missing initial failure section"
     assert "root causes" in folded_report, "report missing section: root causes"
     assert any(
         heading in folded_report

skills/test/lbs-cli.test.ts

@@ -56,6 +56,7 @@ import {
   findNewFailureSignal,
   hasDebugChatOutcome,
   minExpectedOccurrences,
+  waitForDebugChatReady,
 } from "../scripts/e2e/lib/debug-chat.mjs";
 import {
   beginBackendLogCapture,
@@ -1897,6 +1898,26 @@ test("debug chat classifier prefers latest response leaf over body counts", () =
   assert.match(result.reason, /latest visible response leaf/);
 });
 
+test("debug chat readiness waits for the websocket to enable the input", async () => {
+  let enabledChecks = 0;
+  const input = {
+    isVisible: async () => true,
+    isEnabled: async () => {
+      enabledChecks += 1;
+      return enabledChecks >= 3;
+    },
+  };
+  const page = {
+    locator: () => ({ last: () => input }),
+    waitForTimeout: async () => {},
+  };
+
+  const result = await waitForDebugChatReady(page, 1_000);
+
+  assert.deepEqual(result, { ready: true, reason: "" });
+  assert.equal(enabledChecks, 3);
+});
+
 test("debug chat classifier distinguishes new failure signals from old history", () => {
   assert.equal(
     findNewFailureSignal(
@@ -2043,6 +2064,46 @@ test("debug chat classifier passes when expected text appears in a new assistant
   assert.equal(result.new_assistant_message_count, 1);
 });
 
+test("debug chat classifier accepts formatted responses containing every required fragment", () => {
+  const expectedTexts = ["MULTITOOL_COMBO_FINAL", "passcode-6718", "rag-7421", "tool-a", "tool-b"];
+  const result = classifyDebugChatResult({
+    beforeText: "",
+    afterText: "Bot response with formatted details",
+    expectedText: expectedTexts[0],
+    expectedTexts,
+    prompt: "Run the combined task",
+    latestExpectedLeaf: "MULTITOOL_COMBO_FINAL",
+    latestFailureLeaf: "",
+    beforeMessages: [],
+    afterMessages: [{
+      role: "assistant",
+      text: "MULTITOOL_COMBO_FINAL\n- passcode-6718\n- rag-7421\n- tool-a\n- tool-b",
+    }],
+    latestAssistantText: "MULTITOOL_COMBO_FINAL\n- passcode-6718\n- rag-7421\n- tool-a\n- tool-b",
+  });
+
+  assert.equal(result.status, "pass");
+  assert.deepEqual(result.missing_expected_texts, []);
+});
+
+test("debug chat classifier rejects formatted responses missing a required fragment", () => {
+  const result = classifyDebugChatResult({
+    beforeText: "",
+    afterText: "Bot response with incomplete details",
+    expectedText: "MULTITOOL_COMBO_FINAL",
+    expectedTexts: ["MULTITOOL_COMBO_FINAL", "tool-a", "tool-b"],
+    prompt: "Run the combined task",
+    latestExpectedLeaf: "MULTITOOL_COMBO_FINAL",
+    latestFailureLeaf: "",
+    beforeMessages: [],
+    afterMessages: [{ role: "assistant", text: "MULTITOOL_COMBO_FINAL\n- tool-a" }],
+    latestAssistantText: "MULTITOOL_COMBO_FINAL\n- tool-a",
+  });
+
+  assert.equal(result.status, "fail");
+  assert.deepEqual(result.missing_expected_texts, ["tool-b"]);
+});
+
 test("debug chat classifier rejects split non-streaming assistant messages", () => {
   const expectedText = "E2E_OK:skill";
   const result = classifyDebugChatResult({

skills/test/test_complex_agent_task_verify.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+import importlib.util
+import unittest
+from pathlib import Path
+
+
+VERIFY_PATH = (
+    Path(__file__).resolve().parents[1]
+    / "skills"
+    / "langbot-testing"
+    / "fixtures"
+    / "complex-agent-task"
+    / "verify.py"
+)
+SPEC = importlib.util.spec_from_file_location("complex_agent_task_verify", VERIFY_PATH)
+assert SPEC and SPEC.loader
+MODULE = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(MODULE)
+
+
+class ComplexAgentTaskVerifyTests(unittest.TestCase):
+    def test_accepts_initial_failure_heading(self) -> None:
+        self.assertTrue(MODULE.has_initial_failure_section("## Initial failing tests\n- test_price"))
+
+    def test_accepts_baseline_failure_heading(self) -> None:
+        self.assertTrue(MODULE.has_initial_failure_section("## Baseline failing tests\n- test_price"))
+
+    def test_rejects_report_without_failure_section(self) -> None:
+        self.assertFalse(MODULE.has_initial_failure_section("## Verification\nAll tests pass."))
+
+
+if __name__ == "__main__":
+    unittest.main()