test(skills): expand local agent reliability coverage

skills/schemas/case.schema.json

@@ -257,6 +257,22 @@
       "type": "string",
       "enum": ["0", "1", "false", "true"]
     },
+    "automation_debug_chat_load_require_success": {
+      "type": "string",
+      "enum": ["0", "1", "false", "true"]
+    },
+    "automation_debug_chat_load_provider_model_thresholds_json": {
+      "type": "string"
+    },
+    "automation_fake_provider_pipeline_name": {
+      "type": "string"
+    },
+    "automation_fake_provider_model_name": {
+      "type": "string"
+    },
+    "automation_fake_provider_fallback_model_names": {
+      "type": "string"
+    },
     "automation_fake_provider_response_text": {
       "type": "string"
     },
@@ -275,6 +291,9 @@
     "automation_fake_provider_fail_every_n": {
       "type": "string"
     },
+    "automation_fake_provider_fail_models": {
+      "type": "string"
+    },
     "automation_fake_provider_fault_status": {
       "type": "string"
     },
@@ -282,6 +301,16 @@
       "type": "string",
       "enum": ["0", "1", "false", "true"]
     },
+    "automation_fake_provider_fail_after_first_chunk_delay_ms": {
+      "type": "string"
+    },
+    "automation_fake_provider_fail_after_first_chunk_mode": {
+      "type": "string",
+      "enum": ["disconnect", "error_event"]
+    },
+    "automation_fake_provider_fail_after_first_chunk_models": {
+      "type": "string"
+    },
     "automation_fake_provider_dynamic_response": {
       "type": "string",
       "enum": ["0", "1", "false", "true"]

skills/scripts/e2e/ensure-fake-provider-pipeline.mjs

@@ -41,6 +41,8 @@ const pipelineName = env.LANGBOT_FAKE_PROVIDER_PIPELINE_NAME || DEFAULT_PIPELINE
 const providerName = env.LANGBOT_FAKE_PROVIDER_NAME || DEFAULT_PROVIDER_NAME;
 const requester = env.LANGBOT_FAKE_PROVIDER_REQUESTER || DEFAULT_REQUESTER;
 const modelName = env.LANGBOT_FAKE_PROVIDER_MODEL_NAME || DEFAULT_MODEL_NAME;
+const fallbackModelNames = textList(env.LANGBOT_FAKE_PROVIDER_FALLBACK_MODEL_NAMES)
+  .filter((name) => name !== modelName);
 
 const result = {
   source: "automation",
@@ -75,6 +77,7 @@ const result = {
     test_status: "not_run",
     test_reason: "",
   },
+  fallback_models: [],
   pipeline_id: "",
   pipeline_name: pipelineName,
   pipeline_url: "",
@@ -141,11 +144,23 @@ try {
   });
   result.model = model;
 
+  const fallbackModels = [];
+  for (const fallbackModelName of fallbackModelNames) {
+    fallbackModels.push(await ensureModel({
+      backendUrl,
+      token: auth.token,
+      providerUuid: provider.uuid,
+      name: fallbackModelName,
+    }));
+  }
+  result.fallback_models = fallbackModels;
+
   const pipeline = await ensurePipeline({
     backendUrl,
     token: auth.token,
     name: pipelineName,
     modelUuid: model.uuid,
+    fallbackModelUuids: fallbackModels.map((item) => item.uuid),
   });
   Object.assign(result, pipeline);
   result.pipeline_url = `${frontendUrl.replace(/\/$/, "")}/home/agents?id=${encodeURIComponent(pipeline.pipeline_id)}`;
@@ -161,6 +176,7 @@ try {
       LANGBOT_FAKE_PROVIDER_PID: fakeProvider.pid ? String(fakeProvider.pid) : "",
       LANGBOT_FAKE_PROVIDER_PROVIDER_UUID: provider.uuid,
       LANGBOT_FAKE_PROVIDER_MODEL_UUID: model.uuid,
+      LANGBOT_FAKE_PROVIDER_FALLBACK_MODEL_UUIDS: fallbackModels.map((item) => item.uuid).join(","),
       LANGBOT_FAKE_PROVIDER_PIPELINE_URL: result.pipeline_url,
       LANGBOT_FAKE_PROVIDER_PIPELINE_NAME: pipelineName,
     });
@@ -168,7 +184,7 @@ try {
   }
 
   result.status = "pass";
-  result.reason = `Fake provider pipeline is configured with ${requester}/${modelName}.`;
+  result.reason = `Fake provider pipeline is configured with ${requester}/${modelName} and ${fallbackModels.length} fallback(s).`;
 } catch (error) {
   result.status = result.status === "env_issue" ? "env_issue" : "fail";
   result.reason = result.reason || safeReason(error.message);
@@ -328,7 +344,11 @@ function healthyFakeProviderConfig() {
     fault_status: 500,
     fail_first_n: 0,
     fail_every_n: 0,
+    fail_models: [],
     fail_after_first_chunk: false,
+    fail_after_first_chunk_delay_ms: 0,
+    fail_after_first_chunk_mode: "disconnect",
+    fail_after_first_chunk_models: [],
     dynamic_response: true,
   };
 }
@@ -342,7 +362,14 @@ function targetFakeProviderConfig() {
     fault_status: httpFaultStatus(env.LANGBOT_FAKE_PROVIDER_FAULT_STATUS, 500),
     fail_first_n: nonNegativeInteger(env.LANGBOT_FAKE_PROVIDER_FAIL_FIRST_N, 0),
     fail_every_n: nonNegativeInteger(env.LANGBOT_FAKE_PROVIDER_FAIL_EVERY_N, 0),
+    fail_models: textList(env.LANGBOT_FAKE_PROVIDER_FAIL_MODELS),
     fail_after_first_chunk: envBool(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK, false),
+    fail_after_first_chunk_delay_ms: nonNegativeInteger(
+      env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_DELAY_MS,
+      0,
+    ),
+    fail_after_first_chunk_mode: streamFaultMode(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_MODE),
+    fail_after_first_chunk_models: textList(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_MODELS),
     dynamic_response: envBool(env.LANGBOT_FAKE_PROVIDER_DYNAMIC_RESPONSE, true),
   };
 }
@@ -476,7 +503,7 @@ async function ensureModel({ backendUrl, token, providerUuid, name }) {
   };
 }
 
-async function ensurePipeline({ backendUrl, token, name, modelUuid }) {
+async function ensurePipeline({ backendUrl, token, name, modelUuid, fallbackModelUuids = [] }) {
   const list = await apiJson(backendUrl, "/api/v1/pipelines", { token });
   if (isApiFailure(list)) {
     throw new Error(list.json.msg || "Failed to list pipelines.");
@@ -538,7 +565,7 @@ async function ensurePipeline({ backendUrl, token, name, modelUuid }) {
     "max-round": positiveInteger(existingLocalAgentConfig["max-round"], 10),
     model: {
       primary: modelUuid,
-      fallbacks: [],
+      fallbacks: fallbackModelUuids,
     },
   };
   const updatedConfig = {
@@ -605,6 +632,17 @@ function envBool(value, fallback) {
   return fallback;
 }
 
+function textList(value) {
+  return String(value || "")
+    .split(/\r?\n|,/)
+    .map((item) => item.trim())
+    .filter(Boolean);
+}
+
+function streamFaultMode(value) {
+  return String(value || "").trim().toLowerCase() === "error_event" ? "error_event" : "disconnect";
+}
+
 function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }

skills/scripts/e2e/fake-openai-provider.mjs

@@ -54,7 +54,11 @@ const config = {
   fault_status: integer(env.LANGBOT_FAKE_PROVIDER_FAULT_STATUS, 500),
   fail_first_n: integer(env.LANGBOT_FAKE_PROVIDER_FAIL_FIRST_N, 0),
   fail_every_n: integer(env.LANGBOT_FAKE_PROVIDER_FAIL_EVERY_N, 0),
+  fail_models: textList(env.LANGBOT_FAKE_PROVIDER_FAIL_MODELS),
   fail_after_first_chunk: bool(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK, false),
+  fail_after_first_chunk_delay_ms: integer(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_DELAY_MS, 0),
+  fail_after_first_chunk_mode: faultMode(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_MODE),
+  fail_after_first_chunk_models: textList(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_MODELS),
   dynamic_response: !/^(0|false|no|off)$/i.test(env.LANGBOT_FAKE_PROVIDER_DYNAMIC_RESPONSE || ""),
   request_log_limit: integer(env.LANGBOT_FAKE_PROVIDER_REQUEST_LOG_LIMIT, 500),
 };
@@ -135,8 +139,12 @@ const server = createServer(async (request, response) => {
       requestCount += 1;
       const body = await readJson(request);
       const requestId = `chatcmpl-langbot-fake-${requestCount}`;
+      const requestModel = String(body.model || modelName);
       const shouldFail = requestCount <= config.fail_first_n
-        || (config.fail_every_n > 0 && requestCount % config.fail_every_n === 0);
+        || (config.fail_every_n > 0 && requestCount % config.fail_every_n === 0)
+        || config.fail_models.includes(requestModel);
+      const failAfterFirstChunk = config.fail_after_first_chunk
+        || config.fail_after_first_chunk_models.includes(requestModel);
       const replyMessage = buildResponse(body);
       const replyText = replyMessage.content || "";
       requestRecord = recordRequest({
@@ -144,7 +152,7 @@ const server = createServer(async (request, response) => {
         request_number: requestCount,
         path: url.pathname,
         stream: Boolean(body.stream),
-        model: body.model || "",
+        model: requestModel,
         message_count: Array.isArray(body.messages) ? body.messages.length : 0,
         should_fail: shouldFail,
         status: "running",
@@ -179,7 +187,7 @@ const server = createServer(async (request, response) => {
           requestId,
           model: body.model || modelName,
           message: replyMessage,
-          failAfterFirstChunk: config.fail_after_first_chunk,
+          failAfterFirstChunk,
           requestRecord,
           startedPerf,
         });
@@ -416,6 +424,22 @@ async function streamCompletion(response, {
       choices: [{ index: 0, delta: { content: chunks[index] }, finish_reason: null }],
     });
     if (failMidStream && index === 0) {
+      await sleep(config.fail_after_first_chunk_delay_ms);
+      if (config.fail_after_first_chunk_mode === "error_event") {
+        response.write(`data: ${JSON.stringify({
+          error: {
+            message: "LangBot fake provider injected a mid-stream error event",
+            type: "fake_provider_stream_fault",
+            code: "fake_provider_stream_fault",
+          },
+        })}\n\n`);
+        response.end();
+        finishRequestRecord(requestRecord, startedPerf, {
+          status: "mid_stream_error_event",
+          http_status: 200,
+        });
+        return;
+      }
       finishRequestRecord(requestRecord, startedPerf, {
         status: "mid_stream_disconnect",
         http_status: 200,
@@ -815,12 +839,18 @@ function applyConfig(updates) {
   assignNonNegativeInteger(updates, "chunk_count");
   assignNonNegativeInteger(updates, "fail_first_n");
   assignNonNegativeInteger(updates, "fail_every_n");
+  assignTextList(updates, "fail_models");
   assignNonNegativeInteger(updates, "request_log_limit");
   if (updates.fault_status !== undefined) {
     const parsed = Number.parseInt(String(updates.fault_status), 10);
     if (Number.isInteger(parsed) && parsed >= 400 && parsed <= 599) config.fault_status = parsed;
   }
   assignBoolean(updates, "fail_after_first_chunk");
+  assignNonNegativeInteger(updates, "fail_after_first_chunk_delay_ms");
+  if (updates.fail_after_first_chunk_mode !== undefined) {
+    config.fail_after_first_chunk_mode = faultMode(updates.fail_after_first_chunk_mode);
+  }
+  assignTextList(updates, "fail_after_first_chunk_models");
   assignBoolean(updates, "dynamic_response");
 }
 
@@ -838,3 +868,21 @@ function assignBoolean(updates, key) {
   if (updates[key] === undefined) return;
   config[key] = bool(updates[key], config[key]);
 }
+
+function assignTextList(updates, key) {
+  if (updates[key] === undefined) return;
+  config[key] = Array.isArray(updates[key])
+    ? updates[key].map(String).map((item) => item.trim()).filter(Boolean)
+    : textList(updates[key]);
+}
+
+function textList(value) {
+  return String(value || "")
+    .split(/\r?\n|,/)
+    .map((item) => item.trim())
+    .filter(Boolean);
+}
+
+function faultMode(value) {
+  return String(value || "").trim().toLowerCase() === "error_event" ? "error_event" : "disconnect";
+}

skills/skills.index.json

@@ -175,6 +175,7 @@
         "local-agent-complex-coding-task-debug-chat",
         "local-agent-context-compaction-debug-chat",
         "local-agent-effective-prompt-debug-chat",
+        "local-agent-model-fallback-before-first-chunk-debug-chat",
         "local-agent-multimodal-debug-chat",
         "local-agent-multitool-rag-compaction-debug-chat",
         "local-agent-nonstreaming-debug-chat",
@@ -183,6 +184,7 @@
         "local-agent-rag-debug-chat",
         "local-agent-rag-multimodal-debug-chat",
         "local-agent-steering-debug-chat",
+        "local-agent-streaming-post-commit-failure-debug-chat",
         "local-agent-tool-error-recovery-debug-chat",
         "local-agent-tool-loop-limit-debug-chat",
         "mcp-stdio-register",
@@ -1168,6 +1170,38 @@
             "backend_log"
           ]
         },
+        {
+          "id": "local-agent-model-fallback-before-first-chunk-debug-chat",
+          "title": "Local Agent falls back when the primary model fails before streaming starts",
+          "mode": "probe",
+          "area": "pipeline",
+          "type": "chaos",
+          "priority": "p1",
+          "risk": "high",
+          "ci_eligible": false,
+          "tags": [
+            "local-agent",
+            "fallback",
+            "streaming",
+            "fake-provider",
+            "fault-injection"
+          ],
+          "automation": "skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs",
+          "setup_automation": [
+            "node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env"
+          ],
+          "setup_provides_env": [
+            "LANGBOT_FAKE_PROVIDER_URL",
+            "LANGBOT_FAKE_PROVIDER_PIPELINE_URL",
+            "LANGBOT_FAKE_PROVIDER_PIPELINE_NAME"
+          ],
+          "evidence_required": [
+            "metrics",
+            "network",
+            "api_diagnostic",
+            "filesystem"
+          ]
+        },
         {
           "id": "local-agent-multimodal-debug-chat",
           "title": "Local Agent Debug Chat preserves uploaded image input",
@@ -1429,6 +1463,38 @@
             "api_diagnostic"
           ]
         },
+        {
+          "id": "local-agent-streaming-post-commit-failure-debug-chat",
+          "title": "Local Agent does not fall back after a committed stream fails",
+          "mode": "probe",
+          "area": "pipeline",
+          "type": "chaos",
+          "priority": "p1",
+          "risk": "high",
+          "ci_eligible": false,
+          "tags": [
+            "local-agent",
+            "fallback",
+            "streaming",
+            "fake-provider",
+            "fault-injection"
+          ],
+          "automation": "skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs",
+          "setup_automation": [
+            "node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env"
+          ],
+          "setup_provides_env": [
+            "LANGBOT_FAKE_PROVIDER_URL",
+            "LANGBOT_FAKE_PROVIDER_PIPELINE_URL",
+            "LANGBOT_FAKE_PROVIDER_PIPELINE_NAME"
+          ],
+          "evidence_required": [
+            "metrics",
+            "network",
+            "api_diagnostic",
+            "filesystem"
+          ]
+        },
         {
           "id": "local-agent-tool-error-recovery-debug-chat",
           "title": "Local Agent feeds plugin tool errors back to the model",
@@ -2204,6 +2270,8 @@
           ],
           "cases": [
             "local-agent-basic-debug-chat",
+            "local-agent-model-fallback-before-first-chunk-debug-chat",
+            "local-agent-streaming-post-commit-failure-debug-chat",
             "qa-plugin-smoke-live-install",
             "local-agent-effective-prompt-debug-chat",
             "local-agent-context-compaction-debug-chat",

skills/skills/langbot-testing/cases/langbot-live-backend-log-health.yaml

@@ -16,14 +16,14 @@ skills:
 env:
   - LANGBOT_BACKEND_URL
 automation: skills/langbot-testing/probes/langbot-live-backend-log-health.mjs
-metrics_thresholds_json: '{"fail_count":{"max":0}}'
+metrics_thresholds_json: '{"scanned_line_count":{"min":1},"fail_count":{"max":0}}'
 load_profile_json: '{"lookback_seconds":300,"log_source":"LANGBOT_BACKEND_LOG or latest LANGBOT_REPO/data/logs/langbot-*.log"}'
 steps:
   - "Confirm the selected LangBot backend log belongs to the intended test target."
   - "Run `rtk bin/lbs test run langbot-live-backend-log-health --dry-run` first; remove `--dry-run` after checking evidence directory and log source."
   - "Automation scans the recent backend log window for fail-severity runtime findings such as Traceback, ImportError, ERROR, unclosed sessions, and unawaited coroutines."
 checks:
-  - "automation-result.json status is pass only when fail_count is 0."
+  - "automation-result.json status is pass only when at least one timestamped line is scanned and fail_count is 0."
   - "metrics_summary includes scanned_line_count, fail_count, warning_count, and finding_count."
   - "findings.json and scanned-backend.log are written under LBS_EVIDENCE_DIR."
 evidence_required:
@@ -33,6 +33,7 @@ evidence_required:
 diagnostics:
   - "Set LANGBOT_BACKEND_LOG to an explicit log path when the latest log file is not the run target."
   - "Set LANGBOT_BACKEND_LOG_SINCE or LANGBOT_BACKEND_LOG_LOOKBACK_SECONDS to control the scan window."
+  - "A zero-line scan is env_issue, not pass; point LANGBOT_BACKEND_LOG at the active service output or widen the lookback."
   - "This probe measures runtime log health; it does not prove user-facing Debug Chat, plugin, model, or RAG behavior."
 success_patterns:
   - "Live backend log health passed"

skills/skills/langbot-testing/cases/local-agent-model-fallback-before-first-chunk-debug-chat.yaml

@@ -0,0 +1,79 @@
+id: local-agent-model-fallback-before-first-chunk-debug-chat
+title: "Local Agent falls back when the primary model fails before streaming starts"
+mode: probe
+area: pipeline
+type: chaos
+priority: p1
+risk: high
+ci_eligible: false
+tags:
+  - local-agent
+  - fallback
+  - streaming
+  - fake-provider
+  - fault-injection
+skills:
+  - langbot-env-setup
+  - langbot-testing
+env:
+  - LANGBOT_BACKEND_URL
+  - LANGBOT_FRONTEND_URL
+  - LANGBOT_E2E_LOGIN_USER
+automation: skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs
+automation_env:
+  - LANGBOT_BACKEND_URL
+  - LANGBOT_E2E_LOGIN_USER
+  - LANGBOT_FAKE_PROVIDER_PIPELINE_URL
+  - LANGBOT_FAKE_PROVIDER_PIPELINE_NAME
+automation_pipeline_url_env: LANGBOT_FAKE_PROVIDER_PIPELINE_URL
+automation_pipeline_name_env: LANGBOT_FAKE_PROVIDER_PIPELINE_NAME
+automation_debug_chat_load_requests: "1"
+automation_debug_chat_load_concurrency: "1"
+automation_debug_chat_load_timeout_ms: "30000"
+automation_debug_chat_load_response_p95_ms: "10000"
+automation_debug_chat_load_max_error_rate: "0"
+automation_debug_chat_load_min_ok_count: "1"
+automation_debug_chat_load_min_provider_fault_count: "1"
+automation_debug_chat_load_expected_prefix: "FALLBACKQA"
+automation_debug_chat_load_prompt_template: '请只回复 "{expected}"，不要解释，不要添加其他字符。'
+automation_debug_chat_load_stream: "true"
+automation_debug_chat_load_reset: "true"
+automation_debug_chat_load_fail_on_final_mismatch: "true"
+automation_debug_chat_load_provider_model_thresholds_json: '{"qa-fallback-primary":{"min":1},"qa-fallback-secondary":{"min":1}}'
+automation_fake_provider_pipeline_name: "LangBot QA Model Fallback Debug Chat"
+automation_fake_provider_model_name: "qa-fallback-primary"
+automation_fake_provider_fallback_model_names: "qa-fallback-secondary"
+automation_fake_provider_fail_models: "qa-fallback-primary"
+automation_fake_provider_fail_after_first_chunk_models: ""
+automation_fake_provider_first_token_delay_ms: "25"
+automation_fake_provider_chunk_delay_ms: "10"
+automation_fake_provider_fault_status: "503"
+metrics_thresholds_json: '{"response_p95_ms":{"max":10000},"error_rate":{"max":0},"fake_provider_fault_count_min":{"min":1},"primary_model_requests":{"min":1},"fallback_model_requests":{"min":1}}'
+fault_model_json: '{"provider_fault":"HTTP 503 for every qa-fallback-primary request before the first stream chunk","expected_behavior":"Local Agent invokes qa-fallback-secondary and returns the expected Debug Chat response."}'
+setup_automation:
+  - "node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env"
+setup_provides_env:
+  - LANGBOT_FAKE_PROVIDER_URL
+  - LANGBOT_FAKE_PROVIDER_PIPELINE_URL
+  - LANGBOT_FAKE_PROVIDER_PIPELINE_NAME
+steps:
+  - "Create a QA pipeline with a failing primary model and a healthy fallback model on the controlled provider."
+  - "Send one streaming Debug Chat request with a unique response token."
+  - "Inspect provider request records and require both the failed primary request and the successful fallback request."
+checks:
+  - "The user-visible request succeeds with the unique expected response."
+  - "The fake provider records at least one qa-fallback-primary request and at least one qa-fallback-secondary request."
+  - "At least one provider fault is observed without a final Debug Chat error."
+evidence_required:
+  - metrics
+  - network
+  - api_diagnostic
+  - filesystem
+expected_failures:
+  - "fake_provider_fault"
+failure_patterns:
+  - "All models failed during streaming setup"
+  - "Timed out after"
+troubleshooting:
+  - backend-not-listening
+  - debug-chat-history-contaminates-automation

skills/skills/langbot-testing/cases/local-agent-streaming-post-commit-failure-debug-chat.yaml

@@ -0,0 +1,86 @@
+id: local-agent-streaming-post-commit-failure-debug-chat
+title: "Local Agent does not fall back after a committed stream fails"
+mode: probe
+area: pipeline
+type: chaos
+priority: p1
+risk: high
+ci_eligible: false
+tags:
+  - local-agent
+  - fallback
+  - streaming
+  - fake-provider
+  - fault-injection
+skills:
+  - langbot-env-setup
+  - langbot-testing
+env:
+  - LANGBOT_BACKEND_URL
+  - LANGBOT_FRONTEND_URL
+  - LANGBOT_E2E_LOGIN_USER
+automation: skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs
+automation_env:
+  - LANGBOT_BACKEND_URL
+  - LANGBOT_E2E_LOGIN_USER
+  - LANGBOT_FAKE_PROVIDER_PIPELINE_URL
+  - LANGBOT_FAKE_PROVIDER_PIPELINE_NAME
+automation_pipeline_url_env: LANGBOT_FAKE_PROVIDER_PIPELINE_URL
+automation_pipeline_name_env: LANGBOT_FAKE_PROVIDER_PIPELINE_NAME
+automation_debug_chat_load_requests: "1"
+automation_debug_chat_load_concurrency: "1"
+automation_debug_chat_load_timeout_ms: "30000"
+automation_debug_chat_load_max_error_rate: "1"
+automation_debug_chat_load_min_error_rate: "1"
+automation_debug_chat_load_min_error_count: "1"
+automation_debug_chat_load_min_provider_fault_count: "1"
+automation_debug_chat_load_require_success: "false"
+automation_debug_chat_load_expected_prefix: "POSTCOMMITQA"
+automation_debug_chat_load_prompt_template: '请只回复 "{expected}"，不要解释，不要添加其他字符。'
+automation_debug_chat_load_stream: "true"
+automation_debug_chat_load_reset: "true"
+automation_debug_chat_load_fail_on_final_mismatch: "true"
+automation_debug_chat_load_provider_model_thresholds_json: '{"qa-post-commit-primary":{"min":1},"qa-post-commit-fallback":{"max":0}}'
+automation_fake_provider_pipeline_name: "LangBot QA Post Commit Failure Debug Chat"
+automation_fake_provider_model_name: "qa-post-commit-primary"
+automation_fake_provider_fallback_model_names: "qa-post-commit-fallback"
+automation_fake_provider_fail_models: ""
+automation_fake_provider_fail_after_first_chunk_models: "qa-post-commit-primary"
+automation_fake_provider_fail_after_first_chunk_delay_ms: "1000"
+automation_fake_provider_fail_after_first_chunk_mode: "error_event"
+automation_fake_provider_first_token_delay_ms: "25"
+automation_fake_provider_chunk_delay_ms: "10"
+metrics_thresholds_json: '{"error_rate":{"min":1,"max":1},"fake_provider_fault_count_min":{"min":1},"primary_model_requests":{"min":1},"fallback_model_requests":{"max":0}}'
+fault_model_json: '{"provider_fault":"Send a provider error event for qa-post-commit-primary one second after its first content chunk","expected_behavior":"The run terminates with a controlled error and never invokes qa-post-commit-fallback after visible output is committed."}'
+setup_automation:
+  - "node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env"
+setup_provides_env:
+  - LANGBOT_FAKE_PROVIDER_URL
+  - LANGBOT_FAKE_PROVIDER_PIPELINE_URL
+  - LANGBOT_FAKE_PROVIDER_PIPELINE_NAME
+steps:
+  - "Create a QA pipeline with a primary model that sends an error event after one content chunk and a healthy fallback model."
+  - "Send one streaming Debug Chat request and require a terminal error."
+  - "Inspect provider request records and prove the fallback model was never invoked."
+checks:
+  - "The fake provider records a mid-stream error event for qa-post-commit-primary."
+  - "The Debug Chat request terminates as an error instead of timing out or reporting success."
+  - "qa-post-commit-fallback request count remains zero."
+evidence_required:
+  - metrics
+  - network
+  - api_diagnostic
+  - filesystem
+expected_failures:
+  - "mid_stream_error_event"
+  - "failed after first chunk"
+  - "Agent runner execution failed."
+failure_patterns:
+  - "Agent runner execution failed."
+  - "Timed out after"
+  - "qa-post-commit-fallback"
+troubleshooting:
+  - backend-not-listening
+  - debug-chat-history-contaminates-automation
+diagnostics:
+  - "Use error_event rather than a bare TCP disconnect for this contract: LiteLLM may normalize an abrupt EOF into a final chunk before Local Agent can observe the transport boundary."

skills/skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs

@@ -55,6 +55,8 @@ const minErrorCount = nonNegativeInteger(env.LANGBOT_DEBUG_CHAT_LOAD_MIN_ERROR_C
 const minOkCount = nonNegativeInteger(env.LANGBOT_DEBUG_CHAT_LOAD_MIN_OK_COUNT, 0);
 const minProviderFaultCount = nonNegativeInteger(env.LANGBOT_DEBUG_CHAT_LOAD_MIN_PROVIDER_FAULT_COUNT, 0);
 const failOnFinalMismatch = bool(env.LANGBOT_DEBUG_CHAT_LOAD_FAIL_ON_FINAL_MISMATCH, false);
+const requireSuccess = bool(env.LANGBOT_DEBUG_CHAT_LOAD_REQUIRE_SUCCESS, true);
+const providerModelThresholds = jsonObject(env.LANGBOT_DEBUG_CHAT_LOAD_PROVIDER_MODEL_THRESHOLDS_JSON);
 const failureSignals = textList(env.LANGBOT_E2E_FAILURE_SIGNALS || env.LANGBOT_DEBUG_CHAT_LOAD_FAILURE_SIGNALS || "");
 
 const result = {
@@ -80,6 +82,7 @@ const result = {
     stream,
     reset_before_run: resetBeforeRun,
     fail_on_final_mismatch: failOnFinalMismatch,
+    require_success: requireSuccess,
   },
   evidence: {
     network_log: paths.networkLog,
@@ -190,6 +193,8 @@ try {
     status_counts: metrics.status_counts,
     fake_provider_request_count: metrics.fake_provider?.request_count ?? null,
     fake_provider_fault_count: metrics.fake_provider?.fault_count ?? null,
+    fake_provider_mid_stream_disconnect_count: metrics.fake_provider?.mid_stream_disconnect_count ?? null,
+    fake_provider_model_request_counts: metrics.fake_provider?.model_request_counts ?? {},
     fake_provider_duration_p95_ms: metrics.provider_timing?.provider_duration_ms.p95 ?? null,
     langbot_overhead_estimate_p95_ms: metrics.provider_timing?.langbot_overhead_estimate_ms.p95 ?? null,
     send_to_provider_start_p95_ms: metrics.provider_timing?.send_to_provider_start_ms.p95 ?? null,
@@ -782,12 +787,14 @@ function buildMetrics({ samples, totalRequests, concurrency, timeoutMs, loadDura
 function buildThresholds(metrics) {
   const thresholds = {
     error_rate: { actual: metrics.error_rate, max: maxErrorRate, pass: metrics.error_rate <= maxErrorRate },
-    response_p95_ms: {
+  };
+  if (requireSuccess) {
+    thresholds.response_p95_ms = {
       actual: metrics.response_duration_ms.p95,
       max: responseP95BudgetMs,
       pass: metrics.ok_count > 0 && metrics.response_duration_ms.p95 <= responseP95BudgetMs,
-    },
     };
+  }
   if (minErrorRate > 0) {
     thresholds.error_rate_min = {
       actual: metrics.error_rate,
@@ -824,9 +831,36 @@ function buildThresholds(metrics) {
       pass: metrics.ok_count > 0 && metrics.first_response_ms.p95 <= firstResponseP95BudgetMs,
     };
   }
+  for (const [model, bounds] of Object.entries(providerModelThresholds)) {
+    const actual = metrics.fake_provider?.model_request_counts?.[model] ?? 0;
+    if (Number.isFinite(bounds?.min)) {
+      thresholds[`provider_model_${model}_min`] = {
+        actual,
+        min: bounds.min,
+        pass: actual >= bounds.min,
+      };
+    }
+    if (Number.isFinite(bounds?.max)) {
+      thresholds[`provider_model_${model}_max`] = {
+        actual,
+        max: bounds.max,
+        pass: actual <= bounds.max,
+      };
+    }
+  }
   return thresholds;
 }
 
+function jsonObject(value) {
+  if (!value) return {};
+  try {
+    const parsed = JSON.parse(String(value));
+    return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : {};
+  } catch {
+    return {};
+  }
+}
+
 function looksLikeEnvIssue(error) {
   const message = String(error?.message || error || "");
   return /fetch failed|ECONNREFUSED|ENOTFOUND|LANGBOT_.*not configured|Could not read recovery_key|Backend did not respond/i.test(message);

skills/skills/langbot-testing/probes/langbot-live-backend-log-health.mjs

@@ -139,11 +139,16 @@ async function main() {
     const text = await readFile(logPath, "utf8");
     scan = scanLines(text, since, startedAt.getFullYear());
     const failCount = scan.findings.filter((item) => item.severity === "fail").length;
+    if (scan.scanned.length === 0) {
+      status = "env_issue";
+      reason = "Backend log health had no timestamped lines in the selected window; no health verdict is possible.";
+    } else {
       status = failCount === 0 ? "pass" : "fail";
       reason = status === "pass"
         ? "Live backend log health passed; no fail-severity findings in the scanned window."
         : "Live backend log health found fail-severity backend log findings.";
     }
+  }
 
   const warningCount = scan.findings.filter((item) => item.severity === "warning").length;
   const failCount = scan.findings.filter((item) => item.severity === "fail").length;
@@ -158,6 +163,11 @@ async function main() {
     finding_count: scan.findings.length,
   };
   const thresholds = {
+    scanned_line_count_min: {
+      actual: scan.scanned.length,
+      min: 1,
+      pass: scan.scanned.length >= 1,
+    },
     fail_count: { actual: failCount, max: 0, pass: failCount === 0 },
   };
 

skills/skills/langbot-testing/probes/lib/fake-provider-timing.mjs

@@ -6,8 +6,16 @@ export function summarizeFakeProviderState(state) {
   const faultRequests = chatRequests.filter((request) => (
     request?.should_fail === true
       || request?.status === "http_fault"
+      || request?.status === "mid_stream_disconnect"
+      || request?.status === "mid_stream_error_event"
       || (Number.isFinite(request?.http_status) && request.http_status >= 400)
   ));
+  const modelRequestCounts = {};
+  for (const request of chatRequests) {
+    const model = String(request?.model || "");
+    if (!model) continue;
+    modelRequestCounts[model] = (modelRequestCounts[model] || 0) + 1;
+  }
 
   return {
     status: state.status || "unknown",
@@ -16,6 +24,10 @@ export function summarizeFakeProviderState(state) {
     recent_request_count: recentRequests.length,
     chat_request_count: chatRequests.length,
     fault_count: faultRequests.length,
+    mid_stream_disconnect_count: chatRequests.filter(
+      (request) => ["mid_stream_disconnect", "mid_stream_error_event"].includes(request?.status),
+    ).length,
+    model_request_counts: modelRequestCounts,
     streamed_request_count: chatRequests.filter((request) => request?.stream === true).length,
     duration_ms: stats(chatRequests.map((request) => numberOrNull(request?.duration_ms)).filter(Number.isFinite)),
     successful_duration_ms: stats(successfulRequests.map((request) => numberOrNull(request?.duration_ms)).filter(Number.isFinite)),

skills/skills/langbot-testing/references/local-agent-runner-coverage.md

@@ -24,6 +24,8 @@ These browser cases are the minimum gate for a local-agent migration check:
 | Case | Path Covered | Expected Behavior |
 | --- | --- | --- |
 | `local-agent-basic-debug-chat` | Streaming LLM invocation with effective host context | Bot returns deterministic `OK`; backend logs streaming completion. |
+| `local-agent-model-fallback-before-first-chunk-debug-chat` | Primary/fallback selection before stream commitment | Provider records failed primary requests and a successful fallback request; Debug Chat completes. |
+| `local-agent-streaming-post-commit-failure-debug-chat` | Terminal provider failure after stream commitment | Provider records the primary error event, fallback request count stays zero, and Debug Chat reports a controlled failure. |
 | `local-agent-effective-prompt-debug-chat` | PromptPreProcessing and host effective prompt handoff through `ctx.adapter.extra.prompt` | Bot returns `PROMPT_PREPROCESS_OK` from the fixture prompt probe. |
 | `local-agent-context-compaction-debug-chat` | Runner-owned context budgeting and old-history compaction | Automation temporarily shrinks the runner context window, sends multi-turn Debug Chat history, and the bot still recovers the older sentinel. |
 | `local-agent-rag-debug-chat` | Knowledge-base authorization, retrieval, and RAG prompt insertion | Bot returns the KB sentinel, not a generic answer. |
@@ -52,8 +54,8 @@ These browser cases are the minimum gate for a local-agent migration check:
 | Streaming model invocation | Enable Debug Chat streaming and ask for `OK`. | UI receives incremental bot output and backend logs streaming completion. |
 | Non-streaming UI delivery | Disable Debug Chat streaming. | UI receives a final bot message without frontend streaming errors. |
 | Non-streaming model invocation | Use local-agent component tests with `runtime_metadata.streaming_supported=false` or a host adapter that does not support streaming. | Runner calls `invoke_llm` instead of `invoke_llm_stream` and emits `message.completed`. |
-| Model fallback before first chunk | Configure a failing primary and working fallback, preferably with a controlled test provider. | First model failure does not fail the run; fallback model produces the final answer. |
+| Model fallback before first chunk | Run `local-agent-model-fallback-before-first-chunk-debug-chat`. | Provider records both the failed primary and successful fallback model requests; the final user request succeeds. |
-| Failure after streaming commit | Use a controlled provider that emits one chunk and then fails. | Runner reports a terminal run failure and does not fallback after partial output. |
+| Failure after streaming commit | Run `local-agent-streaming-post-commit-failure-debug-chat`. | Provider records a post-content error event, Runner reports terminal failure, and fallback request count remains zero. |
 | No authorized model | Clear model config or configure a model not in run resources. | Runner returns `runner.no_model` instead of calling an unauthorized model. |
 | MCP tool call | Use `qa-local-stdio` and `qa_mcp_echo`. | Bot returns the exact `qa_mcp_echo:<input>` result; `/api/v1/tools` contains `qa_mcp_echo`. |
 | Plugin tool call | Install a fixture plugin exposing a deterministic tool and bind it to the pipeline. | Runner lists the plugin tool and can call it through the same tool loop as MCP tools. |

skills/skills/langbot-testing/references/local-agent-runner.md

@@ -51,6 +51,8 @@ Pair a basic Debug Chat run with a deterministic plugin tool call, for example `
 Run these cases before saying the pluginized local-agent behavior is healthy:
 
 - `local-agent-basic-debug-chat`: basic streaming model invocation.
+- `local-agent-model-fallback-before-first-chunk-debug-chat`: primary model failure before the first visible chunk switches to the configured fallback.
+- `local-agent-streaming-post-commit-failure-debug-chat`: a provider error after a committed content chunk terminates the run without invoking fallback.
 - `local-agent-effective-prompt-debug-chat`: host effective prompt after PromptPreProcessing reaches the runner.
 - `local-agent-rag-debug-chat`: LangRAG retrieval reaches the runner and affects the answer.
 - `mcp-stdio-tool-call`: MCP tool discovery and local-agent tool loop.

skills/skills/langbot-testing/suites/local-agent-gate.yaml

@@ -9,6 +9,8 @@ tags:
   - regression
 cases:
   - local-agent-basic-debug-chat
+  - local-agent-model-fallback-before-first-chunk-debug-chat
+  - local-agent-streaming-post-commit-failure-debug-chat
   - qa-plugin-smoke-live-install
   - local-agent-effective-prompt-debug-chat
   - local-agent-context-compaction-debug-chat

skills/src/commands/validate.ts

@@ -209,6 +209,7 @@ function validateCaseItem(root: string, item: StructuredItem, skillNames: Set<st
     "automation_fake_provider_chunk_count",
     "automation_fake_provider_fail_first_n",
     "automation_fake_provider_fail_every_n",
+    "automation_fake_provider_fail_after_first_chunk_delay_ms",
   ]) {
     const value = scalar(item.fields, key);
     if (value && (!/^\d+$/.test(value) || Number.parseInt(value, 10) < 0)) {
@@ -236,6 +237,7 @@ function validateCaseItem(root: string, item: StructuredItem, skillNames: Set<st
     "automation_debug_chat_load_stream",
     "automation_debug_chat_load_reset",
     "automation_debug_chat_load_fail_on_final_mismatch",
+    "automation_debug_chat_load_require_success",
     "automation_fake_provider_fail_after_first_chunk",
     "automation_fake_provider_dynamic_response",
   ]) {

skills/src/readiness.ts

@@ -133,14 +133,23 @@ export function automationEnvDefaults(item: StructuredItem, env: EnvSource = pro
     ["automation_debug_chat_load_stream", "LANGBOT_DEBUG_CHAT_LOAD_STREAM"],
     ["automation_debug_chat_load_reset", "LANGBOT_DEBUG_CHAT_LOAD_RESET"],
     ["automation_debug_chat_load_fail_on_final_mismatch", "LANGBOT_DEBUG_CHAT_LOAD_FAIL_ON_FINAL_MISMATCH"],
+    ["automation_debug_chat_load_require_success", "LANGBOT_DEBUG_CHAT_LOAD_REQUIRE_SUCCESS"],
+    ["automation_debug_chat_load_provider_model_thresholds_json", "LANGBOT_DEBUG_CHAT_LOAD_PROVIDER_MODEL_THRESHOLDS_JSON"],
+    ["automation_fake_provider_pipeline_name", "LANGBOT_FAKE_PROVIDER_PIPELINE_NAME"],
+    ["automation_fake_provider_model_name", "LANGBOT_FAKE_PROVIDER_MODEL_NAME"],
+    ["automation_fake_provider_fallback_model_names", "LANGBOT_FAKE_PROVIDER_FALLBACK_MODEL_NAMES"],
     ["automation_fake_provider_response_text", "LANGBOT_FAKE_PROVIDER_RESPONSE_TEXT"],
     ["automation_fake_provider_first_token_delay_ms", "LANGBOT_FAKE_PROVIDER_FIRST_TOKEN_DELAY_MS"],
     ["automation_fake_provider_chunk_delay_ms", "LANGBOT_FAKE_PROVIDER_CHUNK_DELAY_MS"],
     ["automation_fake_provider_chunk_count", "LANGBOT_FAKE_PROVIDER_CHUNK_COUNT"],
     ["automation_fake_provider_fail_first_n", "LANGBOT_FAKE_PROVIDER_FAIL_FIRST_N"],
     ["automation_fake_provider_fail_every_n", "LANGBOT_FAKE_PROVIDER_FAIL_EVERY_N"],
+    ["automation_fake_provider_fail_models", "LANGBOT_FAKE_PROVIDER_FAIL_MODELS"],
     ["automation_fake_provider_fault_status", "LANGBOT_FAKE_PROVIDER_FAULT_STATUS"],
     ["automation_fake_provider_fail_after_first_chunk", "LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK"],
+    ["automation_fake_provider_fail_after_first_chunk_delay_ms", "LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_DELAY_MS"],
+    ["automation_fake_provider_fail_after_first_chunk_mode", "LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_MODE"],
+    ["automation_fake_provider_fail_after_first_chunk_models", "LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_MODELS"],
     ["automation_fake_provider_dynamic_response", "LANGBOT_FAKE_PROVIDER_DYNAMIC_RESPONSE"],
     ["automation_filesystem_checks_json", "LANGBOT_E2E_FILESYSTEM_CHECKS_JSON"],
     ["automation_plugin_package", "LANGBOT_E2E_PLUGIN_PACKAGE"],

skills/test/lbs-cli.test.ts

@@ -3582,6 +3582,84 @@ test("fake provider returns IMAGE_OK only when image metadata is present", async
   }
 });
 
+test("fake provider can inject faults for only the selected model", async () => {
+  const provider = await startFakeProviderForTest();
+  const rootUrl = provider.baseUrl.replace(/\/v1\/?$/, "");
+  try {
+    const configured = await fetch(`${rootUrl}/__qa/config`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        config: { fail_models: ["qa-primary"] },
+        reset_request_count: true,
+      }),
+    });
+    assert.equal(configured.status, 200);
+
+    const primary = await fetch(`${provider.baseUrl}/chat/completions`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        model: "qa-primary",
+        stream: false,
+        messages: [{ role: "user", content: "Reply OK" }],
+      }),
+    });
+    assert.equal(primary.status, 500);
+
+    const fallback = await requestFakeProvider(provider, {
+      model: "qa-fallback",
+      messages: [{ role: "user", content: "Reply OK" }],
+    });
+    assert.equal(fakeProviderMessage(fallback).content, "OK");
+
+    const state = await fetch(`${rootUrl}/__qa/config`).then((response) => response.json());
+    assert.deepEqual(
+      state.recent_requests.map((request: { model: string; status: string }) => [request.model, request.status]),
+      [["qa-primary", "http_fault"], ["qa-fallback", "ok"]],
+    );
+  } finally {
+    await provider.stop();
+  }
+});
+
+test("local-agent model failure cases expose fallback fault controls", () => {
+  const beforeFirstChunk = capture(() =>
+    commandTestRun(
+      ctx(["test", "run", "local-agent-model-fallback-before-first-chunk-debug-chat", "--dry-run", "--json"]),
+    ),
+  );
+  assert.equal(beforeFirstChunk.code, 0);
+  const fallbackRun = JSON.parse(beforeFirstChunk.output);
+  assert.equal(fallbackRun.automation.env_defaults.LANGBOT_FAKE_PROVIDER_MODEL_NAME, "qa-fallback-primary");
+  assert.equal(
+    fallbackRun.automation.env_defaults.LANGBOT_FAKE_PROVIDER_FALLBACK_MODEL_NAMES,
+    "qa-fallback-secondary",
+  );
+  assert.equal(fallbackRun.automation.env_defaults.LANGBOT_FAKE_PROVIDER_FAIL_MODELS, "qa-fallback-primary");
+
+  const postCommit = capture(() =>
+    commandTestRun(
+      ctx(["test", "run", "local-agent-streaming-post-commit-failure-debug-chat", "--dry-run", "--json"]),
+    ),
+  );
+  assert.equal(postCommit.code, 0);
+  const postCommitRun = JSON.parse(postCommit.output);
+  assert.equal(
+    postCommitRun.automation.env_defaults.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_MODELS,
+    "qa-post-commit-primary",
+  );
+  assert.equal(
+    postCommitRun.automation.env_defaults.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_DELAY_MS,
+    "1000",
+  );
+  assert.equal(
+    postCommitRun.automation.env_defaults.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_MODE,
+    "error_event",
+  );
+  assert.equal(postCommitRun.automation.env_defaults.LANGBOT_DEBUG_CHAT_LOAD_REQUIRE_SUCCESS, "false");
+});
+
 test("fake provider requires the effective system prompt before returning its sentinel", async () => {
   const provider = await startFakeProviderForTest();
   try {