test(skills): expand local agent reliability coverage

skills/skills/langbot-testing/cases/langbot-live-backend-log-health.yaml

@@ -16,14 +16,14 @@ skills:
 env:
   - LANGBOT_BACKEND_URL
 automation: skills/langbot-testing/probes/langbot-live-backend-log-health.mjs
-metrics_thresholds_json: '{"fail_count":{"max":0}}'
+metrics_thresholds_json: '{"scanned_line_count":{"min":1},"fail_count":{"max":0}}'
 load_profile_json: '{"lookback_seconds":300,"log_source":"LANGBOT_BACKEND_LOG or latest LANGBOT_REPO/data/logs/langbot-*.log"}'
 steps:
   - "Confirm the selected LangBot backend log belongs to the intended test target."
   - "Run `rtk bin/lbs test run langbot-live-backend-log-health --dry-run` first; remove `--dry-run` after checking evidence directory and log source."
   - "Automation scans the recent backend log window for fail-severity runtime findings such as Traceback, ImportError, ERROR, unclosed sessions, and unawaited coroutines."
 checks:
-  - "automation-result.json status is pass only when fail_count is 0."
+  - "automation-result.json status is pass only when at least one timestamped line is scanned and fail_count is 0."
   - "metrics_summary includes scanned_line_count, fail_count, warning_count, and finding_count."
   - "findings.json and scanned-backend.log are written under LBS_EVIDENCE_DIR."
 evidence_required:
@@ -33,6 +33,7 @@ evidence_required:
 diagnostics:
   - "Set LANGBOT_BACKEND_LOG to an explicit log path when the latest log file is not the run target."
   - "Set LANGBOT_BACKEND_LOG_SINCE or LANGBOT_BACKEND_LOG_LOOKBACK_SECONDS to control the scan window."
+  - "A zero-line scan is env_issue, not pass; point LANGBOT_BACKEND_LOG at the active service output or widen the lookback."
   - "This probe measures runtime log health; it does not prove user-facing Debug Chat, plugin, model, or RAG behavior."
 success_patterns:
   - "Live backend log health passed"

skills/skills/langbot-testing/cases/local-agent-model-fallback-before-first-chunk-debug-chat.yaml

@@ -0,0 +1,79 @@
+id: local-agent-model-fallback-before-first-chunk-debug-chat
+title: "Local Agent falls back when the primary model fails before streaming starts"
+mode: probe
+area: pipeline
+type: chaos
+priority: p1
+risk: high
+ci_eligible: false
+tags:
+  - local-agent
+  - fallback
+  - streaming
+  - fake-provider
+  - fault-injection
+skills:
+  - langbot-env-setup
+  - langbot-testing
+env:
+  - LANGBOT_BACKEND_URL
+  - LANGBOT_FRONTEND_URL
+  - LANGBOT_E2E_LOGIN_USER
+automation: skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs
+automation_env:
+  - LANGBOT_BACKEND_URL
+  - LANGBOT_E2E_LOGIN_USER
+  - LANGBOT_FAKE_PROVIDER_PIPELINE_URL
+  - LANGBOT_FAKE_PROVIDER_PIPELINE_NAME
+automation_pipeline_url_env: LANGBOT_FAKE_PROVIDER_PIPELINE_URL
+automation_pipeline_name_env: LANGBOT_FAKE_PROVIDER_PIPELINE_NAME
+automation_debug_chat_load_requests: "1"
+automation_debug_chat_load_concurrency: "1"
+automation_debug_chat_load_timeout_ms: "30000"
+automation_debug_chat_load_response_p95_ms: "10000"
+automation_debug_chat_load_max_error_rate: "0"
+automation_debug_chat_load_min_ok_count: "1"
+automation_debug_chat_load_min_provider_fault_count: "1"
+automation_debug_chat_load_expected_prefix: "FALLBACKQA"
+automation_debug_chat_load_prompt_template: '请只回复 "{expected}"，不要解释，不要添加其他字符。'
+automation_debug_chat_load_stream: "true"
+automation_debug_chat_load_reset: "true"
+automation_debug_chat_load_fail_on_final_mismatch: "true"
+automation_debug_chat_load_provider_model_thresholds_json: '{"qa-fallback-primary":{"min":1},"qa-fallback-secondary":{"min":1}}'
+automation_fake_provider_pipeline_name: "LangBot QA Model Fallback Debug Chat"
+automation_fake_provider_model_name: "qa-fallback-primary"
+automation_fake_provider_fallback_model_names: "qa-fallback-secondary"
+automation_fake_provider_fail_models: "qa-fallback-primary"
+automation_fake_provider_fail_after_first_chunk_models: ""
+automation_fake_provider_first_token_delay_ms: "25"
+automation_fake_provider_chunk_delay_ms: "10"
+automation_fake_provider_fault_status: "503"
+metrics_thresholds_json: '{"response_p95_ms":{"max":10000},"error_rate":{"max":0},"fake_provider_fault_count_min":{"min":1},"primary_model_requests":{"min":1},"fallback_model_requests":{"min":1}}'
+fault_model_json: '{"provider_fault":"HTTP 503 for every qa-fallback-primary request before the first stream chunk","expected_behavior":"Local Agent invokes qa-fallback-secondary and returns the expected Debug Chat response."}'
+setup_automation:
+  - "node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env"
+setup_provides_env:
+  - LANGBOT_FAKE_PROVIDER_URL
+  - LANGBOT_FAKE_PROVIDER_PIPELINE_URL
+  - LANGBOT_FAKE_PROVIDER_PIPELINE_NAME
+steps:
+  - "Create a QA pipeline with a failing primary model and a healthy fallback model on the controlled provider."
+  - "Send one streaming Debug Chat request with a unique response token."
+  - "Inspect provider request records and require both the failed primary request and the successful fallback request."
+checks:
+  - "The user-visible request succeeds with the unique expected response."
+  - "The fake provider records at least one qa-fallback-primary request and at least one qa-fallback-secondary request."
+  - "At least one provider fault is observed without a final Debug Chat error."
+evidence_required:
+  - metrics
+  - network
+  - api_diagnostic
+  - filesystem
+expected_failures:
+  - "fake_provider_fault"
+failure_patterns:
+  - "All models failed during streaming setup"
+  - "Timed out after"
+troubleshooting:
+  - backend-not-listening
+  - debug-chat-history-contaminates-automation

skills/skills/langbot-testing/cases/local-agent-streaming-post-commit-failure-debug-chat.yaml

@@ -0,0 +1,86 @@
+id: local-agent-streaming-post-commit-failure-debug-chat
+title: "Local Agent does not fall back after a committed stream fails"
+mode: probe
+area: pipeline
+type: chaos
+priority: p1
+risk: high
+ci_eligible: false
+tags:
+  - local-agent
+  - fallback
+  - streaming
+  - fake-provider
+  - fault-injection
+skills:
+  - langbot-env-setup
+  - langbot-testing
+env:
+  - LANGBOT_BACKEND_URL
+  - LANGBOT_FRONTEND_URL
+  - LANGBOT_E2E_LOGIN_USER
+automation: skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs
+automation_env:
+  - LANGBOT_BACKEND_URL
+  - LANGBOT_E2E_LOGIN_USER
+  - LANGBOT_FAKE_PROVIDER_PIPELINE_URL
+  - LANGBOT_FAKE_PROVIDER_PIPELINE_NAME
+automation_pipeline_url_env: LANGBOT_FAKE_PROVIDER_PIPELINE_URL
+automation_pipeline_name_env: LANGBOT_FAKE_PROVIDER_PIPELINE_NAME
+automation_debug_chat_load_requests: "1"
+automation_debug_chat_load_concurrency: "1"
+automation_debug_chat_load_timeout_ms: "30000"
+automation_debug_chat_load_max_error_rate: "1"
+automation_debug_chat_load_min_error_rate: "1"
+automation_debug_chat_load_min_error_count: "1"
+automation_debug_chat_load_min_provider_fault_count: "1"
+automation_debug_chat_load_require_success: "false"
+automation_debug_chat_load_expected_prefix: "POSTCOMMITQA"
+automation_debug_chat_load_prompt_template: '请只回复 "{expected}"，不要解释，不要添加其他字符。'
+automation_debug_chat_load_stream: "true"
+automation_debug_chat_load_reset: "true"
+automation_debug_chat_load_fail_on_final_mismatch: "true"
+automation_debug_chat_load_provider_model_thresholds_json: '{"qa-post-commit-primary":{"min":1},"qa-post-commit-fallback":{"max":0}}'
+automation_fake_provider_pipeline_name: "LangBot QA Post Commit Failure Debug Chat"
+automation_fake_provider_model_name: "qa-post-commit-primary"
+automation_fake_provider_fallback_model_names: "qa-post-commit-fallback"
+automation_fake_provider_fail_models: ""
+automation_fake_provider_fail_after_first_chunk_models: "qa-post-commit-primary"
+automation_fake_provider_fail_after_first_chunk_delay_ms: "1000"
+automation_fake_provider_fail_after_first_chunk_mode: "error_event"
+automation_fake_provider_first_token_delay_ms: "25"
+automation_fake_provider_chunk_delay_ms: "10"
+metrics_thresholds_json: '{"error_rate":{"min":1,"max":1},"fake_provider_fault_count_min":{"min":1},"primary_model_requests":{"min":1},"fallback_model_requests":{"max":0}}'
+fault_model_json: '{"provider_fault":"Send a provider error event for qa-post-commit-primary one second after its first content chunk","expected_behavior":"The run terminates with a controlled error and never invokes qa-post-commit-fallback after visible output is committed."}'
+setup_automation:
+  - "node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env"
+setup_provides_env:
+  - LANGBOT_FAKE_PROVIDER_URL
+  - LANGBOT_FAKE_PROVIDER_PIPELINE_URL
+  - LANGBOT_FAKE_PROVIDER_PIPELINE_NAME
+steps:
+  - "Create a QA pipeline with a primary model that sends an error event after one content chunk and a healthy fallback model."
+  - "Send one streaming Debug Chat request and require a terminal error."
+  - "Inspect provider request records and prove the fallback model was never invoked."
+checks:
+  - "The fake provider records a mid-stream error event for qa-post-commit-primary."
+  - "The Debug Chat request terminates as an error instead of timing out or reporting success."
+  - "qa-post-commit-fallback request count remains zero."
+evidence_required:
+  - metrics
+  - network
+  - api_diagnostic
+  - filesystem
+expected_failures:
+  - "mid_stream_error_event"
+  - "failed after first chunk"
+  - "Agent runner execution failed."
+failure_patterns:
+  - "Agent runner execution failed."
+  - "Timed out after"
+  - "qa-post-commit-fallback"
+troubleshooting:
+  - backend-not-listening
+  - debug-chat-history-contaminates-automation
+diagnostics:
+  - "Use error_event rather than a bare TCP disconnect for this contract: LiteLLM may normalize an abrupt EOF into a final chunk before Local Agent can observe the transport boundary."

skills/skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs

@@ -55,6 +55,8 @@ const minErrorCount = nonNegativeInteger(env.LANGBOT_DEBUG_CHAT_LOAD_MIN_ERROR_C
 const minOkCount = nonNegativeInteger(env.LANGBOT_DEBUG_CHAT_LOAD_MIN_OK_COUNT, 0);
 const minProviderFaultCount = nonNegativeInteger(env.LANGBOT_DEBUG_CHAT_LOAD_MIN_PROVIDER_FAULT_COUNT, 0);
 const failOnFinalMismatch = bool(env.LANGBOT_DEBUG_CHAT_LOAD_FAIL_ON_FINAL_MISMATCH, false);
+const requireSuccess = bool(env.LANGBOT_DEBUG_CHAT_LOAD_REQUIRE_SUCCESS, true);
+const providerModelThresholds = jsonObject(env.LANGBOT_DEBUG_CHAT_LOAD_PROVIDER_MODEL_THRESHOLDS_JSON);
 const failureSignals = textList(env.LANGBOT_E2E_FAILURE_SIGNALS || env.LANGBOT_DEBUG_CHAT_LOAD_FAILURE_SIGNALS || "");
 
 const result = {
@@ -80,6 +82,7 @@ const result = {
     stream,
     reset_before_run: resetBeforeRun,
     fail_on_final_mismatch: failOnFinalMismatch,
+    require_success: requireSuccess,
   },
   evidence: {
     network_log: paths.networkLog,
@@ -190,6 +193,8 @@ try {
     status_counts: metrics.status_counts,
     fake_provider_request_count: metrics.fake_provider?.request_count ?? null,
     fake_provider_fault_count: metrics.fake_provider?.fault_count ?? null,
+    fake_provider_mid_stream_disconnect_count: metrics.fake_provider?.mid_stream_disconnect_count ?? null,
+    fake_provider_model_request_counts: metrics.fake_provider?.model_request_counts ?? {},
     fake_provider_duration_p95_ms: metrics.provider_timing?.provider_duration_ms.p95 ?? null,
     langbot_overhead_estimate_p95_ms: metrics.provider_timing?.langbot_overhead_estimate_ms.p95 ?? null,
     send_to_provider_start_p95_ms: metrics.provider_timing?.send_to_provider_start_ms.p95 ?? null,
@@ -782,12 +787,14 @@ function buildMetrics({ samples, totalRequests, concurrency, timeoutMs, loadDura
 function buildThresholds(metrics) {
   const thresholds = {
     error_rate: { actual: metrics.error_rate, max: maxErrorRate, pass: metrics.error_rate <= maxErrorRate },
-    response_p95_ms: {
+  };
+  if (requireSuccess) {
+    thresholds.response_p95_ms = {
       actual: metrics.response_duration_ms.p95,
       max: responseP95BudgetMs,
       pass: metrics.ok_count > 0 && metrics.response_duration_ms.p95 <= responseP95BudgetMs,
-    },
-  };
+    };
+  }
   if (minErrorRate > 0) {
     thresholds.error_rate_min = {
       actual: metrics.error_rate,
@@ -824,9 +831,36 @@ function buildThresholds(metrics) {
       pass: metrics.ok_count > 0 && metrics.first_response_ms.p95 <= firstResponseP95BudgetMs,
     };
   }
+  for (const [model, bounds] of Object.entries(providerModelThresholds)) {
+    const actual = metrics.fake_provider?.model_request_counts?.[model] ?? 0;
+    if (Number.isFinite(bounds?.min)) {
+      thresholds[`provider_model_${model}_min`] = {
+        actual,
+        min: bounds.min,
+        pass: actual >= bounds.min,
+      };
+    }
+    if (Number.isFinite(bounds?.max)) {
+      thresholds[`provider_model_${model}_max`] = {
+        actual,
+        max: bounds.max,
+        pass: actual <= bounds.max,
+      };
+    }
+  }
   return thresholds;
 }
 
+function jsonObject(value) {
+  if (!value) return {};
+  try {
+    const parsed = JSON.parse(String(value));
+    return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : {};
+  } catch {
+    return {};
+  }
+}
+
 function looksLikeEnvIssue(error) {
   const message = String(error?.message || error || "");
   return /fetch failed|ECONNREFUSED|ENOTFOUND|LANGBOT_.*not configured|Could not read recovery_key|Backend did not respond/i.test(message);

skills/skills/langbot-testing/probes/langbot-live-backend-log-health.mjs

@@ -139,10 +139,15 @@ async function main() {
     const text = await readFile(logPath, "utf8");
     scan = scanLines(text, since, startedAt.getFullYear());
     const failCount = scan.findings.filter((item) => item.severity === "fail").length;
-    status = failCount === 0 ? "pass" : "fail";
-    reason = status === "pass"
-      ? "Live backend log health passed; no fail-severity findings in the scanned window."
-      : "Live backend log health found fail-severity backend log findings.";
+    if (scan.scanned.length === 0) {
+      status = "env_issue";
+      reason = "Backend log health had no timestamped lines in the selected window; no health verdict is possible.";
+    } else {
+      status = failCount === 0 ? "pass" : "fail";
+      reason = status === "pass"
+        ? "Live backend log health passed; no fail-severity findings in the scanned window."
+        : "Live backend log health found fail-severity backend log findings.";
+    }
   }
 
   const warningCount = scan.findings.filter((item) => item.severity === "warning").length;
@@ -158,6 +163,11 @@ async function main() {
     finding_count: scan.findings.length,
   };
   const thresholds = {
+    scanned_line_count_min: {
+      actual: scan.scanned.length,
+      min: 1,
+      pass: scan.scanned.length >= 1,
+    },
     fail_count: { actual: failCount, max: 0, pass: failCount === 0 },
   };
 

skills/skills/langbot-testing/probes/lib/fake-provider-timing.mjs

@@ -6,8 +6,16 @@ export function summarizeFakeProviderState(state) {
   const faultRequests = chatRequests.filter((request) => (
     request?.should_fail === true
       || request?.status === "http_fault"
+      || request?.status === "mid_stream_disconnect"
+      || request?.status === "mid_stream_error_event"
       || (Number.isFinite(request?.http_status) && request.http_status >= 400)
   ));
+  const modelRequestCounts = {};
+  for (const request of chatRequests) {
+    const model = String(request?.model || "");
+    if (!model) continue;
+    modelRequestCounts[model] = (modelRequestCounts[model] || 0) + 1;
+  }
 
   return {
     status: state.status || "unknown",
@@ -16,6 +24,10 @@ export function summarizeFakeProviderState(state) {
     recent_request_count: recentRequests.length,
     chat_request_count: chatRequests.length,
     fault_count: faultRequests.length,
+    mid_stream_disconnect_count: chatRequests.filter(
+      (request) => ["mid_stream_disconnect", "mid_stream_error_event"].includes(request?.status),
+    ).length,
+    model_request_counts: modelRequestCounts,
     streamed_request_count: chatRequests.filter((request) => request?.stream === true).length,
     duration_ms: stats(chatRequests.map((request) => numberOrNull(request?.duration_ms)).filter(Number.isFinite)),
     successful_duration_ms: stats(successfulRequests.map((request) => numberOrNull(request?.duration_ms)).filter(Number.isFinite)),

skills/skills/langbot-testing/references/local-agent-runner-coverage.md

@@ -24,6 +24,8 @@ These browser cases are the minimum gate for a local-agent migration check:
 | Case | Path Covered | Expected Behavior |
 | --- | --- | --- |
 | `local-agent-basic-debug-chat` | Streaming LLM invocation with effective host context | Bot returns deterministic `OK`; backend logs streaming completion. |
+| `local-agent-model-fallback-before-first-chunk-debug-chat` | Primary/fallback selection before stream commitment | Provider records failed primary requests and a successful fallback request; Debug Chat completes. |
+| `local-agent-streaming-post-commit-failure-debug-chat` | Terminal provider failure after stream commitment | Provider records the primary error event, fallback request count stays zero, and Debug Chat reports a controlled failure. |
 | `local-agent-effective-prompt-debug-chat` | PromptPreProcessing and host effective prompt handoff through `ctx.adapter.extra.prompt` | Bot returns `PROMPT_PREPROCESS_OK` from the fixture prompt probe. |
 | `local-agent-context-compaction-debug-chat` | Runner-owned context budgeting and old-history compaction | Automation temporarily shrinks the runner context window, sends multi-turn Debug Chat history, and the bot still recovers the older sentinel. |
 | `local-agent-rag-debug-chat` | Knowledge-base authorization, retrieval, and RAG prompt insertion | Bot returns the KB sentinel, not a generic answer. |
@@ -52,8 +54,8 @@ These browser cases are the minimum gate for a local-agent migration check:
 | Streaming model invocation | Enable Debug Chat streaming and ask for `OK`. | UI receives incremental bot output and backend logs streaming completion. |
 | Non-streaming UI delivery | Disable Debug Chat streaming. | UI receives a final bot message without frontend streaming errors. |
 | Non-streaming model invocation | Use local-agent component tests with `runtime_metadata.streaming_supported=false` or a host adapter that does not support streaming. | Runner calls `invoke_llm` instead of `invoke_llm_stream` and emits `message.completed`. |
-| Model fallback before first chunk | Configure a failing primary and working fallback, preferably with a controlled test provider. | First model failure does not fail the run; fallback model produces the final answer. |
-| Failure after streaming commit | Use a controlled provider that emits one chunk and then fails. | Runner reports a terminal run failure and does not fallback after partial output. |
+| Model fallback before first chunk | Run `local-agent-model-fallback-before-first-chunk-debug-chat`. | Provider records both the failed primary and successful fallback model requests; the final user request succeeds. |
+| Failure after streaming commit | Run `local-agent-streaming-post-commit-failure-debug-chat`. | Provider records a post-content error event, Runner reports terminal failure, and fallback request count remains zero. |
 | No authorized model | Clear model config or configure a model not in run resources. | Runner returns `runner.no_model` instead of calling an unauthorized model. |
 | MCP tool call | Use `qa-local-stdio` and `qa_mcp_echo`. | Bot returns the exact `qa_mcp_echo:<input>` result; `/api/v1/tools` contains `qa_mcp_echo`. |
 | Plugin tool call | Install a fixture plugin exposing a deterministic tool and bind it to the pipeline. | Runner lists the plugin tool and can call it through the same tool loop as MCP tools. |

skills/skills/langbot-testing/references/local-agent-runner.md

@@ -51,6 +51,8 @@ Pair a basic Debug Chat run with a deterministic plugin tool call, for example `
 Run these cases before saying the pluginized local-agent behavior is healthy:
 
 - `local-agent-basic-debug-chat`: basic streaming model invocation.
+- `local-agent-model-fallback-before-first-chunk-debug-chat`: primary model failure before the first visible chunk switches to the configured fallback.
+- `local-agent-streaming-post-commit-failure-debug-chat`: a provider error after a committed content chunk terminates the run without invoking fallback.
 - `local-agent-effective-prompt-debug-chat`: host effective prompt after PromptPreProcessing reaches the runner.
 - `local-agent-rag-debug-chat`: LangRAG retrieval reaches the runner and affects the answer.
 - `mcp-stdio-tool-call`: MCP tool discovery and local-agent tool loop.

skills/skills/langbot-testing/suites/local-agent-gate.yaml

@@ -9,6 +9,8 @@ tags:
   - regression
 cases:
   - local-agent-basic-debug-chat
+  - local-agent-model-fallback-before-first-chunk-debug-chat
+  - local-agent-streaming-post-commit-failure-debug-chat
   - qa-plugin-smoke-live-install
   - local-agent-effective-prompt-debug-chat
   - local-agent-context-compaction-debug-chat