diff --git a/skills/schemas/case.schema.json b/skills/schemas/case.schema.json
index 46601142a..4100a1b9c 100644
--- a/skills/schemas/case.schema.json
+++ b/skills/schemas/case.schema.json
@@ -257,6 +257,22 @@
"type": "string",
"enum": ["0", "1", "false", "true"]
},
+ "automation_debug_chat_load_require_success": {
+ "type": "string",
+ "enum": ["0", "1", "false", "true"]
+ },
+ "automation_debug_chat_load_provider_model_thresholds_json": {
+ "type": "string"
+ },
+ "automation_fake_provider_pipeline_name": {
+ "type": "string"
+ },
+ "automation_fake_provider_model_name": {
+ "type": "string"
+ },
+ "automation_fake_provider_fallback_model_names": {
+ "type": "string"
+ },
"automation_fake_provider_response_text": {
"type": "string"
},
@@ -275,6 +291,9 @@
"automation_fake_provider_fail_every_n": {
"type": "string"
},
+ "automation_fake_provider_fail_models": {
+ "type": "string"
+ },
"automation_fake_provider_fault_status": {
"type": "string"
},
@@ -282,6 +301,16 @@
"type": "string",
"enum": ["0", "1", "false", "true"]
},
+ "automation_fake_provider_fail_after_first_chunk_delay_ms": {
+ "type": "string"
+ },
+ "automation_fake_provider_fail_after_first_chunk_mode": {
+ "type": "string",
+ "enum": ["disconnect", "error_event"]
+ },
+ "automation_fake_provider_fail_after_first_chunk_models": {
+ "type": "string"
+ },
"automation_fake_provider_dynamic_response": {
"type": "string",
"enum": ["0", "1", "false", "true"]
diff --git a/skills/scripts/e2e/ensure-fake-provider-pipeline.mjs b/skills/scripts/e2e/ensure-fake-provider-pipeline.mjs
index 01f762319..9960cd0bb 100755
--- a/skills/scripts/e2e/ensure-fake-provider-pipeline.mjs
+++ b/skills/scripts/e2e/ensure-fake-provider-pipeline.mjs
@@ -41,6 +41,8 @@ const pipelineName = env.LANGBOT_FAKE_PROVIDER_PIPELINE_NAME || DEFAULT_PIPELINE
const providerName = env.LANGBOT_FAKE_PROVIDER_NAME || DEFAULT_PROVIDER_NAME;
const requester = env.LANGBOT_FAKE_PROVIDER_REQUESTER || DEFAULT_REQUESTER;
const modelName = env.LANGBOT_FAKE_PROVIDER_MODEL_NAME || DEFAULT_MODEL_NAME;
+const fallbackModelNames = textList(env.LANGBOT_FAKE_PROVIDER_FALLBACK_MODEL_NAMES)
+ .filter((name) => name !== modelName);
const result = {
source: "automation",
@@ -75,6 +77,7 @@ const result = {
test_status: "not_run",
test_reason: "",
},
+ fallback_models: [],
pipeline_id: "",
pipeline_name: pipelineName,
pipeline_url: "",
@@ -141,11 +144,23 @@ try {
});
result.model = model;
+ const fallbackModels = [];
+ for (const fallbackModelName of fallbackModelNames) {
+ fallbackModels.push(await ensureModel({
+ backendUrl,
+ token: auth.token,
+ providerUuid: provider.uuid,
+ name: fallbackModelName,
+ }));
+ }
+ result.fallback_models = fallbackModels;
+
const pipeline = await ensurePipeline({
backendUrl,
token: auth.token,
name: pipelineName,
modelUuid: model.uuid,
+ fallbackModelUuids: fallbackModels.map((item) => item.uuid),
});
Object.assign(result, pipeline);
result.pipeline_url = `${frontendUrl.replace(/\/$/, "")}/home/agents?id=${encodeURIComponent(pipeline.pipeline_id)}`;
@@ -161,6 +176,7 @@ try {
LANGBOT_FAKE_PROVIDER_PID: fakeProvider.pid ? String(fakeProvider.pid) : "",
LANGBOT_FAKE_PROVIDER_PROVIDER_UUID: provider.uuid,
LANGBOT_FAKE_PROVIDER_MODEL_UUID: model.uuid,
+ LANGBOT_FAKE_PROVIDER_FALLBACK_MODEL_UUIDS: fallbackModels.map((item) => item.uuid).join(","),
LANGBOT_FAKE_PROVIDER_PIPELINE_URL: result.pipeline_url,
LANGBOT_FAKE_PROVIDER_PIPELINE_NAME: pipelineName,
});
@@ -168,7 +184,7 @@ try {
}
result.status = "pass";
- result.reason = `Fake provider pipeline is configured with ${requester}/${modelName}.`;
+ result.reason = `Fake provider pipeline is configured with ${requester}/${modelName} and ${fallbackModels.length} fallback(s).`;
} catch (error) {
result.status = result.status === "env_issue" ? "env_issue" : "fail";
result.reason = result.reason || safeReason(error.message);
@@ -328,7 +344,11 @@ function healthyFakeProviderConfig() {
fault_status: 500,
fail_first_n: 0,
fail_every_n: 0,
+ fail_models: [],
fail_after_first_chunk: false,
+ fail_after_first_chunk_delay_ms: 0,
+ fail_after_first_chunk_mode: "disconnect",
+ fail_after_first_chunk_models: [],
dynamic_response: true,
};
}
@@ -342,7 +362,14 @@ function targetFakeProviderConfig() {
fault_status: httpFaultStatus(env.LANGBOT_FAKE_PROVIDER_FAULT_STATUS, 500),
fail_first_n: nonNegativeInteger(env.LANGBOT_FAKE_PROVIDER_FAIL_FIRST_N, 0),
fail_every_n: nonNegativeInteger(env.LANGBOT_FAKE_PROVIDER_FAIL_EVERY_N, 0),
+ fail_models: textList(env.LANGBOT_FAKE_PROVIDER_FAIL_MODELS),
fail_after_first_chunk: envBool(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK, false),
+ fail_after_first_chunk_delay_ms: nonNegativeInteger(
+ env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_DELAY_MS,
+ 0,
+ ),
+ fail_after_first_chunk_mode: streamFaultMode(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_MODE),
+ fail_after_first_chunk_models: textList(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_MODELS),
dynamic_response: envBool(env.LANGBOT_FAKE_PROVIDER_DYNAMIC_RESPONSE, true),
};
}
@@ -476,7 +503,7 @@ async function ensureModel({ backendUrl, token, providerUuid, name }) {
};
}
-async function ensurePipeline({ backendUrl, token, name, modelUuid }) {
+async function ensurePipeline({ backendUrl, token, name, modelUuid, fallbackModelUuids = [] }) {
const list = await apiJson(backendUrl, "/api/v1/pipelines", { token });
if (isApiFailure(list)) {
throw new Error(list.json.msg || "Failed to list pipelines.");
@@ -538,7 +565,7 @@ async function ensurePipeline({ backendUrl, token, name, modelUuid }) {
"max-round": positiveInteger(existingLocalAgentConfig["max-round"], 10),
model: {
primary: modelUuid,
- fallbacks: [],
+ fallbacks: fallbackModelUuids,
},
};
const updatedConfig = {
@@ -605,6 +632,17 @@ function envBool(value, fallback) {
return fallback;
}
+function textList(value) {
+ return String(value || "")
+ .split(/\r?\n|,/)
+ .map((item) => item.trim())
+ .filter(Boolean);
+}
+
+function streamFaultMode(value) {
+ return String(value || "").trim().toLowerCase() === "error_event" ? "error_event" : "disconnect";
+}
+
function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
diff --git a/skills/scripts/e2e/fake-openai-provider.mjs b/skills/scripts/e2e/fake-openai-provider.mjs
index 673f7ddd3..1dd7e9026 100755
--- a/skills/scripts/e2e/fake-openai-provider.mjs
+++ b/skills/scripts/e2e/fake-openai-provider.mjs
@@ -54,7 +54,11 @@ const config = {
fault_status: integer(env.LANGBOT_FAKE_PROVIDER_FAULT_STATUS, 500),
fail_first_n: integer(env.LANGBOT_FAKE_PROVIDER_FAIL_FIRST_N, 0),
fail_every_n: integer(env.LANGBOT_FAKE_PROVIDER_FAIL_EVERY_N, 0),
+ fail_models: textList(env.LANGBOT_FAKE_PROVIDER_FAIL_MODELS),
fail_after_first_chunk: bool(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK, false),
+ fail_after_first_chunk_delay_ms: integer(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_DELAY_MS, 0),
+ fail_after_first_chunk_mode: faultMode(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_MODE),
+ fail_after_first_chunk_models: textList(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_MODELS),
dynamic_response: !/^(0|false|no|off)$/i.test(env.LANGBOT_FAKE_PROVIDER_DYNAMIC_RESPONSE || ""),
request_log_limit: integer(env.LANGBOT_FAKE_PROVIDER_REQUEST_LOG_LIMIT, 500),
};
@@ -135,8 +139,12 @@ const server = createServer(async (request, response) => {
requestCount += 1;
const body = await readJson(request);
const requestId = `chatcmpl-langbot-fake-${requestCount}`;
+ const requestModel = String(body.model || modelName);
const shouldFail = requestCount <= config.fail_first_n
- || (config.fail_every_n > 0 && requestCount % config.fail_every_n === 0);
+ || (config.fail_every_n > 0 && requestCount % config.fail_every_n === 0)
+ || config.fail_models.includes(requestModel);
+ const failAfterFirstChunk = config.fail_after_first_chunk
+ || config.fail_after_first_chunk_models.includes(requestModel);
const replyMessage = buildResponse(body);
const replyText = replyMessage.content || "";
requestRecord = recordRequest({
@@ -144,7 +152,7 @@ const server = createServer(async (request, response) => {
request_number: requestCount,
path: url.pathname,
stream: Boolean(body.stream),
- model: body.model || "",
+ model: requestModel,
message_count: Array.isArray(body.messages) ? body.messages.length : 0,
should_fail: shouldFail,
status: "running",
@@ -179,7 +187,7 @@ const server = createServer(async (request, response) => {
requestId,
model: body.model || modelName,
message: replyMessage,
- failAfterFirstChunk: config.fail_after_first_chunk,
+ failAfterFirstChunk,
requestRecord,
startedPerf,
});
@@ -416,6 +424,22 @@ async function streamCompletion(response, {
choices: [{ index: 0, delta: { content: chunks[index] }, finish_reason: null }],
});
if (failMidStream && index === 0) {
+ await sleep(config.fail_after_first_chunk_delay_ms);
+ if (config.fail_after_first_chunk_mode === "error_event") {
+ response.write(`data: ${JSON.stringify({
+ error: {
+ message: "LangBot fake provider injected a mid-stream error event",
+ type: "fake_provider_stream_fault",
+ code: "fake_provider_stream_fault",
+ },
+ })}\n\n`);
+ response.end();
+ finishRequestRecord(requestRecord, startedPerf, {
+ status: "mid_stream_error_event",
+ http_status: 200,
+ });
+ return;
+ }
finishRequestRecord(requestRecord, startedPerf, {
status: "mid_stream_disconnect",
http_status: 200,
@@ -815,12 +839,18 @@ function applyConfig(updates) {
assignNonNegativeInteger(updates, "chunk_count");
assignNonNegativeInteger(updates, "fail_first_n");
assignNonNegativeInteger(updates, "fail_every_n");
+ assignTextList(updates, "fail_models");
assignNonNegativeInteger(updates, "request_log_limit");
if (updates.fault_status !== undefined) {
const parsed = Number.parseInt(String(updates.fault_status), 10);
if (Number.isInteger(parsed) && parsed >= 400 && parsed <= 599) config.fault_status = parsed;
}
assignBoolean(updates, "fail_after_first_chunk");
+ assignNonNegativeInteger(updates, "fail_after_first_chunk_delay_ms");
+ if (updates.fail_after_first_chunk_mode !== undefined) {
+ config.fail_after_first_chunk_mode = faultMode(updates.fail_after_first_chunk_mode);
+ }
+ assignTextList(updates, "fail_after_first_chunk_models");
assignBoolean(updates, "dynamic_response");
}
@@ -838,3 +868,21 @@ function assignBoolean(updates, key) {
if (updates[key] === undefined) return;
config[key] = bool(updates[key], config[key]);
}
+
+function assignTextList(updates, key) {
+ if (updates[key] === undefined) return;
+ config[key] = Array.isArray(updates[key])
+ ? updates[key].map(String).map((item) => item.trim()).filter(Boolean)
+ : textList(updates[key]);
+}
+
+function textList(value) {
+ return String(value || "")
+ .split(/\r?\n|,/)
+ .map((item) => item.trim())
+ .filter(Boolean);
+}
+
+function faultMode(value) {
+ return String(value || "").trim().toLowerCase() === "error_event" ? "error_event" : "disconnect";
+}
diff --git a/skills/skills.index.json b/skills/skills.index.json
index a0e550c26..b14f77397 100644
--- a/skills/skills.index.json
+++ b/skills/skills.index.json
@@ -175,6 +175,7 @@
"local-agent-complex-coding-task-debug-chat",
"local-agent-context-compaction-debug-chat",
"local-agent-effective-prompt-debug-chat",
+ "local-agent-model-fallback-before-first-chunk-debug-chat",
"local-agent-multimodal-debug-chat",
"local-agent-multitool-rag-compaction-debug-chat",
"local-agent-nonstreaming-debug-chat",
@@ -183,6 +184,7 @@
"local-agent-rag-debug-chat",
"local-agent-rag-multimodal-debug-chat",
"local-agent-steering-debug-chat",
+ "local-agent-streaming-post-commit-failure-debug-chat",
"local-agent-tool-error-recovery-debug-chat",
"local-agent-tool-loop-limit-debug-chat",
"mcp-stdio-register",
@@ -1168,6 +1170,38 @@
"backend_log"
]
},
+ {
+ "id": "local-agent-model-fallback-before-first-chunk-debug-chat",
+ "title": "Local Agent falls back when the primary model fails before streaming starts",
+ "mode": "probe",
+ "area": "pipeline",
+ "type": "chaos",
+ "priority": "p1",
+ "risk": "high",
+ "ci_eligible": false,
+ "tags": [
+ "local-agent",
+ "fallback",
+ "streaming",
+ "fake-provider",
+ "fault-injection"
+ ],
+ "automation": "skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs",
+ "setup_automation": [
+ "node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env"
+ ],
+ "setup_provides_env": [
+ "LANGBOT_FAKE_PROVIDER_URL",
+ "LANGBOT_FAKE_PROVIDER_PIPELINE_URL",
+ "LANGBOT_FAKE_PROVIDER_PIPELINE_NAME"
+ ],
+ "evidence_required": [
+ "metrics",
+ "network",
+ "api_diagnostic",
+ "filesystem"
+ ]
+ },
{
"id": "local-agent-multimodal-debug-chat",
"title": "Local Agent Debug Chat preserves uploaded image input",
@@ -1429,6 +1463,38 @@
"api_diagnostic"
]
},
+ {
+ "id": "local-agent-streaming-post-commit-failure-debug-chat",
+ "title": "Local Agent does not fall back after a committed stream fails",
+ "mode": "probe",
+ "area": "pipeline",
+ "type": "chaos",
+ "priority": "p1",
+ "risk": "high",
+ "ci_eligible": false,
+ "tags": [
+ "local-agent",
+ "fallback",
+ "streaming",
+ "fake-provider",
+ "fault-injection"
+ ],
+ "automation": "skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs",
+ "setup_automation": [
+ "node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env"
+ ],
+ "setup_provides_env": [
+ "LANGBOT_FAKE_PROVIDER_URL",
+ "LANGBOT_FAKE_PROVIDER_PIPELINE_URL",
+ "LANGBOT_FAKE_PROVIDER_PIPELINE_NAME"
+ ],
+ "evidence_required": [
+ "metrics",
+ "network",
+ "api_diagnostic",
+ "filesystem"
+ ]
+ },
{
"id": "local-agent-tool-error-recovery-debug-chat",
"title": "Local Agent feeds plugin tool errors back to the model",
@@ -2204,6 +2270,8 @@
],
"cases": [
"local-agent-basic-debug-chat",
+ "local-agent-model-fallback-before-first-chunk-debug-chat",
+ "local-agent-streaming-post-commit-failure-debug-chat",
"qa-plugin-smoke-live-install",
"local-agent-effective-prompt-debug-chat",
"local-agent-context-compaction-debug-chat",
diff --git a/skills/skills/langbot-testing/cases/langbot-live-backend-log-health.yaml b/skills/skills/langbot-testing/cases/langbot-live-backend-log-health.yaml
index 8ff911371..150ecaf42 100644
--- a/skills/skills/langbot-testing/cases/langbot-live-backend-log-health.yaml
+++ b/skills/skills/langbot-testing/cases/langbot-live-backend-log-health.yaml
@@ -16,14 +16,14 @@ skills:
env:
- LANGBOT_BACKEND_URL
automation: skills/langbot-testing/probes/langbot-live-backend-log-health.mjs
-metrics_thresholds_json: '{"fail_count":{"max":0}}'
+metrics_thresholds_json: '{"scanned_line_count":{"min":1},"fail_count":{"max":0}}'
load_profile_json: '{"lookback_seconds":300,"log_source":"LANGBOT_BACKEND_LOG or latest LANGBOT_REPO/data/logs/langbot-*.log"}'
steps:
- "Confirm the selected LangBot backend log belongs to the intended test target."
- "Run `rtk bin/lbs test run langbot-live-backend-log-health --dry-run` first; remove `--dry-run` after checking evidence directory and log source."
- "Automation scans the recent backend log window for fail-severity runtime findings such as Traceback, ImportError, ERROR, unclosed sessions, and unawaited coroutines."
checks:
- - "automation-result.json status is pass only when fail_count is 0."
+ - "automation-result.json status is pass only when at least one timestamped line is scanned and fail_count is 0."
- "metrics_summary includes scanned_line_count, fail_count, warning_count, and finding_count."
- "findings.json and scanned-backend.log are written under LBS_EVIDENCE_DIR."
evidence_required:
@@ -33,6 +33,7 @@ evidence_required:
diagnostics:
- "Set LANGBOT_BACKEND_LOG to an explicit log path when the latest log file is not the run target."
- "Set LANGBOT_BACKEND_LOG_SINCE or LANGBOT_BACKEND_LOG_LOOKBACK_SECONDS to control the scan window."
+ - "A zero-line scan is env_issue, not pass; point LANGBOT_BACKEND_LOG at the active service output or widen the lookback."
- "This probe measures runtime log health; it does not prove user-facing Debug Chat, plugin, model, or RAG behavior."
success_patterns:
- "Live backend log health passed"
diff --git a/skills/skills/langbot-testing/cases/local-agent-model-fallback-before-first-chunk-debug-chat.yaml b/skills/skills/langbot-testing/cases/local-agent-model-fallback-before-first-chunk-debug-chat.yaml
new file mode 100644
index 000000000..2b598b797
--- /dev/null
+++ b/skills/skills/langbot-testing/cases/local-agent-model-fallback-before-first-chunk-debug-chat.yaml
@@ -0,0 +1,79 @@
+id: local-agent-model-fallback-before-first-chunk-debug-chat
+title: "Local Agent falls back when the primary model fails before streaming starts"
+mode: probe
+area: pipeline
+type: chaos
+priority: p1
+risk: high
+ci_eligible: false
+tags:
+ - local-agent
+ - fallback
+ - streaming
+ - fake-provider
+ - fault-injection
+skills:
+ - langbot-env-setup
+ - langbot-testing
+env:
+ - LANGBOT_BACKEND_URL
+ - LANGBOT_FRONTEND_URL
+ - LANGBOT_E2E_LOGIN_USER
+automation: skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs
+automation_env:
+ - LANGBOT_BACKEND_URL
+ - LANGBOT_E2E_LOGIN_USER
+ - LANGBOT_FAKE_PROVIDER_PIPELINE_URL
+ - LANGBOT_FAKE_PROVIDER_PIPELINE_NAME
+automation_pipeline_url_env: LANGBOT_FAKE_PROVIDER_PIPELINE_URL
+automation_pipeline_name_env: LANGBOT_FAKE_PROVIDER_PIPELINE_NAME
+automation_debug_chat_load_requests: "1"
+automation_debug_chat_load_concurrency: "1"
+automation_debug_chat_load_timeout_ms: "30000"
+automation_debug_chat_load_response_p95_ms: "10000"
+automation_debug_chat_load_max_error_rate: "0"
+automation_debug_chat_load_min_ok_count: "1"
+automation_debug_chat_load_min_provider_fault_count: "1"
+automation_debug_chat_load_expected_prefix: "FALLBACKQA"
+automation_debug_chat_load_prompt_template: '请只回复 "{expected}"，不要解释，不要添加其他字符。'
+automation_debug_chat_load_stream: "true"
+automation_debug_chat_load_reset: "true"
+automation_debug_chat_load_fail_on_final_mismatch: "true"
+automation_debug_chat_load_provider_model_thresholds_json: '{"qa-fallback-primary":{"min":1},"qa-fallback-secondary":{"min":1}}'
+automation_fake_provider_pipeline_name: "LangBot QA Model Fallback Debug Chat"
+automation_fake_provider_model_name: "qa-fallback-primary"
+automation_fake_provider_fallback_model_names: "qa-fallback-secondary"
+automation_fake_provider_fail_models: "qa-fallback-primary"
+automation_fake_provider_fail_after_first_chunk_models: ""
+automation_fake_provider_first_token_delay_ms: "25"
+automation_fake_provider_chunk_delay_ms: "10"
+automation_fake_provider_fault_status: "503"
+metrics_thresholds_json: '{"response_p95_ms":{"max":10000},"error_rate":{"max":0},"fake_provider_fault_count_min":{"min":1},"primary_model_requests":{"min":1},"fallback_model_requests":{"min":1}}'
+fault_model_json: '{"provider_fault":"HTTP 503 for every qa-fallback-primary request before the first stream chunk","expected_behavior":"Local Agent invokes qa-fallback-secondary and returns the expected Debug Chat response."}'
+setup_automation:
+ - "node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env"
+setup_provides_env:
+ - LANGBOT_FAKE_PROVIDER_URL
+ - LANGBOT_FAKE_PROVIDER_PIPELINE_URL
+ - LANGBOT_FAKE_PROVIDER_PIPELINE_NAME
+steps:
+ - "Create a QA pipeline with a failing primary model and a healthy fallback model on the controlled provider."
+ - "Send one streaming Debug Chat request with a unique response token."
+ - "Inspect provider request records and require both the failed primary request and the successful fallback request."
+checks:
+ - "The user-visible request succeeds with the unique expected response."
+ - "The fake provider records at least one qa-fallback-primary request and at least one qa-fallback-secondary request."
+ - "At least one provider fault is observed without a final Debug Chat error."
+evidence_required:
+ - metrics
+ - network
+ - api_diagnostic
+ - filesystem
+expected_failures:
+ - "fake_provider_fault"
+failure_patterns:
+ - "All models failed during streaming setup"
+ - "Timed out after"
+troubleshooting:
+ - backend-not-listening
+ - debug-chat-history-contaminates-automation
diff --git a/skills/skills/langbot-testing/cases/local-agent-streaming-post-commit-failure-debug-chat.yaml b/skills/skills/langbot-testing/cases/local-agent-streaming-post-commit-failure-debug-chat.yaml
new file mode 100644
index 000000000..8bc471ed0
--- /dev/null
+++ b/skills/skills/langbot-testing/cases/local-agent-streaming-post-commit-failure-debug-chat.yaml
@@ -0,0 +1,86 @@
+id: local-agent-streaming-post-commit-failure-debug-chat
+title: "Local Agent does not fall back after a committed stream fails"
+mode: probe
+area: pipeline
+type: chaos
+priority: p1
+risk: high
+ci_eligible: false
+tags:
+ - local-agent
+ - fallback
+ - streaming
+ - fake-provider
+ - fault-injection
+skills:
+ - langbot-env-setup
+ - langbot-testing
+env:
+ - LANGBOT_BACKEND_URL
+ - LANGBOT_FRONTEND_URL
+ - LANGBOT_E2E_LOGIN_USER
+automation: skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs
+automation_env:
+ - LANGBOT_BACKEND_URL
+ - LANGBOT_E2E_LOGIN_USER
+ - LANGBOT_FAKE_PROVIDER_PIPELINE_URL
+ - LANGBOT_FAKE_PROVIDER_PIPELINE_NAME
+automation_pipeline_url_env: LANGBOT_FAKE_PROVIDER_PIPELINE_URL
+automation_pipeline_name_env: LANGBOT_FAKE_PROVIDER_PIPELINE_NAME
+automation_debug_chat_load_requests: "1"
+automation_debug_chat_load_concurrency: "1"
+automation_debug_chat_load_timeout_ms: "30000"
+automation_debug_chat_load_max_error_rate: "1"
+automation_debug_chat_load_min_error_rate: "1"
+automation_debug_chat_load_min_error_count: "1"
+automation_debug_chat_load_min_provider_fault_count: "1"
+automation_debug_chat_load_require_success: "false"
+automation_debug_chat_load_expected_prefix: "POSTCOMMITQA"
+automation_debug_chat_load_prompt_template: '请只回复 "{expected}"，不要解释，不要添加其他字符。'
+automation_debug_chat_load_stream: "true"
+automation_debug_chat_load_reset: "true"
+automation_debug_chat_load_fail_on_final_mismatch: "true"
+automation_debug_chat_load_provider_model_thresholds_json: '{"qa-post-commit-primary":{"min":1},"qa-post-commit-fallback":{"max":0}}'
+automation_fake_provider_pipeline_name: "LangBot QA Post Commit Failure Debug Chat"
+automation_fake_provider_model_name: "qa-post-commit-primary"
+automation_fake_provider_fallback_model_names: "qa-post-commit-fallback"
+automation_fake_provider_fail_models: ""
+automation_fake_provider_fail_after_first_chunk_models: "qa-post-commit-primary"
+automation_fake_provider_fail_after_first_chunk_delay_ms: "1000"
+automation_fake_provider_fail_after_first_chunk_mode: "error_event"
+automation_fake_provider_first_token_delay_ms: "25"
+automation_fake_provider_chunk_delay_ms: "10"
+metrics_thresholds_json: '{"error_rate":{"min":1,"max":1},"fake_provider_fault_count_min":{"min":1},"primary_model_requests":{"min":1},"fallback_model_requests":{"max":0}}'
+fault_model_json: '{"provider_fault":"Send a provider error event for qa-post-commit-primary one second after its first content chunk","expected_behavior":"The run terminates with a controlled error and never invokes qa-post-commit-fallback after visible output is committed."}'
+setup_automation:
+ - "node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env"
+setup_provides_env:
+ - LANGBOT_FAKE_PROVIDER_URL
+ - LANGBOT_FAKE_PROVIDER_PIPELINE_URL
+ - LANGBOT_FAKE_PROVIDER_PIPELINE_NAME
+steps:
+ - "Create a QA pipeline with a primary model that sends an error event after one content chunk and a healthy fallback model."
+ - "Send one streaming Debug Chat request and require a terminal error."
+ - "Inspect provider request records and prove the fallback model was never invoked."
+checks:
+ - "The fake provider records a mid-stream error event for qa-post-commit-primary."
+ - "The Debug Chat request terminates as an error instead of timing out or reporting success."
+ - "qa-post-commit-fallback request count remains zero."
+evidence_required:
+ - metrics
+ - network
+ - api_diagnostic
+ - filesystem
+expected_failures:
+ - "mid_stream_error_event"
+ - "failed after first chunk"
+ - "Agent runner execution failed."
+failure_patterns:
+ - "Agent runner execution failed."
+ - "Timed out after"
+ - "qa-post-commit-fallback"
+troubleshooting:
+ - backend-not-listening
+ - debug-chat-history-contaminates-automation
+diagnostics:
+ - "Use error_event rather than a bare TCP disconnect for this contract: LiteLLM may normalize an abrupt EOF into a final chunk before Local Agent can observe the transport boundary."
diff --git a/skills/skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs b/skills/skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs
index 5b156c470..ec37bf6ba 100755
--- a/skills/skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs
+++ b/skills/skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs
@@ -55,6 +55,8 @@ const minErrorCount = nonNegativeInteger(env.LANGBOT_DEBUG_CHAT_LOAD_MIN_ERROR_C
const minOkCount = nonNegativeInteger(env.LANGBOT_DEBUG_CHAT_LOAD_MIN_OK_COUNT, 0);
const minProviderFaultCount = nonNegativeInteger(env.LANGBOT_DEBUG_CHAT_LOAD_MIN_PROVIDER_FAULT_COUNT, 0);
const failOnFinalMismatch = bool(env.LANGBOT_DEBUG_CHAT_LOAD_FAIL_ON_FINAL_MISMATCH, false);
+const requireSuccess = bool(env.LANGBOT_DEBUG_CHAT_LOAD_REQUIRE_SUCCESS, true);
+const providerModelThresholds = jsonObject(env.LANGBOT_DEBUG_CHAT_LOAD_PROVIDER_MODEL_THRESHOLDS_JSON);
const failureSignals = textList(env.LANGBOT_E2E_FAILURE_SIGNALS || env.LANGBOT_DEBUG_CHAT_LOAD_FAILURE_SIGNALS || "");
const result = {
@@ -80,6 +82,7 @@ const result = {
stream,
reset_before_run: resetBeforeRun,
fail_on_final_mismatch: failOnFinalMismatch,
+ require_success: requireSuccess,
},
evidence: {
network_log: paths.networkLog,
@@ -190,6 +193,8 @@ try {
status_counts: metrics.status_counts,
fake_provider_request_count: metrics.fake_provider?.request_count ?? null,
fake_provider_fault_count: metrics.fake_provider?.fault_count ?? null,
+ fake_provider_mid_stream_disconnect_count: metrics.fake_provider?.mid_stream_disconnect_count ?? null,
+ fake_provider_model_request_counts: metrics.fake_provider?.model_request_counts ?? {},
fake_provider_duration_p95_ms: metrics.provider_timing?.provider_duration_ms.p95 ?? null,
langbot_overhead_estimate_p95_ms: metrics.provider_timing?.langbot_overhead_estimate_ms.p95 ?? null,
send_to_provider_start_p95_ms: metrics.provider_timing?.send_to_provider_start_ms.p95 ?? null,
@@ -782,12 +787,14 @@ function buildMetrics({ samples, totalRequests, concurrency, timeoutMs, loadDura
function buildThresholds(metrics) {
const thresholds = {
error_rate: { actual: metrics.error_rate, max: maxErrorRate, pass: metrics.error_rate <= maxErrorRate },
- response_p95_ms: {
+ };
+ if (requireSuccess) {
+ thresholds.response_p95_ms = {
actual: metrics.response_duration_ms.p95,
max: responseP95BudgetMs,
pass: metrics.ok_count > 0 && metrics.response_duration_ms.p95 <= responseP95BudgetMs,
- },
- };
+ };
+ }
if (minErrorRate > 0) {
thresholds.error_rate_min = {
actual: metrics.error_rate,
@@ -824,9 +831,36 @@ function buildThresholds(metrics) {
pass: metrics.ok_count > 0 && metrics.first_response_ms.p95 <= firstResponseP95BudgetMs,
};
}
+ for (const [model, bounds] of Object.entries(providerModelThresholds)) {
+ const actual = metrics.fake_provider?.model_request_counts?.[model] ?? 0;
+ if (Number.isFinite(bounds?.min)) {
+ thresholds[`provider_model_${model}_min`] = {
+ actual,
+ min: bounds.min,
+ pass: actual >= bounds.min,
+ };
+ }
+ if (Number.isFinite(bounds?.max)) {
+ thresholds[`provider_model_${model}_max`] = {
+ actual,
+ max: bounds.max,
+ pass: actual <= bounds.max,
+ };
+ }
+ }
return thresholds;
}
+function jsonObject(value) {
+ if (!value) return {};
+ try {
+ const parsed = JSON.parse(String(value));
+ return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : {};
+ } catch {
+ return {};
+ }
+}
+
function looksLikeEnvIssue(error) {
const message = String(error?.message || error || "");
return /fetch failed|ECONNREFUSED|ENOTFOUND|LANGBOT_.*not configured|Could not read recovery_key|Backend did not respond/i.test(message);
diff --git a/skills/skills/langbot-testing/probes/langbot-live-backend-log-health.mjs b/skills/skills/langbot-testing/probes/langbot-live-backend-log-health.mjs
index 38a31c389..2fb7bfc9c 100755
--- a/skills/skills/langbot-testing/probes/langbot-live-backend-log-health.mjs
+++ b/skills/skills/langbot-testing/probes/langbot-live-backend-log-health.mjs
@@ -139,10 +139,15 @@ async function main() {
const text = await readFile(logPath, "utf8");
scan = scanLines(text, since, startedAt.getFullYear());
const failCount = scan.findings.filter((item) => item.severity === "fail").length;
- status = failCount === 0 ? "pass" : "fail";
- reason = status === "pass"
- ? "Live backend log health passed; no fail-severity findings in the scanned window."
- : "Live backend log health found fail-severity backend log findings.";
+ if (scan.scanned.length === 0) {
+ status = "env_issue";
+ reason = "Backend log health had no timestamped lines in the selected window; no health verdict is possible.";
+ } else {
+ status = failCount === 0 ? "pass" : "fail";
+ reason = status === "pass"
+ ? "Live backend log health passed; no fail-severity findings in the scanned window."
+ : "Live backend log health found fail-severity backend log findings.";
+ }
}
const warningCount = scan.findings.filter((item) => item.severity === "warning").length;
@@ -158,6 +163,11 @@ async function main() {
finding_count: scan.findings.length,
};
const thresholds = {
+ scanned_line_count_min: {
+ actual: scan.scanned.length,
+ min: 1,
+ pass: scan.scanned.length >= 1,
+ },
fail_count: { actual: failCount, max: 0, pass: failCount === 0 },
};
diff --git a/skills/skills/langbot-testing/probes/lib/fake-provider-timing.mjs b/skills/skills/langbot-testing/probes/lib/fake-provider-timing.mjs
index b383b2663..4dc21f3ee 100755
--- a/skills/skills/langbot-testing/probes/lib/fake-provider-timing.mjs
+++ b/skills/skills/langbot-testing/probes/lib/fake-provider-timing.mjs
@@ -6,8 +6,16 @@ export function summarizeFakeProviderState(state) {
const faultRequests = chatRequests.filter((request) => (
request?.should_fail === true
|| request?.status === "http_fault"
+ || request?.status === "mid_stream_disconnect"
+ || request?.status === "mid_stream_error_event"
|| (Number.isFinite(request?.http_status) && request.http_status >= 400)
));
+ const modelRequestCounts = {};
+ for (const request of chatRequests) {
+ const model = String(request?.model || "");
+ if (!model) continue;
+ modelRequestCounts[model] = (modelRequestCounts[model] || 0) + 1;
+ }
return {
status: state.status || "unknown",
@@ -16,6 +24,10 @@ export function summarizeFakeProviderState(state) {
recent_request_count: recentRequests.length,
chat_request_count: chatRequests.length,
fault_count: faultRequests.length,
+ mid_stream_disconnect_count: chatRequests.filter(
+ (request) => ["mid_stream_disconnect", "mid_stream_error_event"].includes(request?.status),
+ ).length,
+ model_request_counts: modelRequestCounts,
streamed_request_count: chatRequests.filter((request) => request?.stream === true).length,
duration_ms: stats(chatRequests.map((request) => numberOrNull(request?.duration_ms)).filter(Number.isFinite)),
successful_duration_ms: stats(successfulRequests.map((request) => numberOrNull(request?.duration_ms)).filter(Number.isFinite)),
diff --git a/skills/skills/langbot-testing/references/local-agent-runner-coverage.md b/skills/skills/langbot-testing/references/local-agent-runner-coverage.md
index f06b3b31f..b01420a67 100644
--- a/skills/skills/langbot-testing/references/local-agent-runner-coverage.md
+++ b/skills/skills/langbot-testing/references/local-agent-runner-coverage.md
@@ -24,6 +24,8 @@ These browser cases are the minimum gate for a local-agent migration check:
| Case | Path Covered | Expected Behavior |
| --- | --- | --- |
| `local-agent-basic-debug-chat` | Streaming LLM invocation with effective host context | Bot returns deterministic `OK`; backend logs streaming completion. |
+| `local-agent-model-fallback-before-first-chunk-debug-chat` | Primary/fallback selection before stream commitment | Provider records failed primary requests and a successful fallback request; Debug Chat completes. |
+| `local-agent-streaming-post-commit-failure-debug-chat` | Terminal provider failure after stream commitment | Provider records the primary error event, fallback request count stays zero, and Debug Chat reports a controlled failure. |
| `local-agent-effective-prompt-debug-chat` | PromptPreProcessing and host effective prompt handoff through `ctx.adapter.extra.prompt` | Bot returns `PROMPT_PREPROCESS_OK` from the fixture prompt probe. |
| `local-agent-context-compaction-debug-chat` | Runner-owned context budgeting and old-history compaction | Automation temporarily shrinks the runner context window, sends multi-turn Debug Chat history, and the bot still recovers the older sentinel. |
| `local-agent-rag-debug-chat` | Knowledge-base authorization, retrieval, and RAG prompt insertion | Bot returns the KB sentinel, not a generic answer. |
@@ -52,8 +54,8 @@ These browser cases are the minimum gate for a local-agent migration check:
| Streaming model invocation | Enable Debug Chat streaming and ask for `OK`. | UI receives incremental bot output and backend logs streaming completion. |
| Non-streaming UI delivery | Disable Debug Chat streaming. | UI receives a final bot message without frontend streaming errors. |
| Non-streaming model invocation | Use local-agent component tests with `runtime_metadata.streaming_supported=false` or a host adapter that does not support streaming. | Runner calls `invoke_llm` instead of `invoke_llm_stream` and emits `message.completed`. |
-| Model fallback before first chunk | Configure a failing primary and working fallback, preferably with a controlled test provider. | First model failure does not fail the run; fallback model produces the final answer. |
-| Failure after streaming commit | Use a controlled provider that emits one chunk and then fails. | Runner reports a terminal run failure and does not fallback after partial output. |
+| Model fallback before first chunk | Run `local-agent-model-fallback-before-first-chunk-debug-chat`. | Provider records both the failed primary and successful fallback model requests; the final user request succeeds. |
+| Failure after streaming commit | Run `local-agent-streaming-post-commit-failure-debug-chat`. | Provider records a post-content error event, Runner reports terminal failure, and fallback request count remains zero. |
| No authorized model | Clear model config or configure a model not in run resources. | Runner returns `runner.no_model` instead of calling an unauthorized model. |
| MCP tool call | Use `qa-local-stdio` and `qa_mcp_echo`. | Bot returns the exact `qa_mcp_echo:` result; `/api/v1/tools` contains `qa_mcp_echo`. |
| Plugin tool call | Install a fixture plugin exposing a deterministic tool and bind it to the pipeline. | Runner lists the plugin tool and can call it through the same tool loop as MCP tools. |
diff --git a/skills/skills/langbot-testing/references/local-agent-runner.md b/skills/skills/langbot-testing/references/local-agent-runner.md
index 8d829904b..00e239256 100644
--- a/skills/skills/langbot-testing/references/local-agent-runner.md
+++ b/skills/skills/langbot-testing/references/local-agent-runner.md
@@ -51,6 +51,8 @@ Pair a basic Debug Chat run with a deterministic plugin tool call, for example `
Run these cases before saying the pluginized local-agent behavior is healthy:
- `local-agent-basic-debug-chat`: basic streaming model invocation.
+- `local-agent-model-fallback-before-first-chunk-debug-chat`: primary model failure before the first visible chunk switches to the configured fallback.
+- `local-agent-streaming-post-commit-failure-debug-chat`: a provider error after a committed content chunk terminates the run without invoking fallback.
- `local-agent-effective-prompt-debug-chat`: host effective prompt after PromptPreProcessing reaches the runner.
- `local-agent-rag-debug-chat`: LangRAG retrieval reaches the runner and affects the answer.
- `mcp-stdio-tool-call`: MCP tool discovery and local-agent tool loop.
diff --git a/skills/skills/langbot-testing/suites/local-agent-gate.yaml b/skills/skills/langbot-testing/suites/local-agent-gate.yaml
index 40ed15342..8137994b3 100644
--- a/skills/skills/langbot-testing/suites/local-agent-gate.yaml
+++ b/skills/skills/langbot-testing/suites/local-agent-gate.yaml
@@ -9,6 +9,8 @@ tags:
- regression
cases:
- local-agent-basic-debug-chat
+ - local-agent-model-fallback-before-first-chunk-debug-chat
+ - local-agent-streaming-post-commit-failure-debug-chat
- qa-plugin-smoke-live-install
- local-agent-effective-prompt-debug-chat
- local-agent-context-compaction-debug-chat
diff --git a/skills/src/commands/validate.ts b/skills/src/commands/validate.ts
index 109792859..27a4f83e2 100644
--- a/skills/src/commands/validate.ts
+++ b/skills/src/commands/validate.ts
@@ -209,6 +209,7 @@ function validateCaseItem(root: string, item: StructuredItem, skillNames: Set {
+ const provider = await startFakeProviderForTest();
+ const rootUrl = provider.baseUrl.replace(/\/v1\/?$/, "");
+ try {
+ const configured = await fetch(`${rootUrl}/__qa/config`, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({
+ config: { fail_models: ["qa-primary"] },
+ reset_request_count: true,
+ }),
+ });
+ assert.equal(configured.status, 200);
+
+ const primary = await fetch(`${provider.baseUrl}/chat/completions`, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({
+ model: "qa-primary",
+ stream: false,
+ messages: [{ role: "user", content: "Reply OK" }],
+ }),
+ });
+ assert.equal(primary.status, 500);
+
+ const fallback = await requestFakeProvider(provider, {
+ model: "qa-fallback",
+ messages: [{ role: "user", content: "Reply OK" }],
+ });
+ assert.equal(fakeProviderMessage(fallback).content, "OK");
+
+ const state = await fetch(`${rootUrl}/__qa/config`).then((response) => response.json());
+ assert.deepEqual(
+ state.recent_requests.map((request: { model: string; status: string }) => [request.model, request.status]),
+ [["qa-primary", "http_fault"], ["qa-fallback", "ok"]],
+ );
+ } finally {
+ await provider.stop();
+ }
+});
+
+test("local-agent model failure cases expose fallback fault controls", () => {
+ const beforeFirstChunk = capture(() =>
+ commandTestRun(
+ ctx(["test", "run", "local-agent-model-fallback-before-first-chunk-debug-chat", "--dry-run", "--json"]),
+ ),
+ );
+ assert.equal(beforeFirstChunk.code, 0);
+ const fallbackRun = JSON.parse(beforeFirstChunk.output);
+ assert.equal(fallbackRun.automation.env_defaults.LANGBOT_FAKE_PROVIDER_MODEL_NAME, "qa-fallback-primary");
+ assert.equal(
+ fallbackRun.automation.env_defaults.LANGBOT_FAKE_PROVIDER_FALLBACK_MODEL_NAMES,
+ "qa-fallback-secondary",
+ );
+ assert.equal(fallbackRun.automation.env_defaults.LANGBOT_FAKE_PROVIDER_FAIL_MODELS, "qa-fallback-primary");
+
+ const postCommit = capture(() =>
+ commandTestRun(
+ ctx(["test", "run", "local-agent-streaming-post-commit-failure-debug-chat", "--dry-run", "--json"]),
+ ),
+ );
+ assert.equal(postCommit.code, 0);
+ const postCommitRun = JSON.parse(postCommit.output);
+ assert.equal(
+ postCommitRun.automation.env_defaults.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_MODELS,
+ "qa-post-commit-primary",
+ );
+ assert.equal(
+ postCommitRun.automation.env_defaults.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_DELAY_MS,
+ "1000",
+ );
+ assert.equal(
+ postCommitRun.automation.env_defaults.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_MODE,
+ "error_event",
+ );
+ assert.equal(postCommitRun.automation.env_defaults.LANGBOT_DEBUG_CHAT_LOAD_REQUIRE_SUCCESS, "false");
+});
+
test("fake provider requires the effective system prompt before returning its sentinel", async () => {
const provider = await startFakeProviderForTest();
try {