mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-08-07 03:46:38 +00:00
test(skills): expand local agent reliability coverage
This commit is contained in:
huanghuoguoguo
@@ -257,6 +257,22 @@
|
||||
"type": "string",
|
||||
"enum": ["0", "1", "false", "true"]
|
||||
},
|
||||
"automation_debug_chat_load_require_success": {
|
||||
"type": "string",
|
||||
"enum": ["0", "1", "false", "true"]
|
||||
},
|
||||
"automation_debug_chat_load_provider_model_thresholds_json": {
|
||||
"type": "string"
|
||||
},
|
||||
"automation_fake_provider_pipeline_name": {
|
||||
"type": "string"
|
||||
},
|
||||
"automation_fake_provider_model_name": {
|
||||
"type": "string"
|
||||
},
|
||||
"automation_fake_provider_fallback_model_names": {
|
||||
"type": "string"
|
||||
},
|
||||
"automation_fake_provider_response_text": {
|
||||
"type": "string"
|
||||
},
|
||||
@@ -275,6 +291,9 @@
|
||||
"automation_fake_provider_fail_every_n": {
|
||||
"type": "string"
|
||||
},
|
||||
"automation_fake_provider_fail_models": {
|
||||
"type": "string"
|
||||
},
|
||||
"automation_fake_provider_fault_status": {
|
||||
"type": "string"
|
||||
},
|
||||
@@ -282,6 +301,16 @@
|
||||
"type": "string",
|
||||
"enum": ["0", "1", "false", "true"]
|
||||
},
|
||||
"automation_fake_provider_fail_after_first_chunk_delay_ms": {
|
||||
"type": "string"
|
||||
},
|
||||
"automation_fake_provider_fail_after_first_chunk_mode": {
|
||||
"type": "string",
|
||||
"enum": ["disconnect", "error_event"]
|
||||
},
|
||||
"automation_fake_provider_fail_after_first_chunk_models": {
|
||||
"type": "string"
|
||||
},
|
||||
"automation_fake_provider_dynamic_response": {
|
||||
"type": "string",
|
||||
"enum": ["0", "1", "false", "true"]
|
||||
|
||||
@@ -41,6 +41,8 @@ const pipelineName = env.LANGBOT_FAKE_PROVIDER_PIPELINE_NAME || DEFAULT_PIPELINE
|
||||
const providerName = env.LANGBOT_FAKE_PROVIDER_NAME || DEFAULT_PROVIDER_NAME;
|
||||
const requester = env.LANGBOT_FAKE_PROVIDER_REQUESTER || DEFAULT_REQUESTER;
|
||||
const modelName = env.LANGBOT_FAKE_PROVIDER_MODEL_NAME || DEFAULT_MODEL_NAME;
|
||||
const fallbackModelNames = textList(env.LANGBOT_FAKE_PROVIDER_FALLBACK_MODEL_NAMES)
|
||||
.filter((name) => name !== modelName);
|
||||
|
||||
const result = {
|
||||
source: "automation",
|
||||
@@ -75,6 +77,7 @@ const result = {
|
||||
test_status: "not_run",
|
||||
test_reason: "",
|
||||
},
|
||||
fallback_models: [],
|
||||
pipeline_id: "",
|
||||
pipeline_name: pipelineName,
|
||||
pipeline_url: "",
|
||||
@@ -141,11 +144,23 @@ try {
|
||||
});
|
||||
result.model = model;
|
||||
|
||||
const fallbackModels = [];
|
||||
for (const fallbackModelName of fallbackModelNames) {
|
||||
fallbackModels.push(await ensureModel({
|
||||
backendUrl,
|
||||
token: auth.token,
|
||||
providerUuid: provider.uuid,
|
||||
name: fallbackModelName,
|
||||
}));
|
||||
}
|
||||
result.fallback_models = fallbackModels;
|
||||
|
||||
const pipeline = await ensurePipeline({
|
||||
backendUrl,
|
||||
token: auth.token,
|
||||
name: pipelineName,
|
||||
modelUuid: model.uuid,
|
||||
fallbackModelUuids: fallbackModels.map((item) => item.uuid),
|
||||
});
|
||||
Object.assign(result, pipeline);
|
||||
result.pipeline_url = `${frontendUrl.replace(/\/$/, "")}/home/agents?id=${encodeURIComponent(pipeline.pipeline_id)}`;
|
||||
@@ -161,6 +176,7 @@ try {
|
||||
LANGBOT_FAKE_PROVIDER_PID: fakeProvider.pid ? String(fakeProvider.pid) : "",
|
||||
LANGBOT_FAKE_PROVIDER_PROVIDER_UUID: provider.uuid,
|
||||
LANGBOT_FAKE_PROVIDER_MODEL_UUID: model.uuid,
|
||||
LANGBOT_FAKE_PROVIDER_FALLBACK_MODEL_UUIDS: fallbackModels.map((item) => item.uuid).join(","),
|
||||
LANGBOT_FAKE_PROVIDER_PIPELINE_URL: result.pipeline_url,
|
||||
LANGBOT_FAKE_PROVIDER_PIPELINE_NAME: pipelineName,
|
||||
});
|
||||
@@ -168,7 +184,7 @@ try {
|
||||
}
|
||||
|
||||
result.status = "pass";
|
||||
result.reason = `Fake provider pipeline is configured with ${requester}/${modelName}.`;
|
||||
result.reason = `Fake provider pipeline is configured with ${requester}/${modelName} and ${fallbackModels.length} fallback(s).`;
|
||||
} catch (error) {
|
||||
result.status = result.status === "env_issue" ? "env_issue" : "fail";
|
||||
result.reason = result.reason || safeReason(error.message);
|
||||
@@ -328,7 +344,11 @@ function healthyFakeProviderConfig() {
|
||||
fault_status: 500,
|
||||
fail_first_n: 0,
|
||||
fail_every_n: 0,
|
||||
fail_models: [],
|
||||
fail_after_first_chunk: false,
|
||||
fail_after_first_chunk_delay_ms: 0,
|
||||
fail_after_first_chunk_mode: "disconnect",
|
||||
fail_after_first_chunk_models: [],
|
||||
dynamic_response: true,
|
||||
};
|
||||
}
|
||||
@@ -342,7 +362,14 @@ function targetFakeProviderConfig() {
|
||||
fault_status: httpFaultStatus(env.LANGBOT_FAKE_PROVIDER_FAULT_STATUS, 500),
|
||||
fail_first_n: nonNegativeInteger(env.LANGBOT_FAKE_PROVIDER_FAIL_FIRST_N, 0),
|
||||
fail_every_n: nonNegativeInteger(env.LANGBOT_FAKE_PROVIDER_FAIL_EVERY_N, 0),
|
||||
fail_models: textList(env.LANGBOT_FAKE_PROVIDER_FAIL_MODELS),
|
||||
fail_after_first_chunk: envBool(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK, false),
|
||||
fail_after_first_chunk_delay_ms: nonNegativeInteger(
|
||||
env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_DELAY_MS,
|
||||
0,
|
||||
),
|
||||
fail_after_first_chunk_mode: streamFaultMode(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_MODE),
|
||||
fail_after_first_chunk_models: textList(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_MODELS),
|
||||
dynamic_response: envBool(env.LANGBOT_FAKE_PROVIDER_DYNAMIC_RESPONSE, true),
|
||||
};
|
||||
}
|
||||
@@ -476,7 +503,7 @@ async function ensureModel({ backendUrl, token, providerUuid, name }) {
|
||||
};
|
||||
}
|
||||
|
||||
async function ensurePipeline({ backendUrl, token, name, modelUuid }) {
|
||||
async function ensurePipeline({ backendUrl, token, name, modelUuid, fallbackModelUuids = [] }) {
|
||||
const list = await apiJson(backendUrl, "/api/v1/pipelines", { token });
|
||||
if (isApiFailure(list)) {
|
||||
throw new Error(list.json.msg || "Failed to list pipelines.");
|
||||
@@ -538,7 +565,7 @@ async function ensurePipeline({ backendUrl, token, name, modelUuid }) {
|
||||
"max-round": positiveInteger(existingLocalAgentConfig["max-round"], 10),
|
||||
model: {
|
||||
primary: modelUuid,
|
||||
fallbacks: [],
|
||||
fallbacks: fallbackModelUuids,
|
||||
},
|
||||
};
|
||||
const updatedConfig = {
|
||||
@@ -605,6 +632,17 @@ function envBool(value, fallback) {
|
||||
return fallback;
|
||||
}
|
||||
|
||||
function textList(value) {
|
||||
return String(value || "")
|
||||
.split(/\r?\n|,/)
|
||||
.map((item) => item.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function streamFaultMode(value) {
|
||||
return String(value || "").trim().toLowerCase() === "error_event" ? "error_event" : "disconnect";
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
@@ -54,7 +54,11 @@ const config = {
|
||||
fault_status: integer(env.LANGBOT_FAKE_PROVIDER_FAULT_STATUS, 500),
|
||||
fail_first_n: integer(env.LANGBOT_FAKE_PROVIDER_FAIL_FIRST_N, 0),
|
||||
fail_every_n: integer(env.LANGBOT_FAKE_PROVIDER_FAIL_EVERY_N, 0),
|
||||
fail_models: textList(env.LANGBOT_FAKE_PROVIDER_FAIL_MODELS),
|
||||
fail_after_first_chunk: bool(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK, false),
|
||||
fail_after_first_chunk_delay_ms: integer(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_DELAY_MS, 0),
|
||||
fail_after_first_chunk_mode: faultMode(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_MODE),
|
||||
fail_after_first_chunk_models: textList(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_MODELS),
|
||||
dynamic_response: !/^(0|false|no|off)$/i.test(env.LANGBOT_FAKE_PROVIDER_DYNAMIC_RESPONSE || ""),
|
||||
request_log_limit: integer(env.LANGBOT_FAKE_PROVIDER_REQUEST_LOG_LIMIT, 500),
|
||||
};
|
||||
@@ -135,8 +139,12 @@ const server = createServer(async (request, response) => {
|
||||
requestCount += 1;
|
||||
const body = await readJson(request);
|
||||
const requestId = `chatcmpl-langbot-fake-${requestCount}`;
|
||||
const requestModel = String(body.model || modelName);
|
||||
const shouldFail = requestCount <= config.fail_first_n
|
||||
|| (config.fail_every_n > 0 && requestCount % config.fail_every_n === 0);
|
||||
|| (config.fail_every_n > 0 && requestCount % config.fail_every_n === 0)
|
||||
|| config.fail_models.includes(requestModel);
|
||||
const failAfterFirstChunk = config.fail_after_first_chunk
|
||||
|| config.fail_after_first_chunk_models.includes(requestModel);
|
||||
const replyMessage = buildResponse(body);
|
||||
const replyText = replyMessage.content || "";
|
||||
requestRecord = recordRequest({
|
||||
@@ -144,7 +152,7 @@ const server = createServer(async (request, response) => {
|
||||
request_number: requestCount,
|
||||
path: url.pathname,
|
||||
stream: Boolean(body.stream),
|
||||
model: body.model || "",
|
||||
model: requestModel,
|
||||
message_count: Array.isArray(body.messages) ? body.messages.length : 0,
|
||||
should_fail: shouldFail,
|
||||
status: "running",
|
||||
@@ -179,7 +187,7 @@ const server = createServer(async (request, response) => {
|
||||
requestId,
|
||||
model: body.model || modelName,
|
||||
message: replyMessage,
|
||||
failAfterFirstChunk: config.fail_after_first_chunk,
|
||||
failAfterFirstChunk,
|
||||
requestRecord,
|
||||
startedPerf,
|
||||
});
|
||||
@@ -416,6 +424,22 @@ async function streamCompletion(response, {
|
||||
choices: [{ index: 0, delta: { content: chunks[index] }, finish_reason: null }],
|
||||
});
|
||||
if (failMidStream && index === 0) {
|
||||
await sleep(config.fail_after_first_chunk_delay_ms);
|
||||
if (config.fail_after_first_chunk_mode === "error_event") {
|
||||
response.write(`data: ${JSON.stringify({
|
||||
error: {
|
||||
message: "LangBot fake provider injected a mid-stream error event",
|
||||
type: "fake_provider_stream_fault",
|
||||
code: "fake_provider_stream_fault",
|
||||
},
|
||||
})}\n\n`);
|
||||
response.end();
|
||||
finishRequestRecord(requestRecord, startedPerf, {
|
||||
status: "mid_stream_error_event",
|
||||
http_status: 200,
|
||||
});
|
||||
return;
|
||||
}
|
||||
finishRequestRecord(requestRecord, startedPerf, {
|
||||
status: "mid_stream_disconnect",
|
||||
http_status: 200,
|
||||
@@ -815,12 +839,18 @@ function applyConfig(updates) {
|
||||
assignNonNegativeInteger(updates, "chunk_count");
|
||||
assignNonNegativeInteger(updates, "fail_first_n");
|
||||
assignNonNegativeInteger(updates, "fail_every_n");
|
||||
assignTextList(updates, "fail_models");
|
||||
assignNonNegativeInteger(updates, "request_log_limit");
|
||||
if (updates.fault_status !== undefined) {
|
||||
const parsed = Number.parseInt(String(updates.fault_status), 10);
|
||||
if (Number.isInteger(parsed) && parsed >= 400 && parsed <= 599) config.fault_status = parsed;
|
||||
}
|
||||
assignBoolean(updates, "fail_after_first_chunk");
|
||||
assignNonNegativeInteger(updates, "fail_after_first_chunk_delay_ms");
|
||||
if (updates.fail_after_first_chunk_mode !== undefined) {
|
||||
config.fail_after_first_chunk_mode = faultMode(updates.fail_after_first_chunk_mode);
|
||||
}
|
||||
assignTextList(updates, "fail_after_first_chunk_models");
|
||||
assignBoolean(updates, "dynamic_response");
|
||||
}
|
||||
|
||||
@@ -838,3 +868,21 @@ function assignBoolean(updates, key) {
|
||||
if (updates[key] === undefined) return;
|
||||
config[key] = bool(updates[key], config[key]);
|
||||
}
|
||||
|
||||
function assignTextList(updates, key) {
|
||||
if (updates[key] === undefined) return;
|
||||
config[key] = Array.isArray(updates[key])
|
||||
? updates[key].map(String).map((item) => item.trim()).filter(Boolean)
|
||||
: textList(updates[key]);
|
||||
}
|
||||
|
||||
function textList(value) {
|
||||
return String(value || "")
|
||||
.split(/\r?\n|,/)
|
||||
.map((item) => item.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function faultMode(value) {
|
||||
return String(value || "").trim().toLowerCase() === "error_event" ? "error_event" : "disconnect";
|
||||
}
|
||||
|
||||
@@ -175,6 +175,7 @@
|
||||
"local-agent-complex-coding-task-debug-chat",
|
||||
"local-agent-context-compaction-debug-chat",
|
||||
"local-agent-effective-prompt-debug-chat",
|
||||
"local-agent-model-fallback-before-first-chunk-debug-chat",
|
||||
"local-agent-multimodal-debug-chat",
|
||||
"local-agent-multitool-rag-compaction-debug-chat",
|
||||
"local-agent-nonstreaming-debug-chat",
|
||||
@@ -183,6 +184,7 @@
|
||||
"local-agent-rag-debug-chat",
|
||||
"local-agent-rag-multimodal-debug-chat",
|
||||
"local-agent-steering-debug-chat",
|
||||
"local-agent-streaming-post-commit-failure-debug-chat",
|
||||
"local-agent-tool-error-recovery-debug-chat",
|
||||
"local-agent-tool-loop-limit-debug-chat",
|
||||
"mcp-stdio-register",
|
||||
@@ -1168,6 +1170,38 @@
|
||||
"backend_log"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "local-agent-model-fallback-before-first-chunk-debug-chat",
|
||||
"title": "Local Agent falls back when the primary model fails before streaming starts",
|
||||
"mode": "probe",
|
||||
"area": "pipeline",
|
||||
"type": "chaos",
|
||||
"priority": "p1",
|
||||
"risk": "high",
|
||||
"ci_eligible": false,
|
||||
"tags": [
|
||||
"local-agent",
|
||||
"fallback",
|
||||
"streaming",
|
||||
"fake-provider",
|
||||
"fault-injection"
|
||||
],
|
||||
"automation": "skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs",
|
||||
"setup_automation": [
|
||||
"node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env"
|
||||
],
|
||||
"setup_provides_env": [
|
||||
"LANGBOT_FAKE_PROVIDER_URL",
|
||||
"LANGBOT_FAKE_PROVIDER_PIPELINE_URL",
|
||||
"LANGBOT_FAKE_PROVIDER_PIPELINE_NAME"
|
||||
],
|
||||
"evidence_required": [
|
||||
"metrics",
|
||||
"network",
|
||||
"api_diagnostic",
|
||||
"filesystem"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "local-agent-multimodal-debug-chat",
|
||||
"title": "Local Agent Debug Chat preserves uploaded image input",
|
||||
@@ -1429,6 +1463,38 @@
|
||||
"api_diagnostic"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "local-agent-streaming-post-commit-failure-debug-chat",
|
||||
"title": "Local Agent does not fall back after a committed stream fails",
|
||||
"mode": "probe",
|
||||
"area": "pipeline",
|
||||
"type": "chaos",
|
||||
"priority": "p1",
|
||||
"risk": "high",
|
||||
"ci_eligible": false,
|
||||
"tags": [
|
||||
"local-agent",
|
||||
"fallback",
|
||||
"streaming",
|
||||
"fake-provider",
|
||||
"fault-injection"
|
||||
],
|
||||
"automation": "skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs",
|
||||
"setup_automation": [
|
||||
"node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env"
|
||||
],
|
||||
"setup_provides_env": [
|
||||
"LANGBOT_FAKE_PROVIDER_URL",
|
||||
"LANGBOT_FAKE_PROVIDER_PIPELINE_URL",
|
||||
"LANGBOT_FAKE_PROVIDER_PIPELINE_NAME"
|
||||
],
|
||||
"evidence_required": [
|
||||
"metrics",
|
||||
"network",
|
||||
"api_diagnostic",
|
||||
"filesystem"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "local-agent-tool-error-recovery-debug-chat",
|
||||
"title": "Local Agent feeds plugin tool errors back to the model",
|
||||
@@ -2204,6 +2270,8 @@
|
||||
],
|
||||
"cases": [
|
||||
"local-agent-basic-debug-chat",
|
||||
"local-agent-model-fallback-before-first-chunk-debug-chat",
|
||||
"local-agent-streaming-post-commit-failure-debug-chat",
|
||||
"qa-plugin-smoke-live-install",
|
||||
"local-agent-effective-prompt-debug-chat",
|
||||
"local-agent-context-compaction-debug-chat",
|
||||
|
||||
@@ -16,14 +16,14 @@ skills:
|
||||
env:
|
||||
- LANGBOT_BACKEND_URL
|
||||
automation: skills/langbot-testing/probes/langbot-live-backend-log-health.mjs
|
||||
metrics_thresholds_json: '{"fail_count":{"max":0}}'
|
||||
metrics_thresholds_json: '{"scanned_line_count":{"min":1},"fail_count":{"max":0}}'
|
||||
load_profile_json: '{"lookback_seconds":300,"log_source":"LANGBOT_BACKEND_LOG or latest LANGBOT_REPO/data/logs/langbot-*.log"}'
|
||||
steps:
|
||||
- "Confirm the selected LangBot backend log belongs to the intended test target."
|
||||
- "Run `rtk bin/lbs test run langbot-live-backend-log-health --dry-run` first; remove `--dry-run` after checking evidence directory and log source."
|
||||
- "Automation scans the recent backend log window for fail-severity runtime findings such as Traceback, ImportError, ERROR, unclosed sessions, and unawaited coroutines."
|
||||
checks:
|
||||
- "automation-result.json status is pass only when fail_count is 0."
|
||||
- "automation-result.json status is pass only when at least one timestamped line is scanned and fail_count is 0."
|
||||
- "metrics_summary includes scanned_line_count, fail_count, warning_count, and finding_count."
|
||||
- "findings.json and scanned-backend.log are written under LBS_EVIDENCE_DIR."
|
||||
evidence_required:
|
||||
@@ -33,6 +33,7 @@ evidence_required:
|
||||
diagnostics:
|
||||
- "Set LANGBOT_BACKEND_LOG to an explicit log path when the latest log file is not the run target."
|
||||
- "Set LANGBOT_BACKEND_LOG_SINCE or LANGBOT_BACKEND_LOG_LOOKBACK_SECONDS to control the scan window."
|
||||
- "A zero-line scan is env_issue, not pass; point LANGBOT_BACKEND_LOG at the active service output or widen the lookback."
|
||||
- "This probe measures runtime log health; it does not prove user-facing Debug Chat, plugin, model, or RAG behavior."
|
||||
success_patterns:
|
||||
- "Live backend log health passed"
|
||||
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
id: local-agent-model-fallback-before-first-chunk-debug-chat
|
||||
title: "Local Agent falls back when the primary model fails before streaming starts"
|
||||
mode: probe
|
||||
area: pipeline
|
||||
type: chaos
|
||||
priority: p1
|
||||
risk: high
|
||||
ci_eligible: false
|
||||
tags:
|
||||
- local-agent
|
||||
- fallback
|
||||
- streaming
|
||||
- fake-provider
|
||||
- fault-injection
|
||||
skills:
|
||||
- langbot-env-setup
|
||||
- langbot-testing
|
||||
env:
|
||||
- LANGBOT_BACKEND_URL
|
||||
- LANGBOT_FRONTEND_URL
|
||||
- LANGBOT_E2E_LOGIN_USER
|
||||
automation: skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs
|
||||
automation_env:
|
||||
- LANGBOT_BACKEND_URL
|
||||
- LANGBOT_E2E_LOGIN_USER
|
||||
- LANGBOT_FAKE_PROVIDER_PIPELINE_URL
|
||||
- LANGBOT_FAKE_PROVIDER_PIPELINE_NAME
|
||||
automation_pipeline_url_env: LANGBOT_FAKE_PROVIDER_PIPELINE_URL
|
||||
automation_pipeline_name_env: LANGBOT_FAKE_PROVIDER_PIPELINE_NAME
|
||||
automation_debug_chat_load_requests: "1"
|
||||
automation_debug_chat_load_concurrency: "1"
|
||||
automation_debug_chat_load_timeout_ms: "30000"
|
||||
automation_debug_chat_load_response_p95_ms: "10000"
|
||||
automation_debug_chat_load_max_error_rate: "0"
|
||||
automation_debug_chat_load_min_ok_count: "1"
|
||||
automation_debug_chat_load_min_provider_fault_count: "1"
|
||||
automation_debug_chat_load_expected_prefix: "FALLBACKQA"
|
||||
automation_debug_chat_load_prompt_template: '请只回复 "{expected}"，不要解释，不要添加其他字符。'
|
||||
automation_debug_chat_load_stream: "true"
|
||||
automation_debug_chat_load_reset: "true"
|
||||
automation_debug_chat_load_fail_on_final_mismatch: "true"
|
||||
automation_debug_chat_load_provider_model_thresholds_json: '{"qa-fallback-primary":{"min":1},"qa-fallback-secondary":{"min":1}}'
|
||||
automation_fake_provider_pipeline_name: "LangBot QA Model Fallback Debug Chat"
|
||||
automation_fake_provider_model_name: "qa-fallback-primary"
|
||||
automation_fake_provider_fallback_model_names: "qa-fallback-secondary"
|
||||
automation_fake_provider_fail_models: "qa-fallback-primary"
|
||||
automation_fake_provider_fail_after_first_chunk_models: ""
|
||||
automation_fake_provider_first_token_delay_ms: "25"
|
||||
automation_fake_provider_chunk_delay_ms: "10"
|
||||
automation_fake_provider_fault_status: "503"
|
||||
metrics_thresholds_json: '{"response_p95_ms":{"max":10000},"error_rate":{"max":0},"fake_provider_fault_count_min":{"min":1},"primary_model_requests":{"min":1},"fallback_model_requests":{"min":1}}'
|
||||
fault_model_json: '{"provider_fault":"HTTP 503 for every qa-fallback-primary request before the first stream chunk","expected_behavior":"Local Agent invokes qa-fallback-secondary and returns the expected Debug Chat response."}'
|
||||
setup_automation:
|
||||
- "node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env"
|
||||
setup_provides_env:
|
||||
- LANGBOT_FAKE_PROVIDER_URL
|
||||
- LANGBOT_FAKE_PROVIDER_PIPELINE_URL
|
||||
- LANGBOT_FAKE_PROVIDER_PIPELINE_NAME
|
||||
steps:
|
||||
- "Create a QA pipeline with a failing primary model and a healthy fallback model on the controlled provider."
|
||||
- "Send one streaming Debug Chat request with a unique response token."
|
||||
- "Inspect provider request records and require both the failed primary request and the successful fallback request."
|
||||
checks:
|
||||
- "The user-visible request succeeds with the unique expected response."
|
||||
- "The fake provider records at least one qa-fallback-primary request and at least one qa-fallback-secondary request."
|
||||
- "At least one provider fault is observed without a final Debug Chat error."
|
||||
evidence_required:
|
||||
- metrics
|
||||
- network
|
||||
- api_diagnostic
|
||||
- filesystem
|
||||
expected_failures:
|
||||
- "fake_provider_fault"
|
||||
failure_patterns:
|
||||
- "All models failed during streaming setup"
|
||||
- "Timed out after"
|
||||
troubleshooting:
|
||||
- backend-not-listening
|
||||
- debug-chat-history-contaminates-automation
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
id: local-agent-streaming-post-commit-failure-debug-chat
|
||||
title: "Local Agent does not fall back after a committed stream fails"
|
||||
mode: probe
|
||||
area: pipeline
|
||||
type: chaos
|
||||
priority: p1
|
||||
risk: high
|
||||
ci_eligible: false
|
||||
tags:
|
||||
- local-agent
|
||||
- fallback
|
||||
- streaming
|
||||
- fake-provider
|
||||
- fault-injection
|
||||
skills:
|
||||
- langbot-env-setup
|
||||
- langbot-testing
|
||||
env:
|
||||
- LANGBOT_BACKEND_URL
|
||||
- LANGBOT_FRONTEND_URL
|
||||
- LANGBOT_E2E_LOGIN_USER
|
||||
automation: skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs
|
||||
automation_env:
|
||||
- LANGBOT_BACKEND_URL
|
||||
- LANGBOT_E2E_LOGIN_USER
|
||||
- LANGBOT_FAKE_PROVIDER_PIPELINE_URL
|
||||
- LANGBOT_FAKE_PROVIDER_PIPELINE_NAME
|
||||
automation_pipeline_url_env: LANGBOT_FAKE_PROVIDER_PIPELINE_URL
|
||||
automation_pipeline_name_env: LANGBOT_FAKE_PROVIDER_PIPELINE_NAME
|
||||
automation_debug_chat_load_requests: "1"
|
||||
automation_debug_chat_load_concurrency: "1"
|
||||
automation_debug_chat_load_timeout_ms: "30000"
|
||||
automation_debug_chat_load_max_error_rate: "1"
|
||||
automation_debug_chat_load_min_error_rate: "1"
|
||||
automation_debug_chat_load_min_error_count: "1"
|
||||
automation_debug_chat_load_min_provider_fault_count: "1"
|
||||
automation_debug_chat_load_require_success: "false"
|
||||
automation_debug_chat_load_expected_prefix: "POSTCOMMITQA"
|
||||
automation_debug_chat_load_prompt_template: '请只回复 "{expected}"，不要解释，不要添加其他字符。'
|
||||
automation_debug_chat_load_stream: "true"
|
||||
automation_debug_chat_load_reset: "true"
|
||||
automation_debug_chat_load_fail_on_final_mismatch: "true"
|
||||
automation_debug_chat_load_provider_model_thresholds_json: '{"qa-post-commit-primary":{"min":1},"qa-post-commit-fallback":{"max":0}}'
|
||||
automation_fake_provider_pipeline_name: "LangBot QA Post Commit Failure Debug Chat"
|
||||
automation_fake_provider_model_name: "qa-post-commit-primary"
|
||||
automation_fake_provider_fallback_model_names: "qa-post-commit-fallback"
|
||||
automation_fake_provider_fail_models: ""
|
||||
automation_fake_provider_fail_after_first_chunk_models: "qa-post-commit-primary"
|
||||
automation_fake_provider_fail_after_first_chunk_delay_ms: "1000"
|
||||
automation_fake_provider_fail_after_first_chunk_mode: "error_event"
|
||||
automation_fake_provider_first_token_delay_ms: "25"
|
||||
automation_fake_provider_chunk_delay_ms: "10"
|
||||
metrics_thresholds_json: '{"error_rate":{"min":1,"max":1},"fake_provider_fault_count_min":{"min":1},"primary_model_requests":{"min":1},"fallback_model_requests":{"max":0}}'
|
||||
fault_model_json: '{"provider_fault":"Send a provider error event for qa-post-commit-primary one second after its first content chunk","expected_behavior":"The run terminates with a controlled error and never invokes qa-post-commit-fallback after visible output is committed."}'
|
||||
setup_automation:
|
||||
- "node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env"
|
||||
setup_provides_env:
|
||||
- LANGBOT_FAKE_PROVIDER_URL
|
||||
- LANGBOT_FAKE_PROVIDER_PIPELINE_URL
|
||||
- LANGBOT_FAKE_PROVIDER_PIPELINE_NAME
|
||||
steps:
|
||||
- "Create a QA pipeline with a primary model that sends an error event after one content chunk and a healthy fallback model."
|
||||
- "Send one streaming Debug Chat request and require a terminal error."
|
||||
- "Inspect provider request records and prove the fallback model was never invoked."
|
||||
checks:
|
||||
- "The fake provider records a mid-stream error event for qa-post-commit-primary."
|
||||
- "The Debug Chat request terminates as an error instead of timing out or reporting success."
|
||||
- "qa-post-commit-fallback request count remains zero."
|
||||
evidence_required:
|
||||
- metrics
|
||||
- network
|
||||
- api_diagnostic
|
||||
- filesystem
|
||||
expected_failures:
|
||||
- "mid_stream_error_event"
|
||||
- "failed after first chunk"
|
||||
- "Agent runner execution failed."
|
||||
failure_patterns:
|
||||
- "Agent runner execution failed."
|
||||
- "Timed out after"
|
||||
- "qa-post-commit-fallback"
|
||||
troubleshooting:
|
||||
- backend-not-listening
|
||||
- debug-chat-history-contaminates-automation
|
||||
diagnostics:
|
||||
- "Use error_event rather than a bare TCP disconnect for this contract: LiteLLM may normalize an abrupt EOF into a final chunk before Local Agent can observe the transport boundary."
|
||||
@@ -55,6 +55,8 @@ const minErrorCount = nonNegativeInteger(env.LANGBOT_DEBUG_CHAT_LOAD_MIN_ERROR_C
|
||||
const minOkCount = nonNegativeInteger(env.LANGBOT_DEBUG_CHAT_LOAD_MIN_OK_COUNT, 0);
|
||||
const minProviderFaultCount = nonNegativeInteger(env.LANGBOT_DEBUG_CHAT_LOAD_MIN_PROVIDER_FAULT_COUNT, 0);
|
||||
const failOnFinalMismatch = bool(env.LANGBOT_DEBUG_CHAT_LOAD_FAIL_ON_FINAL_MISMATCH, false);
|
||||
const requireSuccess = bool(env.LANGBOT_DEBUG_CHAT_LOAD_REQUIRE_SUCCESS, true);
|
||||
const providerModelThresholds = jsonObject(env.LANGBOT_DEBUG_CHAT_LOAD_PROVIDER_MODEL_THRESHOLDS_JSON);
|
||||
const failureSignals = textList(env.LANGBOT_E2E_FAILURE_SIGNALS || env.LANGBOT_DEBUG_CHAT_LOAD_FAILURE_SIGNALS || "");
|
||||
|
||||
const result = {
|
||||
@@ -80,6 +82,7 @@ const result = {
|
||||
stream,
|
||||
reset_before_run: resetBeforeRun,
|
||||
fail_on_final_mismatch: failOnFinalMismatch,
|
||||
require_success: requireSuccess,
|
||||
},
|
||||
evidence: {
|
||||
network_log: paths.networkLog,
|
||||
@@ -190,6 +193,8 @@ try {
|
||||
status_counts: metrics.status_counts,
|
||||
fake_provider_request_count: metrics.fake_provider?.request_count ?? null,
|
||||
fake_provider_fault_count: metrics.fake_provider?.fault_count ?? null,
|
||||
fake_provider_mid_stream_disconnect_count: metrics.fake_provider?.mid_stream_disconnect_count ?? null,
|
||||
fake_provider_model_request_counts: metrics.fake_provider?.model_request_counts ?? {},
|
||||
fake_provider_duration_p95_ms: metrics.provider_timing?.provider_duration_ms.p95 ?? null,
|
||||
langbot_overhead_estimate_p95_ms: metrics.provider_timing?.langbot_overhead_estimate_ms.p95 ?? null,
|
||||
send_to_provider_start_p95_ms: metrics.provider_timing?.send_to_provider_start_ms.p95 ?? null,
|
||||
@@ -782,12 +787,14 @@ function buildMetrics({ samples, totalRequests, concurrency, timeoutMs, loadDura
|
||||
function buildThresholds(metrics) {
|
||||
const thresholds = {
|
||||
error_rate: { actual: metrics.error_rate, max: maxErrorRate, pass: metrics.error_rate <= maxErrorRate },
|
||||
response_p95_ms: {
|
||||
};
|
||||
if (requireSuccess) {
|
||||
thresholds.response_p95_ms = {
|
||||
actual: metrics.response_duration_ms.p95,
|
||||
max: responseP95BudgetMs,
|
||||
pass: metrics.ok_count > 0 && metrics.response_duration_ms.p95 <= responseP95BudgetMs,
|
||||
},
|
||||
};
|
||||
};
|
||||
}
|
||||
if (minErrorRate > 0) {
|
||||
thresholds.error_rate_min = {
|
||||
actual: metrics.error_rate,
|
||||
@@ -824,9 +831,36 @@ function buildThresholds(metrics) {
|
||||
pass: metrics.ok_count > 0 && metrics.first_response_ms.p95 <= firstResponseP95BudgetMs,
|
||||
};
|
||||
}
|
||||
for (const [model, bounds] of Object.entries(providerModelThresholds)) {
|
||||
const actual = metrics.fake_provider?.model_request_counts?.[model] ?? 0;
|
||||
if (Number.isFinite(bounds?.min)) {
|
||||
thresholds[`provider_model_${model}_min`] = {
|
||||
actual,
|
||||
min: bounds.min,
|
||||
pass: actual >= bounds.min,
|
||||
};
|
||||
}
|
||||
if (Number.isFinite(bounds?.max)) {
|
||||
thresholds[`provider_model_${model}_max`] = {
|
||||
actual,
|
||||
max: bounds.max,
|
||||
pass: actual <= bounds.max,
|
||||
};
|
||||
}
|
||||
}
|
||||
return thresholds;
|
||||
}
|
||||
|
||||
function jsonObject(value) {
|
||||
if (!value) return {};
|
||||
try {
|
||||
const parsed = JSON.parse(String(value));
|
||||
return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
function looksLikeEnvIssue(error) {
|
||||
const message = String(error?.message || error || "");
|
||||
return /fetch failed|ECONNREFUSED|ENOTFOUND|LANGBOT_.*not configured|Could not read recovery_key|Backend did not respond/i.test(message);
|
||||
|
||||
@@ -139,10 +139,15 @@ async function main() {
|
||||
const text = await readFile(logPath, "utf8");
|
||||
scan = scanLines(text, since, startedAt.getFullYear());
|
||||
const failCount = scan.findings.filter((item) => item.severity === "fail").length;
|
||||
status = failCount === 0 ? "pass" : "fail";
|
||||
reason = status === "pass"
|
||||
? "Live backend log health passed; no fail-severity findings in the scanned window."
|
||||
: "Live backend log health found fail-severity backend log findings.";
|
||||
if (scan.scanned.length === 0) {
|
||||
status = "env_issue";
|
||||
reason = "Backend log health had no timestamped lines in the selected window; no health verdict is possible.";
|
||||
} else {
|
||||
status = failCount === 0 ? "pass" : "fail";
|
||||
reason = status === "pass"
|
||||
? "Live backend log health passed; no fail-severity findings in the scanned window."
|
||||
: "Live backend log health found fail-severity backend log findings.";
|
||||
}
|
||||
}
|
||||
|
||||
const warningCount = scan.findings.filter((item) => item.severity === "warning").length;
|
||||
@@ -158,6 +163,11 @@ async function main() {
|
||||
finding_count: scan.findings.length,
|
||||
};
|
||||
const thresholds = {
|
||||
scanned_line_count_min: {
|
||||
actual: scan.scanned.length,
|
||||
min: 1,
|
||||
pass: scan.scanned.length >= 1,
|
||||
},
|
||||
fail_count: { actual: failCount, max: 0, pass: failCount === 0 },
|
||||
};
|
||||
|
||||
|
||||
@@ -6,8 +6,16 @@ export function summarizeFakeProviderState(state) {
|
||||
const faultRequests = chatRequests.filter((request) => (
|
||||
request?.should_fail === true
|
||||
|| request?.status === "http_fault"
|
||||
|| request?.status === "mid_stream_disconnect"
|
||||
|| request?.status === "mid_stream_error_event"
|
||||
|| (Number.isFinite(request?.http_status) && request.http_status >= 400)
|
||||
));
|
||||
const modelRequestCounts = {};
|
||||
for (const request of chatRequests) {
|
||||
const model = String(request?.model || "");
|
||||
if (!model) continue;
|
||||
modelRequestCounts[model] = (modelRequestCounts[model] || 0) + 1;
|
||||
}
|
||||
|
||||
return {
|
||||
status: state.status || "unknown",
|
||||
@@ -16,6 +24,10 @@ export function summarizeFakeProviderState(state) {
|
||||
recent_request_count: recentRequests.length,
|
||||
chat_request_count: chatRequests.length,
|
||||
fault_count: faultRequests.length,
|
||||
mid_stream_disconnect_count: chatRequests.filter(
|
||||
(request) => ["mid_stream_disconnect", "mid_stream_error_event"].includes(request?.status),
|
||||
).length,
|
||||
model_request_counts: modelRequestCounts,
|
||||
streamed_request_count: chatRequests.filter((request) => request?.stream === true).length,
|
||||
duration_ms: stats(chatRequests.map((request) => numberOrNull(request?.duration_ms)).filter(Number.isFinite)),
|
||||
successful_duration_ms: stats(successfulRequests.map((request) => numberOrNull(request?.duration_ms)).filter(Number.isFinite)),
|
||||
|
||||
@@ -24,6 +24,8 @@ These browser cases are the minimum gate for a local-agent migration check:
|
||||
| Case | Path Covered | Expected Behavior |
|
||||
| --- | --- | --- |
|
||||
| `local-agent-basic-debug-chat` | Streaming LLM invocation with effective host context | Bot returns deterministic `OK`; backend logs streaming completion. |
|
||||
| `local-agent-model-fallback-before-first-chunk-debug-chat` | Primary/fallback selection before stream commitment | Provider records failed primary requests and a successful fallback request; Debug Chat completes. |
|
||||
| `local-agent-streaming-post-commit-failure-debug-chat` | Terminal provider failure after stream commitment | Provider records the primary error event, fallback request count stays zero, and Debug Chat reports a controlled failure. |
|
||||
| `local-agent-effective-prompt-debug-chat` | PromptPreProcessing and host effective prompt handoff through `ctx.adapter.extra.prompt` | Bot returns `PROMPT_PREPROCESS_OK` from the fixture prompt probe. |
|
||||
| `local-agent-context-compaction-debug-chat` | Runner-owned context budgeting and old-history compaction | Automation temporarily shrinks the runner context window, sends multi-turn Debug Chat history, and the bot still recovers the older sentinel. |
|
||||
| `local-agent-rag-debug-chat` | Knowledge-base authorization, retrieval, and RAG prompt insertion | Bot returns the KB sentinel, not a generic answer. |
|
||||
@@ -52,8 +54,8 @@ These browser cases are the minimum gate for a local-agent migration check:
|
||||
| Streaming model invocation | Enable Debug Chat streaming and ask for `OK`. | UI receives incremental bot output and backend logs streaming completion. |
|
||||
| Non-streaming UI delivery | Disable Debug Chat streaming. | UI receives a final bot message without frontend streaming errors. |
|
||||
| Non-streaming model invocation | Use local-agent component tests with `runtime_metadata.streaming_supported=false` or a host adapter that does not support streaming. | Runner calls `invoke_llm` instead of `invoke_llm_stream` and emits `message.completed`. |
|
||||
| Model fallback before first chunk | Configure a failing primary and working fallback, preferably with a controlled test provider. | First model failure does not fail the run; fallback model produces the final answer. |
|
||||
| Failure after streaming commit | Use a controlled provider that emits one chunk and then fails. | Runner reports a terminal run failure and does not fallback after partial output. |
|
||||
| Model fallback before first chunk | Run `local-agent-model-fallback-before-first-chunk-debug-chat`. | Provider records both the failed primary and successful fallback model requests; the final user request succeeds. |
|
||||
| Failure after streaming commit | Run `local-agent-streaming-post-commit-failure-debug-chat`. | Provider records a post-content error event, Runner reports terminal failure, and fallback request count remains zero. |
|
||||
| No authorized model | Clear model config or configure a model not in run resources. | Runner returns `runner.no_model` instead of calling an unauthorized model. |
|
||||
| MCP tool call | Use `qa-local-stdio` and `qa_mcp_echo`. | Bot returns the exact `qa_mcp_echo:<input>` result; `/api/v1/tools` contains `qa_mcp_echo`. |
|
||||
| Plugin tool call | Install a fixture plugin exposing a deterministic tool and bind it to the pipeline. | Runner lists the plugin tool and can call it through the same tool loop as MCP tools. |
|
||||
|
||||
@@ -51,6 +51,8 @@ Pair a basic Debug Chat run with a deterministic plugin tool call, for example `
|
||||
Run these cases before saying the pluginized local-agent behavior is healthy:
|
||||
|
||||
- `local-agent-basic-debug-chat`: basic streaming model invocation.
|
||||
- `local-agent-model-fallback-before-first-chunk-debug-chat`: primary model failure before the first visible chunk switches to the configured fallback.
|
||||
- `local-agent-streaming-post-commit-failure-debug-chat`: a provider error after a committed content chunk terminates the run without invoking fallback.
|
||||
- `local-agent-effective-prompt-debug-chat`: host effective prompt after PromptPreProcessing reaches the runner.
|
||||
- `local-agent-rag-debug-chat`: LangRAG retrieval reaches the runner and affects the answer.
|
||||
- `mcp-stdio-tool-call`: MCP tool discovery and local-agent tool loop.
|
||||
|
||||
@@ -9,6 +9,8 @@ tags:
|
||||
- regression
|
||||
cases:
|
||||
- local-agent-basic-debug-chat
|
||||
- local-agent-model-fallback-before-first-chunk-debug-chat
|
||||
- local-agent-streaming-post-commit-failure-debug-chat
|
||||
- qa-plugin-smoke-live-install
|
||||
- local-agent-effective-prompt-debug-chat
|
||||
- local-agent-context-compaction-debug-chat
|
||||
|
||||
@@ -209,6 +209,7 @@ function validateCaseItem(root: string, item: StructuredItem, skillNames: Set<st
|
||||
"automation_fake_provider_chunk_count",
|
||||
"automation_fake_provider_fail_first_n",
|
||||
"automation_fake_provider_fail_every_n",
|
||||
"automation_fake_provider_fail_after_first_chunk_delay_ms",
|
||||
]) {
|
||||
const value = scalar(item.fields, key);
|
||||
if (value && (!/^\d+$/.test(value) || Number.parseInt(value, 10) < 0)) {
|
||||
@@ -236,6 +237,7 @@ function validateCaseItem(root: string, item: StructuredItem, skillNames: Set<st
|
||||
"automation_debug_chat_load_stream",
|
||||
"automation_debug_chat_load_reset",
|
||||
"automation_debug_chat_load_fail_on_final_mismatch",
|
||||
"automation_debug_chat_load_require_success",
|
||||
"automation_fake_provider_fail_after_first_chunk",
|
||||
"automation_fake_provider_dynamic_response",
|
||||
]) {
|
||||
|
||||
@@ -133,14 +133,23 @@ export function automationEnvDefaults(item: StructuredItem, env: EnvSource = pro
|
||||
["automation_debug_chat_load_stream", "LANGBOT_DEBUG_CHAT_LOAD_STREAM"],
|
||||
["automation_debug_chat_load_reset", "LANGBOT_DEBUG_CHAT_LOAD_RESET"],
|
||||
["automation_debug_chat_load_fail_on_final_mismatch", "LANGBOT_DEBUG_CHAT_LOAD_FAIL_ON_FINAL_MISMATCH"],
|
||||
["automation_debug_chat_load_require_success", "LANGBOT_DEBUG_CHAT_LOAD_REQUIRE_SUCCESS"],
|
||||
["automation_debug_chat_load_provider_model_thresholds_json", "LANGBOT_DEBUG_CHAT_LOAD_PROVIDER_MODEL_THRESHOLDS_JSON"],
|
||||
["automation_fake_provider_pipeline_name", "LANGBOT_FAKE_PROVIDER_PIPELINE_NAME"],
|
||||
["automation_fake_provider_model_name", "LANGBOT_FAKE_PROVIDER_MODEL_NAME"],
|
||||
["automation_fake_provider_fallback_model_names", "LANGBOT_FAKE_PROVIDER_FALLBACK_MODEL_NAMES"],
|
||||
["automation_fake_provider_response_text", "LANGBOT_FAKE_PROVIDER_RESPONSE_TEXT"],
|
||||
["automation_fake_provider_first_token_delay_ms", "LANGBOT_FAKE_PROVIDER_FIRST_TOKEN_DELAY_MS"],
|
||||
["automation_fake_provider_chunk_delay_ms", "LANGBOT_FAKE_PROVIDER_CHUNK_DELAY_MS"],
|
||||
["automation_fake_provider_chunk_count", "LANGBOT_FAKE_PROVIDER_CHUNK_COUNT"],
|
||||
["automation_fake_provider_fail_first_n", "LANGBOT_FAKE_PROVIDER_FAIL_FIRST_N"],
|
||||
["automation_fake_provider_fail_every_n", "LANGBOT_FAKE_PROVIDER_FAIL_EVERY_N"],
|
||||
["automation_fake_provider_fail_models", "LANGBOT_FAKE_PROVIDER_FAIL_MODELS"],
|
||||
["automation_fake_provider_fault_status", "LANGBOT_FAKE_PROVIDER_FAULT_STATUS"],
|
||||
["automation_fake_provider_fail_after_first_chunk", "LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK"],
|
||||
["automation_fake_provider_fail_after_first_chunk_delay_ms", "LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_DELAY_MS"],
|
||||
["automation_fake_provider_fail_after_first_chunk_mode", "LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_MODE"],
|
||||
["automation_fake_provider_fail_after_first_chunk_models", "LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_MODELS"],
|
||||
["automation_fake_provider_dynamic_response", "LANGBOT_FAKE_PROVIDER_DYNAMIC_RESPONSE"],
|
||||
["automation_filesystem_checks_json", "LANGBOT_E2E_FILESYSTEM_CHECKS_JSON"],
|
||||
["automation_plugin_package", "LANGBOT_E2E_PLUGIN_PACKAGE"],
|
||||
|
||||
@@ -3582,6 +3582,84 @@ test("fake provider returns IMAGE_OK only when image metadata is present", async
|
||||
}
|
||||
});
|
||||
|
||||
test("fake provider can inject faults for only the selected model", async () => {
|
||||
const provider = await startFakeProviderForTest();
|
||||
const rootUrl = provider.baseUrl.replace(/\/v1\/?$/, "");
|
||||
try {
|
||||
const configured = await fetch(`${rootUrl}/__qa/config`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
config: { fail_models: ["qa-primary"] },
|
||||
reset_request_count: true,
|
||||
}),
|
||||
});
|
||||
assert.equal(configured.status, 200);
|
||||
|
||||
const primary = await fetch(`${provider.baseUrl}/chat/completions`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
model: "qa-primary",
|
||||
stream: false,
|
||||
messages: [{ role: "user", content: "Reply OK" }],
|
||||
}),
|
||||
});
|
||||
assert.equal(primary.status, 500);
|
||||
|
||||
const fallback = await requestFakeProvider(provider, {
|
||||
model: "qa-fallback",
|
||||
messages: [{ role: "user", content: "Reply OK" }],
|
||||
});
|
||||
assert.equal(fakeProviderMessage(fallback).content, "OK");
|
||||
|
||||
const state = await fetch(`${rootUrl}/__qa/config`).then((response) => response.json());
|
||||
assert.deepEqual(
|
||||
state.recent_requests.map((request: { model: string; status: string }) => [request.model, request.status]),
|
||||
[["qa-primary", "http_fault"], ["qa-fallback", "ok"]],
|
||||
);
|
||||
} finally {
|
||||
await provider.stop();
|
||||
}
|
||||
});
|
||||
|
||||
test("local-agent model failure cases expose fallback fault controls", () => {
|
||||
const beforeFirstChunk = capture(() =>
|
||||
commandTestRun(
|
||||
ctx(["test", "run", "local-agent-model-fallback-before-first-chunk-debug-chat", "--dry-run", "--json"]),
|
||||
),
|
||||
);
|
||||
assert.equal(beforeFirstChunk.code, 0);
|
||||
const fallbackRun = JSON.parse(beforeFirstChunk.output);
|
||||
assert.equal(fallbackRun.automation.env_defaults.LANGBOT_FAKE_PROVIDER_MODEL_NAME, "qa-fallback-primary");
|
||||
assert.equal(
|
||||
fallbackRun.automation.env_defaults.LANGBOT_FAKE_PROVIDER_FALLBACK_MODEL_NAMES,
|
||||
"qa-fallback-secondary",
|
||||
);
|
||||
assert.equal(fallbackRun.automation.env_defaults.LANGBOT_FAKE_PROVIDER_FAIL_MODELS, "qa-fallback-primary");
|
||||
|
||||
const postCommit = capture(() =>
|
||||
commandTestRun(
|
||||
ctx(["test", "run", "local-agent-streaming-post-commit-failure-debug-chat", "--dry-run", "--json"]),
|
||||
),
|
||||
);
|
||||
assert.equal(postCommit.code, 0);
|
||||
const postCommitRun = JSON.parse(postCommit.output);
|
||||
assert.equal(
|
||||
postCommitRun.automation.env_defaults.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_MODELS,
|
||||
"qa-post-commit-primary",
|
||||
);
|
||||
assert.equal(
|
||||
postCommitRun.automation.env_defaults.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_DELAY_MS,
|
||||
"1000",
|
||||
);
|
||||
assert.equal(
|
||||
postCommitRun.automation.env_defaults.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_MODE,
|
||||
"error_event",
|
||||
);
|
||||
assert.equal(postCommitRun.automation.env_defaults.LANGBOT_DEBUG_CHAT_LOAD_REQUIRE_SUCCESS, "false");
|
||||
});
|
||||
|
||||
test("fake provider requires the effective system prompt before returning its sentinel", async () => {
|
||||
const provider = await startFakeProviderForTest();
|
||||
try {
|
||||
|
||||
Reference in New Issue
Block a user