xinyin025/LangBot
1
0
Fork 0
mirror of https://github.com/langbot-app/LangBot.git synced 2026-08-09 04:40:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(skills): expand local agent reliability coverage

Browse Source
This commit is contained in:
huanghuoguoguo
2026-07-26 22:27:42 +08:00
parent 4bae8728a6
commit 7d3b53bbce
16 changed files with 517 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
skills/scripts/e2e/ensure-fake-provider-pipeline.mjs
+41 -3
View File
@@ -41,6 +41,8 @@ const pipelineName = env.LANGBOT_FAKE_PROVIDER_PIPELINE_NAME || DEFAULT_PIPELINE
const providerName = env.LANGBOT_FAKE_PROVIDER_NAME || DEFAULT_PROVIDER_NAME;
const requester = env.LANGBOT_FAKE_PROVIDER_REQUESTER || DEFAULT_REQUESTER;
const modelName = env.LANGBOT_FAKE_PROVIDER_MODEL_NAME || DEFAULT_MODEL_NAME;
const fallbackModelNames = textList(env.LANGBOT_FAKE_PROVIDER_FALLBACK_MODEL_NAMES)
.filter((name) => name !== modelName);
const result = {
source: "automation",
@@ -75,6 +77,7 @@ const result = {
test_status: "not_run",
test_reason: "",
},
fallback_models: [],
pipeline_id: "",
pipeline_name: pipelineName,
pipeline_url: "",
@@ -141,11 +144,23 @@ try {
});
result.model = model;
const fallbackModels = [];
for (const fallbackModelName of fallbackModelNames) {
fallbackModels.push(await ensureModel({
backendUrl,
token: auth.token,
providerUuid: provider.uuid,
name: fallbackModelName,
}));
}
result.fallback_models = fallbackModels;
const pipeline = await ensurePipeline({
backendUrl,
token: auth.token,
name: pipelineName,
modelUuid: model.uuid,
fallbackModelUuids: fallbackModels.map((item) => item.uuid),
});
Object.assign(result, pipeline);
result.pipeline_url = `${frontendUrl.replace(/\/$/, "")}/home/agents?id=${encodeURIComponent(pipeline.pipeline_id)}`;
@@ -161,6 +176,7 @@ try {
LANGBOT_FAKE_PROVIDER_PID: fakeProvider.pid ? String(fakeProvider.pid) : "",
LANGBOT_FAKE_PROVIDER_PROVIDER_UUID: provider.uuid,
LANGBOT_FAKE_PROVIDER_MODEL_UUID: model.uuid,
LANGBOT_FAKE_PROVIDER_FALLBACK_MODEL_UUIDS: fallbackModels.map((item) => item.uuid).join(","),
LANGBOT_FAKE_PROVIDER_PIPELINE_URL: result.pipeline_url,
LANGBOT_FAKE_PROVIDER_PIPELINE_NAME: pipelineName,
});
@@ -168,7 +184,7 @@ try {
}
result.status = "pass";
result.reason = `Fake provider pipeline is configured with ${requester}/${modelName}.`;
result.reason = `Fake provider pipeline is configured with ${requester}/${modelName} and ${fallbackModels.length} fallback(s).`;
} catch (error) {
result.status = result.status === "env_issue" ? "env_issue" : "fail";
result.reason = result.reason || safeReason(error.message);
@@ -328,7 +344,11 @@ function healthyFakeProviderConfig() {
fault_status: 500,
fail_first_n: 0,
fail_every_n: 0,
fail_models: [],
fail_after_first_chunk: false,
fail_after_first_chunk_delay_ms: 0,
fail_after_first_chunk_mode: "disconnect",
fail_after_first_chunk_models: [],
dynamic_response: true,
};
}
@@ -342,7 +362,14 @@ function targetFakeProviderConfig() {
fault_status: httpFaultStatus(env.LANGBOT_FAKE_PROVIDER_FAULT_STATUS, 500),
fail_first_n: nonNegativeInteger(env.LANGBOT_FAKE_PROVIDER_FAIL_FIRST_N, 0),
fail_every_n: nonNegativeInteger(env.LANGBOT_FAKE_PROVIDER_FAIL_EVERY_N, 0),
fail_models: textList(env.LANGBOT_FAKE_PROVIDER_FAIL_MODELS),
fail_after_first_chunk: envBool(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK, false),
fail_after_first_chunk_delay_ms: nonNegativeInteger(
env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_DELAY_MS,
0,
),
fail_after_first_chunk_mode: streamFaultMode(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_MODE),
fail_after_first_chunk_models: textList(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_MODELS),
dynamic_response: envBool(env.LANGBOT_FAKE_PROVIDER_DYNAMIC_RESPONSE, true),
};
}
@@ -476,7 +503,7 @@ async function ensureModel({ backendUrl, token, providerUuid, name }) {
};
}
async function ensurePipeline({ backendUrl, token, name, modelUuid }) {
async function ensurePipeline({ backendUrl, token, name, modelUuid, fallbackModelUuids = [] }) {
const list = await apiJson(backendUrl, "/api/v1/pipelines", { token });
if (isApiFailure(list)) {
throw new Error(list.json.msg || "Failed to list pipelines.");
@@ -538,7 +565,7 @@ async function ensurePipeline({ backendUrl, token, name, modelUuid }) {
"max-round": positiveInteger(existingLocalAgentConfig["max-round"], 10),
model: {
primary: modelUuid,
fallbacks: [],
fallbacks: fallbackModelUuids,
},
};
const updatedConfig = {
@@ -605,6 +632,17 @@ function envBool(value, fallback) {
return fallback;
}
function textList(value) {
return String(value || "")
.split(/\r?\n|,/)
.map((item) => item.trim())
.filter(Boolean);
}
function streamFaultMode(value) {
return String(value || "").trim().toLowerCase() === "error_event" ? "error_event" : "disconnect";
}
function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
skills/scripts/e2e/fake-openai-provider.mjs
+51 -3
View File
@@ -54,7 +54,11 @@ const config = {
fault_status: integer(env.LANGBOT_FAKE_PROVIDER_FAULT_STATUS, 500),
fail_first_n: integer(env.LANGBOT_FAKE_PROVIDER_FAIL_FIRST_N, 0),
fail_every_n: integer(env.LANGBOT_FAKE_PROVIDER_FAIL_EVERY_N, 0),
fail_models: textList(env.LANGBOT_FAKE_PROVIDER_FAIL_MODELS),
fail_after_first_chunk: bool(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK, false),
fail_after_first_chunk_delay_ms: integer(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_DELAY_MS, 0),
fail_after_first_chunk_mode: faultMode(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_MODE),
fail_after_first_chunk_models: textList(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_MODELS),
dynamic_response: !/^(0|false|no|off)$/i.test(env.LANGBOT_FAKE_PROVIDER_DYNAMIC_RESPONSE || ""),
request_log_limit: integer(env.LANGBOT_FAKE_PROVIDER_REQUEST_LOG_LIMIT, 500),
};
@@ -135,8 +139,12 @@ const server = createServer(async (request, response) => {
requestCount += 1;
const body = await readJson(request);
const requestId = `chatcmpl-langbot-fake-${requestCount}`;
const requestModel = String(body.model || modelName);
const shouldFail = requestCount <= config.fail_first_n
|| (config.fail_every_n > 0 && requestCount % config.fail_every_n === 0);
|| (config.fail_every_n > 0 && requestCount % config.fail_every_n === 0)
|| config.fail_models.includes(requestModel);
const failAfterFirstChunk = config.fail_after_first_chunk
|| config.fail_after_first_chunk_models.includes(requestModel);
const replyMessage = buildResponse(body);
const replyText = replyMessage.content || "";
requestRecord = recordRequest({
@@ -144,7 +152,7 @@ const server = createServer(async (request, response) => {
request_number: requestCount,
path: url.pathname,
stream: Boolean(body.stream),
model: body.model || "",
model: requestModel,
message_count: Array.isArray(body.messages) ? body.messages.length : 0,
should_fail: shouldFail,
status: "running",
@@ -179,7 +187,7 @@ const server = createServer(async (request, response) => {
requestId,
model: body.model || modelName,
message: replyMessage,
failAfterFirstChunk: config.fail_after_first_chunk,
failAfterFirstChunk,
requestRecord,
startedPerf,
});
@@ -416,6 +424,22 @@ async function streamCompletion(response, {
choices: [{ index: 0, delta: { content: chunks[index] }, finish_reason: null }],
});
if (failMidStream && index === 0) {
await sleep(config.fail_after_first_chunk_delay_ms);
if (config.fail_after_first_chunk_mode === "error_event") {
response.write(`data: ${JSON.stringify({
error: {
message: "LangBot fake provider injected a mid-stream error event",
type: "fake_provider_stream_fault",
code: "fake_provider_stream_fault",
},
})}\n\n`);
response.end();
finishRequestRecord(requestRecord, startedPerf, {
status: "mid_stream_error_event",
http_status: 200,
});
return;
}
finishRequestRecord(requestRecord, startedPerf, {
status: "mid_stream_disconnect",
http_status: 200,
@@ -815,12 +839,18 @@ function applyConfig(updates) {
assignNonNegativeInteger(updates, "chunk_count");
assignNonNegativeInteger(updates, "fail_first_n");
assignNonNegativeInteger(updates, "fail_every_n");
assignTextList(updates, "fail_models");
assignNonNegativeInteger(updates, "request_log_limit");
if (updates.fault_status !== undefined) {
const parsed = Number.parseInt(String(updates.fault_status), 10);
if (Number.isInteger(parsed) && parsed >= 400 && parsed <= 599) config.fault_status = parsed;
}
assignBoolean(updates, "fail_after_first_chunk");
assignNonNegativeInteger(updates, "fail_after_first_chunk_delay_ms");
if (updates.fail_after_first_chunk_mode !== undefined) {
config.fail_after_first_chunk_mode = faultMode(updates.fail_after_first_chunk_mode);
}
assignTextList(updates, "fail_after_first_chunk_models");
assignBoolean(updates, "dynamic_response");
}
@@ -838,3 +868,21 @@ function assignBoolean(updates, key) {
if (updates[key] === undefined) return;
config[key] = bool(updates[key], config[key]);
}
function assignTextList(updates, key) {
if (updates[key] === undefined) return;
config[key] = Array.isArray(updates[key])
? updates[key].map(String).map((item) => item.trim()).filter(Boolean)
: textList(updates[key]);
}
function textList(value) {
return String(value || "")
.split(/\r?\n|,/)
.map((item) => item.trim())
.filter(Boolean);
}
function faultMode(value) {
return String(value || "").trim().toLowerCase() === "error_event" ? "error_event" : "disconnect";
}