xinyin025/LangBot
1
0
Fork 0
mirror of https://github.com/langbot-app/LangBot.git synced 2026-07-26 06:16:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(skills): extend fake provider load profiles

Browse Source
This commit is contained in:
huanghuoguoguo
2026-06-25 12:54:08 +08:00
parent d59b49ec55
commit 7e36869494
13 changed files with 717 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
skills/scripts/e2e/ensure-fake-provider-pipeline.mjs
+101 -8
View File
@@ -54,6 +54,7 @@ const result = {
base_url: "",
pid: null,
reused: false,
config: {},
state_file: fakeStatePath,
stdout_log: fakeStdoutPath,
stderr_log: fakeStderrPath,
@@ -99,9 +100,11 @@ try {
}
const fakeProvider = await ensureFakeProvider();
const setupConfig = await configureFakeProvider(fakeProvider.url, healthyFakeProviderConfig(), true);
result.fake_provider = {
...result.fake_provider,
...fakeProvider,
config: setupConfig.config || healthyFakeProviderConfig(),
};
const user = env.LANGBOT_E2E_LOGIN_USER || "";
@@ -144,6 +147,9 @@ try {
Object.assign(result, pipeline);
result.pipeline_url = `${frontendUrl.replace(/\/$/, "")}/home/pipelines?id=${encodeURIComponent(pipeline.pipeline_id)}`;
const runConfig = await configureFakeProvider(fakeProvider.url, targetFakeProviderConfig(), true);
result.fake_provider.config = runConfig.config || targetFakeProviderConfig();
if (writeEnv) {
await upsertEnvLocal(envLocalPath, {
LANGBOT_E2E_LOGIN_USER: user,
@@ -172,7 +178,7 @@ process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 :
async function ensureFakeProvider() {
const envUrl = normalizeProviderRootUrl(env.LANGBOT_FAKE_PROVIDER_URL || "");
if (envUrl && await fakeProviderHealthy(envUrl)) {
if (envUrl && await fakeProviderHealthy(envUrl) && await fakeProviderConfigurable(envUrl)) {
return {
url: envUrl,
base_url: `${envUrl}/v1`,
@@ -184,12 +190,15 @@ async function ensureFakeProvider() {
const state = await readState(fakeStatePath);
const stateUrl = normalizeProviderRootUrl(state.url || "");
if (stateUrl && await fakeProviderHealthy(stateUrl)) {
return {
url: stateUrl,
base_url: state.base_url || `${stateUrl}/v1`,
pid: Number.isInteger(state.pid) ? state.pid : null,
reused: true,
};
if (await fakeProviderConfigurable(stateUrl)) {
return {
url: stateUrl,
base_url: state.base_url || `${stateUrl}/v1`,
pid: Number.isInteger(state.pid) ? state.pid : null,
reused: true,
};
}
if (Number.isInteger(state.pid)) await stopProcess(state.pid);
}
await mkdir(fakeStateDir, { recursive: true });
@@ -218,7 +227,7 @@ async function ensureFakeProvider() {
await stderr.close();
const started = await waitForFakeProviderState(fakeStatePath, child.pid, 10_000);
if (!started.url || !await fakeProviderHealthy(started.url)) {
if (!started.url || !await fakeProviderHealthy(started.url) || !await fakeProviderConfigurable(started.url)) {
throw new Error(`Fake provider did not become healthy. See ${fakeStderrPath}`);
}
@@ -230,6 +239,23 @@ async function ensureFakeProvider() {
};
}
async function configureFakeProvider(rootUrl, config, resetRequestCount) {
const response = await fetch(`${normalizeProviderRootUrl(rootUrl)}/__qa/config`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
config,
reset_request_count: resetRequestCount,
}),
signal: AbortSignal.timeout(3000),
});
const json = await response.json().catch(() => ({}));
if (!response.ok || json.ok !== true) {
throw new Error(`Fake provider config failed with HTTP ${response.status}.`);
}
return json;
}
async function fakeProviderHealthy(rootUrl) {
try {
const response = await fetch(`${rootUrl.replace(/\/$/, "")}/healthz`, {
@@ -243,6 +269,28 @@ async function fakeProviderHealthy(rootUrl) {
}
}
async function fakeProviderConfigurable(rootUrl) {
try {
const response = await fetch(`${rootUrl.replace(/\/$/, "")}/__qa/config`, {
signal: AbortSignal.timeout(2000),
});
if (!response.ok) return false;
const json = await response.json().catch(() => ({}));
return json.ok === true && json.config && typeof json.config === "object";
} catch {
return false;
}
}
async function stopProcess(pid) {
try {
process.kill(pid, "SIGTERM");
} catch {
return;
}
await sleep(500);
}
async function waitForFakeProviderState(path, expectedPid, timeoutMs) {
const startedAt = Date.now();
let lastState = {};
@@ -268,6 +316,34 @@ function normalizeProviderRootUrl(value) {
return trimmed.endsWith("/v1") ? trimmed.slice(0, -3) : trimmed;
}
function healthyFakeProviderConfig() {
return {
response_text: "OK",
first_token_delay_ms: 25,
chunk_delay_ms: 10,
chunk_count: 0,
fault_status: 500,
fail_first_n: 0,
fail_every_n: 0,
fail_after_first_chunk: false,
dynamic_response: true,
};
}
function targetFakeProviderConfig() {
return {
response_text: env.LANGBOT_FAKE_PROVIDER_RESPONSE_TEXT || "OK",
first_token_delay_ms: nonNegativeInteger(env.LANGBOT_FAKE_PROVIDER_FIRST_TOKEN_DELAY_MS, 25),
chunk_delay_ms: nonNegativeInteger(env.LANGBOT_FAKE_PROVIDER_CHUNK_DELAY_MS, 10),
chunk_count: nonNegativeInteger(env.LANGBOT_FAKE_PROVIDER_CHUNK_COUNT, 0),
fault_status: httpFaultStatus(env.LANGBOT_FAKE_PROVIDER_FAULT_STATUS, 500),
fail_first_n: nonNegativeInteger(env.LANGBOT_FAKE_PROVIDER_FAIL_FIRST_N, 0),
fail_every_n: nonNegativeInteger(env.LANGBOT_FAKE_PROVIDER_FAIL_EVERY_N, 0),
fail_after_first_chunk: envBool(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK, false),
dynamic_response: envBool(env.LANGBOT_FAKE_PROVIDER_DYNAMIC_RESPONSE, true),
};
}
async function skipWizard({ backendUrl, token }) {
const response = await apiJson(backendUrl, "/api/v1/system/wizard/completed", {
method: "POST",
@@ -505,6 +581,23 @@ function positiveInteger(value, fallback) {
return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
}
function nonNegativeInteger(value, fallback) {
const parsed = Number(value);
return Number.isInteger(parsed) && parsed >= 0 ? parsed : fallback;
}
function httpFaultStatus(value, fallback) {
const parsed = Number(value);
return Number.isInteger(parsed) && parsed >= 400 && parsed <= 599 ? parsed : fallback;
}
function envBool(value, fallback) {
if (value === undefined || value === "") return fallback;
if (/^(1|true|yes|on)$/i.test(String(value))) return true;
if (/^(0|false|no|off)$/i.test(String(value))) return false;
return fallback;
}
function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
skills/scripts/e2e/fake-openai-provider.mjs
+100 -22
View File
@@ -10,14 +10,18 @@ const host = args.host || env.LANGBOT_FAKE_PROVIDER_HOST || "127.0.0.1";
const port = integer(args.port ?? env.LANGBOT_FAKE_PROVIDER_PORT, 0);
const stateFile = args["state-file"] || env.LANGBOT_FAKE_PROVIDER_STATE_FILE || "";
const modelName = env.LANGBOT_FAKE_PROVIDER_MODEL_NAME || "gpt-4o-mini";
const responseText = env.LANGBOT_FAKE_PROVIDER_RESPONSE_TEXT || "OK";
const firstTokenDelayMs = integer(env.LANGBOT_FAKE_PROVIDER_FIRST_TOKEN_DELAY_MS, 25);
const chunkDelayMs = integer(env.LANGBOT_FAKE_PROVIDER_CHUNK_DELAY_MS, 10);
const faultStatus = integer(env.LANGBOT_FAKE_PROVIDER_FAULT_STATUS, 500);
const failFirstN = integer(env.LANGBOT_FAKE_PROVIDER_FAIL_FIRST_N, 0);
const failEveryN = integer(env.LANGBOT_FAKE_PROVIDER_FAIL_EVERY_N, 0);
const failAfterFirstChunk = bool(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK, false);
const requestLogLimit = integer(env.LANGBOT_FAKE_PROVIDER_REQUEST_LOG_LIMIT, 500);
const config = {
response_text: env.LANGBOT_FAKE_PROVIDER_RESPONSE_TEXT || "OK",
first_token_delay_ms: integer(env.LANGBOT_FAKE_PROVIDER_FIRST_TOKEN_DELAY_MS, 25),
chunk_delay_ms: integer(env.LANGBOT_FAKE_PROVIDER_CHUNK_DELAY_MS, 10),
chunk_count: integer(env.LANGBOT_FAKE_PROVIDER_CHUNK_COUNT, 0),
fault_status: integer(env.LANGBOT_FAKE_PROVIDER_FAULT_STATUS, 500),
fail_first_n: integer(env.LANGBOT_FAKE_PROVIDER_FAIL_FIRST_N, 0),
fail_every_n: integer(env.LANGBOT_FAKE_PROVIDER_FAIL_EVERY_N, 0),
fail_after_first_chunk: bool(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK, false),
dynamic_response: !/^(0|false|no|off)$/i.test(env.LANGBOT_FAKE_PROVIDER_DYNAMIC_RESPONSE || ""),
request_log_limit: integer(env.LANGBOT_FAKE_PROVIDER_REQUEST_LOG_LIMIT, 500),
};
let requestCount = 0;
const recentRequests = [];
@@ -30,12 +34,48 @@ const server = createServer(async (request, response) => {
sendJson(response, 200, {
ok: true,
model: modelName,
config,
request_count: requestCount,
recent_request_count: recentRequests.length,
});
return;
}
if (request.method === "GET" && url.pathname === "/__qa/config") {
sendJson(response, 200, {
ok: true,
model: modelName,
config,
request_count: requestCount,
recent_requests: recentRequests,
});
return;
}
if (request.method === "POST" && url.pathname === "/__qa/config") {
const body = await readJson(request);
applyConfig(body.config && typeof body.config === "object" ? body.config : body);
if (body.reset_request_count !== false) resetRequestState();
sendJson(response, 200, {
ok: true,
model: modelName,
config,
request_count: requestCount,
});
return;
}
if (request.method === "POST" && url.pathname === "/__qa/reset") {
resetRequestState();
sendJson(response, 200, {
ok: true,
model: modelName,
config,
request_count: requestCount,
});
return;
}
if (request.method === "GET" && ["/models", "/v1/models"].includes(url.pathname)) {
sendJson(response, 200, {
object: "list",
@@ -56,7 +96,8 @@ const server = createServer(async (request, response) => {
requestCount += 1;
const body = await readJson(request);
const requestId = `chatcmpl-langbot-fake-${requestCount}`;
const shouldFail = requestCount <= failFirstN || (failEveryN > 0 && requestCount % failEveryN === 0);
const shouldFail = requestCount <= config.fail_first_n
|| (config.fail_every_n > 0 && requestCount % config.fail_every_n === 0);
recordRequest({
id: requestId,
path: url.pathname,
@@ -67,10 +108,10 @@ const server = createServer(async (request, response) => {
});
if (shouldFail) {
await sleep(firstTokenDelayMs);
sendJson(response, faultStatus, {
await sleep(config.first_token_delay_ms);
sendJson(response, config.fault_status, {
error: {
message: `LangBot fake provider injected HTTP ${faultStatus}`,
message: `LangBot fake provider injected HTTP ${config.fault_status}`,
type: "fake_provider_fault",
code: "fake_provider_fault",
},
@@ -85,10 +126,10 @@ const server = createServer(async (request, response) => {
requestId,
model: body.model || modelName,
content: replyText,
failAfterFirstChunk,
failAfterFirstChunk: config.fail_after_first_chunk,
});
} else {
await sleep(firstTokenDelayMs + chunkDelayMs);
await sleep(config.first_token_delay_ms + config.chunk_delay_ms);
sendJson(response, 200, completionPayload({
requestId,
model: body.model || modelName,
@@ -230,7 +271,7 @@ async function streamCompletion(response, { requestId, model, content, failAfter
"connection": "keep-alive",
});
await sleep(firstTokenDelayMs);
await sleep(config.first_token_delay_ms);
writeSse(response, {
id: requestId,
object: "chat.completion.chunk",
@@ -241,7 +282,7 @@ async function streamCompletion(response, { requestId, model, content, failAfter
const chunks = splitContent(content);
for (let index = 0; index < chunks.length; index += 1) {
await sleep(chunkDelayMs);
await sleep(config.chunk_delay_ms);
writeSse(response, {
id: requestId,
object: "chat.completion.chunk",
@@ -255,7 +296,7 @@ async function streamCompletion(response, { requestId, model, content, failAfter
}
}
await sleep(chunkDelayMs);
await sleep(config.chunk_delay_ms);
const completionTokens = tokenEstimate(content);
writeSse(response, {
id: requestId,
@@ -279,7 +320,7 @@ function writeSse(response, payload) {
function splitContent(content) {
const text = String(content);
const requested = integer(env.LANGBOT_FAKE_PROVIDER_CHUNK_COUNT, 0);
const requested = config.chunk_count;
if (requested <= 1 || text.length <= 1) return [text];
const chunkSize = Math.max(1, Math.ceil(text.length / requested));
const chunks = [];
@@ -294,8 +335,8 @@ function tokenEstimate(content) {
}
function responseTextForBody(body) {
if (/^(0|false|no|off)$/i.test(env.LANGBOT_FAKE_PROVIDER_DYNAMIC_RESPONSE || "")) {
return responseText;
if (!config.dynamic_response) {
return config.response_text;
}
const messages = Array.isArray(body.messages) ? body.messages : [];
const lastUser = [...messages].reverse().find((message) => message?.role === "user");
@@ -306,7 +347,7 @@ function responseTextForBody(body) {
if (exact?.[1]) return exact[1].trim().replace(/[。.!?]+$/, "");
const only = text.match(/只回复\s*([A-Za-z0-9_.:@-]{1,80})/);
if (only?.[1]) return only[1].trim().replace(/[。.!?]+$/, "");
return responseText;
return config.response_text;
}
function flattenContent(content) {
@@ -328,5 +369,42 @@ function recordRequest(entry) {
...entry,
at: new Date().toISOString(),
});
while (recentRequests.length > requestLogLimit) recentRequests.shift();
while (recentRequests.length > config.request_log_limit) recentRequests.shift();
}
function resetRequestState() {
requestCount = 0;
recentRequests.length = 0;
}
function applyConfig(updates) {
if (!updates || typeof updates !== "object") return;
assignString(updates, "response_text");
assignNonNegativeInteger(updates, "first_token_delay_ms");
assignNonNegativeInteger(updates, "chunk_delay_ms");
assignNonNegativeInteger(updates, "chunk_count");
assignNonNegativeInteger(updates, "fail_first_n");
assignNonNegativeInteger(updates, "fail_every_n");
assignNonNegativeInteger(updates, "request_log_limit");
if (updates.fault_status !== undefined) {
const parsed = Number.parseInt(String(updates.fault_status), 10);
if (Number.isInteger(parsed) && parsed >= 400 && parsed <= 599) config.fault_status = parsed;
}
assignBoolean(updates, "fail_after_first_chunk");
assignBoolean(updates, "dynamic_response");
}
function assignString(updates, key) {
if (updates[key] !== undefined) config[key] = String(updates[key]);
}
function assignNonNegativeInteger(updates, key) {
if (updates[key] === undefined) return;
const parsed = Number.parseInt(String(updates[key]), 10);
if (Number.isInteger(parsed) && parsed >= 0) config[key] = parsed;
}
function assignBoolean(updates, key) {
if (updates[key] === undefined) return;
config[key] = bool(updates[key], config[key]);
}