diff --git a/skills/README.md b/skills/README.md index d287f6762..091e3d3b1 100644 --- a/skills/README.md +++ b/skills/README.md @@ -48,6 +48,7 @@ bin/lbs env show # inspect resolved env defaults (redacted) bin/lbs env doctor # diagnose local environment readiness bin/lbs case list --ready bin/lbs test plan +bin/lbs suite plan langbot-debug-chat-load-gate ``` ## Maintenance rule diff --git a/skills/docs/user-guide.md b/skills/docs/user-guide.md index d52a77247..846b7f5c4 100644 --- a/skills/docs/user-guide.md +++ b/skills/docs/user-guide.md @@ -117,6 +117,21 @@ bin/lbs suite plan langbot-user-path-performance-gate bin/lbs suite run langbot-user-path-performance-gate --run-id langbot-user-path-local --include-manual-check ``` +Controlled Debug Chat message-path load gate: + +```bash +bin/lbs suite plan langbot-debug-chat-load-gate +bin/lbs test run langbot-fake-provider-debug-chat-load --run-id langbot-fake-load-local +bin/lbs test run langbot-space-debug-chat-concurrency-smoke --run-id langbot-space-smoke-local +``` + +Start with `langbot-fake-provider-debug-chat-load`. It launches a local +OpenAI-compatible fake provider, creates the matching provider/model/pipeline, +then sends concurrent WebSocket Debug Chat messages through the real backend. +Use `langbot-space-debug-chat-concurrency-smoke` only as a low-volume live +provider smoke; it includes Space/model/network latency and should be compared +against the fake-provider baseline before attributing failures to LangBot. + `manual_check` means the agent must confirm the declared preconditions for that run window. When setup automation is declared, run output may stop early with `env_issue`; fix that environment input before treating the product path as diff --git a/skills/schemas/case.schema.json b/skills/schemas/case.schema.json index 0d63d8dec..9f4da284a 100644 --- a/skills/schemas/case.schema.json +++ b/skills/schemas/case.schema.json @@ -209,6 +209,38 @@ "automation_debug_chat_max_error_rate": { "type": "string" }, + "automation_debug_chat_load_requests": { + "type": "string" + }, + "automation_debug_chat_load_concurrency": { + "type": "string" + }, + "automation_debug_chat_load_timeout_ms": { + "type": "string" + }, + "automation_debug_chat_load_response_p95_ms": { + "type": "string" + }, + "automation_debug_chat_load_first_response_p95_ms": { + "type": "string" + }, + "automation_debug_chat_load_max_error_rate": { + "type": "string" + }, + "automation_debug_chat_load_expected_prefix": { + "type": "string" + }, + "automation_debug_chat_load_prompt_template": { + "type": "string" + }, + "automation_debug_chat_load_stream": { + "type": "string", + "enum": ["0", "1", "false", "true"] + }, + "automation_debug_chat_load_reset": { + "type": "string", + "enum": ["0", "1", "false", "true"] + }, "automation_filesystem_checks_json": { "type": "string" }, diff --git a/skills/scripts/e2e/ensure-fake-provider-pipeline.mjs b/skills/scripts/e2e/ensure-fake-provider-pipeline.mjs new file mode 100644 index 000000000..94a17c08a --- /dev/null +++ b/skills/scripts/e2e/ensure-fake-provider-pipeline.mjs @@ -0,0 +1,539 @@ +#!/usr/bin/env node + +import { spawn } from "node:child_process"; +import { open, readFile, mkdir, writeFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { env } from "node:process"; +import { + apiJson, + ensureEvidence, + evidencePaths, + loadEnvFiles, + redact, + resetAndAuthLocalUser, + writeResult, +} from "./lib/langbot-e2e.mjs"; + +const RUNNER_ID = "local-agent"; +const DEFAULT_LOCAL_PASSWORD = "LangBotE2ELocalPass!2026"; +const DEFAULT_PIPELINE_NAME = "Agent QA Fake Provider Debug Chat"; +const DEFAULT_PROVIDER_NAME = "LangBot QA Fake OpenAI Provider"; +const DEFAULT_MODEL_NAME = "gpt-4o-mini"; +const DEFAULT_REQUESTER = "openai-chat-completions"; + +const caseId = "ensure-fake-provider-pipeline"; + +await loadEnvFiles(); +const paths = evidencePaths(caseId); +await ensureEvidence(paths); + +const writeEnv = process.argv.includes("--write-env"); +const frontendUrl = env.LANGBOT_FRONTEND_URL || ""; +const backendUrl = env.LANGBOT_BACKEND_URL || ""; +const envLocalPath = resolve("skills/.env.local"); +const repoRoot = resolve(env.LANGBOT_REPO || ".."); +const fakeStateDir = resolve(env.LANGBOT_FAKE_PROVIDER_STATE_DIR || resolve(repoRoot, ".qa/fake-provider")); +const fakeStatePath = resolve(fakeStateDir, "state.json"); +const fakeStdoutPath = resolve(fakeStateDir, "fake-provider.stdout.log"); +const fakeStderrPath = resolve(fakeStateDir, "fake-provider.stderr.log"); +const pipelineName = env.LANGBOT_FAKE_PROVIDER_PIPELINE_NAME || DEFAULT_PIPELINE_NAME; +const providerName = env.LANGBOT_FAKE_PROVIDER_NAME || DEFAULT_PROVIDER_NAME; +const requester = env.LANGBOT_FAKE_PROVIDER_REQUESTER || DEFAULT_REQUESTER; +const modelName = env.LANGBOT_FAKE_PROVIDER_MODEL_NAME || DEFAULT_MODEL_NAME; + +const result = { + source: "automation", + case_id: caseId, + run_id: paths.runId, + status: "fail", + reason: "", + frontend_url: frontendUrl, + backend_url: backendUrl, + fake_provider: { + url: "", + base_url: "", + pid: null, + reused: false, + state_file: fakeStatePath, + stdout_log: fakeStdoutPath, + stderr_log: fakeStderrPath, + }, + provider: { + uuid: "", + name: providerName, + requester, + created: false, + updated: false, + }, + model: { + uuid: "", + name: modelName, + created: false, + updated: false, + test_status: "not_run", + test_reason: "", + }, + pipeline_id: "", + pipeline_name: pipelineName, + pipeline_url: "", + created: false, + updated: false, + wrote_env: false, + evidence: { + console_log: paths.consoleLog, + network_log: paths.networkLog, + automation_result_json: paths.automationResultJson, + result_json: paths.resultJson, + }, + evidence_collected: ["api_diagnostic", "network", "filesystem"], +}; + +try { + if (!backendUrl) { + result.status = "env_issue"; + throw new Error("LANGBOT_BACKEND_URL is not configured."); + } + if (!frontendUrl) { + result.status = "env_issue"; + throw new Error("LANGBOT_FRONTEND_URL is not configured."); + } + + const fakeProvider = await ensureFakeProvider(); + result.fake_provider = { + ...result.fake_provider, + ...fakeProvider, + }; + + const user = env.LANGBOT_E2E_LOGIN_USER || ""; + const password = env.LANGBOT_E2E_LOGIN_PASSWORD || DEFAULT_LOCAL_PASSWORD; + if (!user) { + result.status = "env_issue"; + throw new Error("LANGBOT_E2E_LOGIN_USER is required so this setup can create/update the fake provider pipeline."); + } + + const auth = await resetAndAuthLocalUser({ backendUrl, user, password }); + const wizard = await skipWizard({ backendUrl, token: auth.token }); + if (wizard.status !== "pass") { + result.status = "fail"; + throw new Error(wizard.reason || "Failed to mark the local QA wizard as skipped."); + } + + const provider = await ensureProvider({ + backendUrl, + token: auth.token, + name: providerName, + requester, + baseUrl: fakeProvider.base_url, + }); + result.provider = provider; + + const model = await ensureModel({ + backendUrl, + token: auth.token, + providerUuid: provider.uuid, + name: modelName, + }); + result.model = model; + + const pipeline = await ensurePipeline({ + backendUrl, + token: auth.token, + name: pipelineName, + modelUuid: model.uuid, + }); + Object.assign(result, pipeline); + result.pipeline_url = `${frontendUrl.replace(/\/$/, "")}/home/pipelines?id=${encodeURIComponent(pipeline.pipeline_id)}`; + + if (writeEnv) { + await upsertEnvLocal(envLocalPath, { + LANGBOT_E2E_LOGIN_USER: user, + LANGBOT_FAKE_PROVIDER_URL: fakeProvider.url, + LANGBOT_FAKE_PROVIDER_BASE_URL: fakeProvider.base_url, + LANGBOT_FAKE_PROVIDER_PID: fakeProvider.pid ? String(fakeProvider.pid) : "", + LANGBOT_FAKE_PROVIDER_PROVIDER_UUID: provider.uuid, + LANGBOT_FAKE_PROVIDER_MODEL_UUID: model.uuid, + LANGBOT_FAKE_PROVIDER_PIPELINE_URL: result.pipeline_url, + LANGBOT_FAKE_PROVIDER_PIPELINE_NAME: pipelineName, + }); + result.wrote_env = true; + } + + result.status = "pass"; + result.reason = `Fake provider pipeline is configured with ${requester}/${modelName}.`; +} catch (error) { + result.status = result.status === "env_issue" ? "env_issue" : "fail"; + result.reason = result.reason || safeReason(error.message); +} finally { + await writeResult(paths, result); + console.log(JSON.stringify(result, null, 2)); +} + +process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1); + +async function ensureFakeProvider() { + const envUrl = normalizeProviderRootUrl(env.LANGBOT_FAKE_PROVIDER_URL || ""); + if (envUrl && await fakeProviderHealthy(envUrl)) { + return { + url: envUrl, + base_url: `${envUrl}/v1`, + pid: null, + reused: true, + }; + } + + const state = await readState(fakeStatePath); + const stateUrl = normalizeProviderRootUrl(state.url || ""); + if (stateUrl && await fakeProviderHealthy(stateUrl)) { + return { + url: stateUrl, + base_url: state.base_url || `${stateUrl}/v1`, + pid: Number.isInteger(state.pid) ? state.pid : null, + reused: true, + }; + } + + await mkdir(fakeStateDir, { recursive: true }); + await writeFile(fakeStatePath, `${JSON.stringify({ status: "starting", started_at: new Date().toISOString() }, null, 2)}\n`, "utf8"); + const stdout = await open(fakeStdoutPath, "a"); + const stderr = await open(fakeStderrPath, "a"); + const scriptPath = resolve("scripts/e2e/fake-openai-provider.mjs"); + const host = env.LANGBOT_FAKE_PROVIDER_HOST || "127.0.0.1"; + const port = env.LANGBOT_FAKE_PROVIDER_PORT || "0"; + const child = spawn(process.execPath, [ + scriptPath, + `--host=${host}`, + `--port=${port}`, + `--state-file=${fakeStatePath}`, + ], { + cwd: resolve("."), + detached: true, + env: { + ...env, + LANGBOT_FAKE_PROVIDER_MODEL_NAME: modelName, + }, + stdio: ["ignore", stdout.fd, stderr.fd], + }); + child.unref(); + await stdout.close(); + await stderr.close(); + + const started = await waitForFakeProviderState(fakeStatePath, child.pid, 10_000); + if (!started.url || !await fakeProviderHealthy(started.url)) { + throw new Error(`Fake provider did not become healthy. See ${fakeStderrPath}`); + } + + return { + url: started.url, + base_url: started.base_url || `${started.url}/v1`, + pid: child.pid ?? started.pid ?? null, + reused: false, + }; +} + +async function fakeProviderHealthy(rootUrl) { + try { + const response = await fetch(`${rootUrl.replace(/\/$/, "")}/healthz`, { + signal: AbortSignal.timeout(2000), + }); + if (!response.ok) return false; + const json = await response.json().catch(() => ({})); + return json.ok === true; + } catch { + return false; + } +} + +async function waitForFakeProviderState(path, expectedPid, timeoutMs) { + const startedAt = Date.now(); + let lastState = {}; + while (Date.now() - startedAt < timeoutMs) { + const state = await readState(path); + if (state.url && (!expectedPid || state.pid === expectedPid)) return state; + lastState = state; + await sleep(150); + } + return lastState; +} + +async function readState(path) { + try { + return JSON.parse(await readFile(path, "utf8")); + } catch { + return {}; + } +} + +function normalizeProviderRootUrl(value) { + const trimmed = String(value || "").trim().replace(/\/$/, ""); + return trimmed.endsWith("/v1") ? trimmed.slice(0, -3) : trimmed; +} + +async function skipWizard({ backendUrl, token }) { + const response = await apiJson(backendUrl, "/api/v1/system/wizard/completed", { + method: "POST", + token, + body: { status: "skipped" }, + }); + const ok = response.status < 400 && response.json.code === 0; + return { + status: ok ? "pass" : "fail", + http_status: response.status, + code: response.json.code ?? null, + reason: ok ? "Wizard marked skipped for local QA." : response.json.msg || "Wizard status update failed.", + }; +} + +async function ensureProvider({ backendUrl, token, name, requester, baseUrl }) { + const list = await apiJson(backendUrl, "/api/v1/provider/providers", { token }); + if (isApiFailure(list)) { + throw new Error(list.json.msg || "Failed to list providers."); + } + const providers = list.json.data?.providers || []; + const existing = providers.find((provider) => ( + provider.name === name + || (provider.requester === requester && String(provider.base_url || "").replace(/\/$/, "") === baseUrl.replace(/\/$/, "")) + )); + const body = { + name, + requester, + base_url: baseUrl, + api_keys: [env.LANGBOT_FAKE_PROVIDER_API_KEY || "langbot-fake-provider-key"], + }; + + if (existing?.uuid) { + const update = await apiJson(backendUrl, `/api/v1/provider/providers/${encodeURIComponent(existing.uuid)}`, { + method: "PUT", + token, + body, + }); + if (isApiFailure(update)) { + throw new Error(update.json.msg || "Failed to update fake provider."); + } + return { + uuid: existing.uuid, + name, + requester, + created: false, + updated: true, + }; + } + + const create = await apiJson(backendUrl, "/api/v1/provider/providers", { + method: "POST", + token, + body, + }); + const uuid = create.json.data?.uuid || ""; + if (isApiFailure(create) || !uuid) { + throw new Error(create.json.msg || "Failed to create fake provider."); + } + return { + uuid, + name, + requester, + created: true, + updated: false, + }; +} + +async function ensureModel({ backendUrl, token, providerUuid, name }) { + const list = await apiJson(backendUrl, `/api/v1/provider/models/llm?provider_uuid=${encodeURIComponent(providerUuid)}`, { token }); + if (isApiFailure(list)) { + throw new Error(list.json.msg || "Failed to list fake provider models."); + } + const models = list.json.data?.models || []; + const existing = models.find((model) => model.name === name); + const body = { + name, + provider_uuid: providerUuid, + abilities: [], + context_length: positiveInteger(env.LANGBOT_FAKE_PROVIDER_CONTEXT_LENGTH, 8192), + extra_args: {}, + prefered_ranking: 0, + }; + let modelUuid = existing?.uuid || ""; + let created = false; + let updated = false; + + if (modelUuid) { + const update = await apiJson(backendUrl, `/api/v1/provider/models/llm/${encodeURIComponent(modelUuid)}`, { + method: "PUT", + token, + body, + }); + if (isApiFailure(update)) { + throw new Error(update.json.msg || "Failed to update fake provider model."); + } + updated = true; + } else { + const create = await apiJson(backendUrl, "/api/v1/provider/models/llm", { + method: "POST", + token, + body, + }); + modelUuid = create.json.data?.uuid || ""; + if (isApiFailure(create) || !modelUuid) { + throw new Error(create.json.msg || "Failed to create fake provider model."); + } + created = true; + } + + const test = await apiJson(backendUrl, `/api/v1/provider/models/llm/${encodeURIComponent(modelUuid)}/test`, { + method: "POST", + token, + body: { extra_args: {} }, + }); + if (isApiFailure(test)) { + throw new Error(safeReason(test.json.msg || test.json.message || "Fake provider model test failed.")); + } + + return { + uuid: modelUuid, + name, + created, + updated, + test_status: "pass", + test_reason: "", + }; +} + +async function ensurePipeline({ backendUrl, token, name, modelUuid }) { + const list = await apiJson(backendUrl, "/api/v1/pipelines", { token }); + if (isApiFailure(list)) { + throw new Error(list.json.msg || "Failed to list pipelines."); + } + const pipelines = list.json.data?.pipelines || []; + let pipeline = pipelines.find((item) => item.name === name) || null; + let created = false; + + if (!pipeline) { + const create = await apiJson(backendUrl, "/api/v1/pipelines", { + method: "POST", + token, + body: { + name, + description: "Local QA pipeline for controlled fake-provider Debug Chat load tests.", + emoji: "QA", + }, + }); + const pipelineId = create.json.data?.uuid || ""; + if (isApiFailure(create) || !pipelineId) { + throw new Error(create.json.msg || "Failed to create fake provider pipeline."); + } + created = true; + pipeline = { uuid: pipelineId }; + } + + const loaded = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, { token }); + pipeline = loaded.json.data?.pipeline || null; + if (isApiFailure(loaded) || !pipeline?.uuid) { + throw new Error(loaded.json.msg || "Failed to load fake provider pipeline."); + } + + const config = pipeline.config && typeof pipeline.config === "object" ? pipeline.config : {}; + const ai = config.ai && typeof config.ai === "object" ? config.ai : {}; + const existingLocalAgentConfig = ai["local-agent"] && typeof ai["local-agent"] === "object" + ? ai["local-agent"] + : {}; + const localAgentConfig = { + timeout: 60, + prompt: [{ role: "system", content: "You are a deterministic QA assistant. Reply exactly as instructed." }], + "remove-think": false, + "knowledge-bases": [], + "box-session-id-template": "{launcher_type}_{launcher_id}", + "retrieval-top-k": 5, + "rerank-model": "", + "rerank-top-k": 5, + "max-tool-iterations": 20, + "tool-execution-mode": "parallel", + "max-tool-result-chars": 20000, + "context-history-fetch-limit": 20, + "context-window-tokens": 8192, + "context-reserve-tokens": 1024, + "context-keep-recent-tokens": 2048, + "context-summary-tokens": 1024, + ...existingLocalAgentConfig, + // Current backend truncation still reads this field directly. + "max-round": positiveInteger(existingLocalAgentConfig["max-round"], 10), + model: { + primary: modelUuid, + fallbacks: [], + }, + }; + const updatedConfig = { + ...config, + ai: { + ...ai, + runner: { + ...(ai.runner && typeof ai.runner === "object" ? ai.runner : {}), + id: RUNNER_ID, + runner: RUNNER_ID, + "expire-time": 0, + }, + "local-agent": localAgentConfig, + }, + }; + + const update = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, { + method: "PUT", + token, + body: { + name, + description: "Local QA pipeline for controlled fake-provider Debug Chat load tests.", + emoji: "QA", + config: updatedConfig, + }, + }); + if (isApiFailure(update)) { + throw new Error(update.json.msg || "Failed to update fake provider pipeline."); + } + + return { + pipeline_id: pipeline.uuid, + pipeline_name: name, + created, + updated: true, + }; +} + +function isApiFailure(response) { + return response.status >= 400 || (response.json.code !== undefined && response.json.code !== 0); +} + +function positiveInteger(value, fallback) { + const parsed = Number(value); + return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function safeReason(value) { + return redact(String(value || "")).slice(0, 1000); +} + +async function upsertEnvLocal(path, updates) { + await mkdir(dirname(path), { recursive: true }); + let text = ""; + try { + text = await readFile(path, "utf8"); + } catch { + text = ""; + } + const lines = text.split(/\r?\n/); + const seen = new Set(); + const next = lines.map((line) => { + const trimmed = line.trim(); + const equals = trimmed.indexOf("="); + if (equals <= 0 || trimmed.startsWith("#")) return line; + const key = trimmed.slice(0, equals).trim(); + if (!(key in updates)) return line; + seen.add(key); + return `${key}=${updates[key]}`; + }); + for (const [key, value] of Object.entries(updates)) { + if (!seen.has(key)) next.push(`${key}=${value}`); + } + await writeFile(path, `${next.filter((line, index) => line !== "" || index < next.length - 1).join("\n")}\n`, "utf8"); +} diff --git a/skills/scripts/e2e/fake-openai-provider.mjs b/skills/scripts/e2e/fake-openai-provider.mjs new file mode 100644 index 000000000..f75d86ffe --- /dev/null +++ b/skills/scripts/e2e/fake-openai-provider.mjs @@ -0,0 +1,332 @@ +#!/usr/bin/env node + +import { createServer } from "node:http"; +import { mkdir, writeFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { env, exit } from "node:process"; + +const args = parseArgs(process.argv.slice(2)); +const host = args.host || env.LANGBOT_FAKE_PROVIDER_HOST || "127.0.0.1"; +const port = integer(args.port ?? env.LANGBOT_FAKE_PROVIDER_PORT, 0); +const stateFile = args["state-file"] || env.LANGBOT_FAKE_PROVIDER_STATE_FILE || ""; +const modelName = env.LANGBOT_FAKE_PROVIDER_MODEL_NAME || "gpt-4o-mini"; +const responseText = env.LANGBOT_FAKE_PROVIDER_RESPONSE_TEXT || "OK"; +const firstTokenDelayMs = integer(env.LANGBOT_FAKE_PROVIDER_FIRST_TOKEN_DELAY_MS, 25); +const chunkDelayMs = integer(env.LANGBOT_FAKE_PROVIDER_CHUNK_DELAY_MS, 10); +const faultStatus = integer(env.LANGBOT_FAKE_PROVIDER_FAULT_STATUS, 500); +const failFirstN = integer(env.LANGBOT_FAKE_PROVIDER_FAIL_FIRST_N, 0); +const failEveryN = integer(env.LANGBOT_FAKE_PROVIDER_FAIL_EVERY_N, 0); +const failAfterFirstChunk = bool(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK, false); +const requestLogLimit = integer(env.LANGBOT_FAKE_PROVIDER_REQUEST_LOG_LIMIT, 500); + +let requestCount = 0; +const recentRequests = []; + +const server = createServer(async (request, response) => { + const startedAt = Date.now(); + const url = new URL(request.url || "/", `http://${request.headers.host || `${host}:${port}`}`); + try { + if (request.method === "GET" && url.pathname === "/healthz") { + sendJson(response, 200, { + ok: true, + model: modelName, + request_count: requestCount, + recent_request_count: recentRequests.length, + }); + return; + } + + if (request.method === "GET" && ["/models", "/v1/models"].includes(url.pathname)) { + sendJson(response, 200, { + object: "list", + data: [ + { + id: modelName, + object: "model", + created: 1, + owned_by: "langbot-qa", + type: "llm", + }, + ], + }); + return; + } + + if (request.method === "POST" && ["/chat/completions", "/v1/chat/completions"].includes(url.pathname)) { + requestCount += 1; + const body = await readJson(request); + const requestId = `chatcmpl-langbot-fake-${requestCount}`; + const shouldFail = requestCount <= failFirstN || (failEveryN > 0 && requestCount % failEveryN === 0); + recordRequest({ + id: requestId, + path: url.pathname, + stream: Boolean(body.stream), + model: body.model || "", + message_count: Array.isArray(body.messages) ? body.messages.length : 0, + should_fail: shouldFail, + }); + + if (shouldFail) { + await sleep(firstTokenDelayMs); + sendJson(response, faultStatus, { + error: { + message: `LangBot fake provider injected HTTP ${faultStatus}`, + type: "fake_provider_fault", + code: "fake_provider_fault", + }, + }); + return; + } + + const replyText = responseTextForBody(body); + + if (body.stream) { + await streamCompletion(response, { + requestId, + model: body.model || modelName, + content: replyText, + failAfterFirstChunk, + }); + } else { + await sleep(firstTokenDelayMs + chunkDelayMs); + sendJson(response, 200, completionPayload({ + requestId, + model: body.model || modelName, + content: replyText, + })); + } + return; + } + + sendJson(response, 404, { + error: { + message: `No fake provider route for ${request.method} ${url.pathname}`, + type: "not_found", + }, + }); + } catch (error) { + sendJson(response, 500, { + error: { + message: error instanceof Error ? error.message : String(error), + type: "fake_provider_error", + }, + }); + } finally { + const durationMs = Date.now() - startedAt; + if (url.pathname !== "/healthz") { + console.log(JSON.stringify({ + at: new Date().toISOString(), + method: request.method, + path: url.pathname, + duration_ms: durationMs, + })); + } + } +}); + +server.listen(port, host, async () => { + const address = server.address(); + const selectedPort = typeof address === "object" && address ? address.port : port; + const url = `http://${host}:${selectedPort}`; + const state = { + status: "ready", + pid: process.pid, + url, + base_url: `${url}/v1`, + model: modelName, + started_at: new Date().toISOString(), + }; + if (stateFile) { + const path = resolve(stateFile); + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, `${JSON.stringify(state, null, 2)}\n`, "utf8"); + } + console.log(JSON.stringify(state)); +}); + +server.on("error", (error) => { + console.error(JSON.stringify({ + status: "error", + reason: error instanceof Error ? error.message : String(error), + })); + exit(1); +}); + +process.on("SIGTERM", () => { + server.close(() => exit(0)); +}); + +function parseArgs(argv) { + const result = {}; + for (const item of argv) { + const match = item.match(/^--([^=]+)(?:=(.*))?$/); + if (!match) continue; + result[match[1]] = match[2] ?? "1"; + } + return result; +} + +function integer(value, fallback) { + const parsed = Number.parseInt(String(value ?? ""), 10); + return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback; +} + +function bool(value, fallback) { + if (value === undefined || value === "") return fallback; + if (/^(1|true|yes|on)$/i.test(String(value))) return true; + if (/^(0|false|no|off)$/i.test(String(value))) return false; + return fallback; +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, Math.max(0, ms))); +} + +async function readJson(request) { + let text = ""; + for await (const chunk of request) text += chunk.toString(); + if (!text) return {}; + return JSON.parse(text); +} + +function sendJson(response, status, payload) { + const text = `${JSON.stringify(payload)}\n`; + response.writeHead(status, { + "content-type": "application/json", + "content-length": Buffer.byteLength(text), + }); + response.end(text); +} + +function completionPayload({ requestId, model, content }) { + const completionTokens = tokenEstimate(content); + return { + id: requestId, + object: "chat.completion", + created: Math.floor(Date.now() / 1000), + model, + choices: [ + { + index: 0, + message: { + role: "assistant", + content, + }, + finish_reason: "stop", + }, + ], + usage: { + prompt_tokens: 8, + completion_tokens: completionTokens, + total_tokens: 8 + completionTokens, + }, + }; +} + +async function streamCompletion(response, { requestId, model, content, failAfterFirstChunk: failMidStream }) { + response.writeHead(200, { + "content-type": "text/event-stream; charset=utf-8", + "cache-control": "no-cache", + "connection": "keep-alive", + }); + + await sleep(firstTokenDelayMs); + writeSse(response, { + id: requestId, + object: "chat.completion.chunk", + created: Math.floor(Date.now() / 1000), + model, + choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }], + }); + + const chunks = splitContent(content); + for (let index = 0; index < chunks.length; index += 1) { + await sleep(chunkDelayMs); + writeSse(response, { + id: requestId, + object: "chat.completion.chunk", + created: Math.floor(Date.now() / 1000), + model, + choices: [{ index: 0, delta: { content: chunks[index] }, finish_reason: null }], + }); + if (failMidStream && index === 0) { + response.destroy(new Error("LangBot fake provider injected mid-stream disconnect")); + return; + } + } + + await sleep(chunkDelayMs); + const completionTokens = tokenEstimate(content); + writeSse(response, { + id: requestId, + object: "chat.completion.chunk", + created: Math.floor(Date.now() / 1000), + model, + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + usage: { + prompt_tokens: 8, + completion_tokens: completionTokens, + total_tokens: 8 + completionTokens, + }, + }); + response.write("data: [DONE]\n\n"); + response.end(); +} + +function writeSse(response, payload) { + response.write(`data: ${JSON.stringify(payload)}\n\n`); +} + +function splitContent(content) { + const text = String(content); + const requested = integer(env.LANGBOT_FAKE_PROVIDER_CHUNK_COUNT, 0); + if (requested <= 1 || text.length <= 1) return [text]; + const chunkSize = Math.max(1, Math.ceil(text.length / requested)); + const chunks = []; + for (let index = 0; index < text.length; index += chunkSize) { + chunks.push(text.slice(index, index + chunkSize)); + } + return chunks; +} + +function tokenEstimate(content) { + return Math.max(1, Math.ceil(String(content || "").length / 4)); +} + +function responseTextForBody(body) { + if (/^(0|false|no|off)$/i.test(env.LANGBOT_FAKE_PROVIDER_DYNAMIC_RESPONSE || "")) { + return responseText; + } + const messages = Array.isArray(body.messages) ? body.messages : []; + const lastUser = [...messages].reverse().find((message) => message?.role === "user"); + const text = flattenContent(lastUser?.content || ""); + const quoted = text.match(/["'“”](.{1,80}?)["'“”]/); + if (quoted?.[1]) return quoted[1].trim(); + const exact = text.match(/(?:reply|回复|输出|return)\s+(?:exactly\s+)?([A-Za-z0-9_.:@-]{1,80})/i); + if (exact?.[1]) return exact[1].trim().replace(/[。.!?]+$/, ""); + const only = text.match(/只回复\s*([A-Za-z0-9_.:@-]{1,80})/); + if (only?.[1]) return only[1].trim().replace(/[。.!?]+$/, ""); + return responseText; +} + +function flattenContent(content) { + if (typeof content === "string") return content; + if (Array.isArray(content)) { + return content + .map((item) => { + if (typeof item === "string") return item; + if (item && typeof item === "object") return item.text || ""; + return ""; + }) + .join("\n"); + } + return ""; +} + +function recordRequest(entry) { + recentRequests.push({ + ...entry, + at: new Date().toISOString(), + }); + while (recentRequests.length > requestLogLimit) recentRequests.shift(); +} diff --git a/skills/skills.index.json b/skills/skills.index.json index 9a2cbd13d..5bc988ed6 100644 --- a/skills/skills.index.json +++ b/skills/skills.index.json @@ -151,11 +151,13 @@ "agent-runner-release-preflight", "agent-runner-runtime-chaos", "dify-agent-debug-chat", + "langbot-fake-provider-debug-chat-load", "langbot-fault-taxonomy-contract", "langbot-live-backend-latency", "langbot-live-backend-log-health", "langbot-live-control-plane-api", "langbot-overhead-accounting-contract", + "langbot-space-debug-chat-concurrency-smoke", "langrag-kb-retrieve", "langrag-parser-golden-e2e", "langrag-sentinel-kb-discover", @@ -493,6 +495,43 @@ "backend_log" ] }, + { + "id": "langbot-fake-provider-debug-chat-load", + "title": "LangBot Debug Chat controlled fake-provider load probe", + "mode": "probe", + "area": "performance", + "type": "performance", + "priority": "p1", + "risk": "medium", + "ci_eligible": false, + "tags": [ + "performance", + "debug-chat", + "websocket", + "fake-provider", + "load", + "metrics" + ], + "automation": "skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs", + "setup_automation": [ + "node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env" + ], + "setup_provides_env": [ + "LANGBOT_FAKE_PROVIDER_URL", + "LANGBOT_FAKE_PROVIDER_BASE_URL", + "LANGBOT_FAKE_PROVIDER_PID", + "LANGBOT_FAKE_PROVIDER_PROVIDER_UUID", + "LANGBOT_FAKE_PROVIDER_MODEL_UUID", + "LANGBOT_FAKE_PROVIDER_PIPELINE_URL", + "LANGBOT_FAKE_PROVIDER_PIPELINE_NAME" + ], + "evidence_required": [ + "metrics", + "network", + "api_diagnostic", + "filesystem" + ] + }, { "id": "langbot-fault-taxonomy-contract", "title": "LangBot fault taxonomy and cleanup contract", @@ -615,6 +654,43 @@ "filesystem" ] }, + { + "id": "langbot-space-debug-chat-concurrency-smoke", + "title": "LangBot Debug Chat real Space-provider concurrency smoke", + "mode": "probe", + "area": "performance", + "type": "performance", + "priority": "p1", + "risk": "high", + "ci_eligible": false, + "tags": [ + "performance", + "debug-chat", + "websocket", + "space", + "live-provider", + "smoke", + "metrics" + ], + "automation": "skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs", + "setup_automation": [ + "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env" + ], + "setup_provides_env": [ + "LANGBOT_PIPELINE_URL", + "LANGBOT_PIPELINE_NAME", + "LANGBOT_LOCAL_AGENT_PIPELINE_URL", + "LANGBOT_LOCAL_AGENT_PIPELINE_NAME", + "LANGBOT_LOCAL_AGENT_MODEL_UUID", + "LANGBOT_E2E_MODEL_UUID" + ], + "evidence_required": [ + "metrics", + "network", + "api_diagnostic", + "filesystem" + ] + }, { "id": "langrag-kb-retrieve", "title": "LangRAG knowledge base ingests and retrieves a sentinel document", @@ -1220,6 +1296,7 @@ "suites": [ "agent-runner-release-gate", "core-smoke", + "langbot-debug-chat-load-gate", "langbot-live-backend-gate", "langbot-performance-contract-gate", "langbot-performance-reliability-gate", @@ -1286,6 +1363,23 @@ "local-agent-basic-debug-chat" ] }, + { + "id": "langbot-debug-chat-load-gate", + "title": "LangBot Debug Chat load gate", + "description": "Message-path load checks for Pipeline Debug Chat: controlled fake-provider baseline plus optional real Space-provider smoke.", + "type": "performance", + "priority": "p1", + "tags": [ + "performance", + "debug-chat", + "websocket", + "load" + ], + "cases": [ + "langbot-fake-provider-debug-chat-load", + "langbot-space-debug-chat-concurrency-smoke" + ] + }, { "id": "langbot-live-backend-gate", "title": "LangBot live backend reliability gate", @@ -1501,6 +1595,7 @@ "sandbox-native-tools-unavailable", "socks-proxy-without-socksio", "survey-widget-blocks-debug-chat", + "telemetry-proxy-noise", "tool-name-collision-between-mcp-and-plugin", "uv-run-resyncs-local-sdk" ], @@ -1685,6 +1780,14 @@ "mcp-stdio-tool-call" ] }, + { + "id": "telemetry-proxy-noise", + "title": "Telemetry posting fails through the proxy while the target flow succeeds", + "category": "env_issue", + "related_cases": [ + "langbot-space-debug-chat-concurrency-smoke" + ] + }, { "id": "tool-name-collision-between-mcp-and-plugin", "title": "MCP and plugin expose the same tool name", diff --git a/skills/skills/.env.example b/skills/skills/.env.example index a8f5ebf09..d236c199a 100644 --- a/skills/skills/.env.example +++ b/skills/skills/.env.example @@ -26,6 +26,20 @@ LANGBOT_NO_PROXY=localhost,127.0.0.1,::1 LANGBOT_PIPELINE_URL= LANGBOT_PIPELINE_NAME= +# Optional fake OpenAI-compatible provider controls for Debug Chat load tests. +# Leave URL empty to let setup automation start a local provider and write the +# selected URL to skills/.env.local. +LANGBOT_FAKE_PROVIDER_URL= +LANGBOT_FAKE_PROVIDER_HOST=127.0.0.1 +LANGBOT_FAKE_PROVIDER_PORT= +LANGBOT_FAKE_PROVIDER_MODEL_NAME=gpt-4o-mini +LANGBOT_FAKE_PROVIDER_RESPONSE_TEXT=OK +LANGBOT_FAKE_PROVIDER_FIRST_TOKEN_DELAY_MS=25 +LANGBOT_FAKE_PROVIDER_CHUNK_DELAY_MS=10 +LANGBOT_FAKE_PROVIDER_FAIL_FIRST_N=0 +LANGBOT_FAKE_PROVIDER_FAIL_EVERY_N=0 +LANGBOT_FAKE_PROVIDER_FAULT_STATUS=500 + # Optional case-specific runner targets. Prefer these for runner-specific cases # so the automation cannot silently test the wrong runner. LANGBOT_LOCAL_AGENT_PIPELINE_URL= diff --git a/skills/skills/langbot-testing/cases/langbot-fake-provider-debug-chat-load.yaml b/skills/skills/langbot-testing/cases/langbot-fake-provider-debug-chat-load.yaml new file mode 100644 index 000000000..8a71c3558 --- /dev/null +++ b/skills/skills/langbot-testing/cases/langbot-fake-provider-debug-chat-load.yaml @@ -0,0 +1,81 @@ +id: langbot-fake-provider-debug-chat-load +title: "LangBot Debug Chat controlled fake-provider load probe" +mode: probe +area: performance +type: performance +priority: p1 +risk: medium +ci_eligible: false +tags: + - performance + - debug-chat + - websocket + - fake-provider + - load + - metrics +skills: + - langbot-env-setup + - langbot-testing +env: + - LANGBOT_BACKEND_URL + - LANGBOT_FRONTEND_URL + - LANGBOT_E2E_LOGIN_USER +automation: skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs +automation_env: + - LANGBOT_BACKEND_URL + - LANGBOT_E2E_LOGIN_USER + - LANGBOT_FAKE_PROVIDER_PIPELINE_URL + - LANGBOT_FAKE_PROVIDER_PIPELINE_NAME +automation_pipeline_url_env: LANGBOT_FAKE_PROVIDER_PIPELINE_URL +automation_pipeline_name_env: LANGBOT_FAKE_PROVIDER_PIPELINE_NAME +automation_debug_chat_load_requests: "12" +automation_debug_chat_load_concurrency: "4" +automation_debug_chat_load_timeout_ms: "30000" +automation_debug_chat_load_response_p95_ms: "5000" +automation_debug_chat_load_first_response_p95_ms: "3000" +automation_debug_chat_load_max_error_rate: "0" +automation_debug_chat_load_expected_prefix: "FAKEQA" +automation_debug_chat_load_prompt_template: '请只回复 "{expected}",不要解释,不要添加其他字符。' +automation_debug_chat_load_stream: "true" +automation_debug_chat_load_reset: "true" +metrics_thresholds_json: '{"response_p95_ms":{"max":5000},"first_response_p95_ms":{"max":3000},"error_rate":{"max":0}}' +load_profile_json: '{"requests":12,"concurrency":4,"path":"Pipeline Debug Chat WebSocket","provider":"controlled fake OpenAI-compatible provider","metric":"send-to-final-assistant-response"}' +setup_automation: + - "node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env" +setup_provides_env: + - LANGBOT_FAKE_PROVIDER_URL + - LANGBOT_FAKE_PROVIDER_BASE_URL + - LANGBOT_FAKE_PROVIDER_PID + - LANGBOT_FAKE_PROVIDER_PROVIDER_UUID + - LANGBOT_FAKE_PROVIDER_MODEL_UUID + - LANGBOT_FAKE_PROVIDER_PIPELINE_URL + - LANGBOT_FAKE_PROVIDER_PIPELINE_NAME +steps: + - "Start or reuse the local fake OpenAI-compatible provider." + - "Create or update the LangBot provider, model, and local-agent pipeline that points at the fake provider." + - "Reset the target Debug Chat session." + - "Open concurrent WebSocket Debug Chat connections and send unique deterministic prompts through the real backend pipeline." +checks: + - "automation-result.json status is pass when every request receives its own expected assistant response." + - "metrics_summary includes request count, concurrency, p50/p95 response latency, first response latency, throughput, and error rate." + - "thresholds_summary shows response_p95_ms, first_response_p95_ms, and error_rate pass." +evidence_required: + - metrics + - network + - api_diagnostic + - filesystem +diagnostics: + - "This probe removes external model latency from the measurement; it still exercises the live LangBot backend, provider requester, local-agent runner, pipeline, and Debug Chat WebSocket adapter." + - "Use this as the repeatable message-path baseline before comparing against Space or another real provider." +success_patterns: + - "Debug Chat WebSocket concurrency probe passed" + - "Streaming completed" +failure_patterns: + - "WebSocket connection error" + - "Timed out after" + - "Final assistant response did not include" + - "All models failed during streaming setup" +troubleshooting: + - backend-not-listening + - debug-chat-history-contaminates-automation + - local-agent-model-route-unavailable diff --git a/skills/skills/langbot-testing/cases/langbot-space-debug-chat-concurrency-smoke.yaml b/skills/skills/langbot-testing/cases/langbot-space-debug-chat-concurrency-smoke.yaml new file mode 100644 index 000000000..4f9fc779b --- /dev/null +++ b/skills/skills/langbot-testing/cases/langbot-space-debug-chat-concurrency-smoke.yaml @@ -0,0 +1,84 @@ +id: langbot-space-debug-chat-concurrency-smoke +title: "LangBot Debug Chat real Space-provider concurrency smoke" +mode: probe +area: performance +type: performance +priority: p1 +risk: high +ci_eligible: false +tags: + - performance + - debug-chat + - websocket + - space + - live-provider + - smoke + - metrics +skills: + - langbot-env-setup + - langbot-testing +env: + - LANGBOT_BACKEND_URL + - LANGBOT_FRONTEND_URL + - LANGBOT_E2E_LOGIN_USER +automation: skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs +automation_env: + - LANGBOT_BACKEND_URL + - LANGBOT_E2E_LOGIN_USER + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME +automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL +automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME +automation_debug_chat_load_requests: "3" +automation_debug_chat_load_concurrency: "2" +automation_debug_chat_load_timeout_ms: "120000" +automation_debug_chat_load_response_p95_ms: "120000" +automation_debug_chat_load_max_error_rate: "0" +automation_debug_chat_load_expected_prefix: "SPACEQA" +automation_debug_chat_load_prompt_template: '请只回复 "{expected}",不要解释,不要添加其他字符。' +automation_debug_chat_load_stream: "true" +automation_debug_chat_load_reset: "true" +metrics_thresholds_json: '{"response_p95_ms":{"max":120000},"error_rate":{"max":0}}' +load_profile_json: '{"requests":3,"concurrency":2,"path":"Pipeline Debug Chat WebSocket","provider":"LangBot Space model route","metric":"send-to-final-assistant-response","classification":"smoke-not-benchmark"}' +setup_automation: + - "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env" +setup_provides_env: + - LANGBOT_PIPELINE_URL + - LANGBOT_PIPELINE_NAME + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME + - LANGBOT_LOCAL_AGENT_MODEL_UUID + - LANGBOT_E2E_MODEL_UUID +preconditions: + - "The selected local LangBot instance is safe for a low-volume real Space model smoke run." + - "Treat Space/provider/network failures as environment or dependency findings until fake-provider baseline evidence separates LangBot overhead." +steps: + - "Prepare a local-agent pipeline with a tested Space model and fallback models." + - "Reset the target Debug Chat session." + - "Open a small number of concurrent WebSocket Debug Chat connections and send unique deterministic prompts through the live Space provider path." +checks: + - "automation-result.json status is pass when every request receives its own expected assistant response." + - "metrics_summary includes request count, concurrency, p95 response latency, throughput, and error rate." + - "The report classifies the result as a live-provider smoke, not a stable LangBot overhead benchmark." +evidence_required: + - metrics + - network + - api_diagnostic + - filesystem +diagnostics: + - "This probe measures real user-path latency through Space and includes provider latency, model behavior, and network effects." + - "Compare with langbot-fake-provider-debug-chat-load before attributing slow or failed runs to LangBot itself." +success_patterns: + - "Debug Chat WebSocket concurrency probe passed" + - "Streaming completed" +failure_patterns: + - "invalid api key" + - "WebSocket connection error" + - "Timed out after" + - "Final assistant response did not include" + - "All models failed during streaming setup" +troubleshooting: + - local-agent-model-route-unavailable + - marketplace-network-flaky + - proxy-env-mismatch + - telemetry-proxy-noise diff --git a/skills/skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs b/skills/skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs new file mode 100644 index 000000000..af40698a1 --- /dev/null +++ b/skills/skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs @@ -0,0 +1,664 @@ +#!/usr/bin/env node + +import crypto from "node:crypto"; +import net from "node:net"; +import tls from "node:tls"; +import { mkdir, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { env, exit } from "node:process"; +import { + apiJson, + appendLine, + ensureEvidence, + evidencePaths, + loadEnvFiles, + localIsoWithOffset, + redact, + resetAndAuthLocalUser, + writeResult, +} from "../../../scripts/e2e/lib/langbot-e2e.mjs"; + +const DEFAULT_LOCAL_PASSWORD = "LangBotE2ELocalPass!2026"; + +await loadEnvFiles(); +const caseId = env.LBS_CASE_ID || "langbot-debug-chat-concurrency"; +const paths = evidencePaths(caseId); +await ensureEvidence(paths); + +const startedAt = new Date(); +const metricsPath = resolve(paths.evidenceDir, "metrics.json"); +const samplesPath = resolve(paths.evidenceDir, "samples.json"); +const resetDiagnosticPath = resolve(paths.evidenceDir, "debug-chat-reset-diagnostic.json"); +const backendUrl = env.LANGBOT_BACKEND_URL || ""; +const pipelineUrl = env.LANGBOT_E2E_PIPELINE_URL || env.LANGBOT_PIPELINE_URL || ""; +const pipelineName = env.LANGBOT_E2E_PIPELINE_NAME || env.LANGBOT_PIPELINE_NAME || ""; +const sessionType = env.LANGBOT_DEBUG_CHAT_LOAD_SESSION_TYPE || env.LANGBOT_E2E_DEBUG_CHAT_SESSION_TYPE || "person"; +const totalRequests = positiveInteger(env.LANGBOT_DEBUG_CHAT_LOAD_REQUESTS, defaultRequests(caseId)); +const concurrency = Math.min(totalRequests, positiveInteger(env.LANGBOT_DEBUG_CHAT_LOAD_CONCURRENCY, defaultConcurrency(caseId))); +const timeoutMs = positiveInteger(env.LANGBOT_DEBUG_CHAT_LOAD_TIMEOUT_MS, defaultTimeout(caseId)); +const expectedPrefix = env.LANGBOT_DEBUG_CHAT_LOAD_EXPECTED_PREFIX || "LBQA"; +const promptTemplate = env.LANGBOT_DEBUG_CHAT_LOAD_PROMPT_TEMPLATE + || "请只回复 \"{expected}\",不要解释,不要添加其他字符。"; +const stream = bool(env.LANGBOT_DEBUG_CHAT_LOAD_STREAM, true); +const resetBeforeRun = bool(env.LANGBOT_DEBUG_CHAT_LOAD_RESET, true); +const responseP95BudgetMs = positiveNumber(env.LANGBOT_DEBUG_CHAT_LOAD_RESPONSE_P95_MS, defaultP95Budget(caseId)); +const firstResponseP95BudgetMs = positiveNumber(env.LANGBOT_DEBUG_CHAT_LOAD_FIRST_RESPONSE_P95_MS, 0); +const maxErrorRate = positiveNumber(env.LANGBOT_DEBUG_CHAT_LOAD_MAX_ERROR_RATE, 0); + +const result = { + source: "automation", + case_id: caseId, + run_id: paths.runId, + status: "fail", + reason: "", + started_at: startedAt.toISOString(), + started_at_local: localIsoWithOffset(startedAt), + finished_at: "", + finished_at_local: "", + duration_ms: 0, + backend_url: backendUrl, + pipeline_url: pipelineUrl, + pipeline_name: pipelineName, + pipeline_id: "", + session_type: sessionType, + load_profile: { + requests: totalRequests, + concurrency, + timeout_ms: timeoutMs, + stream, + reset_before_run: resetBeforeRun, + }, + evidence: { + network_log: paths.networkLog, + metrics_json: metricsPath, + samples_json: samplesPath, + debug_chat_reset_diagnostic_json: resetDiagnosticPath, + automation_result_json: paths.automationResultJson, + result_json: paths.resultJson, + }, + evidence_collected: ["metrics", "network", "api_diagnostic", "filesystem"], +}; + +try { + if (!backendUrl) { + result.status = "env_issue"; + throw new Error("LANGBOT_BACKEND_URL is not configured."); + } + if (!["person", "group"].includes(sessionType)) { + throw new Error(`LANGBOT_DEBUG_CHAT_LOAD_SESSION_TYPE must be person or group, got ${sessionType}.`); + } + const backendReady = await backendReachable(backendUrl); + if (!backendReady) { + result.status = "env_issue"; + throw new Error(`Backend did not respond at ${backendUrl}.`); + } + + const user = env.LANGBOT_E2E_LOGIN_USER || ""; + const password = env.LANGBOT_E2E_LOGIN_PASSWORD || DEFAULT_LOCAL_PASSWORD; + if (!user) { + result.status = "env_issue"; + throw new Error("LANGBOT_E2E_LOGIN_USER is required so this probe can resolve/reset the Debug Chat session."); + } + const auth = await resetAndAuthLocalUser({ backendUrl, user, password }); + + const pipeline = await resolvePipeline({ backendUrl, token: auth.token, pipelineUrl, pipelineName }); + result.pipeline_id = pipeline.id; + result.pipeline_name = pipeline.name || pipelineName; + if (!result.pipeline_url && env.LANGBOT_FRONTEND_URL) { + result.pipeline_url = `${env.LANGBOT_FRONTEND_URL.replace(/\/$/, "")}/home/pipelines?id=${encodeURIComponent(pipeline.id)}`; + } + + if (resetBeforeRun) { + const reset = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.id)}/ws/reset/${encodeURIComponent(sessionType)}`, { + method: "POST", + token: auth.token, + }); + const resetDiagnostic = { + status: isApiFailure(reset) ? "fail" : "ready", + http_status: reset.status, + code: reset.json.code ?? null, + reason: isApiFailure(reset) ? reset.json.msg || "Debug Chat reset failed." : "Debug Chat session reset.", + }; + await writeFile(resetDiagnosticPath, `${JSON.stringify(resetDiagnostic, null, 2)}\n`, "utf8"); + if (resetDiagnostic.status === "fail") { + throw new Error(resetDiagnostic.reason); + } + } + + const wsUrl = websocketUrl(backendUrl, pipeline.id, sessionType); + const loadStartedAt = performance.now(); + const samples = await runLoad({ + wsUrl, + totalRequests, + concurrency, + timeoutMs, + promptTemplate, + expectedPrefix, + stream, + }); + const loadDurationMs = performance.now() - loadStartedAt; + const metrics = buildMetrics({ + samples, + totalRequests, + concurrency, + timeoutMs, + loadDurationMs, + backendUrl, + pipelineId: pipeline.id, + sessionType, + }); + const thresholds = buildThresholds(metrics); + const passed = Object.values(thresholds).every((item) => item.pass); + result.status = passed ? "pass" : "fail"; + result.reason = passed + ? "Debug Chat WebSocket concurrency probe passed all thresholds." + : "Debug Chat WebSocket concurrency probe breached latency or error-rate thresholds."; + result.metrics_summary = { + requests: metrics.total_requests, + concurrency: metrics.concurrency, + ok_count: metrics.ok_count, + error_count: metrics.error_count, + timeout_count: metrics.timeout_count, + error_rate: metrics.error_rate, + response_p50_ms: metrics.response_duration_ms.p50, + response_p95_ms: metrics.response_duration_ms.p95, + first_response_p95_ms: metrics.first_response_ms.p95, + throughput_rps: metrics.throughput_rps, + status_counts: metrics.status_counts, + }; + result.thresholds_summary = thresholds; + result.artifacts = { + metrics_json: metricsPath, + samples_json: samplesPath, + network_log: paths.networkLog, + automation_result_json: paths.automationResultJson, + result_json: paths.resultJson, + }; + + await writeFile(metricsPath, `${JSON.stringify({ ...metrics, thresholds }, null, 2)}\n`, "utf8"); + await writeFile(samplesPath, `${JSON.stringify(samples, null, 2)}\n`, "utf8"); +} catch (error) { + if (!["env_issue", "blocked"].includes(result.status)) { + result.status = looksLikeEnvIssue(error) ? "env_issue" : "fail"; + } + result.reason = result.reason || safeReason(error.message); +} finally { + const finishedAt = new Date(); + result.finished_at = finishedAt.toISOString(); + result.finished_at_local = localIsoWithOffset(finishedAt); + result.duration_ms = finishedAt.getTime() - startedAt.getTime(); + await mkdir(paths.evidenceDir, { recursive: true }); + await writeResult(paths, result); + console.log(JSON.stringify(result, null, 2)); +} + +exit(result.status === "pass" ? 0 : result.status === "env_issue" || result.status === "blocked" ? 2 : 1); + +function defaultRequests(id) { + return id.includes("space") ? 3 : 12; +} + +function defaultConcurrency(id) { + return id.includes("space") ? 1 : 4; +} + +function defaultTimeout(id) { + return id.includes("space") ? 120_000 : 30_000; +} + +function defaultP95Budget(id) { + return id.includes("space") ? 120_000 : 5_000; +} + +function positiveInteger(value, fallback) { + const parsed = Number.parseInt(String(value || ""), 10); + return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; +} + +function positiveNumber(value, fallback) { + const parsed = Number(value || ""); + return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback; +} + +function bool(value, fallback) { + if (value === undefined || value === "") return fallback; + if (/^(1|true|yes|on)$/i.test(String(value))) return true; + if (/^(0|false|no|off)$/i.test(String(value))) return false; + return fallback; +} + +async function backendReachable(baseUrl) { + try { + const response = await fetch(`${baseUrl.replace(/\/$/, "")}/healthz`, { + signal: AbortSignal.timeout(3000), + }); + return response.status < 500; + } catch { + return false; + } +} + +function pipelineIdFromUrl(url) { + if (!url) return ""; + try { + const parsed = new URL(url); + return parsed.searchParams.get("id") || ""; + } catch { + return ""; + } +} + +async function resolvePipeline({ backendUrl, token, pipelineUrl, pipelineName }) { + const idFromUrl = pipelineIdFromUrl(pipelineUrl); + if (idFromUrl) { + const response = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(idFromUrl)}`, { token }); + const pipeline = response.json.data?.pipeline; + if (isApiFailure(response) || !pipeline?.uuid) { + throw new Error(response.json.msg || `Could not load pipeline ${idFromUrl}.`); + } + return { id: pipeline.uuid, name: pipeline.name || "" }; + } + if (!pipelineName) { + throw new Error("Set LANGBOT_E2E_PIPELINE_URL or LANGBOT_E2E_PIPELINE_NAME before running this probe."); + } + const response = await apiJson(backendUrl, "/api/v1/pipelines", { token }); + if (isApiFailure(response)) { + throw new Error(response.json.msg || "Failed to list pipelines."); + } + const pipeline = (response.json.data?.pipelines || []).find((item) => item.name === pipelineName); + if (!pipeline?.uuid) { + throw new Error(`Could not find pipeline named ${pipelineName}.`); + } + return { id: pipeline.uuid, name: pipeline.name || pipelineName }; +} + +function isApiFailure(response) { + return response.status >= 400 || (response.json.code !== undefined && response.json.code !== 0); +} + +function websocketUrl(baseUrl, pipelineId, sessionType) { + const parsed = new URL(baseUrl); + parsed.protocol = parsed.protocol === "https:" ? "wss:" : "ws:"; + parsed.pathname = `/api/v1/pipelines/${encodeURIComponent(pipelineId)}/ws/connect`; + parsed.search = `?session_type=${encodeURIComponent(sessionType)}`; + return parsed.toString(); +} + +async function runLoad(options) { + const samples = []; + let nextIndex = 0; + const workers = Array.from({ length: options.concurrency }, async () => { + while (nextIndex < options.totalRequests) { + const index = nextIndex; + nextIndex += 1; + const sample = await runSingleRequest({ ...options, index }); + samples.push(sample); + } + }); + await Promise.all(workers); + return samples.sort((left, right) => left.index - right.index); +} + +function expectedForIndex(prefix, index) { + return `${prefix}-${String(index + 1).padStart(4, "0")}`; +} + +function promptForIndex(template, expected) { + return template.replaceAll("{expected}", expected); +} + +function runSingleRequest({ + wsUrl, + index, + timeoutMs, + promptTemplate, + expectedPrefix, + stream, +}) { + return new Promise((resolve) => { + const expected = expectedForIndex(expectedPrefix, index); + const prompt = promptForIndex(promptTemplate, expected); + const sample = { + index, + status: "running", + ok: false, + expected_text: expected, + prompt, + response_text: "", + connected_ms: null, + first_response_ms: null, + response_duration_ms: null, + event_count: 0, + foreign_response_count: 0, + last_foreign_response_text: "", + error: "", + close_code: null, + close_reason: "", + }; + let closed = false; + let connectedAt = 0; + let sentAt = 0; + const startedAt = performance.now(); + let client = null; + const timer = setTimeout(() => { + finish("timeout", `Timed out after ${timeoutMs} ms.`); + }, timeoutMs); + + client = openRawWebSocket(wsUrl, { + onOpen() { + connectedAt = performance.now(); + sample.connected_ms = rounded(connectedAt - startedAt); + }, + onMessage(text) { + sample.event_count += 1; + let data; + try { + data = JSON.parse(String(text || "")); + } catch (error) { + finish("error", `Invalid WebSocket JSON: ${error.message}`); + return; + } + appendLine(paths.networkLog, JSON.stringify({ + request_index: index, + type: data.type, + session_type: data.session_type || "", + role: data.data?.role || "", + is_final: data.data?.is_final ?? null, + content_preview: redact(String(data.data?.content || data.message || "").slice(0, 200)), + })).catch(() => {}); + + if (data.type === "connected") { + sentAt = performance.now(); + client.send(JSON.stringify({ + type: "message", + message: [{ type: "Plain", text: prompt }], + stream, + })); + return; + } + if (data.type === "error") { + finish("error", data.message || "WebSocket error message."); + return; + } + if (data.type !== "response" || data.data?.role !== "assistant") return; + + const content = String(data.data.content || ""); + if (content) sample.response_text = content; + if (data.data.is_final === true) { + const ok = sample.response_text.includes(expected); + if (ok) { + if (sample.first_response_ms === null && sentAt > 0) { + sample.first_response_ms = rounded(performance.now() - sentAt); + } + finish("pass", ""); + } else { + sample.foreign_response_count += 1; + sample.last_foreign_response_text = sample.response_text; + } + } + }, + onError(error) { + finish("connection_error", `WebSocket connection error: ${error.message}`); + }, + onClose(event) { + sample.close_code = event.code; + sample.close_reason = event.reason || ""; + if (!closed) finish("closed", `WebSocket closed before final assistant response: ${event.code}`); + }, + }); + + function finish(status, reason) { + if (closed) return; + closed = true; + clearTimeout(timer); + sample.status = status; + sample.ok = status === "pass"; + sample.error = status === "timeout" && sample.foreign_response_count > 0 + ? `${reason || ""} Saw ${sample.foreign_response_count} foreign assistant response(s); last=${sample.last_foreign_response_text}` + : reason || ""; + if (sentAt > 0) sample.response_duration_ms = rounded(performance.now() - sentAt); + else sample.response_duration_ms = rounded(performance.now() - startedAt); + try { + client?.close(); + } catch { + // Closing a failed socket should not hide the sample result. + } + resolve(sample); + } + }); +} + +function openRawWebSocket(wsUrl, handlers) { + const parsed = new URL(wsUrl); + const secure = parsed.protocol === "wss:"; + const port = Number(parsed.port || (secure ? 443 : 80)); + const host = parsed.hostname; + const path = `${parsed.pathname}${parsed.search}`; + const key = crypto.randomBytes(16).toString("base64"); + const socket = secure + ? tls.connect({ host, port, servername: host }) + : net.connect({ host, port }); + let opened = false; + let closed = false; + let buffer = Buffer.alloc(0); + + socket.setNoDelay(true); + socket.on("connect", () => { + const originProtocol = secure ? "https" : "http"; + const request = [ + `GET ${path} HTTP/1.1`, + `Host: ${parsed.host}`, + "Upgrade: websocket", + "Connection: Upgrade", + `Sec-WebSocket-Key: ${key}`, + "Sec-WebSocket-Version: 13", + `Origin: ${originProtocol}://${parsed.host}`, + "", + "", + ].join("\r\n"); + socket.write(request); + }); + socket.on("data", (chunk) => { + buffer = Buffer.concat([buffer, chunk]); + if (!opened) { + const headerEnd = buffer.indexOf("\r\n\r\n"); + if (headerEnd === -1) return; + const headerText = buffer.slice(0, headerEnd).toString("utf8"); + buffer = buffer.slice(headerEnd + 4); + if (!/^HTTP\/1\.1 101\b/i.test(headerText)) { + handlers.onError(new Error(`Handshake failed: ${headerText.split("\r\n")[0] || "missing status"}`)); + socket.destroy(); + return; + } + opened = true; + handlers.onOpen(); + } + processFrames(); + }); + socket.on("error", (error) => { + if (!closed) handlers.onError(error); + }); + socket.on("close", () => { + if (closed) return; + closed = true; + handlers.onClose({ code: null, reason: "" }); + }); + + function processFrames() { + while (true) { + const frame = readFrame(buffer); + if (!frame) return; + buffer = buffer.slice(frame.consumed); + if (frame.opcode === 0x1) { + handlers.onMessage(frame.payload.toString("utf8")); + } else if (frame.opcode === 0x8) { + const code = frame.payload.length >= 2 ? frame.payload.readUInt16BE(0) : null; + const reason = frame.payload.length > 2 ? frame.payload.slice(2).toString("utf8") : ""; + closed = true; + handlers.onClose({ code, reason }); + socket.end(); + return; + } else if (frame.opcode === 0x9) { + writeFrame(socket, 0xA, frame.payload); + } + } + } + + return { + send(text) { + if (closed || !opened) return; + writeFrame(socket, 0x1, Buffer.from(text, "utf8")); + }, + close() { + if (closed) return; + closed = true; + if (!socket.destroyed) { + if (opened) writeFrame(socket, 0x8, Buffer.alloc(0)); + setTimeout(() => socket.end(), 50).unref(); + } + }, + }; +} + +function readFrame(buffer) { + if (buffer.length < 2) return null; + const first = buffer[0]; + const second = buffer[1]; + const opcode = first & 0x0f; + const masked = Boolean(second & 0x80); + let length = second & 0x7f; + let offset = 2; + if (length === 126) { + if (buffer.length < offset + 2) return null; + length = buffer.readUInt16BE(offset); + offset += 2; + } else if (length === 127) { + if (buffer.length < offset + 8) return null; + const high = buffer.readUInt32BE(offset); + const low = buffer.readUInt32BE(offset + 4); + length = high * 2 ** 32 + low; + offset += 8; + } + let mask = null; + if (masked) { + if (buffer.length < offset + 4) return null; + mask = buffer.slice(offset, offset + 4); + offset += 4; + } + if (buffer.length < offset + length) return null; + let payload = buffer.slice(offset, offset + length); + if (mask) { + payload = Buffer.from(payload); + for (let index = 0; index < payload.length; index += 1) { + payload[index] ^= mask[index % 4]; + } + } + return { + opcode, + payload, + consumed: offset + length, + }; +} + +function writeFrame(socket, opcode, payload) { + const body = Buffer.isBuffer(payload) ? payload : Buffer.from(payload || ""); + const mask = crypto.randomBytes(4); + const headerLength = body.length < 126 ? 2 : body.length <= 0xffff ? 4 : 10; + const header = Buffer.alloc(headerLength); + header[0] = 0x80 | opcode; + if (body.length < 126) { + header[1] = 0x80 | body.length; + } else if (body.length <= 0xffff) { + header[1] = 0x80 | 126; + header.writeUInt16BE(body.length, 2); + } else { + header[1] = 0x80 | 127; + header.writeUInt32BE(Math.floor(body.length / 2 ** 32), 2); + header.writeUInt32BE(body.length >>> 0, 6); + } + const masked = Buffer.from(body); + for (let index = 0; index < masked.length; index += 1) { + masked[index] ^= mask[index % 4]; + } + socket.write(Buffer.concat([header, mask, masked])); +} + +function rounded(value) { + return Number(value.toFixed(3)); +} + +function percentile(values, percentileValue) { + if (values.length === 0) return 0; + const sorted = [...values].sort((a, b) => a - b); + const index = Math.min(sorted.length - 1, Math.ceil((percentileValue / 100) * sorted.length) - 1); + return rounded(sorted[index]); +} + +function stats(values) { + if (values.length === 0) return { min: 0, p50: 0, p95: 0, p99: 0, max: 0 }; + return { + min: rounded(Math.min(...values)), + p50: percentile(values, 50), + p95: percentile(values, 95), + p99: percentile(values, 99), + max: rounded(Math.max(...values)), + }; +} + +function buildMetrics({ samples, totalRequests, concurrency, timeoutMs, loadDurationMs, backendUrl, pipelineId, sessionType }) { + const okSamples = samples.filter((sample) => sample.ok); + const statusCounts = {}; + for (const sample of samples) { + statusCounts[sample.status] = (statusCounts[sample.status] || 0) + 1; + } + const errorCount = samples.length - okSamples.length; + return { + probe: caseId, + backend_url: backendUrl, + pipeline_id: pipelineId, + session_type: sessionType, + total_requests: totalRequests, + completed_requests: samples.length, + concurrency, + timeout_ms: timeoutMs, + ok_count: okSamples.length, + error_count: errorCount, + timeout_count: samples.filter((sample) => sample.status === "timeout").length, + error_rate: samples.length === 0 ? 1 : rounded(errorCount / samples.length), + load_duration_ms: rounded(loadDurationMs), + throughput_rps: loadDurationMs <= 0 ? 0 : rounded(okSamples.length / (loadDurationMs / 1000)), + status_counts: statusCounts, + connected_ms: stats(samples.map((sample) => sample.connected_ms).filter(Number.isFinite)), + first_response_ms: stats(okSamples.map((sample) => sample.first_response_ms).filter(Number.isFinite)), + response_duration_ms: stats(okSamples.map((sample) => sample.response_duration_ms).filter(Number.isFinite)), + samples, + }; +} + +function buildThresholds(metrics) { + const thresholds = { + error_rate: { actual: metrics.error_rate, max: maxErrorRate, pass: metrics.error_rate <= maxErrorRate }, + response_p95_ms: { + actual: metrics.response_duration_ms.p95, + max: responseP95BudgetMs, + pass: metrics.ok_count > 0 && metrics.response_duration_ms.p95 <= responseP95BudgetMs, + }, + }; + if (firstResponseP95BudgetMs > 0) { + thresholds.first_response_p95_ms = { + actual: metrics.first_response_ms.p95, + max: firstResponseP95BudgetMs, + pass: metrics.ok_count > 0 && metrics.first_response_ms.p95 <= firstResponseP95BudgetMs, + }; + } + return thresholds; +} + +function looksLikeEnvIssue(error) { + const message = String(error?.message || error || ""); + return /fetch failed|ECONNREFUSED|ENOTFOUND|LANGBOT_.*not configured|Could not read recovery_key|Backend did not respond/i.test(message); +} + +function safeReason(value) { + return redact(String(value || "")).slice(0, 1000); +} diff --git a/skills/skills/langbot-testing/references/performance-reliability-testing.md b/skills/skills/langbot-testing/references/performance-reliability-testing.md index db325318f..357cf0a74 100644 --- a/skills/skills/langbot-testing/references/performance-reliability-testing.md +++ b/skills/skills/langbot-testing/references/performance-reliability-testing.md @@ -121,6 +121,45 @@ Do not treat these starter live probes as Debug Chat or model-provider performance. They are control-plane readiness checks; user-facing performance needs browser/WebSocket/message-path measurements. +## Debug Chat Load And Fake Provider Baseline + +Use `langbot-fake-provider-debug-chat-load` before real-provider load checks. +The setup automation starts a local OpenAI-compatible fake provider, registers +it as a normal LangBot provider/model, configures a local-agent pipeline, resets +Debug Chat, and then drives concurrent WebSocket messages through the live +backend. + +This is not a mocked backend test. It still exercises: + +- provider/model persistence and runtime reload; +- LiteLLM OpenAI-compatible requester path; +- local-agent runner selection and pipeline execution; +- Debug Chat WebSocket adapter and broadcast behavior; +- backend concurrency, timeout, and error-rate accounting. + +The fake provider is deterministic and can inject controlled latency or faults +with `LANGBOT_FAKE_PROVIDER_*` variables, so it is the baseline for LangBot +message-path overhead. The probe uses unique expected response tokens per +request because Debug Chat broadcasts messages to every connection in the same +session; unique tokens prevent one connection from counting another +connection's response as its own. + +Use `langbot-space-debug-chat-concurrency-smoke` after the fake-provider +baseline. It runs a deliberately small real Space-provider batch and reports +user-visible latency, not pure LangBot overhead. Space/model/network failures +are dependency findings until the fake baseline shows the same symptom. +If a Space smoke passes but log guard finds telemetry posting Tracebacks, +classify that separately as `telemetry-proxy-noise` instead of clearing the +proxy or treating the Debug Chat path as failed. + +Useful commands: + +```bash +rtk bin/lbs test run langbot-fake-provider-debug-chat-load --run-id langbot-fake-load-local +rtk bin/lbs test run langbot-space-debug-chat-concurrency-smoke --run-id langbot-space-smoke-local +rtk bin/lbs suite run langbot-debug-chat-load-gate --run-id langbot-debug-chat-load-local --include-manual-check +``` + ## Gate Layers Use the smallest gate that answers the quality question: @@ -134,6 +173,9 @@ Use the smallest gate that answers the quality question: - `langbot-user-path-performance-gate`: browser-visible user path performance, starting with Pipeline Debug Chat send-to-visible-completion latency. Run it only when the browser profile and target pipeline are ready. +- `langbot-debug-chat-load-gate`: WebSocket Debug Chat load checks, starting + with a controlled fake-provider baseline and optionally a low-volume real + Space-provider smoke. - `langbot-performance-reliability-gate`: combined starter gate for synthetic contracts plus live backend checks. diff --git a/skills/skills/langbot-testing/suites/langbot-debug-chat-load-gate.yaml b/skills/skills/langbot-testing/suites/langbot-debug-chat-load-gate.yaml new file mode 100644 index 000000000..730e87583 --- /dev/null +++ b/skills/skills/langbot-testing/suites/langbot-debug-chat-load-gate.yaml @@ -0,0 +1,13 @@ +id: langbot-debug-chat-load-gate +title: "LangBot Debug Chat load gate" +description: "Message-path load checks for Pipeline Debug Chat: controlled fake-provider baseline plus optional real Space-provider smoke." +type: performance +priority: p1 +tags: + - performance + - debug-chat + - websocket + - load +cases: + - langbot-fake-provider-debug-chat-load + - langbot-space-debug-chat-concurrency-smoke diff --git a/skills/skills/langbot-testing/troubleshooting/telemetry-proxy-noise.yaml b/skills/skills/langbot-testing/troubleshooting/telemetry-proxy-noise.yaml new file mode 100644 index 000000000..945109029 --- /dev/null +++ b/skills/skills/langbot-testing/troubleshooting/telemetry-proxy-noise.yaml @@ -0,0 +1,23 @@ +id: telemetry-proxy-noise +title: "Telemetry posting fails through the proxy while the target flow succeeds" +date: 2026-06-25 +category: env_issue +symptoms: + - "The target Debug Chat or provider smoke request completes successfully." + - "The same log window contains a Traceback for telemetry posting." + - "The traceback references the Space telemetry endpoint." +patterns: + - "Failed to post telemetry" + - "https://space.langbot.app/api/v1/telemetry" + - "httpx.ConnectError" +likely_causes: + - "The backend process inherited proxy settings that are required for model/provider access but unreliable for telemetry posting." + - "The telemetry endpoint is temporarily unreachable through the local proxy route." + - "TLS or proxy negotiation failed for the non-critical telemetry request." +fix_steps: + - "Keep the proxy configuration needed for model/provider access; do not clear it only to hide telemetry noise." + - "Check that uppercase and lowercase proxy variables are consistent before rerunning a live Space smoke." + - "Classify the target flow and log-health result separately: a successful Debug Chat run can still have an environment log-health finding." +verification: "A rerun shows the target case success patterns and no telemetry Traceback in the scanned log window, or the report explicitly records the telemetry issue as environment noise." +related_cases: + - langbot-space-debug-chat-concurrency-smoke diff --git a/skills/src/commands/validate.ts b/skills/src/commands/validate.ts index 1c0ef945d..9176b9f18 100644 --- a/skills/src/commands/validate.ts +++ b/skills/src/commands/validate.ts @@ -186,10 +186,32 @@ function validateCaseItem(root: string, item: StructuredItem, skillNames: Set