diff --git a/skills/scripts/e2e/ensure-langrag-sentinel-kb.mjs b/skills/scripts/e2e/ensure-langrag-sentinel-kb.mjs index b2cf9e7b2..f7c757a6d 100755 --- a/skills/scripts/e2e/ensure-langrag-sentinel-kb.mjs +++ b/skills/scripts/e2e/ensure-langrag-sentinel-kb.mjs @@ -21,7 +21,7 @@ await ensureEvidence(paths); const backendUrl = env.LANGBOT_BACKEND_URL || ""; const user = env.LANGBOT_E2E_LOGIN_USER || ""; const password = env.LANGBOT_E2E_LOGIN_PASSWORD || "LangBotE2ELocalPass!2026"; -const expectedText = env.LANGBOT_E2E_EXPECTED_TEXT || "azalea-cobalt-7421"; +const expectedText = env.LANGBOT_E2E_RAG_EXPECTED_TEXT || "azalea-cobalt-7421"; const query = env.LANGBOT_E2E_RETRIEVE_QUERY || "What is the local agent runner retrieval sentinel?"; const writeEnv = process.argv.includes("--write-env"); const checkOnly = process.argv.includes("--check-only"); diff --git a/skills/scripts/e2e/ensure-local-agent-pipeline.mjs b/skills/scripts/e2e/ensure-local-agent-pipeline.mjs index 70293df87..e72e15109 100755 --- a/skills/scripts/e2e/ensure-local-agent-pipeline.mjs +++ b/skills/scripts/e2e/ensure-local-agent-pipeline.mjs @@ -18,12 +18,14 @@ import { writeResult, } from "./lib/langbot-e2e.mjs"; -const RUNNER_ID = "local-agent"; +const RUNNER_ID = "plugin:langbot/local-agent/default"; const SPACE_PROVIDER_UUID = "00000000-0000-0000-0000-000000000000"; const DEFAULT_PIPELINE_NAME = "Agent QA Local Agent Debug Chat"; const DEFAULT_LOCAL_PASSWORD = "LangBotE2ELocalPass!2026"; const DEFAULT_MODEL_TEST_LIMIT = 8; const DEFAULT_MODEL_FALLBACK_COUNT = 3; +const DEFAULT_FAKE_MODEL_UUID = "langbot-e2e-fake-local-agent-model"; +const DEFAULT_FAKE_PROVIDER_NAME = "LangBot E2E Fake OpenAI Provider"; const caseId = "ensure-local-agent-pipeline"; await loadEnvFiles(); @@ -31,7 +33,10 @@ const paths = evidencePaths(caseId); await ensureEvidence(paths); const writeEnv = process.argv.includes("--write-env"); -const pipelineName = env.LANGBOT_E2E_CREATE_PIPELINE_NAME || env.LANGBOT_LOCAL_AGENT_PIPELINE_NAME || DEFAULT_PIPELINE_NAME; +const pipelineName = + env.LANGBOT_E2E_CREATE_PIPELINE_NAME || + env.LANGBOT_LOCAL_AGENT_PIPELINE_NAME || + DEFAULT_PIPELINE_NAME; const frontendUrl = env.LANGBOT_FRONTEND_URL || ""; const backendUrl = env.LANGBOT_BACKEND_URL || ""; const envLocalPath = resolve("skills/.env.local"); @@ -51,6 +56,7 @@ const result = { selected_model_id: "", selected_model_name: "", fallback_model_ids: [], + fake_provider: null, model_count: 0, space_model_count: 0, scanned_space_model_count: 0, @@ -83,7 +89,9 @@ try { const password = env.LANGBOT_E2E_LOGIN_PASSWORD || DEFAULT_LOCAL_PASSWORD; if (!user) { result.status = "env_issue"; - throw new Error("LANGBOT_E2E_LOGIN_USER is required so this setup can create/update the pipeline via backend API."); + throw new Error( + "LANGBOT_E2E_LOGIN_USER is required so this setup can create/update the pipeline via backend API.", + ); } const auth = await resetAndAuthLocalUser({ backendUrl, user, password }); @@ -97,7 +105,9 @@ try { result.wizard = wizard; if (wizard.status !== "pass") { result.status = "fail"; - throw new Error(wizard.reason || "Failed to mark the local QA wizard as skipped."); + throw new Error( + wizard.reason || "Failed to mark the local QA wizard as skipped.", + ); } const prepared = await ensureLocalAgentPipeline({ @@ -118,10 +128,12 @@ try { LANGBOT_PIPELINE_NAME: result.pipeline_name || pipelineName, LANGBOT_LOCAL_AGENT_PIPELINE_URL: result.pipeline_url, LANGBOT_LOCAL_AGENT_PIPELINE_NAME: result.pipeline_name || pipelineName, - ...(result.selected_model_id ? { - LANGBOT_LOCAL_AGENT_MODEL_UUID: result.selected_model_id, - LANGBOT_E2E_MODEL_UUID: result.selected_model_id, - } : {}), + ...(result.selected_model_id + ? { + LANGBOT_LOCAL_AGENT_MODEL_UUID: result.selected_model_id, + LANGBOT_E2E_MODEL_UUID: result.selected_model_id, + } + : {}), }); result.wrote_env = true; } @@ -132,12 +144,21 @@ try { const browserCheck = await verifyBrowserToken(page, backendUrl); result.browser_token_check = browserCheck; if (!browserCheck.authenticated) { - throw new Error(browserCheck.reason || "Browser token check failed after setup."); + throw new Error( + browserCheck.reason || "Browser token check failed after setup.", + ); } - await page.goto(result.pipeline_url || frontendUrl, { waitUntil: "domcontentloaded" }); - await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {}); + await page.goto(result.pipeline_url || frontendUrl, { + waitUntil: "domcontentloaded", + }); + await page + .waitForLoadState("networkidle", { timeout: 10_000 }) + .catch(() => {}); const text = await bodyText(page); - result.page_signal = ["Pipelines", "流水线", pipelineName].find((signal) => text.includes(signal)) || ""; + result.page_signal = + ["Pipelines", "流水线", pipelineName].find((signal) => + text.includes(signal), + ) || ""; } catch (error) { result.status = result.status === "env_issue" ? "env_issue" : "fail"; result.reason = result.reason || error.message; @@ -148,24 +169,51 @@ try { console.log(JSON.stringify(result, null, 2)); } -process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1); +process.exit( + result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1, +); async function skipWizard({ backendUrl, token }) { - const response = await apiJson(backendUrl, "/api/v1/system/wizard/completed", { - method: "POST", - token, - body: { status: "skipped" }, - }); + const response = await apiJson( + backendUrl, + "/api/v1/system/wizard/completed", + { + method: "POST", + token, + body: { status: "skipped" }, + }, + ); const ok = response.status < 400 && response.json.code === 0; return { status: ok ? "pass" : "fail", http_status: response.status, code: response.json.code ?? null, - reason: ok ? "Wizard marked skipped for local QA." : response.json.msg || "Wizard status update failed.", + reason: ok + ? "Wizard marked skipped for local QA." + : response.json.msg || "Wizard status update failed.", }; } -async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runnerId }) { +async function ensureLocalAgentPipeline({ + backendUrl, + token, + pipelineName, + runnerId, +}) { + const fakeProviderBaseUrl = + env.LANGBOT_E2E_FAKE_PROVIDER_BASE_URL || + env.LANGBOT_FAKE_PROVIDER_BASE_URL || + ""; + let fakeModel = null; + if (fakeProviderBaseUrl) { + fakeModel = await ensureFakeProviderModel({ + backendUrl, + token, + baseUrl: fakeProviderBaseUrl, + }); + if (fakeModel.status !== "pass") return fakeModel; + } + const [pipelineList, modelList] = await Promise.all([ apiJson(backendUrl, "/api/v1/pipelines", { token }), apiJson(backendUrl, "/api/v1/provider/models/llm", { token }), @@ -199,7 +247,9 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne .map((item) => item.trim()) .filter(Boolean), ); - const spaceModels = models.filter((model) => isSpaceModel(model) && !skippedModelIds.has(model.uuid)); + const spaceModels = models.filter( + (model) => isSpaceModel(model) && !skippedModelIds.has(model.uuid), + ); const pipelines = pipelineList.json.data?.pipelines || []; let pipeline = pipelines.find((item) => item.name === pipelineName) || null; let created = false; @@ -210,7 +260,8 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne token, body: { name: pipelineName, - description: "Local QA pipeline for AgentRunner Debug Chat smoke tests.", + description: + "Local QA pipeline for AgentRunner Debug Chat smoke tests.", emoji: "QA", }, }); @@ -224,7 +275,11 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne }; } const pipelineId = createdResponse.json.data?.uuid || ""; - const loaded = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipelineId)}`, { token }); + const loaded = await apiJson( + backendUrl, + `/api/v1/pipelines/${encodeURIComponent(pipelineId)}`, + { token }, + ); pipeline = loaded.json.data?.pipeline || null; created = true; } @@ -238,7 +293,11 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne }; } - const loaded = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, { token }); + const loaded = await apiJson( + backendUrl, + `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, + { token }, + ); if (isApiFailure(loaded) || !loaded.json.data?.pipeline) { return { status: "fail", @@ -251,32 +310,53 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne } pipeline = loaded.json.data.pipeline; - const config = pipeline.config && typeof pipeline.config === "object" ? pipeline.config : {}; + const config = + pipeline.config && typeof pipeline.config === "object" + ? pipeline.config + : {}; const ai = config.ai && typeof config.ai === "object" ? config.ai : {}; - const rawExistingLocalAgentConfig = ai["local-agent"] && typeof ai["local-agent"] === "object" - ? ai["local-agent"] - : {}; + const runnerConfigs = + ai.runner_config && typeof ai.runner_config === "object" + ? ai.runner_config + : {}; + const rawExistingLocalAgentConfig = + runnerConfigs[runnerId] && typeof runnerConfigs[runnerId] === "object" + ? runnerConfigs[runnerId] + : {}; const existingLocalAgentConfig = rawExistingLocalAgentConfig; - const existingModel = existingLocalAgentConfig.model && typeof existingLocalAgentConfig.model === "object" - ? existingLocalAgentConfig.model - : {}; - const requestedModelId = env.LANGBOT_LOCAL_AGENT_MODEL_UUID || env.LANGBOT_E2E_MODEL_UUID || ""; - const selected = await selectWorkingSpaceModel({ - backendUrl, - token, - models, - skippedModelIds, - skippedModelNames, - requestedModelId, - existingModelId: existingModel.primary || "", - }); + const existingModel = + existingLocalAgentConfig.model && + typeof existingLocalAgentConfig.model === "object" + ? existingLocalAgentConfig.model + : {}; + const requestedModelId = + env.LANGBOT_LOCAL_AGENT_MODEL_UUID || env.LANGBOT_E2E_MODEL_UUID || ""; + const selected = fakeModel + ? { + status: "pass", + reason: "", + selected_model_id: fakeModel.model_uuid, + selected_model_name: fakeModel.model_name, + fallback_model_ids: [], + scanned_space_model_count: 0, + tested_model_count: 0, + model_tests: [], + } + : await selectWorkingSpaceModel({ + backendUrl, + token, + models, + skippedModelIds, + skippedModelNames, + requestedModelId, + existingModelId: existingModel.primary || "", + }); const selectedModelId = selected.selected_model_id || ""; const localAgentConfig = { timeout: 300, prompt: [{ role: "system", content: "You are a helpful assistant." }], "remove-think": false, "knowledge-bases": [], - "box-session-id-template": "{launcher_type}_{launcher_id}", "retrieval-top-k": 5, "rerank-model": "", "rerank-top-k": 5, @@ -288,7 +368,6 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne "context-reserve-tokens": 16384, "context-keep-recent-tokens": 20000, "context-summary-tokens": 8000, - ...existingLocalAgentConfig, // Current backend truncation still reads this field directly. "max-round": positiveInteger(existingLocalAgentConfig["max-round"], 10), model: { @@ -303,23 +382,30 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne runner: { ...(ai.runner && typeof ai.runner === "object" ? ai.runner : {}), id: runnerId, - runner: runnerId, "expire-time": 0, }, - "local-agent": localAgentConfig, + runner_config: { + ...runnerConfigs, + [runnerId]: localAgentConfig, + }, }, }; - const updateResponse = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, { - method: "PUT", - token, - body: { - name: pipelineName, - description: "Local QA pipeline for AgentRunner Debug Chat smoke tests.", - emoji: "QA", - config: updatedConfig, + const updateResponse = await apiJson( + backendUrl, + `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, + { + method: "PUT", + token, + body: { + name: pipelineName, + description: + "Local QA pipeline for AgentRunner Debug Chat smoke tests.", + emoji: "QA", + config: updatedConfig, + }, }, - }); + ); if (isApiFailure(updateResponse)) { return { status: "fail", @@ -341,7 +427,8 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne status: selectedModelId ? "pass" : "env_issue", reason: selectedModelId ? `Local-agent pipeline is configured for Debug Chat with Space model ${selected.selected_model_name || selectedModelId} and ${selected.fallback_model_ids.length} fallback(s).` - : selected.reason || "No working Space LLM model is configured in this LangBot instance.", + : selected.reason || + "No working Space LLM model is configured in this LangBot instance.", pipeline_id: pipeline.uuid, pipeline_name: pipelineName, model_count: models.length, @@ -352,21 +439,214 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne selected_model_id: selectedModelId, selected_model_name: selected.selected_model_name, fallback_model_ids: selected.fallback_model_ids, + fake_provider: fakeModel, created, updated: true, }; } +async function ensureFakeProviderModel({ backendUrl, token, baseUrl }) { + const modelUuid = env.LANGBOT_E2E_FAKE_MODEL_UUID || DEFAULT_FAKE_MODEL_UUID; + const modelName = + env.LANGBOT_E2E_FAKE_MODEL_NAME || + env.LANGBOT_FAKE_PROVIDER_MODEL || + env.LANGBOT_FAKE_PROVIDER_MODEL_NAME || + "langbot-e2e-fake-model"; + const providerName = + env.LANGBOT_E2E_FAKE_PROVIDER_NAME || DEFAULT_FAKE_PROVIDER_NAME; + const providerRequester = + env.LANGBOT_E2E_FAKE_PROVIDER_REQUESTER || "openai-chat-completions"; + const apiKey = + env.LANGBOT_E2E_FAKE_PROVIDER_API_KEY || + env.LANGBOT_FAKE_PROVIDER_API_KEY || + "fake-key"; + + const providersResponse = await apiJson( + backendUrl, + "/api/v1/provider/providers", + { token }, + ); + if (isApiFailure(providersResponse)) { + return { + status: "fail", + reason: + providersResponse.json.msg || + "Failed to list providers before creating fake provider.", + provider_status: providersResponse.status, + }; + } + + const normalizedBaseUrl = String(baseUrl || "").replace(/\/$/, ""); + const providers = providersResponse.json.data?.providers || []; + let provider = providers.find( + (item) => + item.name === providerName || + (item.requester === providerRequester && + String(item.base_url || "").replace(/\/$/, "") === normalizedBaseUrl), + ); + const providerBody = { + name: providerName, + requester: providerRequester, + base_url: normalizedBaseUrl, + api_keys: [apiKey], + }; + + if (provider?.uuid) { + const updateResponse = await apiJson( + backendUrl, + `/api/v1/provider/providers/${encodeURIComponent(provider.uuid)}`, + { + method: "PUT", + token, + body: providerBody, + }, + ); + if (isApiFailure(updateResponse)) { + return { + status: "fail", + reason: updateResponse.json.msg || "Failed to update fake provider.", + provider_status: updateResponse.status, + }; + } + } else { + const createProviderResponse = await apiJson( + backendUrl, + "/api/v1/provider/providers", + { + method: "POST", + token, + body: providerBody, + }, + ); + if (isApiFailure(createProviderResponse)) { + return { + status: "fail", + reason: + createProviderResponse.json.msg || "Failed to create fake provider.", + provider_status: createProviderResponse.status, + }; + } + provider = { uuid: createProviderResponse.json.data?.uuid || "" }; + } + + if (!provider?.uuid) { + return { + status: "fail", + reason: "Fake provider did not return a provider uuid.", + }; + } + + let resolvedModelUuid = modelUuid; + let modelResponse = await apiJson( + backendUrl, + `/api/v1/provider/models/llm/${encodeURIComponent(modelUuid)}`, + { + token, + }, + ); + if (modelResponse.status === 404) { + const providerModelsResponse = await apiJson( + backendUrl, + `/api/v1/provider/models/llm?provider_uuid=${encodeURIComponent(provider.uuid)}`, + { token }, + ); + if (!isApiFailure(providerModelsResponse)) { + const existingModel = ( + providerModelsResponse.json.data?.models || [] + ).find((item) => item.name === modelName); + if (existingModel?.uuid) { + resolvedModelUuid = existingModel.uuid; + modelResponse = await apiJson( + backendUrl, + `/api/v1/provider/models/llm/${encodeURIComponent(resolvedModelUuid)}`, + { + token, + }, + ); + } + } + } + + const modelBody = { + name: modelName, + provider_uuid: provider.uuid, + abilities: ["func_call", "vision"], + context_length: 8192, + extra_args: {}, + prefered_ranking: 0, + }; + if (modelResponse.status === 404) { + const createModelResponse = await apiJson( + backendUrl, + "/api/v1/provider/models/llm", + { + method: "POST", + token, + body: { + uuid: modelUuid, + ...modelBody, + }, + }, + ); + if (isApiFailure(createModelResponse)) { + return { + status: "fail", + reason: createModelResponse.json.msg || "Failed to create fake model.", + model_status: createModelResponse.status, + }; + } + resolvedModelUuid = createModelResponse.json.data?.uuid || modelUuid; + } else if (isApiFailure(modelResponse)) { + return { + status: "fail", + reason: modelResponse.json.msg || "Failed to load fake model.", + model_status: modelResponse.status, + }; + } else { + const updateModelResponse = await apiJson( + backendUrl, + `/api/v1/provider/models/llm/${encodeURIComponent(resolvedModelUuid)}`, + { + method: "PUT", + token, + body: modelBody, + }, + ); + if (isApiFailure(updateModelResponse)) { + return { + status: "fail", + reason: updateModelResponse.json.msg || "Failed to update fake model.", + model_status: updateModelResponse.status, + }; + } + } + + return { + status: "pass", + provider_uuid: provider.uuid, + provider_requester: providerRequester, + base_url: normalizedBaseUrl, + model_uuid: resolvedModelUuid, + model_name: modelName, + }; +} + function isApiFailure(response) { - return response.status >= 400 || (response.json.code !== undefined && response.json.code !== 0); + return ( + response.status >= 400 || + (response.json.code !== undefined && response.json.code !== 0) + ); } function isSpaceModel(model) { - const provider = model?.provider && typeof model.provider === "object" ? model.provider : {}; - return model?.provider_uuid === SPACE_PROVIDER_UUID - || provider.uuid === SPACE_PROVIDER_UUID - || provider.requester === "space-chat-completions" - || provider.name === "LangBot Models"; + const provider = + model?.provider && typeof model.provider === "object" ? model.provider : {}; + return ( + model?.provider_uuid === SPACE_PROVIDER_UUID || + provider.uuid === SPACE_PROVIDER_UUID || + provider.requester === "space-chat-completions" || + provider.name === "LangBot Models" + ); } async function selectWorkingSpaceModel({ @@ -379,15 +659,24 @@ async function selectWorkingSpaceModel({ existingModelId, }) { const modelTests = []; - const testLimit = positiveInteger(env.LANGBOT_E2E_MODEL_TEST_LIMIT, DEFAULT_MODEL_TEST_LIMIT); - const fallbackCount = positiveInteger(env.LANGBOT_E2E_MODEL_FALLBACK_COUNT, DEFAULT_MODEL_FALLBACK_COUNT); + const testLimit = positiveInteger( + env.LANGBOT_E2E_MODEL_TEST_LIMIT, + DEFAULT_MODEL_TEST_LIMIT, + ); + const fallbackCount = positiveInteger( + env.LANGBOT_E2E_MODEL_FALLBACK_COUNT, + DEFAULT_MODEL_FALLBACK_COUNT, + ); const workingModels = []; - const spaceModels = rankModels(models.filter((model) => ( - model.uuid - && isSpaceModel(model) - && !skippedModelIds.has(model.uuid) - && !skippedModelNames.has(model.name) - ))); + const spaceModels = rankModels( + models.filter( + (model) => + model.uuid && + isSpaceModel(model) && + !skippedModelIds.has(model.uuid) && + !skippedModelNames.has(model.name), + ), + ); const requestedModel = requestedModelId ? spaceModels.find((model) => model.uuid === requestedModelId) || null : null; @@ -396,7 +685,9 @@ async function selectWorkingSpaceModel({ : null; const candidates = uniqueCandidates([ ...(requestedModel ? [existingCandidate(requestedModel, "requested")] : []), - ...(existingModel ? [existingCandidate(existingModel, "existing-pipeline")] : []), + ...(existingModel + ? [existingCandidate(existingModel, "existing-pipeline")] + : []), ...spaceModels.map((model) => existingCandidate(model, "configured-space")), ]); @@ -405,9 +696,16 @@ async function selectWorkingSpaceModel({ scanResult = await scanSpaceModels({ backendUrl, token }); if (scanResult.status === "pass") { const knownNames = new Set(spaceModels.map((model) => model.name)); - candidates.push(...scanResult.models - .filter((model) => model.name && !knownNames.has(model.name) && !skippedModelNames.has(model.name)) - .map((model) => scannedCandidate(model))); + candidates.push( + ...scanResult.models + .filter( + (model) => + model.name && + !knownNames.has(model.name) && + !skippedModelNames.has(model.name), + ) + .map((model) => scannedCandidate(model)), + ); } } @@ -435,14 +733,16 @@ async function selectWorkingSpaceModel({ }; } - const baseReason = unique.length === 0 - ? scanResult.reason || "No Space LLM model candidates are available." - : `No working Space LLM model found after testing ${modelTests.length} candidate(s).`; + const baseReason = + unique.length === 0 + ? scanResult.reason || "No Space LLM model candidates are available." + : `No working Space LLM model found after testing ${modelTests.length} candidate(s).`; return { status: "env_issue", - reason: requestedModelId && !requestedModel - ? `Requested Space LLM model ${requestedModelId} is missing or skipped; ${baseReason}` - : baseReason, + reason: + requestedModelId && !requestedModel + ? `Requested Space LLM model ${requestedModelId} is missing or skipped; ${baseReason}` + : baseReason, selected_model_id: "", selected_model_name: "", fallback_model_ids: [], @@ -462,7 +762,11 @@ async function scanSpaceModels({ backendUrl, token }) { return { status: "env_issue", models: [], - reason: safeReason(response.json.msg || response.json.message || "Failed to scan Space LLM models."), + reason: safeReason( + response.json.msg || + response.json.message || + "Failed to scan Space LLM models.", + ), }; } return { @@ -492,28 +796,42 @@ async function ensureAndTestModel({ backendUrl, token, candidate }) { if (isApiFailure(create) || !modelUuid) { return modelTestResult(candidate, { status: "fail", - reason: safeReason(create.json.msg || "Failed to create scanned Space model."), + reason: safeReason( + create.json.msg || "Failed to create scanned Space model.", + ), http_status: create.status, }); } created = true; } - const test = await apiJson(backendUrl, `/api/v1/provider/models/llm/${encodeURIComponent(modelUuid)}/test`, { - method: "POST", - token, - body: { extra_args: {} }, - }); + const test = await apiJson( + backendUrl, + `/api/v1/provider/models/llm/${encodeURIComponent(modelUuid)}/test`, + { + method: "POST", + token, + body: { extra_args: {} }, + }, + ); const passed = !isApiFailure(test); if (!passed && created) { - await apiJson(backendUrl, `/api/v1/provider/models/llm/${encodeURIComponent(modelUuid)}`, { - method: "DELETE", - token, - }).catch(() => {}); + await apiJson( + backendUrl, + `/api/v1/provider/models/llm/${encodeURIComponent(modelUuid)}`, + { + method: "DELETE", + token, + }, + ).catch(() => {}); } return modelTestResult(candidate, { status: passed ? "pass" : "fail", - reason: passed ? "" : safeReason(test.json.msg || test.json.message || "Space model test failed."), + reason: passed + ? "" + : safeReason( + test.json.msg || test.json.message || "Space model test failed.", + ), http_status: test.status, model_uuid: modelUuid, created, @@ -558,7 +876,9 @@ function uniqueCandidates(candidates) { const seen = new Set(); const result = []; for (const candidate of candidates) { - const key = candidate.uuid ? `uuid:${candidate.uuid}` : `name:${candidate.name}`; + const key = candidate.uuid + ? `uuid:${candidate.uuid}` + : `name:${candidate.name}`; if (!candidate.name || seen.has(key)) continue; seen.add(key); result.push(candidate); @@ -568,8 +888,12 @@ function uniqueCandidates(candidates) { function rankModels(models) { return [...models].sort((left, right) => { - const leftRank = Number.isFinite(Number(left.prefered_ranking)) ? Number(left.prefered_ranking) : 9999; - const rightRank = Number.isFinite(Number(right.prefered_ranking)) ? Number(right.prefered_ranking) : 9999; + const leftRank = Number.isFinite(Number(left.prefered_ranking)) + ? Number(left.prefered_ranking) + : 9999; + const rightRank = Number.isFinite(Number(right.prefered_ranking)) + ? Number(right.prefered_ranking) + : 9999; if (leftRank !== rightRank) return leftRank - rightRank; return String(left.name || "").localeCompare(String(right.name || "")); }); @@ -605,5 +929,9 @@ async function upsertEnvLocal(path, updates) { for (const [key, value] of Object.entries(updates)) { if (!seen.has(key)) next.push(`${key}=${value}`); } - await writeFile(path, `${next.filter((line, index) => line !== "" || index < next.length - 1).join("\n")}\n`, "utf8"); + await writeFile( + path, + `${next.filter((line, index) => line !== "" || index < next.length - 1).join("\n")}\n`, + "utf8", + ); } diff --git a/skills/scripts/e2e/fake-openai-provider.mjs b/skills/scripts/e2e/fake-openai-provider.mjs index 1cca9c46b..da0d88c9c 100755 --- a/skills/scripts/e2e/fake-openai-provider.mjs +++ b/skills/scripts/e2e/fake-openai-provider.mjs @@ -9,7 +9,43 @@ const args = parseArgs(process.argv.slice(2)); const host = args.host || env.LANGBOT_FAKE_PROVIDER_HOST || "127.0.0.1"; const port = integer(args.port ?? env.LANGBOT_FAKE_PROVIDER_PORT, 0); const stateFile = args["state-file"] || env.LANGBOT_FAKE_PROVIDER_STATE_FILE || ""; -const modelName = env.LANGBOT_FAKE_PROVIDER_MODEL_NAME || "gpt-4o-mini"; +const modelName = args.model || env.LANGBOT_FAKE_PROVIDER_MODEL_NAME || env.LANGBOT_FAKE_PROVIDER_MODEL || "gpt-4o-mini"; +const COMPACTION_SENTINEL = "qa_compaction_sentinel_7391"; +const COMBO_SENTINEL = "qa_combo_compaction_sentinel_2406"; +const RAG_SENTINEL = "azalea-cobalt-7421"; +const COMBO_TOOL_INPUT = "combo-tool-ok-local-agent"; +const COMBO_TOOL_RESULT = `qa-plugin-smoke:${COMBO_TOOL_INPUT}`; +const COMBO_FINAL = `COMBO_FINAL ${COMBO_SENTINEL} ${RAG_SENTINEL} ${COMBO_TOOL_RESULT}`; +const MULTITOOL_SENTINEL = "qa_multitool_compaction_sentinel_6718"; +const MULTITOOL_TOOL_A_INPUT = "multi-tool-a-local-agent"; +const MULTITOOL_TOOL_B_INPUT = "multi-tool-b-local-agent"; +const MULTITOOL_TOOL_A_RESULT = `qa-plugin-smoke:${MULTITOOL_TOOL_A_INPUT}`; +const MULTITOOL_TOOL_B_RESULT = `qa-plugin-smoke:${MULTITOOL_TOOL_B_INPUT}`; +const MULTITOOL_FINAL = [ + "MULTITOOL_COMBO_FINAL", + MULTITOOL_SENTINEL, + RAG_SENTINEL, + MULTITOOL_TOOL_A_RESULT, + MULTITOOL_TOOL_B_RESULT, +].join(" "); +const PARALLEL_SENTINEL = "qa_parallel_compaction_sentinel_8142"; +const PARALLEL_TOOL_A_INPUT = "parallel-tool-a-local-agent"; +const PARALLEL_TOOL_B_INPUT = "parallel-tool-b-local-agent"; +const PARALLEL_TOOL_A_RESULT = `qa-plugin-smoke:${PARALLEL_TOOL_A_INPUT}`; +const PARALLEL_TOOL_B_RESULT = `qa-plugin-smoke:${PARALLEL_TOOL_B_INPUT}`; +const PARALLEL_FINAL = [ + "PARALLEL_COMBO_FINAL", + PARALLEL_SENTINEL, + RAG_SENTINEL, + PARALLEL_TOOL_A_RESULT, + PARALLEL_TOOL_B_RESULT, +].join(" "); +const LOOP_LIMIT_INPUT = "loop-limit-repeat-local-agent"; +const TOOL_ERROR_INPUT = "tool-error-recovery-local-agent"; +const TOOL_ERROR_FINAL = "TOOL_ERROR_RECOVERY_FINAL qa-plugin-smoke forced failure observed"; +const STEERING_FOLLOWUP_SENTINEL = "qa_steering_sentinel_6194"; +const STEERING_SLEEP_INPUT = "steering-e2e-anchor"; +const STEERING_SLEEP_RESULT = `qa-plugin-smoke:sleep:8:${STEERING_SLEEP_INPUT}`; const config = { response_text: env.LANGBOT_FAKE_PROVIDER_RESPONSE_TEXT || "OK", first_token_delay_ms: integer(env.LANGBOT_FAKE_PROVIDER_FIRST_TOKEN_DELAY_MS, 25), @@ -88,6 +124,7 @@ const server = createServer(async (request, response) => { created: 1, owned_by: "langbot-qa", type: "llm", + context_length: 8192, }, ], }); @@ -100,7 +137,8 @@ const server = createServer(async (request, response) => { const requestId = `chatcmpl-langbot-fake-${requestCount}`; const shouldFail = requestCount <= config.fail_first_n || (config.fail_every_n > 0 && requestCount % config.fail_every_n === 0); - const replyText = responseTextForBody(body); + const replyMessage = buildResponse(body); + const replyText = replyMessage.content || ""; requestRecord = recordRequest({ id: requestId, request_number: requestCount, @@ -140,7 +178,7 @@ const server = createServer(async (request, response) => { await streamCompletion(response, { requestId, model: body.model || modelName, - content: replyText, + message: replyMessage, failAfterFirstChunk: config.fail_after_first_chunk, requestRecord, startedPerf, @@ -150,7 +188,7 @@ const server = createServer(async (request, response) => { sendJson(response, 200, completionPayload({ requestId, model: body.model || modelName, - content: replyText, + message: replyMessage, })); markRequestTiming(requestRecord, "first_chunk", startedPerf); markRequestTiming(requestRecord, "first_content_chunk", startedPerf); @@ -270,8 +308,8 @@ function sendJson(response, status, payload) { response.end(text); } -function completionPayload({ requestId, model, content }) { - const completionTokens = tokenEstimate(content); +function completionPayload({ requestId, model, message }) { + const completionTokens = tokenEstimate(message.content || JSON.stringify(message.tool_calls || [])); return { id: requestId, object: "chat.completion", @@ -280,11 +318,8 @@ function completionPayload({ requestId, model, content }) { choices: [ { index: 0, - message: { - role: "assistant", - content, - }, - finish_reason: "stop", + message, + finish_reason: message.tool_calls?.length ? "tool_calls" : "stop", }, ], usage: { @@ -298,7 +333,7 @@ function completionPayload({ requestId, model, content }) { async function streamCompletion(response, { requestId, model, - content, + message, failAfterFirstChunk: failMidStream, requestRecord, startedPerf, @@ -319,7 +354,56 @@ async function streamCompletion(response, { choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }], }); - const chunks = splitContent(content); + const chunks = message.tool_calls?.length ? [] : splitContent(message.content || ""); + if (message.tool_calls?.length) { + await sleep(config.chunk_delay_ms); + markRequestTiming(requestRecord, "first_content_chunk", startedPerf); + requestRecord.content_chunk_count = (requestRecord.content_chunk_count || 0) + 1; + writeSse(response, { + id: requestId, + object: "chat.completion.chunk", + created: Math.floor(Date.now() / 1000), + model, + choices: [ + { + index: 0, + delta: { + tool_calls: message.tool_calls.map((call, index) => ({ + index, + id: call.id, + type: call.type, + function: { + name: call.function.name, + arguments: call.function.arguments, + }, + })), + }, + finish_reason: null, + }, + ], + }); + await sleep(config.chunk_delay_ms); + writeSse(response, { + id: requestId, + object: "chat.completion.chunk", + created: Math.floor(Date.now() / 1000), + model, + choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }], + usage: { + prompt_tokens: 8, + completion_tokens: tokenEstimate(JSON.stringify(message.tool_calls)), + total_tokens: 8 + tokenEstimate(JSON.stringify(message.tool_calls)), + }, + }); + response.write("data: [DONE]\n\n"); + response.end(); + finishRequestRecord(requestRecord, startedPerf, { + status: "ok", + http_status: 200, + }); + return; + } + for (let index = 0; index < chunks.length; index += 1) { await sleep(config.chunk_delay_ms); if (index === 0) markRequestTiming(requestRecord, "first_content_chunk", startedPerf); @@ -342,7 +426,7 @@ async function streamCompletion(response, { } await sleep(config.chunk_delay_ms); - const completionTokens = tokenEstimate(content); + const completionTokens = tokenEstimate(message.content || ""); writeSse(response, { id: requestId, object: "chat.completion.chunk", @@ -399,6 +483,258 @@ function responseTextForBody(body) { return config.response_text; } +function messageText(messages = []) { + return messages + .map((message) => { + const parts = [contentText(message?.content)]; + if (Array.isArray(message?.tool_calls)) { + parts.push( + message.tool_calls + .map((call) => `${call?.function?.name || call?.name || ""} ${call?.function?.arguments || ""}`) + .join("\n"), + ); + } + return parts.filter(Boolean).join("\n"); + }) + .join("\n"); +} + +function contentText(content) { + if (typeof content === "string") return content; + if (Array.isArray(content)) return content.map((part) => contentText(part)).join(""); + if (content && typeof content === "object") { + for (const key of ["text", "content", "message", "error", "value"]) { + if (content[key] !== undefined && content[key] !== null) { + return contentText(content[key]); + } + } + return JSON.stringify(content); + } + return ""; +} + +function latestUserText(messages = []) { + const latest = [...messages].reverse().find((message) => message?.role === "user"); + return contentText(latest?.content); +} + +function toolNames(tools = []) { + return tools + .map((tool) => tool?.function?.name || tool?.name || "") + .filter(Boolean); +} + +function firstToolName(tools = [], candidates = []) { + const names = toolNames(tools); + return candidates.find((candidate) => names.includes(candidate)) || ""; +} + +function isSummarizationRequest(messages = []) { + const text = messageText(messages); + return /context summarization assistant|conversation to summarize|/i.test(text); +} + +function buildSummaryResponse(text) { + const references = []; + if (text.includes(COMPACTION_SENTINEL)) references.push(COMPACTION_SENTINEL); + if (text.includes(COMBO_SENTINEL)) references.push(COMBO_SENTINEL); + if (text.includes(MULTITOOL_SENTINEL)) references.push(MULTITOOL_SENTINEL); + if (text.includes(PARALLEL_SENTINEL)) references.push(PARALLEL_SENTINEL); + if (/RAG_TOOL_COMBO_GOAL/i.test(text)) references.push("RAG_TOOL_COMBO_GOAL"); + if (/MULTITOOL_RAG_GOAL/i.test(text)) references.push("MULTITOOL_RAG_GOAL"); + if (/PARALLEL_RAG_GOAL/i.test(text)) references.push("PARALLEL_RAG_GOAL"); + const critical = references.length + ? references.map((reference) => `- ${reference}`).join("\n") + : "- (none)"; + return { + role: "assistant", + content: [ + "## Goal", + "Preserve deterministic LangBot E2E context across compaction.", + "", + "## Constraints & Preferences", + "- Keep exact sentinel values verbatim.", + "", + "## Progress", + "### Done", + "- [x] Earlier conversation was compacted by the fake provider.", + "", + "### In Progress", + "- [ ] Continue the current Debug Chat run.", + "", + "### Blocked", + "- (none)", + "", + "## Key Decisions", + "- **Deterministic QA**: Return only known sentinel values from compacted input.", + "", + "## Next Steps", + "1. Answer the current user request using preserved context.", + "", + "## Critical Context", + critical, + ].join("\n"), + }; +} + +function buildResponse(payload) { + const text = messageText(payload.messages || []); + const current = latestUserText(payload.messages || []); + const tools = payload.tools || []; + if (isSummarizationRequest(payload.messages || [])) { + return buildSummaryResponse(text); + } + + const pluginTool = firstToolName(tools, ["qa_plugin_echo"]); + const pluginFailTool = firstToolName(tools, ["qa_plugin_fail"]); + const pluginSleepTool = firstToolName(tools, ["qa_plugin_sleep"]); + const mcpTool = firstToolName(tools, ["qa_mcp_echo"]); + + if (/STEERING_NO_FOLLOWUP|qa_plugin_sleep|steering-e2e-anchor|qa_steering_sentinel_6194/i.test(current || text)) { + if (text.includes(STEERING_FOLLOWUP_SENTINEL) && text.includes(STEERING_SLEEP_RESULT)) { + return { role: "assistant", content: STEERING_FOLLOWUP_SENTINEL }; + } + if (text.includes(STEERING_SLEEP_RESULT) && !text.includes(STEERING_FOLLOWUP_SENTINEL)) { + return { role: "assistant", content: "STEERING_NO_FOLLOWUP" }; + } + if (pluginSleepTool && !text.includes(STEERING_SLEEP_RESULT)) { + return toolCall("call_qa_plugin_sleep_steering", pluginSleepTool, { seconds: 8, text: STEERING_SLEEP_INPUT }); + } + } + + if (/测试暗号是什么|original sentinel|first.*sentinel/i.test(current)) { + return { + role: "assistant", + content: text.includes(COMPACTION_SENTINEL) ? COMPACTION_SENTINEL : "COMPACTION_SENTINEL_MISSING", + }; + } + if ((current.includes(COMPACTION_SENTINEL) + || current.includes(COMBO_SENTINEL) + || current.includes(MULTITOOL_SENTINEL) + || current.includes(PARALLEL_SENTINEL)) + && /请只回复 MEMORY_SET|only reply MEMORY_SET/i.test(current)) { + return { role: "assistant", content: "MEMORY_SET" }; + } + if (/PARALLEL_CONTEXT_PRESSURE_READY/.test(current)) return { role: "assistant", content: "PARALLEL_CONTEXT_PRESSURE_READY" }; + if (/MULTITOOL_CONTEXT_PRESSURE_READY/.test(current)) return { role: "assistant", content: "MULTITOOL_CONTEXT_PRESSURE_READY" }; + if (/COMBO_CONTEXT_PRESSURE_READY/.test(current)) return { role: "assistant", content: "COMBO_CONTEXT_PRESSURE_READY" }; + if (/CONTEXT_PRESSURE_READY/.test(current)) return { role: "assistant", content: "CONTEXT_PRESSURE_READY" }; + + if (/qa_mcp_echo:mcp-ok-local-agent/.test(text)) return { role: "assistant", content: "qa_mcp_echo:mcp-ok-local-agent" }; + if (/qa_mcp_echo|mcp-ok-local-agent/i.test(current || text) && mcpTool && !/qa_mcp_echo:mcp-ok-local-agent/.test(text)) { + return toolCall("call_qa_mcp_echo", mcpTool, { text: "mcp-ok-local-agent" }); + } + + if (/LOOP_LIMIT|loop-limit-repeat-local-agent|iteration limit/i.test(current || text) && pluginTool) { + return toolCall(`call_qa_plugin_echo_loop_limit_${Date.now()}`, pluginTool, { text: LOOP_LIMIT_INPUT }); + } + + if (/TOOL_ERROR_RECOVERY|tool-error-recovery-local-agent|qa_plugin_fail/i.test(current || text)) { + if (/(?:Error:|Tool execution failed:|ActionCallError:|RuntimeError:)/i.test(text) + && /(?:qa-plugin-smoke forced failure|qa_plugin_fail|tool-error-recovery-local-agent)/i.test(text)) { + return { role: "assistant", content: TOOL_ERROR_FINAL }; + } + if (pluginFailTool) { + return toolCall("call_qa_plugin_fail_recovery", pluginFailTool, { text: TOOL_ERROR_INPUT }); + } + } + + const isParallelComboRequest = /PARALLEL_COMBO|parallel-tool-a-local-agent|parallel-tool-b-local-agent|PARALLEL_RAG_GOAL/i.test(current || text); + if (isParallelComboRequest && pluginTool && !text.includes(PARALLEL_TOOL_A_RESULT) && !text.includes(PARALLEL_TOOL_B_RESULT)) { + return toolCalls([ + ["call_qa_plugin_echo_parallel_a", pluginTool, { text: PARALLEL_TOOL_A_INPUT }], + ["call_qa_plugin_echo_parallel_b", pluginTool, { text: PARALLEL_TOOL_B_INPUT }], + ]); + } + if (isParallelComboRequest && (text.includes(PARALLEL_TOOL_A_RESULT) || text.includes(PARALLEL_TOOL_B_RESULT))) { + return missingOrFinal(text, [ + [PARALLEL_SENTINEL, "parallel-memory"], + [RAG_SENTINEL, "rag"], + [PARALLEL_TOOL_A_RESULT, "tool-a"], + [PARALLEL_TOOL_B_RESULT, "tool-b"], + ], "PARALLEL_COMBO_MISSING", PARALLEL_FINAL); + } + + const isMultiToolComboRequest = /MULTITOOL_COMBO|multi-tool-a-local-agent|multi-tool-b-local-agent|MULTITOOL_RAG_GOAL/i.test(current || text); + if (isMultiToolComboRequest && pluginTool && !text.includes(MULTITOOL_TOOL_A_RESULT)) { + return toolCall("call_qa_plugin_echo_multi_a", pluginTool, { text: MULTITOOL_TOOL_A_INPUT }); + } + if (isMultiToolComboRequest && pluginTool && text.includes(MULTITOOL_TOOL_A_RESULT) && !text.includes(MULTITOOL_TOOL_B_RESULT)) { + return toolCall("call_qa_plugin_echo_multi_b", pluginTool, { text: MULTITOOL_TOOL_B_INPUT }); + } + if (isMultiToolComboRequest && text.includes(MULTITOOL_TOOL_A_RESULT) && text.includes(MULTITOOL_TOOL_B_RESULT)) { + return missingOrFinal(text, [ + [MULTITOOL_SENTINEL, "multi-memory"], + [RAG_SENTINEL, "rag"], + [MULTITOOL_TOOL_A_RESULT, "tool-a"], + [MULTITOOL_TOOL_B_RESULT, "tool-b"], + ], "MULTITOOL_COMBO_MISSING", MULTITOOL_FINAL); + } + + const isComboRequest = /qa_combo|\bCOMBO_FINAL\b|combo-tool-ok-local-agent|RAG_TOOL_COMBO_GOAL/i.test(current || text); + if (isComboRequest && pluginTool && !text.includes(COMBO_TOOL_RESULT)) { + return toolCall("call_qa_plugin_echo_combo", pluginTool, { text: COMBO_TOOL_INPUT }); + } + if (isComboRequest && text.includes(COMBO_TOOL_RESULT)) { + return missingOrFinal(text, [ + [COMBO_SENTINEL, "combo-memory"], + [RAG_SENTINEL, "rag"], + [COMBO_TOOL_RESULT, "tool-result"], + ], "COMBO_MISSING", COMBO_FINAL); + } + + if (/qa-plugin-smoke:plugin-tool-ok-local-agent/.test(text)) { + return { role: "assistant", content: "qa-plugin-smoke:plugin-tool-ok-local-agent" }; + } + if (/qa_plugin_echo|plugin-tool-ok-local-agent/i.test(current || text) && pluginTool && !/qa-plugin-smoke:plugin-tool-ok-local-agent/.test(text)) { + return toolCall("call_qa_plugin_echo", pluginTool, { text: "plugin-tool-ok-local-agent" }); + } + + const e2eTool = firstToolName(tools, ["e2e_lookup"]); + if (/Use the e2e lookup tool|e2e lookup tool|tool loop/i.test(current || text) && e2eTool && !/tool-result:alpha/.test(text)) { + return toolCall("call_e2e_lookup", e2eTool, { query: "alpha" }); + } + if (/tool-result:alpha/.test(text)) return { role: "assistant", content: "Tool loop final answer after tool-result:alpha" }; + if (/RAG_SENTINEL/.test(text)) return { role: "assistant", content: "RAG final answer with RAG_SENTINEL" }; + if (/NONSTREAM_OK/i.test(current || text)) return { role: "assistant", content: "NONSTREAM_OK" }; + if (/IMAGE_OK/i.test(current || text) + && /(?:\[Image\]|langbot_input_attachments|data:image\/|\"type\":\s*\"image\")/i.test(current || text)) { + return { role: "assistant", content: "IMAGE_OK" }; + } + if (/azalea-cobalt-7421/.test(text)) return { role: "assistant", content: "azalea-cobalt-7421" }; + + return { role: "assistant", content: responseTextForBody(payload) }; +} + +function toolCall(id, name, args) { + return toolCalls([[id, name, args]]); +} + +function toolCalls(calls) { + return { + role: "assistant", + content: "", + tool_calls: calls.map(([id, name, args]) => ({ + id, + type: "function", + function: { + name, + arguments: JSON.stringify(args), + }, + })), + }; +} + +function missingOrFinal(text, requirements, prefix, finalText) { + const missing = requirements + .filter(([needle]) => !text.includes(needle)) + .map(([, label]) => label); + return { + role: "assistant", + content: missing.length ? `${prefix}_${missing.join("_")}` : finalText, + }; +} + function flattenContent(content) { if (typeof content === "string") return content; if (Array.isArray(content)) { diff --git a/skills/scripts/e2e/install-qa-plugin-smoke.mjs b/skills/scripts/e2e/install-qa-plugin-smoke.mjs index 73b89af69..523588232 100755 --- a/skills/scripts/e2e/install-qa-plugin-smoke.mjs +++ b/skills/scripts/e2e/install-qa-plugin-smoke.mjs @@ -26,7 +26,12 @@ const packagePath = resolve( || "skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/dist/qa-plugin-smoke-0.1.0.lbpkg", ); const expectedPluginId = env.LANGBOT_E2E_EXPECTED_PLUGIN_ID || "qa/plugin-smoke"; -const expectedTool = env.LANGBOT_E2E_EXPECTED_TOOL || (expectedPluginId === "qa/plugin-smoke" ? "qa_plugin_echo" : ""); +const expectedTools = (env.LANGBOT_E2E_EXPECTED_TOOLS || env.LANGBOT_E2E_EXPECTED_TOOL || ( + expectedPluginId === "qa/plugin-smoke" ? "qa_plugin_echo" : "" +)) + .split(",") + .map((item) => item.trim()) + .filter(Boolean); const expectedRunnerId = env.LANGBOT_E2E_EXPECTED_RUNNER_ID || ""; const result = { @@ -40,6 +45,7 @@ const result = { package_preview: null, task_id: null, task: null, + reinstall_reason: "", plugin_present_before: false, plugin_present_after: false, tool_names: [], @@ -64,21 +70,23 @@ try { } result.plugin_present_before = await hasPlugin(backendUrl, auth.token); - if (!result.plugin_present_before) { - const form = new FormData(); - form.set("file", new Blob([bytes]), packagePath.split("/").pop()); - const response = await fetch(`${backendUrl.replace(/\/$/, "")}/api/v1/plugins/install/local`, { - method: "POST", - headers: { Authorization: `Bearer ${auth.token}` }, - body: form, - }); - const json = await response.json().catch(() => ({})); - if (response.status >= 400 || json.code !== 0) { - throw new Error(json.msg || `Plugin install request failed with HTTP ${response.status}.`); + if (expectedTools.length > 0) { + result.tool_names = await listToolNames(backendUrl, auth.token); + } + const missingToolsBefore = expectedTools.filter((tool) => !result.tool_names.includes(tool)); + if (result.plugin_present_before && missingToolsBefore.length > 0) { + result.reinstall_reason = `Installed plugin is missing expected tools: ${missingToolsBefore.join(", ")}`; + const removeTask = await removePlugin(backendUrl, auth.token); + if (!isTaskComplete(removeTask)) { + throw new Error(`Plugin reinstall cleanup did not complete successfully: ${JSON.stringify(removeTask)}`); } - result.task_id = json.data?.task_id ?? null; - if (!result.task_id) throw new Error("Plugin install response did not include task_id."); - result.task = await waitForTask(backendUrl, auth.token, result.task_id); + result.plugin_present_before = false; + await sleep(1000); + } + + if (!result.plugin_present_before) { + result.task = await installPlugin(backendUrl, auth.token, bytes, packagePath); + result.task_id = result.task?.id || result.task_id; if (!isTaskComplete(result.task)) { throw new Error(`Plugin install task did not complete successfully: ${JSON.stringify(result.task)}`); } @@ -87,10 +95,11 @@ try { await sleep(1000); result.plugin_present_after = await hasPlugin(backendUrl, auth.token); if (!result.plugin_present_after) throw new Error(`${expectedPluginId} is not listed by /api/v1/plugins after install.`); - if (expectedTool) { + if (expectedTools.length > 0) { result.tool_names = await listToolNames(backendUrl, auth.token); - if (!result.tool_names.includes(expectedTool)) { - throw new Error(`${expectedTool} is not listed by /api/v1/tools after install.`); + const missingTools = expectedTools.filter((tool) => !result.tool_names.includes(tool)); + if (missingTools.length > 0) { + throw new Error(`${missingTools.join(", ")} is not listed by /api/v1/tools after install.`); } } if (expectedRunnerId) { @@ -121,6 +130,41 @@ async function hasPlugin(backendUrl, token) { }); } +async function installPlugin(backendUrl, token, bytes, packagePath) { + const form = new FormData(); + form.set("file", new Blob([bytes]), packagePath.split("/").pop()); + const response = await fetch(`${backendUrl.replace(/\/$/, "")}/api/v1/plugins/install/local`, { + method: "POST", + headers: { Authorization: `Bearer ${token}` }, + body: form, + }); + const json = await response.json().catch(() => ({})); + if (response.status >= 400 || json.code !== 0) { + throw new Error(json.msg || `Plugin install request failed with HTTP ${response.status}.`); + } + const taskId = json.data?.task_id ?? null; + if (!taskId) throw new Error("Plugin install response did not include task_id."); + const task = await waitForTask(backendUrl, token, taskId); + return { id: taskId, ...task }; +} + +async function removePlugin(backendUrl, token) { + const [author, pluginName] = expectedPluginId.split("/"); + if (!author || !pluginName) throw new Error(`Invalid expected plugin id: ${expectedPluginId}`); + const response = await apiJson( + backendUrl, + `/api/v1/plugins/${encodeURIComponent(author)}/${encodeURIComponent(pluginName)}?delete_data=false`, + { method: "DELETE", token }, + ); + if (response.status >= 400 || response.json.code !== 0) { + throw new Error(response.json.msg || `Plugin delete request failed with HTTP ${response.status}.`); + } + const taskId = response.json.data?.task_id ?? null; + if (!taskId) throw new Error("Plugin delete response did not include task_id."); + const task = await waitForTask(backendUrl, token, taskId); + return { id: taskId, ...task }; +} + async function previewPackage(backendUrl, token, bytes, packagePath) { const form = new FormData(); form.set("file", new Blob([bytes]), packagePath.split("/").pop()); diff --git a/skills/scripts/e2e/lib/langbot-e2e.mjs b/skills/scripts/e2e/lib/langbot-e2e.mjs index a7584c904..e733a58d6 100755 --- a/skills/scripts/e2e/lib/langbot-e2e.mjs +++ b/skills/scripts/e2e/lib/langbot-e2e.mjs @@ -1,5 +1,5 @@ import { appendFile, mkdir, readFile, stat, writeFile } from "node:fs/promises"; -import { join, resolve } from "node:path"; +import { basename, join, resolve } from "node:path"; import { env } from "node:process"; const secretRe = /(?:authorization|bearer|token|secret|password|api[_-]?key|jwt|oauth)\s*[:=]\s*["']?[^"',\s]+/gi; @@ -71,6 +71,86 @@ export async function writeResult(paths, result) { } } +function browserDiagnosticFindings(source, text) { + const findings = []; + const lines = String(text || "").split(/\r?\n/); + for (let index = 0; index < lines.length; index += 1) { + const line = lines[index]; + if (!line) continue; + const lineNumber = index + 1; + + if (source === "console") { + const checks = [ + ["pageerror", /\[pageerror\]/i], + ["frontend_uncaught_error", /\[error\].*(?:\bUncaught\b|Unhandled(?: promise rejection|Rejection)|TypeError|ReferenceError)/i], + ["http_5xx", /Failed to load resource: the server responded with a status of 5\d\d/i], + ["api_server_error", /\[error\].*Server error:/i], + ["plugin_runtime_timeout", /\[error\].*Action [A-Za-z0-9_]+ call timed out/i], + ]; + for (const [kind, regex] of checks) { + if (!regex.test(line)) continue; + findings.push({ + source, + severity: "fail", + kind, + line: lineNumber, + excerpt: redact(line.trim()), + }); + break; + } + continue; + } + + if (source === "network") { + if (/\[response\]\s+5\d\d\b/i.test(line)) { + findings.push({ + source, + severity: "fail", + kind: "http_5xx", + line: lineNumber, + excerpt: redact(line.trim()), + }); + continue; + } + if (/\[requestfailed\]/i.test(line) && !/net::ERR_ABORTED/i.test(line)) { + findings.push({ + source, + severity: "warning", + kind: "request_failed", + line: lineNumber, + excerpt: redact(line.trim()), + }); + } + } + } + return findings; +} + +export async function scanBrowserDiagnostics(paths) { + const sources = [ + ["console", paths.consoleLog], + ["network", paths.networkLog], + ]; + const findings = []; + for (const [source, path] of sources) { + let text = ""; + try { + text = await readFile(path, "utf8"); + } catch { + continue; + } + findings.push(...browserDiagnosticFindings(source, text)); + } + const hasFailure = findings.some((finding) => finding.severity === "fail"); + return { + status: hasFailure ? "fail" : "pass", + findings, + reason: hasFailure + ? `Browser diagnostics found ${findings.filter((finding) => finding.severity === "fail").length} failing signal(s).` + : "No failing browser diagnostics found.", + }; +} + export async function loadEnvFiles(paths = ["skills/.env", "skills/.env.local"]) { const processEnvKeys = new Set(Object.keys(env)); for (const path of paths) { @@ -92,8 +172,28 @@ export async function loadEnvFiles(paths = ["skills/.env", "skills/.env.local"]) } } -export async function readRecoveryKey(repo = env.LANGBOT_REPO || "../LangBot") { - const configPath = resolve(repo, "data/config.yaml"); +export async function resolveLangBotRepo(repo = env.LANGBOT_REPO || "", cwd = process.cwd()) { + if (repo) return resolve(repo); + + const candidates = [ + resolve(cwd), + basename(cwd) === "skills" ? resolve(cwd, "..") : "", + resolve(cwd, "../LangBot"), + resolve(cwd, "LangBot"), + ].filter(Boolean); + + const seen = new Set(); + for (const candidate of candidates) { + if (seen.has(candidate)) continue; + seen.add(candidate); + if (await pathExists(resolve(candidate, "data/config.yaml"))) return candidate; + } + + return resolve(cwd, "../LangBot"); +} + +export async function readRecoveryKey(repo = env.LANGBOT_REPO || "") { + const configPath = resolve(await resolveLangBotRepo(repo), "data/config.yaml"); const config = await readFile(configPath, "utf8"); const match = config.match(/^\s*recovery_key:\s*['"]?([^'"\s#]+)['"]?\s*$/m); return match?.[1] || ""; @@ -202,6 +302,62 @@ export async function verifyBrowserToken(page, backendUrl) { }, backendUrl); } +export async function ensureAuthenticatedBrowser(page, { + frontendUrl = env.LANGBOT_FRONTEND_URL || "", + backendUrl = env.LANGBOT_BACKEND_URL || "", + user = env.LANGBOT_E2E_LOGIN_USER || "", + password = env.LANGBOT_E2E_LOGIN_PASSWORD || "LangBotE2ELocalPass!2026", + recoveryKey = "", +} = {}) { + if (!frontendUrl) return { status: "env_issue", reason: "LANGBOT_FRONTEND_URL is not configured." }; + if (!backendUrl) return { status: "env_issue", reason: "LANGBOT_BACKEND_URL is not configured." }; + + const current = await verifyBrowserToken(page, backendUrl).catch((error) => ({ + authenticated: false, + reason: error.message, + })); + if (current.authenticated) { + return { + status: "pass", + reason: "Existing browser token is valid.", + backend_token_check: null, + browser_token_check: current, + injected: false, + }; + } + + if (!user) { + return { + status: "blocked", + reason: "Browser profile is not authenticated for LANGBOT_FRONTEND_URL, and LANGBOT_E2E_LOGIN_USER is not configured for automatic local login.", + backend_token_check: null, + browser_token_check: current, + injected: false, + }; + } + + const auth = await resetAndAuthLocalUser({ backendUrl, user, password, recoveryKey }); + await setBrowserToken(page, frontendUrl, auth.token); + const browserCheck = await verifyBrowserToken(page, backendUrl); + if (!browserCheck.authenticated) { + return { + status: "blocked", + reason: browserCheck.reason || "Browser token check failed after automatic local login.", + backend_token_check: auth.check, + browser_token_check: browserCheck, + injected: true, + }; + } + + return { + status: "pass", + reason: "Browser token injected from local recovery login.", + backend_token_check: auth.check, + browser_token_check: browserCheck, + injected: true, + }; +} + export function exitCode(status) { if (status === "pass") return 0; if (status === "blocked" || status === "env_issue") return 2; diff --git a/skills/scripts/e2e/local-agent-steering-debug-chat.mjs b/skills/scripts/e2e/local-agent-steering-debug-chat.mjs index dae2e265c..1a9bcfee7 100755 --- a/skills/scripts/e2e/local-agent-steering-debug-chat.mjs +++ b/skills/scripts/e2e/local-agent-steering-debug-chat.mjs @@ -11,6 +11,7 @@ import { } from "./lib/debug-chat.mjs"; import { createBrowser, + ensureAuthenticatedBrowser, ensureEvidence, evidencePaths, exitCode, @@ -74,6 +75,7 @@ const result = { pipeline_config: null, debug_chat_reset: null, tool_diagnostic: null, + browser_auth: null, steering: null, evidence: { console_log: paths.consoleLog, @@ -95,6 +97,18 @@ try { browser = await createBrowser(paths); const { page } = browser; + const authDiagnostic = await ensureAuthenticatedBrowser(page, { + frontendUrl: env.LANGBOT_FRONTEND_URL || "", + backendUrl, + }); + result.browser_auth = authDiagnostic; + if (!result.evidence_collected.includes("api_diagnostic")) result.evidence_collected.push("api_diagnostic"); + if (authDiagnostic.status === "env_issue" || authDiagnostic.status === "blocked" || authDiagnostic.status === "fail") { + result.status = authDiagnostic.status; + result.reason = authDiagnostic.reason || "Browser authentication failed."; + throw new Error(result.reason); + } + const openResult = await openPipelineDebugChat(page, { pipelineUrl, pipelineName, diff --git a/skills/scripts/e2e/mcp-stdio-register.mjs b/skills/scripts/e2e/mcp-stdio-register.mjs index e2e31a9ad..07747ac55 100755 --- a/skills/scripts/e2e/mcp-stdio-register.mjs +++ b/skills/scripts/e2e/mcp-stdio-register.mjs @@ -1,7 +1,7 @@ #!/usr/bin/env node import { existsSync, readFileSync } from "node:fs"; -import { writeFile } from "node:fs/promises"; +import { readFile, writeFile } from "node:fs/promises"; import { resolve } from "node:path"; import { env } from "node:process"; import { @@ -46,6 +46,8 @@ const startupTimeoutSec = Number(env.LANGBOT_MCP_STARTUP_TIMEOUT_SEC || "300"); const readyTimeoutMs = Number(env.LANGBOT_MCP_READY_TIMEOUT_MS || "360000"); const backendUrl = env.LANGBOT_BACKEND_URL || ""; const apiDiagnosticPath = resolve(paths.evidenceDir, "api-diagnostic.json"); +const envLocalPath = resolve("skills/.env.local"); +const serverUuidEnvKey = env.LANGBOT_MCP_SERVER_UUID_ENV_KEY || "LANGBOT_MCP_QA_STDIO_SERVER_UUID"; let browser; const result = { @@ -70,6 +72,7 @@ const result = { result_json: paths.resultJson, }, evidence_collected: ["api_diagnostic"], + wrote_env: false, }; async function run() { @@ -159,6 +162,7 @@ async function run() { const deadline = Date.now() + readyTimeoutMs; let lastTools = []; let lastRuntime = null; + let lastServer = null; while (Date.now() < deadline) { await new Promise((resolveReady) => setTimeout(resolveReady, 500)); const tools = await getJson("/api/v1/tools"); @@ -167,7 +171,8 @@ async function run() { .map((tool) => tool.name || tool.tool_name || tool.function?.name || "") .filter(Boolean) .sort(); - lastRuntime = server.json.data?.server?.runtime_info || null; + lastServer = server.json.data?.server || null; + lastRuntime = lastServer?.runtime_info || null; if (lastTools.includes(expectedTool)) break; } @@ -177,6 +182,7 @@ async function run() { save_status: save.status, save_code: save.json.code ?? null, save_msg: save.json.msg || "", + server_uuid: lastServer?.uuid || save.json.data?.uuid || "", tool_names: lastTools, has_expected_tool: lastTools.includes(expectedTool), runtime_status: lastRuntime?.status || null, @@ -212,11 +218,47 @@ async function run() { result.reason = `MCP server ${serverName} did not expose ${expectedTool}. See ${apiDiagnosticPath}.`; return; } + if (!diagnostic.server_uuid) { + result.status = "fail"; + result.reason = `MCP server ${serverName} exposed ${expectedTool}, but the server UUID was not returned. See ${apiDiagnosticPath}.`; + return; + } + + await upsertEnvLocal(envLocalPath, { + [serverUuidEnvKey]: diagnostic.server_uuid, + }); + result.wrote_env = true; + result.server_uuid = diagnostic.server_uuid; + result.server_uuid_env_key = serverUuidEnvKey; result.status = "pass"; result.reason = `MCP server ${serverName} is connected and exposes ${expectedTool} through LangBot /api/v1/tools.`; } +async function upsertEnvLocal(path, updates) { + let text = ""; + try { + text = await readFile(path, "utf8"); + } catch { + text = ""; + } + + const lines = text ? text.split(/\r?\n/) : []; + const seen = new Set(); + const updated = lines.map((line) => { + const match = line.match(/^([A-Z][A-Z0-9_]*)=/); + if (!match || !Object.prototype.hasOwnProperty.call(updates, match[1])) return line; + seen.add(match[1]); + return `${match[1]}=${updates[match[1]]}`; + }); + + for (const [key, value] of Object.entries(updates)) { + if (!seen.has(key)) updated.push(`${key}=${value}`); + } + + await writeFile(path, `${updated.filter((line, index) => line || index < updated.length - 1).join("\n")}\n`, "utf8"); +} + try { await run(); } catch (error) { diff --git a/skills/scripts/e2e/pipeline-debug-chat.mjs b/skills/scripts/e2e/pipeline-debug-chat.mjs index 4b20f7757..2fcf81405 100755 --- a/skills/scripts/e2e/pipeline-debug-chat.mjs +++ b/skills/scripts/e2e/pipeline-debug-chat.mjs @@ -11,12 +11,14 @@ import { } from "./lib/debug-chat.mjs"; import { createBrowser, + ensureAuthenticatedBrowser, ensureEvidence, evidencePaths, exitCode, localIsoWithOffset, pathExists, safeScreenshot, + scanBrowserDiagnostics, writeResult, } from "./lib/langbot-e2e.mjs"; @@ -50,15 +52,19 @@ const pipelineName = pipelineRequired const expectedRunnerId = env.LANGBOT_E2E_EXPECTED_RUNNER_ID || ""; const resetDebugChat = boolFromEnv(env.LANGBOT_E2E_RESET_DEBUG_CHAT, false); const restoreRunnerConfig = boolFromEnv(env.LANGBOT_E2E_RESTORE_RUNNER_CONFIG, true); +const restoreExtensions = boolFromEnv(env.LANGBOT_E2E_RESTORE_EXTENSIONS, true); const debugChatSessionType = env.LANGBOT_E2E_DEBUG_CHAT_SESSION_TYPE || "person"; const pipelineConfigDiagnosticPath = resolve(paths.evidenceDir, "pipeline-config-diagnostic.json"); +const pipelineExtensionsDiagnosticPath = resolve(paths.evidenceDir, "pipeline-extensions-diagnostic.json"); const debugChatResetDiagnosticPath = resolve(paths.evidenceDir, "debug-chat-reset-diagnostic.json"); const pipelineConfigRestoreDiagnosticPath = resolve(paths.evidenceDir, "pipeline-config-restore-diagnostic.json"); +const pipelineExtensionsRestoreDiagnosticPath = resolve(paths.evidenceDir, "pipeline-extensions-restore-diagnostic.json"); const metricsPath = resolve(paths.evidenceDir, "metrics.json"); const startedAt = new Date(); let browser; -let restorePlan = null; +let restoreConfigPlan = null; +let restoreExtensionsPlan = null; let result = { source: "automation", case_id: caseId, @@ -317,6 +323,11 @@ function sanitizePipelineDiagnostic(diagnostic) { return safe; } +function sanitizePipelineExtensionsDiagnostic(diagnostic) { + const { restore_extensions: _restoreExtensions, ...safe } = diagnostic || {}; + return safe; +} + async function prepareImageFixture(paths) { if (imagePathEnv) return resolve(imagePathEnv); if (!imageBase64Path) return ""; @@ -451,6 +462,31 @@ async function inspectAndPatchPipelineConfig(page, { ? runnerConfigs[runnerId] : {}; const patchKeys = Object.keys(runnerConfigPatch || {}); + const sensitiveConfigKeyRe = /(?:api[_-]?key|authorization|bearer|credential|jwt|oauth|password|secret)/i; + const sanitizeConfigValue = (key, value, depth = 0) => { + if (sensitiveConfigKeyRe.test(String(key))) return "[redacted]"; + if (typeof value === "string") { + return value.length > 1200 ? `${value.slice(0, 1200)}...[truncated ${value.length - 1200} chars]` : value; + } + if (Array.isArray(value)) { + const items = value.slice(0, 50).map((item) => sanitizeConfigValue(key, item, depth + 1)); + if (value.length > 50) items.push(`[truncated ${value.length - 50} items]`); + return items; + } + if (value && typeof value === "object") { + if (depth >= 3) return "[object truncated]"; + return Object.fromEntries( + Object.entries(value).map(([childKey, childValue]) => [ + childKey, + sanitizeConfigValue(childKey, childValue, depth + 1), + ]), + ); + } + return value; + }; + const pickPatchValues = (config) => Object.fromEntries( + patchKeys.map((key) => [key, sanitizeConfigValue(key, config?.[key])]), + ); const baseDiagnostic = { status: "ready", authenticated: true, @@ -462,6 +498,7 @@ async function inspectAndPatchPipelineConfig(page, { expected_runner_id: expectedRunnerId || "", patch_keys: patchKeys, runner_config_before_keys: Object.keys(currentRunnerConfig), + runner_config_patch_before: pickPatchValues(currentRunnerConfig), patched: patchKeys.length > 0, }; @@ -506,6 +543,7 @@ async function inspectAndPatchPipelineConfig(page, { put_status: update.status, put_code: update.json.code ?? null, runner_config_after_keys: Object.keys(updatedRunnerConfig), + runner_config_patch_after: pickPatchValues(updatedRunnerConfig), restore_config: config, }; }, { @@ -548,6 +586,201 @@ async function restorePipelineConfig(page, { backendUrl, pipelineId, config }) { }, { backendUrl, pipelineId, config }); } +async function inspectAndPatchPipelineExtensions(page, { + backendUrl, + pipelineId, + extensionsPatch, +}) { + return await page.evaluate(async ({ + backendUrl, + pipelineId, + extensionsPatch, + }) => { + const token = localStorage.getItem("token"); + if (!token) { + return { + status: "blocked", + authenticated: false, + pipeline_id: pipelineId, + reason: "Browser profile has no localStorage token.", + }; + } + + const headers = { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + }; + const getJson = async (path) => { + const response = await fetch(`${backendUrl}${path}`, { headers }); + return { + status: response.status, + json: await response.json().catch(() => ({})), + }; + }; + const putJson = async (path, body) => { + const response = await fetch(`${backendUrl}${path}`, { + method: "PUT", + headers, + body: JSON.stringify(body), + }); + return { + status: response.status, + json: await response.json().catch(() => ({})), + }; + }; + + if (!pipelineId) { + return { + status: "blocked", + authenticated: true, + pipeline_resolved: false, + reason: "Pipeline id is required before patching extensions.", + }; + } + + const before = await getJson(`/api/v1/pipelines/${encodeURIComponent(pipelineId)}/extensions`); + let extensions = before.json.data || {}; + let getStatus = before.status; + let getCode = before.json.code ?? null; + let fallbackReason = ""; + if (before.status >= 400 || before.json.code !== 0) { + const pipelineResponse = await getJson(`/api/v1/pipelines/${encodeURIComponent(pipelineId)}`); + const pipeline = pipelineResponse.json.data?.pipeline || {}; + const prefs = pipeline.extensions_preferences || {}; + if (pipelineResponse.status >= 400 || pipelineResponse.json.code !== 0 || !pipeline.uuid) { + return { + status: "fail", + authenticated: true, + pipeline_id: pipelineId, + get_status: before.status, + get_code: before.json.code ?? null, + fallback_pipeline_status: pipelineResponse.status, + fallback_pipeline_code: pipelineResponse.json.code ?? null, + reason: before.json.msg || "Could not load pipeline extensions.", + }; + } + fallbackReason = before.json.msg || "Could not load pipeline extensions; restored from pipeline preferences."; + extensions = { + enable_all_plugins: prefs.enable_all_plugins ?? true, + enable_all_mcp_servers: prefs.enable_all_mcp_servers ?? true, + enable_all_skills: prefs.enable_all_skills ?? true, + bound_plugins: prefs.plugins || [], + bound_mcp_servers: prefs.mcp_servers || [], + bound_skills: prefs.skills || [], + }; + } + + const patchKeys = Object.keys(extensionsPatch || {}); + const restoreExtensions = { + enable_all_plugins: extensions.enable_all_plugins ?? true, + enable_all_mcp_servers: extensions.enable_all_mcp_servers ?? true, + enable_all_skills: extensions.enable_all_skills ?? true, + bound_plugins: extensions.bound_plugins || [], + bound_mcp_servers: extensions.bound_mcp_servers || [], + bound_skills: extensions.bound_skills || [], + }; + const baseDiagnostic = { + status: "ready", + authenticated: true, + pipeline_id: pipelineId, + patch_keys: patchKeys, + patched: patchKeys.length > 0, + get_status: getStatus, + get_code: getCode, + fallback_reason: fallbackReason, + extensions_before: { + enable_all_plugins: restoreExtensions.enable_all_plugins, + enable_all_mcp_servers: restoreExtensions.enable_all_mcp_servers, + enable_all_skills: restoreExtensions.enable_all_skills, + bound_plugins: restoreExtensions.bound_plugins, + bound_mcp_servers: restoreExtensions.bound_mcp_servers, + bound_skills: restoreExtensions.bound_skills, + }, + }; + + if (patchKeys.length === 0) { + return baseDiagnostic; + } + + const updateBody = { + enable_all_plugins: Object.prototype.hasOwnProperty.call(extensionsPatch, "enable_all_plugins") + ? Boolean(extensionsPatch.enable_all_plugins) + : restoreExtensions.enable_all_plugins, + enable_all_mcp_servers: Object.prototype.hasOwnProperty.call(extensionsPatch, "enable_all_mcp_servers") + ? Boolean(extensionsPatch.enable_all_mcp_servers) + : restoreExtensions.enable_all_mcp_servers, + enable_all_skills: Object.prototype.hasOwnProperty.call(extensionsPatch, "enable_all_skills") + ? Boolean(extensionsPatch.enable_all_skills) + : restoreExtensions.enable_all_skills, + bound_plugins: Object.prototype.hasOwnProperty.call(extensionsPatch, "bound_plugins") + ? extensionsPatch.bound_plugins + : restoreExtensions.bound_plugins, + bound_mcp_servers: Object.prototype.hasOwnProperty.call(extensionsPatch, "bound_mcp_servers") + ? extensionsPatch.bound_mcp_servers + : restoreExtensions.bound_mcp_servers, + bound_skills: Object.prototype.hasOwnProperty.call(extensionsPatch, "bound_skills") + ? extensionsPatch.bound_skills + : restoreExtensions.bound_skills, + }; + + const update = await putJson(`/api/v1/pipelines/${encodeURIComponent(pipelineId)}/extensions`, updateBody); + if (update.status >= 400 || update.json.code !== 0) { + return { + ...baseDiagnostic, + status: "fail", + put_status: update.status, + put_code: update.json.code ?? null, + reason: update.json.msg || "Pipeline extensions update failed.", + }; + } + + return { + ...baseDiagnostic, + put_status: update.status, + put_code: update.json.code ?? null, + extensions_after: updateBody, + restore_extensions: restoreExtensions, + }; + }, { + backendUrl, + pipelineId, + extensionsPatch, + }); +} + +async function restorePipelineExtensions(page, { backendUrl, pipelineId, extensions }) { + return await page.evaluate(async ({ backendUrl, pipelineId, extensions }) => { + const token = localStorage.getItem("token"); + if (!token) { + return { + status: "blocked", + authenticated: false, + pipeline_id: pipelineId, + reason: "Browser profile has no localStorage token.", + }; + } + const response = await fetch(`${backendUrl}/api/v1/pipelines/${encodeURIComponent(pipelineId)}/extensions`, { + method: "PUT", + headers: { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(extensions), + }); + const json = await response.json().catch(() => ({})); + return { + status: response.status >= 400 || json.code !== 0 ? "fail" : "ready", + authenticated: true, + pipeline_id: pipelineId, + put_status: response.status, + put_code: json.code ?? null, + reason: response.status >= 400 || json.code !== 0 + ? json.msg || "Pipeline extensions restore failed." + : "Pipeline extensions restored.", + }; + }, { backendUrl, pipelineId, extensions }); +} + async function resetPipelineDebugChat(page, { backendUrl, pipelineId, sessionType }) { return await page.evaluate(async ({ backendUrl, pipelineId, sessionType }) => { const token = localStorage.getItem("token"); @@ -590,11 +823,13 @@ try { const promptSteps = promptStepsFromEnv(); const filesystemChecks = parseJsonEnv("LANGBOT_E2E_FILESYSTEM_CHECKS_JSON", []); const runnerConfigPatch = parseJsonEnv("LANGBOT_E2E_RUNNER_CONFIG_PATCH_JSON", {}); + const extensionsPatch = parseJsonEnv("LANGBOT_E2E_EXTENSIONS_PATCH_JSON", {}); const runnerPatchKeys = Object.keys(runnerConfigPatch); - if (runnerPatchKeys.length > 0 || resetDebugChat || expectedRunnerId) { + const extensionsPatchKeys = Object.keys(extensionsPatch); + if (runnerPatchKeys.length > 0 || extensionsPatchKeys.length > 0 || resetDebugChat || expectedRunnerId) { if (!backendUrl) { result.status = "env_issue"; - result.reason = "LANGBOT_BACKEND_URL is required for runner config patch, runner assertion, or Debug Chat reset."; + result.reason = "LANGBOT_BACKEND_URL is required for runner config patch, extensions patch, runner assertion, or Debug Chat reset."; throw new Error(result.reason); } } @@ -602,13 +837,30 @@ try { result.prompt = promptSteps.length === 1 ? promptSteps[0].prompt : `${promptSteps.length} prompts`; result.expected_text = promptSteps.at(-1)?.expectedText || expectedText; - const openResult = await openPipelineDebugChat(page, { - pipelineUrl, - pipelineName, - envHint: pipelineRequired - ? "case-specific pipeline env mapped to LANGBOT_E2E_PIPELINE_URL or LANGBOT_E2E_PIPELINE_NAME" - : "LANGBOT_PIPELINE_URL or LANGBOT_PIPELINE_NAME", + const authDiagnostic = await ensureAuthenticatedBrowser(page, { + frontendUrl: env.LANGBOT_FRONTEND_URL || "", + backendUrl, }); + result.browser_auth = authDiagnostic; + if (!result.evidence_collected.includes("api_diagnostic")) result.evidence_collected.push("api_diagnostic"); + const authFailed = authDiagnostic.status === "env_issue" || authDiagnostic.status === "blocked" || authDiagnostic.status === "fail"; + if (authFailed) { + result.status = authDiagnostic.status; + result.reason = authDiagnostic.reason || "Browser authentication failed."; + } else { + result.status = "running"; + result.reason = ""; + } + + const openResult = authFailed + ? { opened: false, status: result.status, reason: result.reason } + : await openPipelineDebugChat(page, { + pipelineUrl, + pipelineName, + envHint: pipelineRequired + ? "case-specific pipeline env mapped to LANGBOT_E2E_PIPELINE_URL or LANGBOT_E2E_PIPELINE_NAME" + : "LANGBOT_PIPELINE_URL or LANGBOT_PIPELINE_NAME", + }); result.url = page.url(); if (!openResult.opened) { @@ -617,7 +869,7 @@ try { } else { result.status = "running"; result.reason = ""; - if (runnerPatchKeys.length > 0 || resetDebugChat || expectedRunnerId) { + if (runnerPatchKeys.length > 0 || extensionsPatchKeys.length > 0 || resetDebugChat || expectedRunnerId) { const pipelineDiagnostic = await inspectAndPatchPipelineConfig(page, { backendUrl, pipelineUrl, @@ -636,13 +888,35 @@ try { result.reason = pipelineDiagnostic.reason || "Pipeline config preparation failed."; } else { if (pipelineDiagnostic.restore_config && restoreRunnerConfig) { - restorePlan = { + restoreConfigPlan = { backendUrl, pipelineId: pipelineDiagnostic.pipeline_id, config: pipelineDiagnostic.restore_config, }; } - if (resetDebugChat) { + if (extensionsPatchKeys.length > 0) { + const extensionsDiagnostic = await inspectAndPatchPipelineExtensions(page, { + backendUrl, + pipelineId: pipelineDiagnostic.pipeline_id, + extensionsPatch, + }); + const safeExtensionsDiagnostic = sanitizePipelineExtensionsDiagnostic(extensionsDiagnostic); + await writeFile(pipelineExtensionsDiagnosticPath, `${JSON.stringify(safeExtensionsDiagnostic, null, 2)}\n`, "utf8"); + result.evidence.pipeline_extensions_diagnostic_json = pipelineExtensionsDiagnosticPath; + result.pipeline_extensions = safeExtensionsDiagnostic; + if (!result.evidence_collected.includes("api_diagnostic")) result.evidence_collected.push("api_diagnostic"); + if (extensionsDiagnostic.status === "fail" || extensionsDiagnostic.status === "blocked") { + result.status = extensionsDiagnostic.status; + result.reason = extensionsDiagnostic.reason || "Pipeline extensions preparation failed."; + } else if (extensionsDiagnostic.restore_extensions && restoreExtensions) { + restoreExtensionsPlan = { + backendUrl, + pipelineId: extensionsDiagnostic.pipeline_id, + extensions: extensionsDiagnostic.restore_extensions, + }; + } + } + if (!["fail", "blocked", "env_issue"].includes(result.status) && resetDebugChat) { const resetDiagnostic = await resetPipelineDebugChat(page, { backendUrl, pipelineId: pipelineDiagnostic.pipeline_id, @@ -720,6 +994,15 @@ try { result.reason = filesystemResult.reason || "Filesystem checks failed."; } } + + if (result.status === "pass" && !boolFromEnv(env.LANGBOT_E2E_ALLOW_BROWSER_ERRORS, false)) { + const browserDiagnostics = await scanBrowserDiagnostics(paths); + result.browser_diagnostics = browserDiagnostics; + if (browserDiagnostics.status === "fail") { + result.status = "fail"; + result.reason = browserDiagnostics.reason; + } + } } } catch (error) { if (!["env_issue", "blocked", "fail", "pass"].includes(result.status) || !result.reason) { @@ -728,10 +1011,20 @@ try { result.reason = result.reason || error.message; } finally { if (browser?.page) await safeScreenshot(browser.page, paths.screenshot); - if (browser?.page && restorePlan) { - const restoreDiagnostic = await restorePipelineConfig(browser.page, restorePlan).catch((error) => ({ + if (browser?.page && restoreExtensionsPlan) { + const restoreDiagnostic = await restorePipelineExtensions(browser.page, restoreExtensionsPlan).catch((error) => ({ status: "fail", - pipeline_id: restorePlan.pipelineId, + pipeline_id: restoreExtensionsPlan.pipelineId, + reason: error.message, + })); + await writeFile(pipelineExtensionsRestoreDiagnosticPath, `${JSON.stringify(restoreDiagnostic, null, 2)}\n`, "utf8"); + result.evidence.pipeline_extensions_restore_diagnostic_json = pipelineExtensionsRestoreDiagnosticPath; + result.pipeline_extensions_restore = restoreDiagnostic; + } + if (browser?.page && restoreConfigPlan) { + const restoreDiagnostic = await restorePipelineConfig(browser.page, restoreConfigPlan).catch((error) => ({ + status: "fail", + pipeline_id: restoreConfigPlan.pipelineId, reason: error.message, })); await writeFile(pipelineConfigRestoreDiagnosticPath, `${JSON.stringify(restoreDiagnostic, null, 2)}\n`, "utf8"); diff --git a/skills/skills.index.json b/skills/skills.index.json index fa571e440..72a76cb72 100644 --- a/skills/skills.index.json +++ b/skills/skills.index.json @@ -166,14 +166,19 @@ "langrag-parser-golden-e2e", "langrag-sentinel-kb-discover", "local-agent-basic-debug-chat", + "local-agent-combo-rag-compaction-tool-debug-chat", "local-agent-context-compaction-debug-chat", "local-agent-effective-prompt-debug-chat", "local-agent-multimodal-debug-chat", + "local-agent-multitool-rag-compaction-debug-chat", "local-agent-nonstreaming-debug-chat", + "local-agent-parallel-tools-rag-compaction-debug-chat", "local-agent-plugin-tool-call-debug-chat", "local-agent-rag-debug-chat", "local-agent-rag-multimodal-debug-chat", "local-agent-steering-debug-chat", + "local-agent-tool-error-recovery-debug-chat", + "local-agent-tool-loop-limit-debug-chat", "mcp-stdio-register", "mcp-stdio-tool-call", "pipeline-debug-chat", @@ -911,6 +916,43 @@ "backend_log" ] }, + { + "id": "local-agent-combo-rag-compaction-tool-debug-chat", + "title": "Local Agent preserves RAG, compacted history, and plugin tool result in one Debug Chat run", + "mode": "agent-browser", + "area": "pipeline", + "type": "regression", + "priority": "p1", + "risk": "high", + "ci_eligible": false, + "tags": [ + "local-agent", + "langrag", + "context", + "compaction", + "plugin", + "tools", + "pipeline" + ], + "automation": "scripts/e2e/pipeline-debug-chat.mjs", + "setup_automation": [ + "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env", + "node:scripts/e2e/ensure-langrag-sentinel-kb.mjs --write-env", + "case:qa-plugin-smoke-live-install" + ], + "setup_provides_env": [ + "LANGBOT_LOCAL_AGENT_PIPELINE_URL", + "LANGBOT_LOCAL_AGENT_PIPELINE_NAME", + "LANGBOT_LOCAL_AGENT_RAG_KB_UUID" + ], + "evidence_required": [ + "ui", + "screenshot", + "console", + "backend_log", + "api_diagnostic" + ] + }, { "id": "local-agent-context-compaction-debug-chat", "title": "Local Agent compacts long Debug Chat history and preserves older facts", @@ -1002,9 +1044,47 @@ "backend_log" ] }, + { + "id": "local-agent-multitool-rag-compaction-debug-chat", + "title": "Local Agent preserves RAG, compacted history, and two-step plugin tool loop", + "mode": "agent-browser", + "area": "pipeline", + "type": "regression", + "priority": "p1", + "risk": "high", + "ci_eligible": false, + "tags": [ + "local-agent", + "langrag", + "context", + "compaction", + "plugin", + "tools", + "tool-loop", + "pipeline" + ], + "automation": "scripts/e2e/pipeline-debug-chat.mjs", + "setup_automation": [ + "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env", + "node:scripts/e2e/ensure-langrag-sentinel-kb.mjs --write-env", + "case:qa-plugin-smoke-live-install" + ], + "setup_provides_env": [ + "LANGBOT_LOCAL_AGENT_PIPELINE_URL", + "LANGBOT_LOCAL_AGENT_PIPELINE_NAME", + "LANGBOT_LOCAL_AGENT_RAG_KB_UUID" + ], + "evidence_required": [ + "ui", + "screenshot", + "console", + "backend_log", + "api_diagnostic" + ] + }, { "id": "local-agent-nonstreaming-debug-chat", - "title": "Local Agent Debug Chat returns a deterministic non-streaming response", + "title": "Local Agent Debug Chat returns a deterministic response with UI streaming disabled", "mode": "agent-browser", "area": "pipeline", "type": "regression", @@ -1030,6 +1110,44 @@ "backend_log" ] }, + { + "id": "local-agent-parallel-tools-rag-compaction-debug-chat", + "title": "Local Agent preserves RAG, compacted history, and parallel plugin tools", + "mode": "agent-browser", + "area": "pipeline", + "type": "regression", + "priority": "p1", + "risk": "high", + "ci_eligible": false, + "tags": [ + "local-agent", + "langrag", + "context", + "compaction", + "plugin", + "tools", + "parallel-tools", + "pipeline" + ], + "automation": "scripts/e2e/pipeline-debug-chat.mjs", + "setup_automation": [ + "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env", + "node:scripts/e2e/ensure-langrag-sentinel-kb.mjs --write-env", + "case:qa-plugin-smoke-live-install" + ], + "setup_provides_env": [ + "LANGBOT_LOCAL_AGENT_PIPELINE_URL", + "LANGBOT_LOCAL_AGENT_PIPELINE_NAME", + "LANGBOT_LOCAL_AGENT_RAG_KB_UUID" + ], + "evidence_required": [ + "ui", + "screenshot", + "console", + "backend_log", + "api_diagnostic" + ] + }, { "id": "local-agent-plugin-tool-call-debug-chat", "title": "Local Agent can call a plugin-provided tool", @@ -1157,6 +1275,73 @@ "api_diagnostic" ] }, + { + "id": "local-agent-tool-error-recovery-debug-chat", + "title": "Local Agent feeds plugin tool errors back to the model", + "mode": "agent-browser", + "area": "pipeline", + "type": "regression", + "priority": "p1", + "risk": "high", + "ci_eligible": false, + "tags": [ + "local-agent", + "plugin", + "tools", + "tool-error", + "pipeline" + ], + "automation": "scripts/e2e/pipeline-debug-chat.mjs", + "setup_automation": [ + "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env", + "case:qa-plugin-smoke-live-install" + ], + "setup_provides_env": [ + "LANGBOT_LOCAL_AGENT_PIPELINE_URL", + "LANGBOT_LOCAL_AGENT_PIPELINE_NAME" + ], + "evidence_required": [ + "ui", + "screenshot", + "console", + "backend_log", + "api_diagnostic" + ] + }, + { + "id": "local-agent-tool-loop-limit-debug-chat", + "title": "Local Agent stops repeated tool calls at max-tool-iterations", + "mode": "agent-browser", + "area": "pipeline", + "type": "regression", + "priority": "p1", + "risk": "high", + "ci_eligible": false, + "tags": [ + "local-agent", + "plugin", + "tools", + "tool-loop", + "limit", + "pipeline" + ], + "automation": "scripts/e2e/pipeline-debug-chat.mjs", + "setup_automation": [ + "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env", + "case:qa-plugin-smoke-live-install" + ], + "setup_provides_env": [ + "LANGBOT_LOCAL_AGENT_PIPELINE_URL", + "LANGBOT_LOCAL_AGENT_PIPELINE_NAME" + ], + "evidence_required": [ + "ui", + "screenshot", + "console", + "backend_log", + "api_diagnostic" + ] + }, { "id": "mcp-stdio-register", "title": "MCP stdio fixture is registered and exposes qa_mcp_echo", @@ -1203,7 +1388,8 @@ ], "setup_provides_env": [ "LANGBOT_LOCAL_AGENT_PIPELINE_URL", - "LANGBOT_LOCAL_AGENT_PIPELINE_NAME" + "LANGBOT_LOCAL_AGENT_PIPELINE_NAME", + "LANGBOT_MCP_QA_STDIO_SERVER_UUID" ], "evidence_required": [ "ui", @@ -1483,6 +1669,11 @@ "local-agent-context-compaction-debug-chat", "local-agent-rag-debug-chat", "local-agent-plugin-tool-call-debug-chat", + "local-agent-tool-error-recovery-debug-chat", + "local-agent-tool-loop-limit-debug-chat", + "local-agent-combo-rag-compaction-tool-debug-chat", + "local-agent-multitool-rag-compaction-debug-chat", + "local-agent-parallel-tools-rag-compaction-debug-chat", "mcp-stdio-register", "mcp-stdio-tool-call", "local-agent-nonstreaming-debug-chat", @@ -1633,6 +1824,11 @@ "local-agent-context-compaction-debug-chat", "local-agent-rag-debug-chat", "local-agent-plugin-tool-call-debug-chat", + "local-agent-tool-error-recovery-debug-chat", + "local-agent-tool-loop-limit-debug-chat", + "local-agent-combo-rag-compaction-tool-debug-chat", + "local-agent-multitool-rag-compaction-debug-chat", + "local-agent-parallel-tools-rag-compaction-debug-chat", "local-agent-steering-debug-chat", "mcp-stdio-tool-call", "local-agent-nonstreaming-debug-chat", @@ -1692,7 +1888,10 @@ "path": "fixtures/rag/sentinel-doc.txt", "related_cases": [ "langrag-kb-retrieve", - "local-agent-rag-debug-chat" + "local-agent-rag-debug-chat", + "local-agent-combo-rag-compaction-tool-debug-chat", + "local-agent-multitool-rag-compaction-debug-chat", + "local-agent-parallel-tools-rag-compaction-debug-chat" ] }, { @@ -1724,6 +1923,11 @@ "plugin-e2e-smoke", "local-agent-effective-prompt-debug-chat", "local-agent-plugin-tool-call-debug-chat", + "local-agent-tool-error-recovery-debug-chat", + "local-agent-tool-loop-limit-debug-chat", + "local-agent-combo-rag-compaction-tool-debug-chat", + "local-agent-multitool-rag-compaction-debug-chat", + "local-agent-parallel-tools-rag-compaction-debug-chat", "local-agent-steering-debug-chat" ] }, @@ -1734,7 +1938,8 @@ "path": "fixtures/plugins/qa-plugin-smoke/dist/qa-plugin-smoke-0.1.0.lbpkg", "related_cases": [ "qa-plugin-smoke-live-install", - "plugin-e2e-smoke" + "plugin-e2e-smoke", + "local-agent-tool-error-recovery-debug-chat" ] } ], diff --git a/skills/skills/langbot-testing/cases/local-agent-combo-rag-compaction-tool-debug-chat.yaml b/skills/skills/langbot-testing/cases/local-agent-combo-rag-compaction-tool-debug-chat.yaml new file mode 100644 index 000000000..637445f85 --- /dev/null +++ b/skills/skills/langbot-testing/cases/local-agent-combo-rag-compaction-tool-debug-chat.yaml @@ -0,0 +1,103 @@ +id: local-agent-combo-rag-compaction-tool-debug-chat +title: "Local Agent preserves RAG, compacted history, and plugin tool result in one Debug Chat run" +mode: agent-browser +area: pipeline +type: regression +priority: p1 +risk: high +ci_eligible: false +tags: + - local-agent + - langrag + - context + - compaction + - plugin + - tools + - pipeline +skills: + - langbot-env-setup + - langbot-testing +env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME +env_optional: + - LANGBOT_E2E_FAKE_PROVIDER_BASE_URL +automation: scripts/e2e/pipeline-debug-chat.mjs +automation_env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_BROWSER_PROFILE + - LANGBOT_CHROMIUM_EXECUTABLE + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME + - LANGBOT_LOCAL_AGENT_RAG_KB_UUID +automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL +automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME +automation_expected_runner_id: "plugin:langbot/local-agent/default" +automation_runner_config_patch_json: '{"knowledge-bases":["${LANGBOT_LOCAL_AGENT_RAG_KB_UUID}"],"retrieval-top-k":1,"context-window-tokens":650,"context-reserve-tokens":180,"context-keep-recent-tokens":120,"context-summary-tokens":220,"max-tool-iterations":4,"tool-execution-mode":"serial"}' +automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot","name":"local-agent"},{"author":"qa","name":"plugin-smoke"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}' +automation_restore_runner_config: "1" +automation_restore_extensions: "1" +automation_reset_debug_chat: "1" +automation_debug_chat_session_type: "person" +automation_expected_text: "COMBO_FINAL qa_combo_compaction_sentinel_2406 azalea-cobalt-7421 qa-plugin-smoke:combo-tool-ok-local-agent" +automation_response_timeout_ms: "180000" +automation_prompts_json: '[{"prompt":"请记住这个用于 local-agent 组合回归测试的暗号:qa_combo_compaction_sentinel_2406。请只回复 MEMORY_SET。","expected_text":"MEMORY_SET","response_timeout_ms":"180000"},{"prompt":"下面这轮只用于制造长历史压力,目标标记 RAG_TOOL_COMBO_GOAL 必须被压缩保留。请忽略填充内容,最后只回复 COMBO_CONTEXT_PRESSURE_READY。填充片段 C001 combo context padding. C002 combo context padding. C003 combo context padding. C004 combo context padding. C005 combo context padding. C006 combo context padding. C007 combo context padding. C008 combo context padding. C009 combo context padding. C010 combo context padding. C011 combo context padding. C012 combo context padding. C013 combo context padding. C014 combo context padding. C015 combo context padding. C016 combo context padding. C017 combo context padding. C018 combo context padding. C019 combo context padding. C020 combo context padding. C021 combo context padding. C022 combo context padding. C023 combo context padding. C024 combo context padding. C025 combo context padding. C026 combo context padding. C027 combo context padding. C028 combo context padding. C029 combo context padding. C030 combo context padding. C031 combo context padding. C032 combo context padding. C033 combo context padding. C034 combo context padding. C035 combo context padding. C036 combo context padding. C037 combo context padding. C038 combo context padding. C039 combo context padding. C040 combo context padding.","expected_text":"COMBO_CONTEXT_PRESSURE_READY","response_timeout_ms":"180000"},{"prompt":"qa_combo final check: using the knowledge base, the compacted earlier passcode, and qa_plugin_echo with exactly combo-tool-ok-local-agent, return only COMBO_FINAL plus the passcode, the RAG sentinel, and the plugin tool result.","expected_text":"COMBO_FINAL qa_combo_compaction_sentinel_2406 azalea-cobalt-7421 qa-plugin-smoke:combo-tool-ok-local-agent","response_timeout_ms":"180000"}]' +setup_automation: + - "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env" + - "node:scripts/e2e/ensure-langrag-sentinel-kb.mjs --write-env" + - "case:qa-plugin-smoke-live-install" +setup_provides_env: + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME + - LANGBOT_LOCAL_AGENT_RAG_KB_UUID +preconditions: + - "The selected model route supports function/tool calling and deterministic multi-turn answers, or LANGBOT_E2E_FAKE_PROVIDER_BASE_URL points to scripts/e2e/fake-openai-provider.mjs." + - "The LangRAG sentinel knowledge base contains azalea-cobalt-7421 and qa-plugin-smoke exposes qa_plugin_echo." +steps: + - "Ensure the local-agent pipeline, LangRAG sentinel KB, and qa-plugin-smoke fixture plugin are available." + - "Temporarily bind the LangRAG KB, shrink the runner context budget, and allow serial plugin tool execution." + - "Temporarily bind only langbot/local-agent and qa/plugin-smoke; disable MCP servers and skills to isolate the tool surface." + - "Reset the person Debug Chat session for the target pipeline." + - "Send the passcode memory prompt and wait for MEMORY_SET." + - "Send the long pressure prompt and wait for COMBO_CONTEXT_PRESSURE_READY." + - "Ask the runner to combine compacted passcode, RAG sentinel, and qa_plugin_echo result in one final answer." + - "Restore the original runner config and pipeline extension bindings." +checks: + - "UI: All three user messages appear in Debug Chat." + - "UI: The final Bot message contains COMBO_FINAL qa_combo_compaction_sentinel_2406 azalea-cobalt-7421 qa-plugin-smoke:combo-tool-ok-local-agent." + - "API diagnostic: pipeline-config-diagnostic.json shows knowledge-bases, context token settings, and tool-loop settings were patched." + - "API diagnostic: pipeline-extensions-diagnostic.json shows only langbot/local-agent and qa/plugin-smoke were bound." + - "API diagnostic: restore diagnostics show the original runner config and extension bindings were restored." + - "Logs: Backend completes RAG retrieval, plugin tool call, and Debug Chat response without runner timeout or model setup failure." + - "Console: No unexpected frontend runtime errors appear during the run." +evidence_required: + - ui + - screenshot + - console + - backend_log + - api_diagnostic +diagnostics: + - "For token-free deterministic UI coverage, start scripts/e2e/fake-openai-provider.mjs and pass LANGBOT_E2E_FAKE_PROVIDER_BASE_URL to this case." + - "The ensure-langrag-sentinel-kb setup script verifies LANGBOT_E2E_RAG_EXPECTED_TEXT, defaulting to azalea-cobalt-7421, while this case's Debug Chat expected text is the full COMBO_FINAL string." + - "The fake provider intentionally returns COMBO_MISSING_* if the final model input lacks compacted memory, RAG context, or plugin tool result." + - "If the final answer lacks azalea-cobalt-7421, first rerun ensure-langrag-sentinel-kb and local-agent-rag-debug-chat." + - "If the final answer lacks qa-plugin-smoke:combo-tool-ok-local-agent, inspect /api/v1/tools and the pipeline extensions diagnostic." +success_patterns: + - "Processing request from person_websocket" + - "Streaming completed" +failure_patterns: + - "Action invoke_llm_stream call timed out" + - "All models failed during streaming setup" + - "Task exception was never retrieved" + - "COMBO_MISSING_" + - "survey widget blocks debug chat" +troubleshooting: + - local-agent-model-route-unavailable + - tool-name-collision-between-mcp-and-plugin + - plugin-runtime-timeout + - proxy-env-mismatch + - survey-widget-blocks-debug-chat + - debug-chat-history-contaminates-automation diff --git a/skills/skills/langbot-testing/cases/local-agent-context-compaction-debug-chat.yaml b/skills/skills/langbot-testing/cases/local-agent-context-compaction-debug-chat.yaml index 7c0d6a32a..ee84d5a58 100644 --- a/skills/skills/langbot-testing/cases/local-agent-context-compaction-debug-chat.yaml +++ b/skills/skills/langbot-testing/cases/local-agent-context-compaction-debug-chat.yaml @@ -19,6 +19,8 @@ env: - LANGBOT_BACKEND_URL - LANGBOT_LOCAL_AGENT_PIPELINE_URL - LANGBOT_LOCAL_AGENT_PIPELINE_NAME +env_optional: + - LANGBOT_E2E_FAKE_PROVIDER_BASE_URL automation: scripts/e2e/pipeline-debug-chat.mjs automation_env: - LANGBOT_FRONTEND_URL @@ -31,7 +33,9 @@ automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME automation_expected_runner_id: "plugin:langbot/local-agent/default" automation_runner_config_patch_json: '{"context-window-tokens":225,"context-reserve-tokens":50,"context-keep-recent-tokens":30,"context-summary-tokens":105,"knowledge-bases":[]}' +automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot","name":"local-agent"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}' automation_restore_runner_config: "1" +automation_restore_extensions: "1" automation_reset_debug_chat: "1" automation_debug_chat_session_type: "person" automation_expected_text: "qa_compaction_sentinel_7391" @@ -43,12 +47,13 @@ setup_provides_env: - LANGBOT_LOCAL_AGENT_PIPELINE_URL - LANGBOT_LOCAL_AGENT_PIPELINE_NAME preconditions: - - "The selected model route can follow short deterministic instructions across multiple Debug Chat turns." + - "The selected model route can follow short deterministic instructions across multiple Debug Chat turns, or LANGBOT_E2E_FAKE_PROVIDER_BASE_URL points to scripts/e2e/fake-openai-provider.mjs." steps: - "Open the target local-agent pipeline through LANGBOT_FRONTEND_URL." - "Use the authenticated browser token only inside automation to GET and PUT /api/v1/pipelines/{uuid}." - "Assert the saved runner is plugin:langbot/local-agent/default." - "Temporarily set context-window-tokens, context-reserve-tokens, context-keep-recent-tokens, and context-summary-tokens to force compaction, and clear knowledge-bases so RAG does not answer the memory question." + - "Temporarily bind only the local-agent plugin, disable MCP servers, and disable skills so unrelated visible tools cannot answer or distract the memory question." - "Reset the person Debug Chat session for the target pipeline." - "Send the sentinel memory prompt and wait for MEMORY_SET." - "Send the long padding prompt and wait for CONTEXT_PRESSURE_READY." @@ -59,6 +64,8 @@ checks: - "UI: The final Bot message contains qa_compaction_sentinel_7391." - "API diagnostic: pipeline-config-diagnostic.json shows patched=true and patch_keys include the four token context compaction fields plus knowledge-bases." - "API diagnostic: pipeline-config-restore-diagnostic.json shows the original runner config was restored." + - "API diagnostic: pipeline-extensions-diagnostic.json shows enable_all_plugins=false, bound_plugins contains langbot/local-agent, enable_all_mcp_servers=false, and enable_all_skills=false." + - "API diagnostic: pipeline-extensions-restore-diagnostic.json shows the original extension bindings were restored." - "Logs: Backend completes the multi-turn Debug Chat path without runner timeout or model setup failure." - "Console: No unexpected frontend runtime errors appear during the run." evidence_required: @@ -68,7 +75,8 @@ evidence_required: - backend_log - api_diagnostic diagnostics: - - "If the final sentinel is missing, inspect whether pipeline-config-diagnostic.json targeted ai.runner_config[runnerId], cleared knowledge-bases, and whether the backend log shows the local-agent runner loading the small context settings." + - "For token-free deterministic UI coverage, start scripts/e2e/fake-openai-provider.mjs and pass LANGBOT_E2E_FAKE_PROVIDER_BASE_URL to this case; setup will bind the local-agent pipeline to that fake OpenAI-compatible model." + - "If the final sentinel is missing, inspect whether pipeline-config-diagnostic.json targeted ai.runner_config[runnerId], cleared knowledge-bases, whether pipeline-extensions-diagnostic.json isolated unrelated tools, and whether the backend log shows the local-agent runner loading the small context settings." - "If the model ignores deterministic replies, rerun with a known-good model route before diagnosing ContextAssembler." - "If restore fails, use pipeline-config-restore-diagnostic.json and GET /api/v1/pipelines/{uuid} to confirm the current saved config before retrying." success_patterns: diff --git a/skills/skills/langbot-testing/cases/local-agent-multitool-rag-compaction-debug-chat.yaml b/skills/skills/langbot-testing/cases/local-agent-multitool-rag-compaction-debug-chat.yaml new file mode 100644 index 000000000..85d24a07c --- /dev/null +++ b/skills/skills/langbot-testing/cases/local-agent-multitool-rag-compaction-debug-chat.yaml @@ -0,0 +1,102 @@ +id: local-agent-multitool-rag-compaction-debug-chat +title: "Local Agent preserves RAG, compacted history, and two-step plugin tool loop" +mode: agent-browser +area: pipeline +type: regression +priority: p1 +risk: high +ci_eligible: false +tags: + - local-agent + - langrag + - context + - compaction + - plugin + - tools + - tool-loop + - pipeline +skills: + - langbot-env-setup + - langbot-testing +env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME +env_optional: + - LANGBOT_E2E_FAKE_PROVIDER_BASE_URL +automation: scripts/e2e/pipeline-debug-chat.mjs +automation_env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_BROWSER_PROFILE + - LANGBOT_CHROMIUM_EXECUTABLE + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME + - LANGBOT_LOCAL_AGENT_RAG_KB_UUID +automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL +automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME +automation_expected_runner_id: "plugin:langbot/local-agent/default" +automation_runner_config_patch_json: '{"knowledge-bases":["${LANGBOT_LOCAL_AGENT_RAG_KB_UUID}"],"retrieval-top-k":1,"context-window-tokens":900,"context-reserve-tokens":220,"context-keep-recent-tokens":160,"context-summary-tokens":260,"max-tool-iterations":5,"tool-execution-mode":"serial"}' +automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot","name":"local-agent"},{"author":"qa","name":"plugin-smoke"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}' +automation_restore_runner_config: "1" +automation_restore_extensions: "1" +automation_reset_debug_chat: "1" +automation_debug_chat_session_type: "person" +automation_expected_text: "MULTITOOL_COMBO_FINAL qa_multitool_compaction_sentinel_6718 azalea-cobalt-7421 qa-plugin-smoke:multi-tool-a-local-agent qa-plugin-smoke:multi-tool-b-local-agent" +automation_response_timeout_ms: "180000" +automation_prompts_json: '[{"prompt":"请记住这个用于 local-agent 多工具组合回归测试的暗号:qa_multitool_compaction_sentinel_6718。请只回复 MEMORY_SET。","expected_text":"MEMORY_SET","response_timeout_ms":"180000"},{"prompt":"下面这轮只用于制造长历史压力,目标标记 MULTITOOL_RAG_GOAL 必须被压缩保留。请忽略填充内容,最后只回复 MULTITOOL_CONTEXT_PRESSURE_READY。填充片段 M001 multitool context padding. M002 multitool context padding. M003 multitool context padding. M004 multitool context padding. M005 multitool context padding. M006 multitool context padding. M007 multitool context padding. M008 multitool context padding. M009 multitool context padding. M010 multitool context padding. M011 multitool context padding. M012 multitool context padding. M013 multitool context padding. M014 multitool context padding. M015 multitool context padding. M016 multitool context padding. M017 multitool context padding. M018 multitool context padding. M019 multitool context padding. M020 multitool context padding. M021 multitool context padding. M022 multitool context padding. M023 multitool context padding. M024 multitool context padding. M025 multitool context padding. M026 multitool context padding. M027 multitool context padding. M028 multitool context padding. M029 multitool context padding. M030 multitool context padding. M031 multitool context padding. M032 multitool context padding.","expected_text":"MULTITOOL_CONTEXT_PRESSURE_READY","response_timeout_ms":"180000"},{"prompt":"MULTITOOL_COMBO final check: using the knowledge base, the compacted earlier passcode, and qa_plugin_echo twice with exactly multi-tool-a-local-agent then multi-tool-b-local-agent, return only MULTITOOL_COMBO_FINAL plus the passcode, the RAG sentinel, and both plugin tool results.","expected_text":"MULTITOOL_COMBO_FINAL qa_multitool_compaction_sentinel_6718 azalea-cobalt-7421 qa-plugin-smoke:multi-tool-a-local-agent qa-plugin-smoke:multi-tool-b-local-agent","response_timeout_ms":"180000"}]' +setup_automation: + - "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env" + - "node:scripts/e2e/ensure-langrag-sentinel-kb.mjs --write-env" + - "case:qa-plugin-smoke-live-install" +setup_provides_env: + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME + - LANGBOT_LOCAL_AGENT_RAG_KB_UUID +preconditions: + - "The selected model route supports function/tool calling and repeated tool loop turns, or LANGBOT_E2E_FAKE_PROVIDER_BASE_URL points to scripts/e2e/fake-openai-provider.mjs." + - "The LangRAG sentinel knowledge base contains azalea-cobalt-7421 and qa-plugin-smoke exposes qa_plugin_echo." +steps: + - "Ensure the local-agent pipeline, LangRAG sentinel KB, and qa-plugin-smoke fixture plugin are available." + - "Temporarily bind the LangRAG KB, shrink the runner context budget, and set max-tool-iterations high enough for two serial plugin tool calls." + - "Temporarily bind only langbot/local-agent and qa/plugin-smoke; disable MCP servers and skills to isolate the tool surface." + - "Reset the person Debug Chat session for the target pipeline." + - "Send the passcode memory prompt and wait for MEMORY_SET." + - "Send the long pressure prompt and wait for MULTITOOL_CONTEXT_PRESSURE_READY." + - "Ask the runner to combine compacted passcode, RAG sentinel, and two qa_plugin_echo results in one final answer." + - "Restore the original runner config and pipeline extension bindings." +checks: + - "UI: All three user messages appear in Debug Chat." + - "UI: The final Bot message contains MULTITOOL_COMBO_FINAL with the compacted sentinel, azalea-cobalt-7421, and both qa-plugin-smoke tool results." + - "API diagnostic: pipeline-config-diagnostic.json shows knowledge-bases, context token settings, and max-tool-iterations were patched." + - "API diagnostic: pipeline-extensions-diagnostic.json shows only langbot/local-agent and qa/plugin-smoke were bound." + - "Logs: Backend completes RAG retrieval, two plugin tool calls, and Debug Chat response without runner timeout or model setup failure." + - "Console: No unexpected frontend runtime errors appear during the run." +evidence_required: + - ui + - screenshot + - console + - backend_log + - api_diagnostic +diagnostics: + - "For token-free deterministic UI coverage, start scripts/e2e/fake-openai-provider.mjs and pass LANGBOT_E2E_FAKE_PROVIDER_BASE_URL to this case." + - "The fake provider intentionally returns MULTITOOL_COMBO_MISSING_* if the final model input lacks compacted memory, RAG context, or either plugin tool result." + - "If only one tool result is present, inspect max-tool-iterations and the runner tool-loop events." + - "If the final answer lacks azalea-cobalt-7421, first rerun ensure-langrag-sentinel-kb and local-agent-rag-debug-chat." +success_patterns: + - "Processing request from person_websocket" + - "Streaming completed" +failure_patterns: + - "Action invoke_llm_stream call timed out" + - "All models failed during streaming setup" + - "Task exception was never retrieved" + - "MULTITOOL_COMBO_MISSING_" + - "survey widget blocks debug chat" +troubleshooting: + - local-agent-model-route-unavailable + - tool-name-collision-between-mcp-and-plugin + - plugin-runtime-timeout + - proxy-env-mismatch + - survey-widget-blocks-debug-chat + - debug-chat-history-contaminates-automation diff --git a/skills/skills/langbot-testing/cases/local-agent-nonstreaming-debug-chat.yaml b/skills/skills/langbot-testing/cases/local-agent-nonstreaming-debug-chat.yaml index 910032d79..18e79d4e1 100644 --- a/skills/skills/langbot-testing/cases/local-agent-nonstreaming-debug-chat.yaml +++ b/skills/skills/langbot-testing/cases/local-agent-nonstreaming-debug-chat.yaml @@ -1,5 +1,5 @@ id: local-agent-nonstreaming-debug-chat -title: "Local Agent Debug Chat returns a deterministic non-streaming response" +title: "Local Agent Debug Chat returns a deterministic response with UI streaming disabled" mode: agent-browser area: pipeline type: regression @@ -48,14 +48,16 @@ steps: checks: - "UI: The user message appears in Debug Chat." - "UI: A Bot message appears and contains NONSTREAM_OK." - - "Logs: Backend completes the request as a normal response rather than only relying on the streaming-completed path." + - "UI: The Debug Chat stream switch is disabled for this send path." + - "Logs: Backend completes the request without plugin/runtime timeout or frontend streaming UI errors." - "Console: No unexpected frontend runtime errors appear during the send/receive path." evidence_required: - ui - console - backend_log diagnostics: - - "If the UI still streams after the switch is disabled, inspect the adapter streaming capability and runner config before diagnosing the model." + - "This browser case validates the Debug Chat non-streaming UI delivery path. The runner-internal invoke_llm path is covered by langbot-local-agent component tests using runtime_metadata.streaming_supported=false." + - "If the UI still streams after the switch is disabled, inspect the Debug Chat WebSocket stream flag before diagnosing the model." - "Use GET /api/v1/pipelines/{uuid} only to confirm the saved runner and model config." troubleshooting: - local-agent-model-route-unavailable diff --git a/skills/skills/langbot-testing/cases/local-agent-parallel-tools-rag-compaction-debug-chat.yaml b/skills/skills/langbot-testing/cases/local-agent-parallel-tools-rag-compaction-debug-chat.yaml new file mode 100644 index 000000000..7f6489134 --- /dev/null +++ b/skills/skills/langbot-testing/cases/local-agent-parallel-tools-rag-compaction-debug-chat.yaml @@ -0,0 +1,103 @@ +id: local-agent-parallel-tools-rag-compaction-debug-chat +title: "Local Agent preserves RAG, compacted history, and parallel plugin tools" +mode: agent-browser +area: pipeline +type: regression +priority: p1 +risk: high +ci_eligible: false +tags: + - local-agent + - langrag + - context + - compaction + - plugin + - tools + - parallel-tools + - pipeline +skills: + - langbot-env-setup + - langbot-testing +env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME +env_optional: + - LANGBOT_E2E_FAKE_PROVIDER_BASE_URL +automation: scripts/e2e/pipeline-debug-chat.mjs +automation_env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_BROWSER_PROFILE + - LANGBOT_CHROMIUM_EXECUTABLE + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME + - LANGBOT_LOCAL_AGENT_RAG_KB_UUID +automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL +automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME +automation_expected_runner_id: "plugin:langbot/local-agent/default" +automation_runner_config_patch_json: '{"knowledge-bases":["${LANGBOT_LOCAL_AGENT_RAG_KB_UUID}"],"retrieval-top-k":1,"context-window-tokens":900,"context-reserve-tokens":220,"context-keep-recent-tokens":160,"context-summary-tokens":260,"max-tool-iterations":3,"tool-execution-mode":"parallel"}' +automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot","name":"local-agent"},{"author":"qa","name":"plugin-smoke"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}' +automation_restore_runner_config: "1" +automation_restore_extensions: "1" +automation_reset_debug_chat: "1" +automation_debug_chat_session_type: "person" +automation_expected_text: "PARALLEL_COMBO_FINAL qa_parallel_compaction_sentinel_8142 azalea-cobalt-7421 qa-plugin-smoke:parallel-tool-a-local-agent qa-plugin-smoke:parallel-tool-b-local-agent" +automation_response_timeout_ms: "180000" +automation_prompts_json: '[{"prompt":"请记住这个用于 local-agent 并行工具组合回归测试的暗号:qa_parallel_compaction_sentinel_8142。请只回复 MEMORY_SET。","expected_text":"MEMORY_SET","response_timeout_ms":"180000"},{"prompt":"下面这轮只用于制造长历史压力,目标标记 PARALLEL_RAG_GOAL 必须被压缩保留。请忽略填充内容,最后只回复 PARALLEL_CONTEXT_PRESSURE_READY。填充片段 P001 parallel context padding. P002 parallel context padding. P003 parallel context padding. P004 parallel context padding. P005 parallel context padding. P006 parallel context padding. P007 parallel context padding. P008 parallel context padding. P009 parallel context padding. P010 parallel context padding. P011 parallel context padding. P012 parallel context padding. P013 parallel context padding. P014 parallel context padding. P015 parallel context padding. P016 parallel context padding. P017 parallel context padding. P018 parallel context padding. P019 parallel context padding. P020 parallel context padding. P021 parallel context padding. P022 parallel context padding. P023 parallel context padding. P024 parallel context padding. P025 parallel context padding. P026 parallel context padding. P027 parallel context padding. P028 parallel context padding. P029 parallel context padding. P030 parallel context padding. P031 parallel context padding. P032 parallel context padding.","expected_text":"PARALLEL_CONTEXT_PRESSURE_READY","response_timeout_ms":"180000"},{"prompt":"PARALLEL_COMBO final check: using the knowledge base, the compacted earlier passcode, and one model turn that calls qa_plugin_echo for both parallel-tool-a-local-agent and parallel-tool-b-local-agent, return only PARALLEL_COMBO_FINAL plus the passcode, the RAG sentinel, and both plugin tool results.","expected_text":"PARALLEL_COMBO_FINAL qa_parallel_compaction_sentinel_8142 azalea-cobalt-7421 qa-plugin-smoke:parallel-tool-a-local-agent qa-plugin-smoke:parallel-tool-b-local-agent","response_timeout_ms":"180000"}]' +setup_automation: + - "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env" + - "node:scripts/e2e/ensure-langrag-sentinel-kb.mjs --write-env" + - "case:qa-plugin-smoke-live-install" +setup_provides_env: + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME + - LANGBOT_LOCAL_AGENT_RAG_KB_UUID +preconditions: + - "The selected model route supports function/tool calling with multiple tool calls in one turn, or LANGBOT_E2E_FAKE_PROVIDER_BASE_URL points to scripts/e2e/fake-openai-provider.mjs." + - "The LangRAG sentinel knowledge base contains azalea-cobalt-7421 and qa-plugin-smoke exposes qa_plugin_echo." +steps: + - "Ensure the local-agent pipeline, LangRAG sentinel KB, and qa-plugin-smoke fixture plugin are available." + - "Temporarily bind the LangRAG KB, shrink the runner context budget, and set tool-execution-mode to parallel." + - "Temporarily bind only langbot/local-agent and qa/plugin-smoke; disable MCP servers and skills to isolate the tool surface." + - "Reset the person Debug Chat session for the target pipeline." + - "Send the passcode memory prompt and wait for MEMORY_SET." + - "Send the long pressure prompt and wait for PARALLEL_CONTEXT_PRESSURE_READY." + - "Ask the runner to combine compacted passcode, RAG sentinel, and two same-turn qa_plugin_echo results in one final answer." + - "Restore the original runner config and pipeline extension bindings." +checks: + - "UI: All three user messages appear in Debug Chat." + - "UI: The final Bot message contains PARALLEL_COMBO_FINAL with the compacted sentinel, azalea-cobalt-7421, and both qa-plugin-smoke tool results." + - "API diagnostic: pipeline-config-diagnostic.json shows knowledge-bases, context token settings, max-tool-iterations, and tool-execution-mode=parallel were patched." + - "API diagnostic: pipeline-extensions-diagnostic.json shows only langbot/local-agent and qa/plugin-smoke were bound." + - "Logs: Backend completes RAG retrieval, both plugin tool calls, and Debug Chat response without runner timeout or model setup failure." + - "Console: No unexpected frontend runtime errors appear during the run." +evidence_required: + - ui + - screenshot + - console + - backend_log + - api_diagnostic +diagnostics: + - "For token-free deterministic UI coverage, start scripts/e2e/fake-openai-provider.mjs and pass LANGBOT_E2E_FAKE_PROVIDER_BASE_URL to this case." + - "The fake provider intentionally returns two qa_plugin_echo tool_calls in the same streamed model turn for this prompt." + - "The fake provider intentionally returns PARALLEL_COMBO_MISSING_* if the final model input lacks compacted memory, RAG context, or either plugin tool result." + - "If only one tool result is present, inspect the provider trace and runner parallel tool batch handling." + - "If the final answer lacks azalea-cobalt-7421, first rerun ensure-langrag-sentinel-kb and local-agent-rag-debug-chat." +success_patterns: + - "Processing request from person_websocket" + - "Streaming completed" +failure_patterns: + - "Action invoke_llm_stream call timed out" + - "All models failed during streaming setup" + - "Task exception was never retrieved" + - "PARALLEL_COMBO_MISSING_" + - "survey widget blocks debug chat" +troubleshooting: + - local-agent-model-route-unavailable + - tool-name-collision-between-mcp-and-plugin + - plugin-runtime-timeout + - proxy-env-mismatch + - survey-widget-blocks-debug-chat + - debug-chat-history-contaminates-automation diff --git a/skills/skills/langbot-testing/cases/local-agent-plugin-tool-call-debug-chat.yaml b/skills/skills/langbot-testing/cases/local-agent-plugin-tool-call-debug-chat.yaml index 1c37e2fb1..bffa3f3e5 100644 --- a/skills/skills/langbot-testing/cases/local-agent-plugin-tool-call-debug-chat.yaml +++ b/skills/skills/langbot-testing/cases/local-agent-plugin-tool-call-debug-chat.yaml @@ -19,6 +19,8 @@ env: - LANGBOT_BACKEND_URL - LANGBOT_LOCAL_AGENT_PIPELINE_URL - LANGBOT_LOCAL_AGENT_PIPELINE_NAME +env_optional: + - LANGBOT_E2E_FAKE_PROVIDER_BASE_URL automation: scripts/e2e/pipeline-debug-chat.mjs automation_env: - LANGBOT_FRONTEND_URL @@ -38,7 +40,7 @@ setup_provides_env: - LANGBOT_LOCAL_AGENT_PIPELINE_URL - LANGBOT_LOCAL_AGENT_PIPELINE_NAME preconditions: - - "The selected model route supports function/tool calling." + - "The selected model route supports function/tool calling, or LANGBOT_E2E_FAKE_PROVIDER_BASE_URL points to scripts/e2e/fake-openai-provider.mjs." steps: - "Install or enable the bundled qa-plugin-smoke fixture plugin." - "Confirm /api/v1/tools or the plugin detail shows qa_plugin_echo." @@ -58,6 +60,7 @@ evidence_required: - backend_log - api_diagnostic diagnostics: + - "For token-free deterministic UI coverage, start scripts/e2e/fake-openai-provider.mjs and pass LANGBOT_E2E_FAKE_PROVIDER_BASE_URL to this case; setup will bind the local-agent pipeline to that fake OpenAI-compatible model." - "If qa_plugin_echo is not listed, rebuild and reinstall the qa-plugin-smoke fixture plugin." - "If the selected model returns model_not_found or no available channel only when tools are provided, switch to a known-good function-calling model before diagnosing plugin tools or local-agent." troubleshooting: diff --git a/skills/skills/langbot-testing/cases/local-agent-rag-debug-chat.yaml b/skills/skills/langbot-testing/cases/local-agent-rag-debug-chat.yaml index b7e715b16..0057b1844 100644 --- a/skills/skills/langbot-testing/cases/local-agent-rag-debug-chat.yaml +++ b/skills/skills/langbot-testing/cases/local-agent-rag-debug-chat.yaml @@ -18,6 +18,8 @@ env: - LANGBOT_BACKEND_URL - LANGBOT_LOCAL_AGENT_PIPELINE_URL - LANGBOT_LOCAL_AGENT_PIPELINE_NAME +env_optional: + - LANGBOT_E2E_FAKE_PROVIDER_BASE_URL automation: scripts/e2e/pipeline-debug-chat.mjs automation_env: - LANGBOT_FRONTEND_URL @@ -43,7 +45,7 @@ setup_provides_env: - LANGBOT_LOCAL_AGENT_PIPELINE_NAME - LANGBOT_LOCAL_AGENT_RAG_KB_UUID preconditions: - - "The target pipeline already has a text-capable model route that is available for this run." + - "The target pipeline already has a text-capable model route that is available for this run, or LANGBOT_E2E_FAKE_PROVIDER_BASE_URL points to scripts/e2e/fake-openai-provider.mjs." steps: - "Ensure case langrag-kb-retrieve has produced a knowledge base containing sentinel azalea-cobalt-7421." - "Open LANGBOT_FRONTEND_URL." @@ -68,6 +70,8 @@ evidence_required: - backend_log - api_diagnostic diagnostics: + - "For token-free deterministic UI coverage, start scripts/e2e/fake-openai-provider.mjs and pass LANGBOT_E2E_FAKE_PROVIDER_BASE_URL to this case; setup will bind the local-agent pipeline to that fake OpenAI-compatible model." + - "The ensure-langrag-sentinel-kb setup script verifies LANGBOT_E2E_RAG_EXPECTED_TEXT, defaulting to azalea-cobalt-7421, so the Debug Chat expected text does not leak into KB setup." - "Use GET /api/v1/pipelines/{uuid} only to confirm the saved runner_config contains the knowledge base uuid." - "If the bot ignores the knowledge base, rerun Retrieve Test before debugging the runner." troubleshooting: diff --git a/skills/skills/langbot-testing/cases/local-agent-steering-debug-chat.yaml b/skills/skills/langbot-testing/cases/local-agent-steering-debug-chat.yaml index 973822f2b..b769ed8c1 100644 --- a/skills/skills/langbot-testing/cases/local-agent-steering-debug-chat.yaml +++ b/skills/skills/langbot-testing/cases/local-agent-steering-debug-chat.yaml @@ -74,7 +74,6 @@ success_patterns: - "Steering" - "Streaming completed" failure_patterns: - - "STEERING_NO_FOLLOWUP" - "Action invoke_llm_stream call timed out" - "All models failed during streaming setup" - "Task exception was never retrieved" diff --git a/skills/skills/langbot-testing/cases/local-agent-tool-error-recovery-debug-chat.yaml b/skills/skills/langbot-testing/cases/local-agent-tool-error-recovery-debug-chat.yaml new file mode 100644 index 000000000..6a8a97f8c --- /dev/null +++ b/skills/skills/langbot-testing/cases/local-agent-tool-error-recovery-debug-chat.yaml @@ -0,0 +1,94 @@ +id: local-agent-tool-error-recovery-debug-chat +title: "Local Agent feeds plugin tool errors back to the model" +mode: agent-browser +area: pipeline +type: regression +priority: p1 +risk: high +ci_eligible: false +tags: + - local-agent + - plugin + - tools + - tool-error + - pipeline +skills: + - langbot-env-setup + - langbot-testing +env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME +env_optional: + - LANGBOT_E2E_FAKE_PROVIDER_BASE_URL +automation: scripts/e2e/pipeline-debug-chat.mjs +automation_env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_BROWSER_PROFILE + - LANGBOT_CHROMIUM_EXECUTABLE + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME +automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL +automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME +automation_expected_runner_id: "plugin:langbot/local-agent/default" +automation_runner_config_patch_json: '{"knowledge-bases":[],"max-tool-iterations":3,"tool-execution-mode":"serial"}' +automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot","name":"local-agent"},{"author":"qa","name":"plugin-smoke"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}' +automation_restore_runner_config: "1" +automation_restore_extensions: "1" +automation_reset_debug_chat: "1" +automation_debug_chat_session_type: "person" +automation_prompt: "TOOL_ERROR_RECOVERY regression: call qa_plugin_fail with exactly tool-error-recovery-local-agent. After observing the tool error, return only TOOL_ERROR_RECOVERY_FINAL qa-plugin-smoke forced failure observed." +automation_expected_text: "TOOL_ERROR_RECOVERY_FINAL qa-plugin-smoke forced failure observed" +automation_response_timeout_ms: "180000" +setup_automation: + - "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env" + - "case:qa-plugin-smoke-live-install" +setup_provides_env: + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME +preconditions: + - "The selected model route supports function/tool calling, or LANGBOT_E2E_FAKE_PROVIDER_BASE_URL points to scripts/e2e/fake-openai-provider.mjs." + - "qa-plugin-smoke exposes qa_plugin_fail; the setup case reinstalls the fixture if an older installed package is missing it." +steps: + - "Ensure the local-agent pipeline and qa-plugin-smoke fixture plugin are available." + - "Temporarily bind only langbot/local-agent plus qa/plugin-smoke and clear RAG knowledge bases." + - "Reset the person Debug Chat session for the target pipeline." + - "Send the TOOL_ERROR_RECOVERY prompt." + - "Verify the failing plugin tool result is fed back into the model and the final answer contains the recovery sentinel." + - "Restore the original runner config and pipeline extension bindings." +checks: + - "UI: The final Bot message contains TOOL_ERROR_RECOVERY_FINAL qa-plugin-smoke forced failure observed." + - "API diagnostic: pipeline-config-diagnostic.json shows max-tool-iterations=3 and knowledge-bases cleared." + - "API diagnostic: pipeline-extensions-diagnostic.json shows only langbot/local-agent and qa/plugin-smoke were bound." + - "Logs: Backend shows the request completed without runner.timeout, runner.tool_error run failure, or model setup failure." + - "Console: No unexpected frontend runtime errors appear during Debug Chat." +evidence_required: + - ui + - screenshot + - console + - backend_log + - api_diagnostic +diagnostics: + - "For token-free deterministic UI coverage, start scripts/e2e/fake-openai-provider.mjs and pass LANGBOT_E2E_FAKE_PROVIDER_BASE_URL to this case." + - "The fake provider calls qa_plugin_fail once, then only returns the final sentinel after it sees an Error: tool result." + - "If the run returns a generic answer, inspect whether qa_plugin_fail was exposed by /api/v1/tools and whether the installed fixture plugin was refreshed." +success_patterns: + - "TOOL_ERROR_RECOVERY_FINAL" + - "Streaming completed" +failure_patterns: + - "Action invoke_llm_stream call timed out" + - "All models failed during streaming setup" + - "Task exception was never retrieved" + - "runner.timeout" + - "runner.tool_error" + - "Tool call iteration limit reached" + - "survey widget blocks debug chat" +troubleshooting: + - local-agent-model-route-unavailable + - tool-name-collision-between-mcp-and-plugin + - plugin-runtime-timeout + - proxy-env-mismatch + - survey-widget-blocks-debug-chat + - debug-chat-history-contaminates-automation diff --git a/skills/skills/langbot-testing/cases/local-agent-tool-loop-limit-debug-chat.yaml b/skills/skills/langbot-testing/cases/local-agent-tool-loop-limit-debug-chat.yaml new file mode 100644 index 000000000..e1d336062 --- /dev/null +++ b/skills/skills/langbot-testing/cases/local-agent-tool-loop-limit-debug-chat.yaml @@ -0,0 +1,92 @@ +id: local-agent-tool-loop-limit-debug-chat +title: "Local Agent stops repeated tool calls at max-tool-iterations" +mode: agent-browser +area: pipeline +type: regression +priority: p1 +risk: high +ci_eligible: false +tags: + - local-agent + - plugin + - tools + - tool-loop + - limit + - pipeline +skills: + - langbot-env-setup + - langbot-testing +env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME +env_optional: + - LANGBOT_E2E_FAKE_PROVIDER_BASE_URL +automation: scripts/e2e/pipeline-debug-chat.mjs +automation_env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_BROWSER_PROFILE + - LANGBOT_CHROMIUM_EXECUTABLE + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME +automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL +automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME +automation_expected_runner_id: "plugin:langbot/local-agent/default" +automation_runner_config_patch_json: '{"knowledge-bases":[],"max-tool-iterations":2,"tool-execution-mode":"serial"}' +automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot","name":"local-agent"},{"author":"qa","name":"plugin-smoke"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}' +automation_restore_runner_config: "1" +automation_restore_extensions: "1" +automation_reset_debug_chat: "1" +automation_debug_chat_session_type: "person" +automation_prompt: "LOOP_LIMIT regression: keep requesting qa_plugin_echo with exactly loop-limit-repeat-local-agent until the runner stops you. The correct final visible answer is the runner limit message." +automation_expected_text: "Tool call iteration limit reached. I stopped before calling more tools." +automation_response_timeout_ms: "180000" +setup_automation: + - "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env" + - "case:qa-plugin-smoke-live-install" +setup_provides_env: + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME +preconditions: + - "The selected model route supports function/tool calling, or LANGBOT_E2E_FAKE_PROVIDER_BASE_URL points to scripts/e2e/fake-openai-provider.mjs." + - "qa-plugin-smoke exposes qa_plugin_echo." +steps: + - "Ensure the local-agent pipeline and qa-plugin-smoke fixture plugin are available." + - "Temporarily set max-tool-iterations to 2 and bind only langbot/local-agent plus qa/plugin-smoke." + - "Reset the person Debug Chat session for the target pipeline." + - "Send the LOOP_LIMIT prompt and wait for the runner limit message." + - "Restore the original runner config and pipeline extension bindings." +checks: + - "UI: The final Bot message contains Tool call iteration limit reached. I stopped before calling more tools." + - "API diagnostic: pipeline-config-diagnostic.json shows max-tool-iterations was patched to 2." + - "API diagnostic: pipeline-extensions-diagnostic.json shows only langbot/local-agent and qa/plugin-smoke were bound." + - "Logs: Backend shows the request completed without infinite loop, runner timeout, or model setup failure." + - "Console: No unexpected frontend runtime errors appear during Debug Chat." +evidence_required: + - ui + - screenshot + - console + - backend_log + - api_diagnostic +diagnostics: + - "For token-free deterministic UI coverage, start scripts/e2e/fake-openai-provider.mjs and pass LANGBOT_E2E_FAKE_PROVIDER_BASE_URL to this case." + - "The fake provider intentionally keeps requesting qa_plugin_echo; this case passes only when the runner stops at max-tool-iterations." + - "If the run times out, inspect whether max-tool-iterations was patched or whether the runner ignored tool loop limits." +success_patterns: + - "Tool call iteration limit reached" + - "Streaming completed" +failure_patterns: + - "Action invoke_llm_stream call timed out" + - "All models failed during streaming setup" + - "Task exception was never retrieved" + - "runner.timeout" + - "survey widget blocks debug chat" +troubleshooting: + - local-agent-model-route-unavailable + - tool-name-collision-between-mcp-and-plugin + - plugin-runtime-timeout + - proxy-env-mismatch + - survey-widget-blocks-debug-chat + - debug-chat-history-contaminates-automation diff --git a/skills/skills/langbot-testing/cases/mcp-stdio-register.yaml b/skills/skills/langbot-testing/cases/mcp-stdio-register.yaml index 6c9857ba9..3184e8ffe 100644 --- a/skills/skills/langbot-testing/cases/mcp-stdio-register.yaml +++ b/skills/skills/langbot-testing/cases/mcp-stdio-register.yaml @@ -34,8 +34,10 @@ steps: - "Poll /api/v1/tools and /api/v1/mcp/servers/qa-local-stdio until qa_mcp_echo is visible." checks: - "API diagnostic: qa-local-stdio runtime_status is connected." + - "API diagnostic: qa-local-stdio server_uuid is present for pipeline extension binding." - "API diagnostic: runtime_tool_names includes qa_mcp_echo." - "API diagnostic: /api/v1/tools includes qa_mcp_echo." + - "Automation writes LANGBOT_MCP_QA_STDIO_SERVER_UUID into skills/.env.local for downstream tool-call cases." - "Console and network logs contain no unexpected frontend/runtime failures." evidence_required: - screenshot diff --git a/skills/skills/langbot-testing/cases/mcp-stdio-tool-call.yaml b/skills/skills/langbot-testing/cases/mcp-stdio-tool-call.yaml index 593eecba3..30f7170cf 100644 --- a/skills/skills/langbot-testing/cases/mcp-stdio-tool-call.yaml +++ b/skills/skills/langbot-testing/cases/mcp-stdio-tool-call.yaml @@ -18,15 +18,23 @@ env: - LANGBOT_BACKEND_URL - LANGBOT_LOCAL_AGENT_PIPELINE_URL - LANGBOT_LOCAL_AGENT_PIPELINE_NAME +env_optional: + - LANGBOT_E2E_FAKE_PROVIDER_BASE_URL automation: scripts/e2e/pipeline-debug-chat.mjs automation_env: - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL - LANGBOT_BROWSER_PROFILE - LANGBOT_CHROMIUM_EXECUTABLE - LANGBOT_LOCAL_AGENT_PIPELINE_URL - LANGBOT_LOCAL_AGENT_PIPELINE_NAME + - LANGBOT_MCP_QA_STDIO_SERVER_UUID automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME +automation_expected_runner_id: "plugin:langbot/local-agent/default" +automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot","name":"local-agent"}],"enable_all_mcp_servers":false,"bound_mcp_servers":["${LANGBOT_MCP_QA_STDIO_SERVER_UUID}"],"enable_all_skills":false,"bound_skills":[]}' +automation_restore_extensions: "1" +automation_reset_debug_chat: "1" automation_prompt: "Call the qa_mcp_echo MCP tool with exactly this text: mcp-ok-local-agent. Return only the tool result." automation_expected_text: "qa_mcp_echo:mcp-ok-local-agent" automation_response_timeout_ms: "180000" @@ -36,6 +44,7 @@ setup_automation: setup_provides_env: - LANGBOT_LOCAL_AGENT_PIPELINE_URL - LANGBOT_LOCAL_AGENT_PIPELINE_NAME + - LANGBOT_MCP_QA_STDIO_SERVER_UUID failure_patterns: - "qa-plugin-smoke:mcp-ok-local-agent" - "qa_echo:mcp-ok-local-agent" @@ -45,7 +54,7 @@ failure_patterns: - "no available channel for model" preconditions: - "box.local.allowed_mount_roots includes the bundled MCP fixture directory when LangBot runs stdio MCP servers through Box." - - "The selected model route supports function/tool calling." + - "The selected model route supports function/tool calling, or LANGBOT_E2E_FAKE_PROVIDER_BASE_URL points to scripts/e2e/fake-openai-provider.mjs." steps: - "Open LANGBOT_FRONTEND_URL." - "Navigate to MCP Servers." @@ -66,6 +75,7 @@ checks: - "API diagnostic: /api/v1/tools contains qa_mcp_echo." - "UI: Debug Chat bot response contains qa_mcp_echo:mcp-ok-local-agent." - "Logs: Backend logs show the MCP tool call was executed, not only listed." + - "API diagnostic: pipeline-extensions-diagnostic.json shows the qa-local-stdio server UUID is the only bound MCP server for the run." - "Console: No unexpected frontend errors appear during MCP form use or Debug Chat." evidence_required: - ui @@ -73,6 +83,7 @@ evidence_required: - backend_log - api_diagnostic diagnostics: + - "For token-free deterministic UI coverage, start scripts/e2e/fake-openai-provider.mjs and pass LANGBOT_E2E_FAKE_PROVIDER_BASE_URL to this case; setup will bind the local-agent pipeline to that fake OpenAI-compatible model." - "Run node scripts/e2e/mcp-stdio-fixture.mjs to verify the bundled stdio fixture can list and call qa_mcp_echo without involving a model provider." - "Run node scripts/e2e/mcp-stdio-register.mjs to upsert qa-local-stdio in LangBot and verify /api/v1/tools exposes qa_mcp_echo." - "If backend logs show host_path is outside allowed_mount_roots, add the fixture directory to box.local.allowed_mount_roots in the local LangBot data config." diff --git a/skills/skills/langbot-testing/cases/qa-plugin-smoke-live-install.yaml b/skills/skills/langbot-testing/cases/qa-plugin-smoke-live-install.yaml index f0ec3c336..ff01f68c0 100644 --- a/skills/skills/langbot-testing/cases/qa-plugin-smoke-live-install.yaml +++ b/skills/skills/langbot-testing/cases/qa-plugin-smoke-live-install.yaml @@ -20,14 +20,14 @@ env: automation: scripts/e2e/install-qa-plugin-smoke.mjs automation_plugin_package: "skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/dist/qa-plugin-smoke-0.1.0.lbpkg" automation_expected_plugin_id: "qa/plugin-smoke" -automation_expected_tool: "qa_plugin_echo" +automation_expected_tool: "qa_plugin_echo,qa_plugin_fail" steps: - "Run `rtk bin/lbs test run qa-plugin-smoke-live-install --dry-run` first; remove `--dry-run` only after readiness points at a local test LangBot instance." - "Automation authenticates the local test user, uploads the QA plugin smoke .lbpkg package when missing, waits for install, and checks tool exposure." checks: - "automation-result.json status is pass." - "/api/v1/plugins lists qa/plugin-smoke after install." - - "/api/v1/tools lists qa_plugin_echo after install." + - "/api/v1/tools lists qa_plugin_echo and qa_plugin_fail after install." evidence_required: - api_diagnostic - filesystem @@ -39,6 +39,7 @@ success_patterns: failure_patterns: - "Plugin install task did not complete successfully" - "qa_plugin_echo is not listed" + - "qa_plugin_fail is not listed" troubleshooting: - plugin-runtime-timeout - plugin-dependency-install-offline diff --git a/skills/skills/langbot-testing/fixtures/fixtures.json b/skills/skills/langbot-testing/fixtures/fixtures.json index 70badac14..24d4437fd 100644 --- a/skills/skills/langbot-testing/fixtures/fixtures.json +++ b/skills/skills/langbot-testing/fixtures/fixtures.json @@ -49,7 +49,13 @@ "title": "LangRAG sentinel text document", "kind": "text", "path": "fixtures/rag/sentinel-doc.txt", - "related_cases": ["langrag-kb-retrieve", "local-agent-rag-debug-chat"], + "related_cases": [ + "langrag-kb-retrieve", + "local-agent-rag-debug-chat", + "local-agent-combo-rag-compaction-tool-debug-chat", + "local-agent-multitool-rag-compaction-debug-chat", + "local-agent-parallel-tools-rag-compaction-debug-chat" + ], "checks": ["exists"] }, { @@ -78,6 +84,11 @@ "plugin-e2e-smoke", "local-agent-effective-prompt-debug-chat", "local-agent-plugin-tool-call-debug-chat", + "local-agent-tool-error-recovery-debug-chat", + "local-agent-tool-loop-limit-debug-chat", + "local-agent-combo-rag-compaction-tool-debug-chat", + "local-agent-multitool-rag-compaction-debug-chat", + "local-agent-parallel-tools-rag-compaction-debug-chat", "local-agent-steering-debug-chat" ], "checks": ["exists"] @@ -87,7 +98,7 @@ "title": "QA plugin smoke prebuilt package", "kind": "plugin_package", "path": "fixtures/plugins/qa-plugin-smoke/dist/qa-plugin-smoke-0.1.0.lbpkg", - "related_cases": ["qa-plugin-smoke-live-install", "plugin-e2e-smoke"], + "related_cases": ["qa-plugin-smoke-live-install", "plugin-e2e-smoke", "local-agent-tool-error-recovery-debug-chat"], "checks": ["exists", "zip_package"] } ] diff --git a/skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/README.md b/skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/README.md index e276f5774..c3b7f2057 100644 --- a/skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/README.md +++ b/skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/README.md @@ -7,3 +7,4 @@ Tools: - `qa_echo(text)` returns `qa-plugin-smoke:`. - `qa_plugin_echo(text)` returns `qa-plugin-smoke:`. - `qa_plugin_sleep(seconds, text)` waits up to 15 seconds and returns `qa-plugin-smoke:sleep::`. +- `qa_plugin_fail(text)` raises a deterministic error containing `text`. diff --git a/skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/components/tools/qa_plugin_fail.py b/skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/components/tools/qa_plugin_fail.py new file mode 100644 index 000000000..2c6e48f24 --- /dev/null +++ b/skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/components/tools/qa_plugin_fail.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from typing import Any + +from langbot_plugin.api.definition.components.tool.tool import Tool +from langbot_plugin.api.entities.builtin.provider import session as provider_session + + +class QAPluginFailTool(Tool): + async def call( + self, + params: dict[str, Any], + session: provider_session.Session, + query_id: int, + ) -> str: + text = str(params.get("text", "qa-plugin-fail")) + raise RuntimeError(f"qa-plugin-smoke forced failure: {text}") diff --git a/skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/components/tools/qa_plugin_fail.yaml b/skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/components/tools/qa_plugin_fail.yaml new file mode 100644 index 000000000..7bcf36bd4 --- /dev/null +++ b/skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/components/tools/qa_plugin_fail.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Tool +metadata: + name: qa_plugin_fail + label: + en_US: QA Plugin Fail + zh_Hans: QA 插件失败工具 + description: + en_US: Raises a deterministic error for local-agent tool error recovery tests. + zh_Hans: 为 local-agent 工具错误恢复测试抛出确定性错误。 +spec: + parameters: + type: object + properties: + text: + type: string + description: Text included in the deterministic failure message. + required: + - text + llm_prompt: Always fail with a deterministic qa-plugin-smoke error for tool error recovery tests. +execution: + python: + path: qa_plugin_fail.py + attr: QAPluginFailTool diff --git a/skills/skills/langbot-testing/references/agent-runner-release-gate.md b/skills/skills/langbot-testing/references/agent-runner-release-gate.md index 89a75fcfa..eda0284bd 100644 --- a/skills/skills/langbot-testing/references/agent-runner-release-gate.md +++ b/skills/skills/langbot-testing/references/agent-runner-release-gate.md @@ -111,13 +111,15 @@ Each probe writes `automation-result.json` and probe logs under | Deterministic QA runner install | `agent-runner-live-install` | A local `.lbpkg` AgentRunner package can install and register a runner. | | Deterministic QA runner Debug Chat | `agent-runner-qa-debug-chat` | The installed QA runner executes through WebUI Debug Chat without a model provider. | | Required runner plugins | `agent-runner-release-preflight` | `langbot/local-agent` and `langbot/acp-agent-runner` are visible to the host. | -| Required QA plugin tool | `plugin-e2e-smoke`, `agent-runner-release-preflight` | The deterministic `qa_plugin_echo` tool is exposed before tool-loop cases start. | +| Required QA plugin tools | `plugin-e2e-smoke`, `agent-runner-release-preflight`, `qa-plugin-smoke-live-install` | The deterministic `qa_plugin_echo` and `qa_plugin_fail` tools are exposed before tool-loop and tool-error cases start. | | Knowledge base fixture | `langrag-kb-retrieve`, `local-agent-rag-debug-chat` | LangRAG data is queryable and the runner inserts retrieved context. | | Effective prompt bridge | `local-agent-effective-prompt-debug-chat` | Host prompt preprocessing reaches the runner. | | History and compaction | `local-agent-context-compaction-debug-chat` | Runner-owned history budgeting keeps recoverable older context. | | Streaming LLM | `local-agent-basic-debug-chat` | The default streaming path returns a visible answer. | | Non-streaming LLM | `local-agent-nonstreaming-debug-chat` | The non-streaming adapter path returns a visible answer. | | Plugin tool loop | `local-agent-plugin-tool-call-debug-chat` | Function-call capable models can call host plugin tools through authorization. | +| Plugin tool error recovery | `local-agent-tool-error-recovery-debug-chat` | Tool execution errors are serialized into model-facing tool results and the model can produce a final answer instead of failing the run. | +| Parallel plugin tool batch | `local-agent-parallel-tools-rag-compaction-debug-chat` | Local-agent executes multiple same-turn plugin tool calls and returns both results with RAG and compacted history. | | MCP registration | `mcp-stdio-register` | The deterministic stdio MCP server is registered and exposes `qa_mcp_echo`. | | MCP tool loop | `mcp-stdio-tool-call` | Local-agent can call the registered MCP tool through the same tool loop. | | Multimodal input | `local-agent-multimodal-debug-chat` | Image upload and structured input reach the runner. | diff --git a/skills/skills/langbot-testing/references/local-agent-runner-coverage.md b/skills/skills/langbot-testing/references/local-agent-runner-coverage.md index 04495f8d5..429cd5cd4 100644 --- a/skills/skills/langbot-testing/references/local-agent-runner-coverage.md +++ b/skills/skills/langbot-testing/references/local-agent-runner-coverage.md @@ -29,10 +29,15 @@ These browser cases are the minimum gate for a local-agent migration check: | `local-agent-rag-debug-chat` | Knowledge-base authorization, retrieval, and RAG prompt insertion | Bot returns the KB sentinel, not a generic answer. | | `mcp-stdio-tool-call` | MCP stdio discovery, tool detail, model function calling, and tool execution | Bot returns `qa_mcp_echo:` and backend logs the MCP tool call. | | `local-agent-plugin-tool-call-debug-chat` | Plugin tool discovery, tool detail, model function calling, and tool execution | Bot returns `qa-plugin-smoke:` and backend logs the plugin tool call. | +| `local-agent-tool-error-recovery-debug-chat` | Plugin tool execution failure, model-facing error tool result, and final recovery turn | Bot returns `TOOL_ERROR_RECOVERY_FINAL...` only after the model sees the tool error result. | +| `local-agent-tool-loop-limit-debug-chat` | Repeated plugin tool requests and max tool iteration enforcement | Bot returns the runner limit message instead of looping indefinitely. | +| `local-agent-combo-rag-compaction-tool-debug-chat` | RAG, compacted history, and plugin tool execution in the same Debug Chat run | Bot returns `COMBO_FINAL` with the compacted sentinel, KB sentinel, and plugin tool result together. | +| `local-agent-multitool-rag-compaction-debug-chat` | RAG, compacted history, and repeated serial plugin tool loop turns in the same Debug Chat run | Bot returns `MULTITOOL_COMBO_FINAL` with the compacted sentinel, KB sentinel, and both plugin tool results together. | +| `local-agent-parallel-tools-rag-compaction-debug-chat` | RAG, compacted history, and two plugin tool calls emitted in the same model turn | Bot returns `PARALLEL_COMBO_FINAL` with the compacted sentinel, KB sentinel, and both plugin tool results together. | | `local-agent-steering-debug-chat` | Host steering claim, runner pull at turn boundary, and follow-up injection during an active tool loop | Two user messages produce one assistant response containing the steering sentinel. | | `local-agent-multimodal-debug-chat` | Image upload, structured input contents, and multimodal runner consumption | UI shows uploaded image and bot returns `IMAGE_OK`; backend receives an image input. | | `local-agent-rag-multimodal-debug-chat` | RAG insertion while structured image input is present | UI shows uploaded image, bot returns the KB sentinel, and backend logs the same request with `[Image]`. | -| `local-agent-nonstreaming-debug-chat` | Host non-streaming adapter path and runner non-streaming invocation | Bot returns `NONSTREAM_OK`; backend completes without the streaming-completed path. | +| `local-agent-nonstreaming-debug-chat` | Debug Chat non-streaming UI delivery path | Bot returns `NONSTREAM_OK` with the Debug Chat stream switch disabled; runner-internal `invoke_llm` mode is covered by local-agent component tests using `runtime_metadata.streaming_supported=false`. | ## Full Coverage Matrix @@ -44,14 +49,18 @@ These browser cases are the minimum gate for a local-agent migration check: | Multimodal plus RAG | Run `local-agent-rag-multimodal-debug-chat`. | RAG sentinel is still retrievable and the image is not dropped from the user message; exact image-preservation inside the model message is covered by unit tests. | | History and context compaction | Run `local-agent-context-compaction-debug-chat` with a small temporary `context-window-tokens` budget. | The runner compacts older history into `` and the final answer still recovers the older sentinel from the compacted context. | | Streaming model invocation | Enable Debug Chat streaming and ask for `OK`. | UI receives incremental bot output and backend logs streaming completion. | -| Non-streaming model invocation | Disable Debug Chat streaming or use a non-streaming adapter path. | UI receives a final bot message and backend logs a normal response completion. | +| Non-streaming UI delivery | Disable Debug Chat streaming. | UI receives a final bot message without frontend streaming errors. | +| Non-streaming model invocation | Use local-agent component tests with `runtime_metadata.streaming_supported=false` or a host adapter that does not support streaming. | Runner calls `invoke_llm` instead of `invoke_llm_stream` and emits `message.completed`. | | Model fallback before first chunk | Configure a failing primary and working fallback, preferably with a controlled test provider. | First model failure does not fail the run; fallback model produces the final answer. | | Failure after streaming commit | Use a controlled provider that emits one chunk and then fails. | Runner reports a terminal run failure and does not fallback after partial output. | | No authorized model | Clear model config or configure a model not in run resources. | Runner returns `runner.no_model` instead of calling an unauthorized model. | | MCP tool call | Use `qa-local-stdio` and `qa_mcp_echo`. | Bot returns the exact `qa_mcp_echo:` result; `/api/v1/tools` contains `qa_mcp_echo`. | | Plugin tool call | Install a fixture plugin exposing a deterministic tool and bind it to the pipeline. | Runner lists the plugin tool and can call it through the same tool loop as MCP tools. | +| Repeated tool loop | Run `local-agent-multitool-rag-compaction-debug-chat` with `max-tool-iterations` high enough for two serial calls. | Runner calls `qa_plugin_echo` twice and the final answer contains both distinct tool results with RAG and compacted history. | +| Parallel tool batch | Run `local-agent-parallel-tools-rag-compaction-debug-chat` with `tool-execution-mode: parallel`. | Runner executes two same-turn `qa_plugin_echo` calls and the final answer contains both results with RAG and compacted history. | +| Tool iteration limit | Run `local-agent-tool-loop-limit-debug-chat` with `max-tool-iterations: 2` and a fake provider that keeps requesting tools. | Runner stops and returns `Tool call iteration limit reached...`; it does not keep invoking tools indefinitely. | | Run steering | Use `local-agent-steering-debug-chat` with the fixture `qa_plugin_sleep` tool. | A follow-up sent while the sleep tool keeps the run active is claimed into the same run: two user messages, one assistant response, sentinel included. | -| Tool errors | Make the model request an unauthorized tool or invalid arguments in a controlled unit/component test. | Tool result contains an error message and the run does not bypass authorization. | +| Tool errors | Run `local-agent-tool-error-recovery-debug-chat` for plugin tool execution errors; keep unauthorized-tool branches in unit/component tests. | The model receives an `Error:` tool result and returns the recovery sentinel; unauthorized calls remain blocked in component tests. | | Tool iteration limit | Use a controlled model/tool fixture that repeatedly requests more tool calls. | Runner stops with `runner.tool_loop_limit` at the configured limit. | | Knowledge retrieval | Bind a KB containing a unique sentinel. | Bot returns the sentinel and backend logs LangRAG retrieval. | | Legacy `knowledge-base` config | Load a pipeline config using the old single-KB field. | Runner still retrieves from the KB. | diff --git a/skills/skills/langbot-testing/references/local-agent-runner.md b/skills/skills/langbot-testing/references/local-agent-runner.md index 63a2b7e94..823693c5a 100644 --- a/skills/skills/langbot-testing/references/local-agent-runner.md +++ b/skills/skills/langbot-testing/references/local-agent-runner.md @@ -36,7 +36,7 @@ If the direct MCP fixture passes but `/api/v1/tools` still shows the old MCP nam For a multimodal check, upload a small image and ask for a deterministic acknowledgement. Prefer the bundled 64x64 red-square fixture over a 1x1 image because some model providers reject tiny images before the runner path is exercised. -For a non-streaming check, disable the Debug Chat stream switch before sending the prompt. +For a Debug Chat non-streaming delivery check, disable the Debug Chat stream switch before sending the prompt. This validates the UI/adapter delivery path. Runner-internal non-streaming model invocation is covered by component tests that set `runtime_metadata.streaming_supported=false`. ## Timeout And Tool Regression Checks @@ -53,9 +53,14 @@ Run these cases before saying the pluginized local-agent behavior is healthy: - `local-agent-rag-debug-chat`: LangRAG retrieval reaches the runner and affects the answer. - `mcp-stdio-tool-call`: MCP tool discovery and local-agent tool loop. - `local-agent-plugin-tool-call-debug-chat`: plugin tool discovery and local-agent tool loop. +- `local-agent-tool-error-recovery-debug-chat`: plugin tool execution errors are returned to the model as tool results and can produce a final recovery answer. +- `local-agent-tool-loop-limit-debug-chat`: repeated plugin tool requests stop at the configured max tool iteration limit. +- `local-agent-combo-rag-compaction-tool-debug-chat`: one Debug Chat run combines compacted history, LangRAG context, and a plugin tool result. +- `local-agent-multitool-rag-compaction-debug-chat`: one Debug Chat run combines compacted history, LangRAG context, and two serial plugin tool calls. +- `local-agent-parallel-tools-rag-compaction-debug-chat`: one Debug Chat run combines compacted history, LangRAG context, and two same-turn parallel plugin tool calls. - `local-agent-multimodal-debug-chat`: uploaded image reaches `ctx.input.contents`. - `local-agent-rag-multimodal-debug-chat`: RAG retrieval still works when the same user message carries an image. -- `local-agent-nonstreaming-debug-chat`: runner works when the host adapter cannot or should not stream. +- `local-agent-nonstreaming-debug-chat`: Debug Chat still returns a complete visible response when UI streaming is disabled. ## Pass Criteria diff --git a/skills/skills/langbot-testing/references/mcp-stdio-testing.md b/skills/skills/langbot-testing/references/mcp-stdio-testing.md index c1930b81a..31b3ab546 100644 --- a/skills/skills/langbot-testing/references/mcp-stdio-testing.md +++ b/skills/skills/langbot-testing/references/mcp-stdio-testing.md @@ -79,7 +79,9 @@ node scripts/e2e/mcp-stdio-register.mjs It upserts `qa-local-stdio` through the authenticated WebUI session, points it at the bundled `qa_mcp_echo_server.py`, then checks `/api/v1/tools` and the MCP runtime info. A pass proves LangBot has refreshed the saved server and exposes -`qa_mcp_echo` before any model provider is involved. +`qa_mcp_echo` before any model provider is involved. It also writes the resolved +server UUID to `LANGBOT_MCP_QA_STDIO_SERVER_UUID` in `skills/.env.local`; pipeline +extension binding uses this UUID, not the human-readable server name. ## Local-Agent Tool Call Check diff --git a/skills/skills/langbot-testing/suites/agent-runner-release-gate.yaml b/skills/skills/langbot-testing/suites/agent-runner-release-gate.yaml index a2c150669..733b2c37e 100644 --- a/skills/skills/langbot-testing/suites/agent-runner-release-gate.yaml +++ b/skills/skills/langbot-testing/suites/agent-runner-release-gate.yaml @@ -30,6 +30,11 @@ cases: - local-agent-context-compaction-debug-chat - local-agent-rag-debug-chat - local-agent-plugin-tool-call-debug-chat + - local-agent-tool-error-recovery-debug-chat + - local-agent-tool-loop-limit-debug-chat + - local-agent-combo-rag-compaction-tool-debug-chat + - local-agent-multitool-rag-compaction-debug-chat + - local-agent-parallel-tools-rag-compaction-debug-chat - mcp-stdio-register - mcp-stdio-tool-call - local-agent-nonstreaming-debug-chat diff --git a/skills/skills/langbot-testing/suites/local-agent-gate.yaml b/skills/skills/langbot-testing/suites/local-agent-gate.yaml index 42eee0228..40ed15342 100644 --- a/skills/skills/langbot-testing/suites/local-agent-gate.yaml +++ b/skills/skills/langbot-testing/suites/local-agent-gate.yaml @@ -14,6 +14,11 @@ cases: - local-agent-context-compaction-debug-chat - local-agent-rag-debug-chat - local-agent-plugin-tool-call-debug-chat + - local-agent-tool-error-recovery-debug-chat + - local-agent-tool-loop-limit-debug-chat + - local-agent-combo-rag-compaction-tool-debug-chat + - local-agent-multitool-rag-compaction-debug-chat + - local-agent-parallel-tools-rag-compaction-debug-chat - local-agent-steering-debug-chat - mcp-stdio-tool-call - local-agent-nonstreaming-debug-chat diff --git a/skills/src/commands/log.ts b/skills/src/commands/log.ts index aad9eaa66..fbadd44d5 100644 --- a/skills/src/commands/log.ts +++ b/skills/src/commands/log.ts @@ -117,7 +117,7 @@ function patternContextFromOptions(root: string, options: Record): string { const explicit = optionString(options, "backend-log"); if (explicit) return resolve(explicit); - const auto = latestLangBotLogPath(loadEnv(root)); + const auto = latestLangBotLogPath(loadEnv(root), root); return auto ? resolve(auto) : ""; } diff --git a/skills/src/commands/test.ts b/skills/src/commands/test.ts index 67ddc3122..97356840b 100644 --- a/skills/src/commands/test.ts +++ b/skills/src/commands/test.ts @@ -343,6 +343,78 @@ function manualEvidenceTemplate(mode: string): ManualEvidenceTemplate { }; } +function existingEvidencePath(evidence: Record | undefined, key: string): string { + const path = evidence?.[key]; + return path && existsSync(path) ? path : ""; +} + +function renderAutomationEvidence( + mode: string, + automation: AutomationResultEvidence, + logGuard: LogGuardResult, +): ManualEvidenceTemplate { + if (automation.status !== "loaded") return manualEvidenceTemplate(mode); + + const isProbe = isProbeMode(mode); + const screenshot = existingEvidencePath(automation.evidence, "screenshot"); + const consoleLog = existingEvidencePath(automation.evidence, "console_log"); + const networkLog = existingEvidencePath(automation.evidence, "network_log"); + const apiDiagnostics = Object.entries(automation.evidence ?? {}) + .filter(([key, value]) => key.endsWith("_diagnostic_json") && value && existsSync(value)) + .map(([, value]) => value); + const successLines = logGuard.success_signals + .map((signal) => signal.excerpt || `${signal.source}:${signal.line ?? ""} ${signal.pattern}`.trim()) + .slice(0, 3); + const findings = logGuard.findings + .map((finding) => `${finding.severity}/${finding.kind}${finding.excerpt ? `: ${finding.excerpt}` : ""}`) + .slice(0, 3); + const chatSummary = automation.chat_results?.length + ? automation.chat_results + .map((item) => `#${(item.index ?? 0) + 1} ${item.status ?? "unknown"} ${item.expected_text ?? ""}`.trim()) + .join("; ") + : automation.reason || ""; + const diagnostics = [ + automation.browser_diagnostics?.reason, + apiDiagnostics.length ? `api diagnostics: ${apiDiagnostics.join(", ")}` : "", + ].filter(Boolean).join("; ") || "No extra diagnostics recorded."; + + if (isProbe) { + return { + result: automation.result || "unknown", + target_tested: String(automation.url || automation.path || "automation target"), + execution_path: "automation script", + probe_result: automation.reason || chatSummary || "Automation completed.", + logs_or_artifacts: [ + consoleLog ? `console=${consoleLog}` : "", + networkLog ? `network=${networkLog}` : "", + automation.path ? `automation=${automation.path}` : "", + ].filter(Boolean).join(", ") || "No log artifacts recorded.", + diagnostics, + matched_troubleshooting: findings.length ? findings.join(" | ") : "None.", + assets_to_update: automation.result === "pass" ? "None." : "Review case/troubleshooting assets if this failure is expected to recur.", + }; + } + + return { + result: automation.result || "unknown", + url_tested: automation.url || "Not recorded.", + browser_path: "direct Playwright automation", + ui_result: chatSummary || automation.reason || "Automation completed.", + console_errors: automation.browser_diagnostics?.status === "fail" + ? automation.browser_diagnostics.reason || "Browser diagnostics failed." + : "None found by browser diagnostics.", + network_symptoms: networkLog ? `Captured in ${networkLog}` : "No network log artifact recorded.", + backend_logs: successLines.length + ? successLines.join(" | ") + : `Log Guard status: ${logGuard.status}${findings.length ? `; ${findings.join(" | ")}` : ""}`, + frontend_logs: consoleLog ? `Captured in ${consoleLog}` : "No frontend console log artifact recorded.", + screenshots: screenshot || "No screenshot artifact recorded.", + diagnostics, + matched_troubleshooting: findings.length ? findings.join(" | ") : "None.", + assets_to_update: automation.result === "pass" ? "None." : "Review case/troubleshooting assets if this failure is expected to recur.", + }; +} + function envSummary(item: StructuredItem, env: Record): Record { const keys = [ ...listValue(item.fields, "env"), @@ -1321,6 +1393,8 @@ export function commandTestRun(ctx: CommandContext): number { function buildReport(root: string, item: StructuredItem, options: Record): TestReport { const env = loadEnv(root); const mode = caseMode(item); + const automationResult = readAutomationResultEvidence(options); + const logGuard = scanStructuredLogSources(root, item, options); const related = relatedTroubleshooting(root, item).map((entry) => ({ id: scalar(entry.fields, "id"), title: scalar(entry.fields, "title"), @@ -1332,8 +1406,8 @@ function buildReport(root: string, item: StructuredItem, options: Record; + evidence_collected?: string[]; + browser_diagnostics?: { + status?: string; + reason?: string; + finding_count?: number; + }; + chat_results?: Array<{ + index?: number; + expected_text?: string; + status?: string; + reason?: string; + }>; metrics_summary?: Record; thresholds_summary?: Record; artifacts?: Record; @@ -216,13 +232,13 @@ export function scanLogSources( for (const configured of configuredSources) { const explicitPath = options[configured.option]; const autoPath = configured.source === "backend" && options["no-auto-log"] !== true - ? latestLangBotLogPath(env) + ? latestLangBotLogPath(env, root) : null; const rawPath = typeof explicitPath === "string" ? explicitPath : autoPath; const autoDetected = typeof explicitPath !== "string" && rawPath === autoPath; if (!rawPath) { if (configured.source === "backend" && options["no-auto-log"] !== true) { - const logsDir = env.LANGBOT_REPO ? join(env.LANGBOT_REPO, "data", "logs") : "LANGBOT_REPO/data/logs"; + const logsDir = langBotLogsDir(env, root) ?? "LANGBOT_REPO/data/logs"; sources.push({ source: "backend", path: join(logsDir, "langbot-*.log"), status: "auto_not_found", line_count: 0, auto_detected: true }); } continue; @@ -610,6 +626,47 @@ function objectField(data: Record, key: string): Record, key: string): Record | undefined { + const value = data[key]; + if (!value || typeof value !== "object" || Array.isArray(value)) return undefined; + const entries = Object.entries(value as Record) + .filter((entry): entry is [string, string] => typeof entry[1] === "string" && entry[1].trim().length > 0); + return entries.length ? Object.fromEntries(entries) : undefined; +} + +function stringListField(data: Record, key: string): string[] | undefined { + const value = data[key]; + if (!Array.isArray(value)) return undefined; + const items = value.map((item) => typeof item === "string" ? item : "").filter(Boolean); + return items.length ? items : undefined; +} + +function chatResultsField(data: Record): AutomationResultEvidence["chat_results"] { + const value = data.chat_results; + if (!Array.isArray(value)) return undefined; + const results = value + .filter((item): item is Record => Boolean(item) && typeof item === "object" && !Array.isArray(item)) + .map((item) => ({ + index: numberField(item, "index"), + expected_text: stringField(item, "expected_text"), + status: stringField(item, "status"), + reason: stringField(item, "reason"), + })); + return results.length ? results : undefined; +} + +function browserDiagnosticsField(data: Record): AutomationResultEvidence["browser_diagnostics"] { + const value = data.browser_diagnostics; + if (!value || typeof value !== "object" || Array.isArray(value)) return undefined; + const diagnostics = value as Record; + const findings = Array.isArray(diagnostics.findings) ? diagnostics.findings.length : undefined; + return { + status: stringField(diagnostics, "status"), + reason: stringField(diagnostics, "reason"), + finding_count: findings, + }; +} + function evidenceDirFromOptions(options: Record): string | undefined { const explicit = typeof options["evidence-dir"] === "string" ? options["evidence-dir"] : undefined; if (explicit) return resolve(explicit); @@ -652,6 +709,13 @@ export function readAutomationResultEvidence(options: Record): string | null { - const repo = env.LANGBOT_REPO; - if (!repo) return null; - const logsDir = join(repo, "data", "logs"); +function langBotRepoFromRoot(root: string): string | null { + const candidates = [ + resolve(root), + basename(resolve(root)) === "skills" ? resolve(root, "..") : "", + resolve(root, "../LangBot"), + resolve(root, "LangBot"), + ].filter(Boolean); + + const seen = new Set(); + for (const candidate of candidates) { + if (seen.has(candidate)) continue; + seen.add(candidate); + if (existsSync(join(candidate, "data", "config.yaml"))) return candidate; + } + return null; +} + +function langBotLogsDir(env: Record, root: string): string | null { + const repo = env.LANGBOT_REPO ? resolve(env.LANGBOT_REPO) : langBotRepoFromRoot(root); + return repo ? join(repo, "data", "logs") : null; +} + +export function latestLangBotLogPath(env: Record, root = process.cwd()): string | null { + const logsDir = langBotLogsDir(env, root); + if (!logsDir) return null; if (!existsSync(logsDir)) return null; const candidates = readdirSync(logsDir) diff --git a/skills/src/readiness.ts b/skills/src/readiness.ts index 945fcb598..b20a23ecc 100644 --- a/skills/src/readiness.ts +++ b/skills/src/readiness.ts @@ -111,6 +111,8 @@ export function automationEnvDefaults(item: StructuredItem, env: EnvSource = pro ["automation_image_base64_fixture", "LANGBOT_E2E_IMAGE_BASE64_PATH"], ["automation_runner_config_patch_json", "LANGBOT_E2E_RUNNER_CONFIG_PATCH_JSON"], ["automation_restore_runner_config", "LANGBOT_E2E_RESTORE_RUNNER_CONFIG"], + ["automation_extensions_patch_json", "LANGBOT_E2E_EXTENSIONS_PATCH_JSON"], + ["automation_restore_extensions", "LANGBOT_E2E_RESTORE_EXTENSIONS"], ["automation_expected_runner_id", "LANGBOT_E2E_EXPECTED_RUNNER_ID"], ["automation_reset_debug_chat", "LANGBOT_E2E_RESET_DEBUG_CHAT"], ["automation_debug_chat_session_type", "LANGBOT_E2E_DEBUG_CHAT_SESSION_TYPE"], diff --git a/skills/test/lbs-cli.test.ts b/skills/test/lbs-cli.test.ts index 29cec450e..41d4e0e65 100644 --- a/skills/test/lbs-cli.test.ts +++ b/skills/test/lbs-cli.test.ts @@ -1,16 +1,51 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import { appendFileSync, chmodSync, existsSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; -import { spawnSync } from "node:child_process"; +import { + appendFileSync, + chmodSync, + existsSync, + mkdtempSync, + mkdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { spawn, spawnSync } from "node:child_process"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { CommandContext } from "../src/types.ts"; -import { commandCaseList, commandCaseNew, commandCaseShow } from "../src/commands/case.ts"; +import { + commandCaseList, + commandCaseNew, + commandCaseShow, +} from "../src/commands/case.ts"; import { commandEnvDoctor, commandEnvShow } from "../src/commands/env.ts"; -import { commandFixtureCheck, commandFixtureList } from "../src/commands/fixture.ts"; -import { commandLogGuard, commandLogScan, commandLogWatch } from "../src/commands/log.ts"; -import { commandSuiteList, commandSuiteNew, commandSuitePlan, commandSuiteReport, commandSuiteRun, commandSuiteShow, commandSuiteStart } from "../src/commands/suite.ts"; -import { commandTestPlan, commandTestRecommend, commandTestReport, commandTestResult, commandTestRun, commandTestStart } from "../src/commands/test.ts"; +import { + commandFixtureCheck, + commandFixtureList, +} from "../src/commands/fixture.ts"; +import { + commandLogGuard, + commandLogScan, + commandLogWatch, +} from "../src/commands/log.ts"; +import { + commandSuiteList, + commandSuiteNew, + commandSuitePlan, + commandSuiteReport, + commandSuiteRun, + commandSuiteShow, + commandSuiteStart, +} from "../src/commands/suite.ts"; +import { + commandTestPlan, + commandTestRecommend, + commandTestReport, + commandTestResult, + commandTestRun, + commandTestStart, +} from "../src/commands/test.ts"; import { commandTroubleSearch } from "../src/commands/trouble.ts"; import { commandValidate } from "../src/commands/validate.ts"; import { commandIndex } from "../src/commands/skill.ts"; @@ -21,6 +56,11 @@ import { findNewFailureSignal, minExpectedOccurrences, } from "../scripts/e2e/lib/debug-chat.mjs"; +import { + ensureAuthenticatedBrowser, + resolveLangBotRepo, + scanBrowserDiagnostics, +} from "../scripts/e2e/lib/langbot-e2e.mjs"; const root = process.cwd(); @@ -37,6 +77,149 @@ test("repo root detects the skills tree before generated bin exists", () => { } }); +test("e2e helpers resolve embedded LangBot repo from skills cwd", async () => { + const tmp = mkdtempSync(join(tmpdir(), "lbs-langbot-repo-detect-")); + try { + const repo = join(tmp, "LangBot"); + const skills = join(repo, "skills"); + mkdirSync(join(repo, "data"), { recursive: true }); + mkdirSync(skills, { recursive: true }); + writeFileSync( + join(repo, "data", "config.yaml"), + "system:\n recovery_key: test\n", + ); + + assert.equal(await resolveLangBotRepo("", skills), repo); + } finally { + rmSync(tmp, { recursive: true, force: true }); + } +}); + +test("e2e browser diagnostics fail on console server errors", async () => { + const tmp = mkdtempSync(join(tmpdir(), "lbs-browser-diagnostics-")); + try { + const consoleLog = join(tmp, "console.log"); + const networkLog = join(tmp, "network.log"); + writeFileSync( + consoleLog, + [ + "[2026-06-30T03:02:49.702+08:00] [error] Failed to load resource: the server responded with a status of 500 ()", + "[2026-06-30T03:02:49.703+08:00] [error] Server error: Action list_plugins call timed out", + ].join("\n"), + ); + writeFileSync( + networkLog, + "[2026-06-30T03:02:49.701+08:00] [response] 500 http://127.0.0.1:5300/api/v1/plugins\n", + ); + + const result = await scanBrowserDiagnostics({ consoleLog, networkLog }); + + assert.equal(result.status, "fail"); + assert.match(result.reason, /Browser diagnostics found/); + assert.ok( + result.findings.some( + (finding: { kind: string }) => finding.kind === "api_server_error", + ), + ); + assert.ok( + result.findings.some( + (finding: { kind: string }) => finding.kind === "http_5xx", + ), + ); + } finally { + rmSync(tmp, { recursive: true, force: true }); + } +}); + +test("e2e helper can inject a local login token into a fresh browser context", async () => { + const tmp = mkdtempSync(join(tmpdir(), "lbs-auth-helper-")); + const previousFetch = globalThis.fetch; + const previousRepo = process.env.LANGBOT_REPO; + try { + const repo = join(tmp, "LangBot"); + mkdirSync(join(repo, "data"), { recursive: true }); + writeFileSync( + join(repo, "data", "config.yaml"), + "system:\n recovery_key: recovery-test\n", + ); + process.env.LANGBOT_REPO = repo; + + globalThis.fetch = (async (url: string, init?: RequestInit) => { + if (url.endsWith("/api/v1/user/reset-password")) { + return new Response(JSON.stringify({ code: 0 }), { status: 200 }); + } + if (url.endsWith("/api/v1/user/auth")) { + return new Response( + JSON.stringify({ code: 0, data: { token: "fresh-token" } }), + { status: 200 }, + ); + } + if (url.endsWith("/api/v1/user/check-token")) { + const auth = + init?.headers && typeof init.headers === "object" + ? (init.headers as Record).Authorization + : ""; + return new Response( + JSON.stringify({ + code: auth === "Bearer fresh-token" ? 0 : -1, + msg: auth === "Bearer fresh-token" ? "ok" : "missing token", + }), + { status: auth === "Bearer fresh-token" ? 200 : 401 }, + ); + } + return new Response(JSON.stringify({ code: -1, msg: "unexpected" }), { + status: 404, + }); + }) as typeof fetch; + + let token = ""; + const page = { + addInitScript: async () => {}, + goto: async () => {}, + evaluate: async (fn: unknown, arg: string) => { + const text = String(fn); + if (text.includes("localStorage.setItem")) { + token = arg; + return undefined; + } + if (text.includes("localStorage.getItem")) { + if (!token) { + return { + authenticated: false, + http_status: 0, + code: null, + reason: "No localStorage token.", + }; + } + return { + authenticated: true, + http_status: 200, + code: 0, + reason: "Token accepted by backend.", + }; + } + return undefined; + }, + }; + + const result = await ensureAuthenticatedBrowser(page, { + frontendUrl: "http://127.0.0.1:3000", + backendUrl: "http://127.0.0.1:5300", + user: "qa@example.invalid", + password: "password", + }); + + assert.equal(result.status, "pass"); + assert.equal(result.injected, true); + assert.equal(token, "fresh-token"); + } finally { + globalThis.fetch = previousFetch; + if (previousRepo === undefined) delete process.env.LANGBOT_REPO; + else process.env.LANGBOT_REPO = previousRepo; + rmSync(tmp, { recursive: true, force: true }); + } +}); + function ctx(args: string[]): CommandContext { return { root, args }; } @@ -55,7 +238,11 @@ function capture(fn: () => number): { code: number; output: string } { } } -function captureAll(fn: () => number): { code: number; output: string; error: string } { +function captureAll(fn: () => number): { + code: number; + output: string; + error: string; +} { const originalLog = console.log; const originalWrite = process.stderr.write; const lines: string[] = []; @@ -76,7 +263,12 @@ function captureAll(fn: () => number): { code: number; output: string; error: st } } -function suiteResult(caseId: string, runId: string, status = "pass", evidence = ["ui", "screenshot", "console", "backend_log"]): string { +function suiteResult( + caseId: string, + runId: string, + status = "pass", + evidence = ["ui", "screenshot", "console", "backend_log"], +): string { return JSON.stringify({ source: "final", case_id: caseId, @@ -90,7 +282,9 @@ function suiteResult(caseId: string, runId: string, status = "pass", evidence = } function withEnv(values: Record, fn: () => T): T { - const previous = new Map(Object.keys(values).map((key) => [key, process.env[key]])); + const previous = new Map( + Object.keys(values).map((key) => [key, process.env[key]]), + ); try { for (const [key, value] of Object.entries(values)) process.env[key] = value; return fn(); @@ -102,7 +296,9 @@ function withEnv(values: Record, fn: () => T): T { } } -async function captureAsync(fn: () => Promise): Promise<{ code: number; output: string }> { +async function captureAsync( + fn: () => Promise, +): Promise<{ code: number; output: string }> { const originalLog = console.log; const lines: string[] = []; console.log = (...args: unknown[]) => { @@ -116,6 +312,88 @@ async function captureAsync(fn: () => Promise): Promise<{ code: number; } } +async function startFakeProviderForTest(): Promise<{ + baseUrl: string; + stop: () => Promise; +}> { + const child = spawn( + process.execPath, + [ + join(root, "scripts/e2e/fake-openai-provider.mjs"), + "--host=127.0.0.1", + "--port=0", + ], + { + cwd: root, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + + const started = await new Promise<{ baseUrl: string }>((resolve, reject) => { + const timer = setTimeout(() => { + reject(new Error("Timed out waiting for fake provider startup.")); + }, 5000); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk.toString(); + const newline = stdout.indexOf("\n"); + if (newline < 0) return; + clearTimeout(timer); + try { + const payload = JSON.parse(stdout.slice(0, newline)); + resolve({ baseUrl: payload.base_url }); + } catch (error) { + reject(error); + } + }); + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + child.on("error", (error) => { + clearTimeout(timer); + reject(error); + }); + child.on("exit", (code) => { + if (code === null) return; + clearTimeout(timer); + reject( + new Error(`Fake provider exited before startup: ${code}; ${stderr}`), + ); + }); + }); + + return { + baseUrl: started.baseUrl, + stop: async () => { + if (child.exitCode !== null) return; + child.kill("SIGTERM"); + await new Promise((resolve) => child.once("exit", () => resolve())); + }, + }; +} + +async function requestFakeProvider( + provider: { baseUrl: string }, + payload: Record, +): Promise> { + const response = await fetch(`${provider.baseUrl}/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + model: "langbot-e2e-fake-model", + stream: false, + ...payload, + }), + }); + assert.equal(response.status, 200); + return await response.json(); +} + +function fakeProviderMessage(json: Record): Record { + return json.choices?.[0]?.message || {}; +} + test("validate accepts the repository assets", () => { const result = capture(() => commandValidate(root)); assert.equal(result.code, 0); @@ -130,10 +408,18 @@ test("validate allows blank shared env values but requires declared keys", () => const testingDir = join(skillsDir, "langbot-testing"); mkdirSync(schemasDir, { recursive: true }); mkdirSync(testingDir, { recursive: true }); - for (const schemaName of ["case.schema.json", "suite.schema.json", "troubleshooting.schema.json", "skill-index.schema.json"]) { + for (const schemaName of [ + "case.schema.json", + "suite.schema.json", + "troubleshooting.schema.json", + "skill-index.schema.json", + ]) { writeFileSync(join(schemasDir, schemaName), "{}"); } - writeFileSync(join(testingDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n"); + writeFileSync( + join(testingDir, "SKILL.md"), + "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", + ); const envText = [ "LANGBOT_FRONTEND_URL=http://127.0.0.1:3000", "LANGBOT_BACKEND_URL=http://127.0.0.1:5300", @@ -158,23 +444,48 @@ test("validate allows blank shared env values but requires declared keys", () => test("index includes case summaries for agent discovery", () => { const result = capture(() => commandIndex({ root, args: ["index"] })); assert.equal(result.code, 0); - const index = JSON.parse(readFileSync(join(root, "skills.index.json"), "utf8")); - const testing = index.skills.find((skill: { name: string }) => skill.name === "langbot-testing"); + const index = JSON.parse( + readFileSync(join(root, "skills.index.json"), "utf8"), + ); + const testing = index.skills.find( + (skill: { name: string }) => skill.name === "langbot-testing", + ); assert.ok(testing); - assert.ok(testing.case_summaries.some((item: { id: string; priority: string; evidence_required: string[] }) => ( - item.id === "pipeline-debug-chat" && item.priority === "p0" && item.evidence_required.includes("backend_log") - ))); - assert.ok(testing.case_summaries.some((item: { id: string; setup_automation: string[]; setup_provides_env: string[] }) => ( - item.id === "agent-runner-qa-debug-chat" && - item.setup_automation.includes("case:agent-runner-live-install") && - item.setup_provides_env.includes("LANGBOT_QA_AGENT_RUNNER_PIPELINE_URL") - ))); - assert.ok(testing.suite_summaries.some((item: { id: string; cases: string[] }) => ( - item.id === "core-smoke" && item.cases.includes("pipeline-debug-chat") - ))); - assert.ok(testing.fixtures.some((item: { id: string; related_cases: string[] }) => ( - item.id === "mcp-stdio-echo-server" && item.related_cases.includes("mcp-stdio-tool-call") - ))); + assert.ok( + testing.case_summaries.some( + (item: { id: string; priority: string; evidence_required: string[] }) => + item.id === "pipeline-debug-chat" && + item.priority === "p0" && + item.evidence_required.includes("backend_log"), + ), + ); + assert.ok( + testing.case_summaries.some( + (item: { + id: string; + setup_automation: string[]; + setup_provides_env: string[]; + }) => + item.id === "agent-runner-qa-debug-chat" && + item.setup_automation.includes("case:agent-runner-live-install") && + item.setup_provides_env.includes( + "LANGBOT_QA_AGENT_RUNNER_PIPELINE_URL", + ), + ), + ); + assert.ok( + testing.suite_summaries.some( + (item: { id: string; cases: string[] }) => + item.id === "core-smoke" && item.cases.includes("pipeline-debug-chat"), + ), + ); + assert.ok( + testing.fixtures.some( + (item: { id: string; related_cases: string[] }) => + item.id === "mcp-stdio-echo-server" && + item.related_cases.includes("mcp-stdio-tool-call"), + ), + ); }); test("index check detects stale index without writing", () => { @@ -184,12 +495,16 @@ test("index check detects stale index without writing", () => { const fresh = readFileSync(path, "utf8"); try { - const ok = capture(() => commandIndex({ root, args: ["index", "--check"] })); + const ok = capture(() => + commandIndex({ root, args: ["index", "--check"] }), + ); assert.equal(ok.code, 0); assert.match(ok.output, /^OK /); writeFileSync(path, "{}\n"); - const stale = captureAll(() => commandIndex({ root, args: ["index", "--check"] })); + const stale = captureAll(() => + commandIndex({ root, args: ["index", "--check"] }), + ); assert.equal(stale.code, 1); assert.match(stale.error, /index is stale/); assert.equal(readFileSync(path, "utf8"), "{}\n"); @@ -207,22 +522,24 @@ test("case list exposes seeded QA cases", () => { }); test("case list JSON filters by reusable agent-selection metadata", () => { - const result = capture(() => commandCaseList(ctx([ - "case", - "list", - "--json", - "--priority", - "p0", - "--automation", - ]))); + const result = capture(() => + commandCaseList( + ctx(["case", "list", "--json", "--priority", "p0", "--automation"]), + ), + ); assert.equal(result.code, 0); const rows = JSON.parse(result.output); assert.ok(rows.length >= 2); assert.ok(rows.every((row: { priority: string }) => row.priority === "p0")); assert.ok(rows.every((row: { automation: string }) => row.automation)); - assert.ok(rows.some((row: { id: string; evidence_required: string[]; readiness: string }) => ( - row.id === "pipeline-debug-chat" && row.evidence_required.includes("backend_log") && row.readiness - ))); + assert.ok( + rows.some( + (row: { id: string; evidence_required: string[]; readiness: string }) => + row.id === "pipeline-debug-chat" && + row.evidence_required.includes("backend_log") && + row.readiness, + ), + ); }); test("case list distinguishes machine readiness from manual precondition checks", () => { @@ -234,7 +551,10 @@ test("case list distinguishes machine readiness from manual precondition checks" join(skillDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", ); - writeFileSync(join(tmp, "skills", ".env"), "LANGBOT_FRONTEND_URL=http://127.0.0.1:3000\n"); + writeFileSync( + join(tmp, "skills", ".env"), + "LANGBOT_FRONTEND_URL=http://127.0.0.1:3000\n", + ); writeFileSync( join(skillDir, "cases", "manual-case.yaml"), [ @@ -263,12 +583,16 @@ test("case list distinguishes machine readiness from manual precondition checks" ].join("\n"), ); - const machineReady = capture(() => commandCaseList({ root: tmp, args: ["case", "list", "--machine-ready"] })); + const machineReady = capture(() => + commandCaseList({ root: tmp, args: ["case", "list", "--machine-ready"] }), + ); assert.equal(machineReady.code, 0); assert.match(machineReady.output, /manual-case/); assert.match(machineReady.output, /manual-check/); - const ready = capture(() => commandCaseList({ root: tmp, args: ["case", "list", "--ready"] })); + const ready = capture(() => + commandCaseList({ root: tmp, args: ["case", "list", "--ready"] }), + ); assert.equal(ready.code, 0); assert.doesNotMatch(ready.output, /manual-case/); } finally { @@ -277,7 +601,9 @@ test("case list distinguishes machine readiness from manual precondition checks" }); test("case show prints structured agent-browser case", () => { - const result = capture(() => commandCaseShow(ctx(["case", "show", "pipeline-debug-chat"]))); + const result = capture(() => + commandCaseShow(ctx(["case", "show", "pipeline-debug-chat"])), + ); assert.equal(result.code, 0); assert.match(result.output, /^id: pipeline-debug-chat/m); assert.match(result.output, /^mode: agent-browser/m); @@ -294,10 +620,12 @@ test("case new writes required selection metadata", () => { "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", ); - const result = capture(() => commandCaseNew({ - root: tmp, - args: ["case", "new", "new-case", "--title", "New Case"], - })); + const result = capture(() => + commandCaseNew({ + root: tmp, + args: ["case", "new", "new-case", "--title", "New Case"], + }), + ); assert.equal(result.code, 0); const text = readFileSync(join(skillDir, "cases", "new-case.yaml"), "utf8"); @@ -311,34 +639,59 @@ test("case new writes required selection metadata", () => { }); test("suite list and plan expose reusable case groups", () => { - const list = capture(() => commandSuiteList(ctx(["suite", "list", "--json", "--priority", "p0"]))); + const list = capture(() => + commandSuiteList(ctx(["suite", "list", "--json", "--priority", "p0"])), + ); assert.equal(list.code, 0); const suites = JSON.parse(list.output); - assert.ok(suites.some((suite: { id: string; cases: string[] }) => ( - suite.id === "core-smoke" && suite.cases.includes("webui-login-state") - ))); + assert.ok( + suites.some( + (suite: { id: string; cases: string[] }) => + suite.id === "core-smoke" && suite.cases.includes("webui-login-state"), + ), + ); - const plan = capture(() => commandSuitePlan(ctx(["suite", "plan", "core-smoke", "--json"]))); + const plan = capture(() => + commandSuitePlan(ctx(["suite", "plan", "core-smoke", "--json"])), + ); assert.equal(plan.code, 0); const suitePlan = JSON.parse(plan.output); assert.equal(suitePlan.id, "core-smoke"); - assert.ok(suitePlan.cases.some((item: { id: string; evidence_required: string[] }) => ( - item.id === "pipeline-debug-chat" && item.evidence_required.includes("backend_log") - ))); - assert.ok(suitePlan.commands.some((item: { id: string; automation: string }) => ( - item.id === "pipeline-debug-chat" && item.automation.includes("test run") - ))); + assert.ok( + suitePlan.cases.some( + (item: { id: string; evidence_required: string[] }) => + item.id === "pipeline-debug-chat" && + item.evidence_required.includes("backend_log"), + ), + ); + assert.ok( + suitePlan.commands.some( + (item: { id: string; automation: string }) => + item.id === "pipeline-debug-chat" && + item.automation.includes("test run"), + ), + ); - const localAgent = capture(() => commandSuitePlan(ctx(["suite", "plan", "local-agent-gate", "--json"]))); + const localAgent = capture(() => + commandSuitePlan(ctx(["suite", "plan", "local-agent-gate", "--json"])), + ); assert.equal(localAgent.code, 0); const localAgentPlan = JSON.parse(localAgent.output); - assert.ok(["ready", "missing", "manual_check"].includes(localAgentPlan.readiness.status)); - const basic = localAgentPlan.cases.find((item: { id: string }) => item.id === "local-agent-basic-debug-chat"); + assert.ok( + ["ready", "missing", "manual_check"].includes( + localAgentPlan.readiness.status, + ), + ); + const basic = localAgentPlan.cases.find( + (item: { id: string }) => item.id === "local-agent-basic-debug-chat", + ); assert.equal(basic.automation_readiness.pipeline_env_required, true); }); test("suite show prints structured suite YAML", () => { - const result = capture(() => commandSuiteShow(ctx(["suite", "show", "local-agent-gate"]))); + const result = capture(() => + commandSuiteShow(ctx(["suite", "show", "local-agent-gate"])), + ); assert.equal(result.code, 0); assert.match(result.output, /^id: local-agent-gate/m); assert.match(result.output, /^cases:/m); @@ -349,16 +702,20 @@ test("suite start creates a run handoff with per-case evidence commands", () => const tmp = mkdtempSync(join(tmpdir(), "lbs-suite-start-")); try { const evidenceRoot = join(tmp, "evidence"); - const result = capture(() => commandSuiteStart(ctx([ - "suite", - "start", - "core-smoke", - "--run-id", - "core-smoke-local", - "--evidence-dir", - evidenceRoot, - "--json", - ]))); + const result = capture(() => + commandSuiteStart( + ctx([ + "suite", + "start", + "core-smoke", + "--run-id", + "core-smoke-local", + "--evidence-dir", + evidenceRoot, + "--json", + ]), + ), + ); assert.equal(result.code, 0); const start = JSON.parse(result.output); assert.equal(start.suite.id, "core-smoke"); @@ -369,12 +726,23 @@ test("suite start creates a run handoff with per-case evidence commands", () => assert.match(start.report_command, /bin\/lbs suite report core-smoke/); assert.ok(existsSync(join(evidenceRoot, "suite-start.json"))); assert.ok(existsSync(join(evidenceRoot, "suite-start.md"))); - const pipeline = start.cases.find((item: { id: string }) => item.id === "pipeline-debug-chat"); + const pipeline = start.cases.find( + (item: { id: string }) => item.id === "pipeline-debug-chat", + ); assert.ok(pipeline); assert.ok(existsSync(join(evidenceRoot, "pipeline-debug-chat"))); - assert.match(pipeline.automation_command, /bin\/lbs test run pipeline-debug-chat/); - assert.match(pipeline.report_command, /--evidence-dir .+pipeline-debug-chat/); - assert.match(pipeline.result_command_template, /bin\/lbs test result pipeline-debug-chat/); + assert.match( + pipeline.automation_command, + /bin\/lbs test run pipeline-debug-chat/, + ); + assert.match( + pipeline.report_command, + /--evidence-dir .+pipeline-debug-chat/, + ); + assert.match( + pipeline.result_command_template, + /bin\/lbs test result pipeline-debug-chat/, + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -398,25 +766,33 @@ test("suite report aggregates case result JSON files", () => { ); } - const result = capture(() => commandSuiteReport(ctx([ - "suite", - "report", - "core-smoke", - "--run-id", - runId, - "--evidence-dir", - evidenceRoot, - "--json", - ]))); + const result = capture(() => + commandSuiteReport( + ctx([ + "suite", + "report", + "core-smoke", + "--run-id", + runId, + "--evidence-dir", + evidenceRoot, + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); assert.equal(report.status, "env_issue"); assert.equal(report.counts.pass, 2); assert.equal(report.counts.env_issue, 1); - assert.ok(report.cases.some((item: { id: string; result: { status: string } }) => ( - item.id === "local-agent-basic-debug-chat" && item.result.status === "env_issue" - ))); + assert.ok( + report.cases.some( + (item: { id: string; result: { status: string } }) => + item.id === "local-agent-basic-debug-chat" && + item.result.status === "env_issue", + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -427,7 +803,11 @@ test("suite report treats pass without required evidence as incomplete", () => { try { const evidenceRoot = join(tmp, "suite-evidence"); const runId = "suite-report-evidence"; - for (const caseId of ["webui-login-state", "pipeline-debug-chat", "local-agent-basic-debug-chat"]) { + for (const caseId of [ + "webui-login-state", + "pipeline-debug-chat", + "local-agent-basic-debug-chat", + ]) { const dir = join(evidenceRoot, caseId); mkdirSync(dir, { recursive: true }); writeFileSync( @@ -436,23 +816,31 @@ test("suite report treats pass without required evidence as incomplete", () => { ); } - const result = capture(() => commandSuiteReport(ctx([ - "suite", - "report", - "core-smoke", - "--run-id", - runId, - "--evidence-dir", - evidenceRoot, - "--json", - ]))); + const result = capture(() => + commandSuiteReport( + ctx([ + "suite", + "report", + "core-smoke", + "--run-id", + runId, + "--evidence-dir", + evidenceRoot, + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); assert.equal(report.status, "incomplete"); - assert.ok(report.cases.some((item: { id: string; result: { evidence_missing: string[] } }) => ( - item.id === "pipeline-debug-chat" && item.result.evidence_missing.includes("backend_log") - ))); + assert.ok( + report.cases.some( + (item: { id: string; result: { evidence_missing: string[] } }) => + item.id === "pipeline-debug-chat" && + item.result.evidence_missing.includes("backend_log"), + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -464,25 +852,35 @@ test("suite report marks missing case evidence as incomplete", () => { const evidenceRoot = join(tmp, "suite-evidence"); const runId = "suite-report-missing"; mkdirSync(join(evidenceRoot, "webui-login-state"), { recursive: true }); - writeFileSync(join(evidenceRoot, "webui-login-state", "result.json"), suiteResult("webui-login-state", runId, "pass")); + writeFileSync( + join(evidenceRoot, "webui-login-state", "result.json"), + suiteResult("webui-login-state", runId, "pass"), + ); - const result = capture(() => commandSuiteReport(ctx([ - "suite", - "report", - "core-smoke", - "--run-id", - runId, - "--evidence-dir", - evidenceRoot, - "--json", - ]))); + const result = capture(() => + commandSuiteReport( + ctx([ + "suite", + "report", + "core-smoke", + "--run-id", + runId, + "--evidence-dir", + evidenceRoot, + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); assert.equal(report.status, "incomplete"); - assert.ok(report.cases.some((item: { id: string; result: { status: string } }) => ( - item.id === "pipeline-debug-chat" && item.result.status === "missing" - ))); + assert.ok( + report.cases.some( + (item: { id: string; result: { status: string } }) => + item.id === "pipeline-debug-chat" && item.result.status === "missing", + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -493,34 +891,61 @@ test("suite report rejects result files from the wrong case or run", () => { try { const evidenceRoot = join(tmp, "suite-evidence"); const runId = "suite-report-mismatch"; - for (const caseId of ["webui-login-state", "pipeline-debug-chat", "local-agent-basic-debug-chat"]) { + for (const caseId of [ + "webui-login-state", + "pipeline-debug-chat", + "local-agent-basic-debug-chat", + ]) { const dir = join(evidenceRoot, caseId); mkdirSync(dir, { recursive: true }); - writeFileSync(join(dir, "result.json"), suiteResult(caseId, runId, "pass")); + writeFileSync( + join(dir, "result.json"), + suiteResult(caseId, runId, "pass"), + ); } - writeFileSync(join(evidenceRoot, "pipeline-debug-chat", "result.json"), suiteResult("webui-login-state", runId, "pass")); - writeFileSync(join(evidenceRoot, "local-agent-basic-debug-chat", "result.json"), suiteResult("local-agent-basic-debug-chat", "old-run", "pass")); + writeFileSync( + join(evidenceRoot, "pipeline-debug-chat", "result.json"), + suiteResult("webui-login-state", runId, "pass"), + ); + writeFileSync( + join(evidenceRoot, "local-agent-basic-debug-chat", "result.json"), + suiteResult("local-agent-basic-debug-chat", "old-run", "pass"), + ); - const result = capture(() => commandSuiteReport(ctx([ - "suite", - "report", - "core-smoke", - "--run-id", - runId, - "--evidence-dir", - evidenceRoot, - "--json", - ]))); + const result = capture(() => + commandSuiteReport( + ctx([ + "suite", + "report", + "core-smoke", + "--run-id", + runId, + "--evidence-dir", + evidenceRoot, + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); assert.equal(report.status, "fail"); - assert.ok(report.cases.some((item: { id: string; result: { status: string; reason: string } }) => ( - item.id === "pipeline-debug-chat" && item.result.status === "invalid" && item.result.reason.includes("case_id mismatch") - ))); - assert.ok(report.cases.some((item: { id: string; result: { status: string; reason: string } }) => ( - item.id === "local-agent-basic-debug-chat" && item.result.status === "invalid" && item.result.reason.includes("run_id mismatch") - ))); + assert.ok( + report.cases.some( + (item: { id: string; result: { status: string; reason: string } }) => + item.id === "pipeline-debug-chat" && + item.result.status === "invalid" && + item.result.reason.includes("case_id mismatch"), + ), + ); + assert.ok( + report.cases.some( + (item: { id: string; result: { status: string; reason: string } }) => + item.id === "local-agent-basic-debug-chat" && + item.result.status === "invalid" && + item.result.reason.includes("run_id mismatch"), + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -536,7 +961,10 @@ test("suite run executes automated cases and aggregates a verdict", () => { mkdirSync(casesDir, { recursive: true }); mkdirSync(suitesDir, { recursive: true }); mkdirSync(scriptsDir, { recursive: true }); - writeFileSync(join(skillDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n"); + writeFileSync( + join(skillDir, "SKILL.md"), + "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", + ); writeFileSync(join(tmp, "skills", ".env"), ""); writeFileSync( join(casesDir, "one.yaml"), @@ -602,16 +1030,30 @@ test("suite run executes automated cases and aggregates a verdict", () => { ].join("\n"), ); - const result = capture(() => commandSuiteRun({ - root: tmp, - args: ["suite", "run", "mini", "--run-id", "mini-run", "--evidence-dir", join(tmp, "evidence"), "--json"], - })); + const result = capture(() => + commandSuiteRun({ + root: tmp, + args: [ + "suite", + "run", + "mini", + "--run-id", + "mini-run", + "--evidence-dir", + join(tmp, "evidence"), + "--json", + ], + }), + ); assert.equal(result.code, 0); const payload = JSON.parse(result.output); assert.equal(payload.report.status, "pass"); assert.equal(payload.report.counts.pass, 2); - assert.deepEqual(payload.executions.map((item: { status: string }) => item.status), ["ok", "ok"]); + assert.deepEqual( + payload.executions.map((item: { status: string }) => item.status), + ["ok", "ok"], + ); assert.ok(existsSync(join(tmp, "evidence", "one", "result.json"))); assert.ok(existsSync(join(tmp, "evidence", "two", "result.json"))); } finally { @@ -629,7 +1071,10 @@ test("suite run JSON captures failed case output", () => { mkdirSync(casesDir, { recursive: true }); mkdirSync(suitesDir, { recursive: true }); mkdirSync(scriptsDir, { recursive: true }); - writeFileSync(join(skillDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n"); + writeFileSync( + join(skillDir, "SKILL.md"), + "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", + ); writeFileSync(join(tmp, "skills", ".env"), ""); writeFileSync( join(casesDir, "fail-case.yaml"), @@ -659,12 +1104,26 @@ test("suite run JSON captures failed case output", () => { " - fail-case", ].join("\n"), ); - writeFileSync(join(scriptsDir, "fail.mjs"), "console.error('child failure detail'); process.exit(1);\n"); + writeFileSync( + join(scriptsDir, "fail.mjs"), + "console.error('child failure detail'); process.exit(1);\n", + ); - const result = capture(() => commandSuiteRun({ - root: tmp, - args: ["suite", "run", "mini", "--run-id", "mini-run", "--evidence-dir", join(tmp, "evidence"), "--json"], - })); + const result = capture(() => + commandSuiteRun({ + root: tmp, + args: [ + "suite", + "run", + "mini", + "--run-id", + "mini-run", + "--evidence-dir", + join(tmp, "evidence"), + "--json", + ], + }), + ); assert.equal(result.code, 1); const payload = JSON.parse(result.output); @@ -686,7 +1145,10 @@ test("suite run preserves classified env_issue automation results", () => { mkdirSync(casesDir, { recursive: true }); mkdirSync(suitesDir, { recursive: true }); mkdirSync(scriptsDir, { recursive: true }); - writeFileSync(join(skillDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n"); + writeFileSync( + join(skillDir, "SKILL.md"), + "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", + ); writeFileSync(join(tmp, "skills", ".env"), ""); writeFileSync( join(casesDir, "env-case.yaml"), @@ -737,10 +1199,21 @@ test("suite run preserves classified env_issue automation results", () => { ].join("\n"), ); - const result = capture(() => commandSuiteRun({ - root: tmp, - args: ["suite", "run", "mini", "--run-id", "mini-run", "--evidence-dir", join(tmp, "evidence"), "--json"], - })); + const result = capture(() => + commandSuiteRun({ + root: tmp, + args: [ + "suite", + "run", + "mini", + "--run-id", + "mini-run", + "--evidence-dir", + join(tmp, "evidence"), + "--json", + ], + }), + ); assert.equal(result.code, 2); const payload = JSON.parse(result.output); @@ -764,7 +1237,10 @@ test("suite run failure cannot be masked by stale pass result", () => { mkdirSync(suitesDir, { recursive: true }); mkdirSync(scriptsDir, { recursive: true }); mkdirSync(join(evidenceDir, "fail-case"), { recursive: true }); - writeFileSync(join(skillDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n"); + writeFileSync( + join(skillDir, "SKILL.md"), + "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", + ); writeFileSync(join(tmp, "skills", ".env"), ""); writeFileSync( join(casesDir, "fail-case.yaml"), @@ -797,17 +1273,31 @@ test("suite run failure cannot be masked by stale pass result", () => { ].join("\n"), ); writeFileSync(join(scriptsDir, "fail.mjs"), "process.exit(1);\n"); - writeFileSync(join(evidenceDir, "fail-case", "result.json"), JSON.stringify({ - case_id: "fail-case", - run_id: "stale-run-fail-case", - status: "pass", - evidence_collected: ["filesystem"], - })); + writeFileSync( + join(evidenceDir, "fail-case", "result.json"), + JSON.stringify({ + case_id: "fail-case", + run_id: "stale-run-fail-case", + status: "pass", + evidence_collected: ["filesystem"], + }), + ); - const result = capture(() => commandSuiteRun({ - root: tmp, - args: ["suite", "run", "mini", "--run-id", "stale-run", "--evidence-dir", evidenceDir, "--json"], - })); + const result = capture(() => + commandSuiteRun({ + root: tmp, + args: [ + "suite", + "run", + "mini", + "--run-id", + "stale-run", + "--evidence-dir", + evidenceDir, + "--json", + ], + }), + ); assert.equal(result.code, 1); const payload = JSON.parse(result.output); @@ -829,7 +1319,10 @@ test("suite run dry-run plans automation without creating evidence", () => { mkdirSync(casesDir, { recursive: true }); mkdirSync(suitesDir, { recursive: true }); mkdirSync(scriptsDir, { recursive: true }); - writeFileSync(join(skillDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n"); + writeFileSync( + join(skillDir, "SKILL.md"), + "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", + ); writeFileSync(join(tmp, "skills", ".env"), ""); writeFileSync( join(casesDir, "dry-case.yaml"), @@ -862,10 +1355,22 @@ test("suite run dry-run plans automation without creating evidence", () => { writeFileSync(join(scriptsDir, "fail-if-run.mjs"), "process.exit(9);\n"); const evidenceDir = join(tmp, "evidence"); - const result = capture(() => commandSuiteRun({ - root: tmp, - args: ["suite", "run", "dry-suite", "--run-id", "dry-run", "--evidence-dir", evidenceDir, "--dry-run", "--json"], - })); + const result = capture(() => + commandSuiteRun({ + root: tmp, + args: [ + "suite", + "run", + "dry-suite", + "--run-id", + "dry-run", + "--evidence-dir", + evidenceDir, + "--dry-run", + "--json", + ], + }), + ); assert.equal(result.code, 0); const payload = JSON.parse(result.output); @@ -875,13 +1380,27 @@ test("suite run dry-run plans automation without creating evidence", () => { assert.equal(existsSync(evidenceDir), false); assert.equal(existsSync(join(tmp, "reports", "dry-run.md")), false); - const markdown = capture(() => commandSuiteRun({ - root: tmp, - args: ["suite", "run", "dry-suite", "--run-id", "dry-run-markdown", "--evidence-dir", join(tmp, "evidence-md"), "--dry-run"], - })); + const markdown = capture(() => + commandSuiteRun({ + root: tmp, + args: [ + "suite", + "run", + "dry-suite", + "--run-id", + "dry-run-markdown", + "--evidence-dir", + join(tmp, "evidence-md"), + "--dry-run", + ], + }), + ); assert.equal(markdown.code, 0); assert.match(markdown.output, /# Suite Report: dry-suite/); - assert.equal(existsSync(join(tmp, "reports", "dry-run-markdown.md")), false); + assert.equal( + existsSync(join(tmp, "reports", "dry-run-markdown.md")), + false, + ); assert.equal(existsSync(join(tmp, "evidence-md")), false); } finally { rmSync(tmp, { recursive: true, force: true }); @@ -898,7 +1417,10 @@ test("suite run skips manual-check cases unless explicitly included", () => { mkdirSync(casesDir, { recursive: true }); mkdirSync(suitesDir, { recursive: true }); mkdirSync(scriptsDir, { recursive: true }); - writeFileSync(join(skillDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n"); + writeFileSync( + join(skillDir, "SKILL.md"), + "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", + ); writeFileSync(join(tmp, "skills", ".env"), ""); writeFileSync( join(casesDir, "manual-case.yaml"), @@ -942,25 +1464,53 @@ test("suite run skips manual-check cases unless explicitly included", () => { ].join("\n"), ); - const skipped = capture(() => commandSuiteRun({ - root: tmp, - args: ["suite", "run", "manual-suite", "--run-id", "manual-run", "--evidence-dir", join(tmp, "evidence"), "--json"], - })); + const skipped = capture(() => + commandSuiteRun({ + root: tmp, + args: [ + "suite", + "run", + "manual-suite", + "--run-id", + "manual-run", + "--evidence-dir", + join(tmp, "evidence"), + "--json", + ], + }), + ); assert.equal(skipped.code, 1); const skippedPayload = JSON.parse(skipped.output); assert.equal(skippedPayload.executions[0].status, "skipped"); assert.match(skippedPayload.executions[0].reason, /manual_check/); - assert.equal(existsSync(join(tmp, "evidence", "manual-case", "result.json")), false); + assert.equal( + existsSync(join(tmp, "evidence", "manual-case", "result.json")), + false, + ); - const included = capture(() => commandSuiteRun({ - root: tmp, - args: ["suite", "run", "manual-suite", "--run-id", "manual-run-included", "--evidence-dir", join(tmp, "evidence-included"), "--include-manual-check", "--json"], - })); + const included = capture(() => + commandSuiteRun({ + root: tmp, + args: [ + "suite", + "run", + "manual-suite", + "--run-id", + "manual-run-included", + "--evidence-dir", + join(tmp, "evidence-included"), + "--include-manual-check", + "--json", + ], + }), + ); assert.equal(included.code, 0); const includedPayload = JSON.parse(included.output); assert.equal(includedPayload.executions[0].status, "ok"); assert.equal(includedPayload.report.status, "pass"); - assert.ok(existsSync(join(tmp, "evidence-included", "manual-case", "result.json"))); + assert.ok( + existsSync(join(tmp, "evidence-included", "manual-case", "result.json")), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -978,7 +1528,10 @@ test("suite run skips cases with missing machine readiness unless explicitly inc mkdirSync(suitesDir, { recursive: true }); mkdirSync(fixturesDir, { recursive: true }); mkdirSync(scriptsDir, { recursive: true }); - writeFileSync(join(skillDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n"); + writeFileSync( + join(skillDir, "SKILL.md"), + "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", + ); writeFileSync(join(tmp, "skills", ".env"), ""); writeFileSync( join(casesDir, "not-ready-case.yaml"), @@ -1002,14 +1555,20 @@ test("suite run skips cases with missing machine readiness unless explicitly inc ); writeFileSync( join(fixturesDir, "fixtures.json"), - `${JSON.stringify([{ - id: "missing-fixture", - title: "Missing fixture", - kind: "file", - path: "fixtures/missing.txt", - related_cases: ["not-ready-case"], - checks: ["exists"], - }], null, 2)}\n`, + `${JSON.stringify( + [ + { + id: "missing-fixture", + title: "Missing fixture", + kind: "file", + path: "fixtures/missing.txt", + related_cases: ["not-ready-case"], + checks: ["exists"], + }, + ], + null, + 2, + )}\n`, ); writeFileSync( join(suitesDir, "readiness-suite.yaml"), @@ -1035,28 +1594,64 @@ test("suite run skips cases with missing machine readiness unless explicitly inc ].join("\n"), ); - const skipped = capture(() => commandSuiteRun({ - root: tmp, - args: ["suite", "run", "readiness-suite", "--run-id", "readiness-run", "--evidence-dir", join(tmp, "evidence"), "--json"], - })); + const skipped = capture(() => + commandSuiteRun({ + root: tmp, + args: [ + "suite", + "run", + "readiness-suite", + "--run-id", + "readiness-run", + "--evidence-dir", + join(tmp, "evidence"), + "--json", + ], + }), + ); assert.equal(skipped.code, 1); const skippedPayload = JSON.parse(skipped.output); assert.equal(skippedPayload.executions[0].status, "skipped"); assert.match(skippedPayload.executions[0].reason, /readiness missing/); - assert.match(skippedPayload.executions[0].reason, /LBS_TEST_SUITE_RUN_MISSING_ENV/); - assert.match(skippedPayload.executions[0].reason, /LBS_TEST_SUITE_RUN_MISSING_AUTOMATION_ENV/); + assert.match( + skippedPayload.executions[0].reason, + /LBS_TEST_SUITE_RUN_MISSING_ENV/, + ); + assert.match( + skippedPayload.executions[0].reason, + /LBS_TEST_SUITE_RUN_MISSING_AUTOMATION_ENV/, + ); assert.match(skippedPayload.executions[0].reason, /missing-fixture/); - assert.equal(existsSync(join(tmp, "evidence", "not-ready-case", "result.json")), false); + assert.equal( + existsSync(join(tmp, "evidence", "not-ready-case", "result.json")), + false, + ); - const included = capture(() => commandSuiteRun({ - root: tmp, - args: ["suite", "run", "readiness-suite", "--run-id", "readiness-run-included", "--evidence-dir", join(tmp, "evidence-included"), "--include-not-ready", "--json"], - })); + const included = capture(() => + commandSuiteRun({ + root: tmp, + args: [ + "suite", + "run", + "readiness-suite", + "--run-id", + "readiness-run-included", + "--evidence-dir", + join(tmp, "evidence-included"), + "--include-not-ready", + "--json", + ], + }), + ); assert.equal(included.code, 0); const includedPayload = JSON.parse(included.output); assert.equal(includedPayload.executions[0].status, "ok"); assert.equal(includedPayload.report.status, "pass"); - assert.ok(existsSync(join(tmp, "evidence-included", "not-ready-case", "result.json"))); + assert.ok( + existsSync( + join(tmp, "evidence-included", "not-ready-case", "result.json"), + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -1072,13 +1667,18 @@ test("suite new writes a reusable suite skeleton", () => { "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", ); - const result = capture(() => commandSuiteNew({ - root: tmp, - args: ["suite", "new", "new-suite", "--title", "New Suite"], - })); + const result = capture(() => + commandSuiteNew({ + root: tmp, + args: ["suite", "new", "new-suite", "--title", "New Suite"], + }), + ); assert.equal(result.code, 0); - const text = readFileSync(join(skillDir, "suites", "new-suite.yaml"), "utf8"); + const text = readFileSync( + join(skillDir, "suites", "new-suite.yaml"), + "utf8", + ); assert.match(text, /^description:/m); assert.match(text, /^priority: p2/m); assert.match(text, /^cases:/m); @@ -1088,18 +1688,29 @@ test("suite new writes a reusable suite skeleton", () => { }); test("fixture list and check expose reusable fixture readiness", () => { - const list = capture(() => commandFixtureList(ctx(["fixture", "list", "langbot-testing", "--json"]))); + const list = capture(() => + commandFixtureList(ctx(["fixture", "list", "langbot-testing", "--json"])), + ); assert.equal(list.code, 0); const fixtures = JSON.parse(list.output); - assert.ok(fixtures.some((item: { id: string; exists: boolean }) => ( - item.id === "mcp-stdio-echo-server" && item.exists === true - ))); + assert.ok( + fixtures.some( + (item: { id: string; exists: boolean }) => + item.id === "mcp-stdio-echo-server" && item.exists === true, + ), + ); - const check = capture(() => commandFixtureCheck(ctx(["fixture", "check", "langbot-testing", "--json"]))); + const check = capture(() => + commandFixtureCheck(ctx(["fixture", "check", "langbot-testing", "--json"])), + ); assert.equal(check.code, 0); const report = JSON.parse(check.output); assert.equal(report.status, "pass"); - assert.ok(report.fixtures.some((item: { id: string }) => item.id === "qa-plugin-smoke-package")); + assert.ok( + report.fixtures.some( + (item: { id: string }) => item.id === "qa-plugin-smoke-package", + ), + ); }); test("fixture check reports missing manifest paths", () => { @@ -1113,15 +1724,30 @@ test("fixture check reports missing manifest paths", () => { ); writeFileSync( join(skillDir, "fixtures", "fixtures.json"), - JSON.stringify([{ id: "missing-fixture", title: "Missing Fixture", path: "fixtures/missing.txt" }]), + JSON.stringify([ + { + id: "missing-fixture", + title: "Missing Fixture", + path: "fixtures/missing.txt", + }, + ]), ); - const result = capture(() => commandFixtureCheck({ root: tmp, args: ["fixture", "check", "langbot-testing", "--json"] })); + const result = capture(() => + commandFixtureCheck({ + root: tmp, + args: ["fixture", "check", "langbot-testing", "--json"], + }), + ); assert.equal(result.code, 1); const report = JSON.parse(result.output); assert.equal(report.status, "fail"); - assert.ok(report.findings.some((finding: { id?: string }) => finding.id === "missing-fixture")); + assert.ok( + report.findings.some( + (finding: { id?: string }) => finding.id === "missing-fixture", + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -1132,42 +1758,63 @@ test("fixture check verifies QA AgentRunner source shape", () => { try { const skillDir = join(tmp, "skills", "langbot-testing"); const fixtureDir = join(skillDir, "fixtures", "plugins", "qa-agent-runner"); - mkdirSync(join(fixtureDir, "components", "agent_runner"), { recursive: true }); + mkdirSync(join(fixtureDir, "components", "agent_runner"), { + recursive: true, + }); writeFileSync( join(skillDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", ); writeFileSync( join(skillDir, "fixtures", "fixtures.json"), - JSON.stringify([{ - id: "qa-agent-runner-source", - title: "QA AgentRunner", - path: "fixtures/plugins/qa-agent-runner/manifest.yaml", - checks: ["exists", "qa_agent_runner_source"], - }]), + JSON.stringify([ + { + id: "qa-agent-runner-source", + title: "QA AgentRunner", + path: "fixtures/plugins/qa-agent-runner/manifest.yaml", + checks: ["exists", "qa_agent_runner_source"], + }, + ]), + ); + writeFileSync( + join(fixtureDir, "manifest.yaml"), + "spec:\n components:\n AgentRunner: {}\nexecution:\n python:\n attr: QAAgentRunnerPlugin\n", ); - writeFileSync(join(fixtureDir, "manifest.yaml"), "spec:\n components:\n AgentRunner: {}\nexecution:\n python:\n attr: QAAgentRunnerPlugin\n"); - const result = capture(() => commandFixtureCheck({ root: tmp, args: ["fixture", "check", "langbot-testing", "--json"] })); + const result = capture(() => + commandFixtureCheck({ + root: tmp, + args: ["fixture", "check", "langbot-testing", "--json"], + }), + ); assert.equal(result.code, 1); const report = JSON.parse(result.output); - assert.ok(report.findings.some((finding: { kind?: string; path?: string }) => ( - finding.kind === "fixture_check_missing_file" - && finding.path?.endsWith("components/agent_runner/default.py") - ))); + assert.ok( + report.findings.some( + (finding: { kind?: string; path?: string }) => + finding.kind === "fixture_check_missing_file" && + finding.path?.endsWith("components/agent_runner/default.py"), + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } }); test("fixture check accepts complete QA AgentRunner source shape", () => { - const result = capture(() => commandFixtureCheck(ctx(["fixture", "check", "langbot-testing", "--json"]))); + const result = capture(() => + commandFixtureCheck(ctx(["fixture", "check", "langbot-testing", "--json"])), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); - assert.ok(report.fixtures.some((item: { id: string; checks: string[] }) => ( - item.id === "qa-agent-runner-source" && item.checks.includes("qa_agent_runner_source") - ))); + assert.ok( + report.fixtures.some( + (item: { id: string; checks: string[] }) => + item.id === "qa-agent-runner-source" && + item.checks.includes("qa_agent_runner_source"), + ), + ); }); test("fixture check rejects invalid plugin package files", () => { @@ -1182,21 +1829,32 @@ test("fixture check rejects invalid plugin package files", () => { writeFileSync(join(skillDir, "fixtures", "bad.lbpkg"), "not a zip"); writeFileSync( join(skillDir, "fixtures", "fixtures.json"), - JSON.stringify([{ - id: "bad-package", - title: "Bad Package", - path: "fixtures/bad.lbpkg", - checks: ["exists", "zip_package"], - }]), + JSON.stringify([ + { + id: "bad-package", + title: "Bad Package", + path: "fixtures/bad.lbpkg", + checks: ["exists", "zip_package"], + }, + ]), ); - const result = capture(() => commandFixtureCheck({ root: tmp, args: ["fixture", "check", "langbot-testing", "--json"] })); + const result = capture(() => + commandFixtureCheck({ + root: tmp, + args: ["fixture", "check", "langbot-testing", "--json"], + }), + ); assert.equal(result.code, 1); const report = JSON.parse(result.output); - assert.ok(report.findings.some((finding: { kind?: string; id?: string }) => ( - finding.kind === "fixture_check_invalid_zip" && finding.id === "bad-package" - ))); + assert.ok( + report.findings.some( + (finding: { kind?: string; id?: string }) => + finding.kind === "fixture_check_invalid_zip" && + finding.id === "bad-package", + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -1218,7 +1876,10 @@ test("debug chat classifier prefers latest response leaf over body counts", () = test("debug chat classifier distinguishes new failure signals from old history", () => { assert.equal( - findNewFailureSignal("Agent runner temporarily unavailable", "Agent runner temporarily unavailable"), + findNewFailureSignal( + "Agent runner temporarily unavailable", + "Agent runner temporarily unavailable", + ), "", ); assert.equal( @@ -1400,12 +2061,27 @@ test("env doctor explains a missing backend listener with a startup hint", async ].join("\n"), ); - const result = await captureAsync(() => commandEnvDoctor({ root: tmp, args: ["env", "doctor"] })); + const result = await captureAsync(() => + commandEnvDoctor({ root: tmp, args: ["env", "doctor"] }), + ); assert.equal(result.code, 1); - assert.match(result.output, /FAIL: LANGBOT_BACKEND_URL: no HTTP service reachable because 127\.0\.0\.1:59998 is not listening/); - assert.match(result.output, new RegExp(`WARN: LANGBOT_BACKEND_URL: start backend: cd ${repoDir.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")} && uv run main.py`)); - assert.match(result.output, new RegExp(`WARN: LANGBOT_FRONTEND_URL: start frontend: cd ${webDir.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")} && pnpm dev`)); + assert.match( + result.output, + /FAIL: LANGBOT_BACKEND_URL: no HTTP service reachable because 127\.0\.0\.1:59998 is not listening/, + ); + assert.match( + result.output, + new RegExp( + `WARN: LANGBOT_BACKEND_URL: start backend: cd ${repoDir.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")} && uv run main.py`, + ), + ); + assert.match( + result.output, + new RegExp( + `WARN: LANGBOT_FRONTEND_URL: start frontend: cd ${webDir.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")} && pnpm dev`, + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -1436,10 +2112,15 @@ test("env doctor does not require proxy variables", async () => { ].join("\n"), ); - const result = await captureAsync(() => commandEnvDoctor({ root: tmp, args: ["env", "doctor"] })); + const result = await captureAsync(() => + commandEnvDoctor({ root: tmp, args: ["env", "doctor"] }), + ); assert.equal(result.code, 1); - assert.doesNotMatch(result.output, /missing LANGBOT_PROXY|missing LANGBOT_NO_PROXY/); + assert.doesNotMatch( + result.output, + /missing LANGBOT_PROXY|missing LANGBOT_NO_PROXY/, + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -1480,7 +2161,9 @@ test("env doctor reports missing socksio for active SOCKS proxy", async () => { ].join("\n"), ); - const result = await captureAsync(() => commandEnvDoctor({ root: tmp, args: ["env", "doctor"] })); + const result = await captureAsync(() => + commandEnvDoctor({ root: tmp, args: ["env", "doctor"] }), + ); assert.equal(result.code, 1); assert.match(result.output, /FAIL: SOCKS proxy ALL_PROXY is configured/); @@ -1508,13 +2191,20 @@ test("env show redacts secret-like values by default", () => { ].join("\n"), ); - const text = capture(() => commandEnvShow({ root: tmp, args: ["env", "show"] })); + const text = capture(() => + commandEnvShow({ root: tmp, args: ["env", "show"] }), + ); assert.equal(text.code, 0); assert.match(text.output, /LANGBOT_API_KEY=\[redacted\]/); - assert.match(text.output, /LANGBOT_PROXY_HTTP=http:\/\/\[redacted\]@127\.0\.0\.1:7890/); + assert.match( + text.output, + /LANGBOT_PROXY_HTTP=http:\/\/\[redacted\]@127\.0\.0\.1:7890/, + ); assert.doesNotMatch(text.output, /sk-test-secret|user:pass/); - const json = capture(() => commandEnvShow({ root: tmp, args: ["env", "show", "--json"] })); + const json = capture(() => + commandEnvShow({ root: tmp, args: ["env", "show", "--json"] }), + ); assert.equal(json.code, 0); const parsed = JSON.parse(json.output); assert.equal(parsed.LANGBOT_API_KEY, "[redacted]"); @@ -1525,10 +2215,15 @@ test("env show redacts secret-like values by default", () => { }); test("test plan renders agent-browser QA guidance", () => { - const result = capture(() => commandTestPlan(ctx(["test", "plan", "pipeline-debug-chat"]))); + const result = capture(() => + commandTestPlan(ctx(["test", "plan", "pipeline-debug-chat"])), + ); assert.equal(result.code, 0); assert.match(result.output, /Mode: agent-browser/); - assert.match(result.output, /Use browser\/UI interaction as the primary QA path/); + assert.match( + result.output, + /Use browser\/UI interaction as the primary QA path/, + ); assert.match(result.output, /API\/curl\/log checks are diagnostic only/); assert.match(result.output, /## Browser Steps/); assert.match(result.output, /## Success Signals/); @@ -1540,29 +2235,45 @@ test("test plan renders agent-browser QA guidance", () => { }); test("test plan JSON is parseable and includes troubleshooting patterns", () => { - const result = capture(() => commandTestPlan(ctx(["test", "plan", "pipeline-debug-chat", "--json"]))); + const result = capture(() => + commandTestPlan(ctx(["test", "plan", "pipeline-debug-chat", "--json"])), + ); assert.equal(result.code, 0); const plan = JSON.parse(result.output); assert.equal(plan.id, "pipeline-debug-chat"); assert.equal(plan.mode, "agent-browser"); assert.ok(["ready", "missing"].includes(plan.automation_readiness.status)); assert.ok(plan.automation_readiness.defaulted.includes("LANGBOT_E2E_PROMPT")); - assert.ok(plan.automation_readiness.defaulted.includes("LANGBOT_E2E_EXPECTED_TEXT")); + assert.ok( + plan.automation_readiness.defaulted.includes("LANGBOT_E2E_EXPECTED_TEXT"), + ); assert.equal(plan.manual_readiness.status, "manual_check"); assert.ok(plan.success_patterns.includes("Streaming completed")); - assert.ok(plan.troubleshooting.some((entry: { id: string }) => entry.id === "plugin-runtime-timeout")); + assert.ok( + plan.troubleshooting.some( + (entry: { id: string }) => entry.id === "plugin-runtime-timeout", + ), + ); }); test("test plan JSON exposes missing case-specific pipeline readiness", () => { - const result = capture(() => commandTestPlan(ctx(["test", "plan", "local-agent-basic-debug-chat", "--json"]))); + const result = capture(() => + commandTestPlan( + ctx(["test", "plan", "local-agent-basic-debug-chat", "--json"]), + ), + ); assert.equal(result.code, 0); const plan = JSON.parse(result.output); assert.equal(plan.env_readiness.status, "ready"); assert.ok(["ready", "missing"].includes(plan.automation_readiness.status)); assert.ok(plan.automation_readiness.pipeline_env_required); assert.ok( - plan.automation_readiness.missing.includes("LANGBOT_LOCAL_AGENT_PIPELINE_URL|LANGBOT_LOCAL_AGENT_PIPELINE_NAME") - || plan.automation_readiness.configured.some((key: string) => key.startsWith("LANGBOT_LOCAL_AGENT_PIPELINE_")), + plan.automation_readiness.missing.includes( + "LANGBOT_LOCAL_AGENT_PIPELINE_URL|LANGBOT_LOCAL_AGENT_PIPELINE_NAME", + ) || + plan.automation_readiness.configured.some((key: string) => + key.startsWith("LANGBOT_LOCAL_AGENT_PIPELINE_"), + ), ); }); @@ -1570,31 +2281,53 @@ test("generic pipeline readiness accepts either URL or name target", () => { const originalUrl = process.env.LANGBOT_PIPELINE_URL; const originalName = process.env.LANGBOT_PIPELINE_NAME; try { - withEnv({ - LANGBOT_BROWSER_PROFILE: "/tmp/langbot-test-profile", - LANGBOT_CHROMIUM_EXECUTABLE: "/tmp/langbot-test-chromium", - }, () => { - process.env.LANGBOT_PIPELINE_URL = "http://127.0.0.1:3000/home/agents?id=only-url"; - process.env.LANGBOT_PIPELINE_NAME = ""; + withEnv( + { + LANGBOT_BROWSER_PROFILE: "/tmp/langbot-test-profile", + LANGBOT_CHROMIUM_EXECUTABLE: "/tmp/langbot-test-chromium", + }, + () => { + process.env.LANGBOT_PIPELINE_URL = + "http://127.0.0.1:3000/home/agents?id=only-url"; + process.env.LANGBOT_PIPELINE_NAME = ""; - const ready = capture(() => commandTestPlan(ctx(["test", "plan", "pipeline-debug-chat", "--json"]))); - assert.equal(ready.code, 0); - const plan = JSON.parse(ready.output); - assert.equal(plan.env_readiness.status, "ready"); - assert.equal(plan.automation_readiness.status, "ready"); - assert.ok(plan.automation_readiness.required.includes("LANGBOT_PIPELINE_URL|LANGBOT_PIPELINE_NAME")); - }); + const ready = capture(() => + commandTestPlan( + ctx(["test", "plan", "pipeline-debug-chat", "--json"]), + ), + ); + assert.equal(ready.code, 0); + const plan = JSON.parse(ready.output); + assert.equal(plan.env_readiness.status, "ready"); + assert.equal(plan.automation_readiness.status, "ready"); + assert.ok( + plan.automation_readiness.required.includes( + "LANGBOT_PIPELINE_URL|LANGBOT_PIPELINE_NAME", + ), + ); + }, + ); process.env.LANGBOT_PIPELINE_URL = ""; process.env.LANGBOT_PIPELINE_NAME = ""; - const missing = capture(() => commandTestPlan(ctx(["test", "plan", "pipeline-debug-chat", "--json"]))); + const missing = capture(() => + commandTestPlan(ctx(["test", "plan", "pipeline-debug-chat", "--json"])), + ); assert.equal(missing.code, 0); const missingPlan = JSON.parse(missing.output); assert.equal(missingPlan.env_readiness.status, "missing"); - assert.ok(missingPlan.env_readiness.missing.includes("LANGBOT_PIPELINE_URL|LANGBOT_PIPELINE_NAME")); + assert.ok( + missingPlan.env_readiness.missing.includes( + "LANGBOT_PIPELINE_URL|LANGBOT_PIPELINE_NAME", + ), + ); assert.equal(missingPlan.automation_readiness.status, "missing"); - assert.ok(missingPlan.automation_readiness.missing.includes("LANGBOT_PIPELINE_URL|LANGBOT_PIPELINE_NAME")); + assert.ok( + missingPlan.automation_readiness.missing.includes( + "LANGBOT_PIPELINE_URL|LANGBOT_PIPELINE_NAME", + ), + ); } finally { if (originalUrl === undefined) delete process.env.LANGBOT_PIPELINE_URL; else process.env.LANGBOT_PIPELINE_URL = originalUrl; @@ -1604,15 +2337,19 @@ test("generic pipeline readiness accepts either URL or name target", () => { }); test("test recommend maps AgentRunner ledger changes to focused probes", () => { - const result = capture(() => commandTestRecommend(ctx([ - "test", - "recommend", - "--file", - "LangBot/src/langbot/pkg/agent/runner/run_ledger_store.py", - "--file", - "LangBot/tests/unit_tests/agent/test_run_ledger_store.py", - "--json", - ]))); + const result = capture(() => + commandTestRecommend( + ctx([ + "test", + "recommend", + "--file", + "LangBot/src/langbot/pkg/agent/runner/run_ledger_store.py", + "--file", + "LangBot/tests/unit_tests/agent/test_run_ledger_store.py", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); const ids = report.recommendations.map((item: { id: string }) => item.id); @@ -1621,18 +2358,30 @@ test("test recommend maps AgentRunner ledger changes to focused probes", () => { assert.ok(ids.includes("agent-runner-ledger-contention")); assert.ok(ids.includes("agent-runner-async-db-readiness")); assert.ok(ids.includes("agent-runner-ledger-concurrency")); - assert.ok(report.commands.every((command: string) => !command.startsWith("bin/lbs test run ") || command.endsWith(" --dry-run"))); - assert.ok(report.notes.some((note: string) => note.includes("Remove --dry-run"))); + assert.ok( + report.commands.every( + (command: string) => + !command.startsWith("bin/lbs test run ") || + command.endsWith(" --dry-run"), + ), + ); + assert.ok( + report.notes.some((note: string) => note.includes("Remove --dry-run")), + ); }); test("test recommend maps AgentRunner result changes to fixture contract", () => { - const result = capture(() => commandTestRecommend(ctx([ - "test", - "recommend", - "--file", - "langbot-plugin-sdk/src/langbot_plugin/api/entities/builtin/agent_runner/result.py", - "--json", - ]))); + const result = capture(() => + commandTestRecommend( + ctx([ + "test", + "recommend", + "--file", + "langbot-plugin-sdk/src/langbot_plugin/api/entities/builtin/agent_runner/result.py", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); const ids = report.recommendations.map((item: { id: string }) => item.id); @@ -1642,13 +2391,17 @@ test("test recommend maps AgentRunner result changes to fixture contract", () => }); test("test recommend maps QA AgentRunner fixture changes to live install", () => { - const result = capture(() => commandTestRecommend(ctx([ - "test", - "recommend", - "--file", - "langbot-skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/components/agent_runner/default.py", - "--json", - ]))); + const result = capture(() => + commandTestRecommend( + ctx([ + "test", + "recommend", + "--file", + "langbot-skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/components/agent_runner/default.py", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); const ids = report.recommendations.map((item: { id: string }) => item.id); @@ -1658,13 +2411,17 @@ test("test recommend maps QA AgentRunner fixture changes to live install", () => }); test("test recommend maps QA plugin smoke fixture changes to live install", () => { - const result = capture(() => commandTestRecommend(ctx([ - "test", - "recommend", - "--file", - "langbot-skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/main.py", - "--json", - ]))); + const result = capture(() => + commandTestRecommend( + ctx([ + "test", + "recommend", + "--file", + "langbot-skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/main.py", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); const ids = report.recommendations.map((item: { id: string }) => item.id); @@ -1681,26 +2438,66 @@ test("test recommend keeps git status paths intact", () => { }; try { const repo = join(tmp, "LangBot"); - mkdirSync(join(repo, "src", "langbot", "pkg", "agent", "runner"), { recursive: true }); + mkdirSync(join(repo, "src", "langbot", "pkg", "agent", "runner"), { + recursive: true, + }); spawnSync("git", ["init"], { cwd: repo }); - spawnSync("git", ["config", "user.email", "qa@example.test"], { cwd: repo }); + spawnSync("git", ["config", "user.email", "qa@example.test"], { + cwd: repo, + }); spawnSync("git", ["config", "user.name", "QA"], { cwd: repo }); writeFileSync(join(repo, "README.md"), "test\n"); - writeFileSync(join(repo, "src", "langbot", "pkg", "agent", "runner", "run_ledger_store.py"), "# test\n"); - spawnSync("git", ["add", "README.md", "src/langbot/pkg/agent/runner/run_ledger_store.py"], { cwd: repo }); + writeFileSync( + join( + repo, + "src", + "langbot", + "pkg", + "agent", + "runner", + "run_ledger_store.py", + ), + "# test\n", + ); + spawnSync( + "git", + ["add", "README.md", "src/langbot/pkg/agent/runner/run_ledger_store.py"], + { cwd: repo }, + ); spawnSync("git", ["commit", "-m", "init"], { cwd: repo }); - writeFileSync(join(repo, "src", "langbot", "pkg", "agent", "runner", "run_ledger_store.py"), "# changed\n"); + writeFileSync( + join( + repo, + "src", + "langbot", + "pkg", + "agent", + "runner", + "run_ledger_store.py", + ), + "# changed\n", + ); process.env.LANGBOT_REPO = repo; process.env.LANGBOT_PLUGIN_SDK_REPO = join(tmp, "missing-sdk"); process.env.LANGBOT_AGENT_RUNNER_REPO = join(tmp, "missing-runner"); process.env.LANGBOT_LOCAL_AGENT_REPO = join(tmp, "missing-local"); - const result = capture(() => commandTestRecommend({ root, args: ["test", "recommend", "--json"] })); + const result = capture(() => + commandTestRecommend({ root, args: ["test", "recommend", "--json"] }), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); - assert.ok(report.changed_files.includes("LangBot/src/langbot/pkg/agent/runner/run_ledger_store.py")); - assert.ok(!report.changed_files.some((file: string) => file.includes("LangBot/rc/"))); + assert.ok( + report.changed_files.includes( + "LangBot/src/langbot/pkg/agent/runner/run_ledger_store.py", + ), + ); + assert.ok( + !report.changed_files.some((file: string) => + file.includes("LangBot/rc/"), + ), + ); } finally { for (const [key, value] of Object.entries(originalRepos)) { if (value === undefined) delete process.env[key]; @@ -1711,25 +2508,41 @@ test("test recommend keeps git status paths intact", () => { }); test("test start creates a run handoff with a bounded report command", () => { - const result = capture(() => commandTestStart(ctx(["test", "start", "pipeline-debug-chat"]))); + const result = capture(() => + commandTestStart(ctx(["test", "start", "pipeline-debug-chat"])), + ); assert.equal(result.code, 0); assert.match(result.output, /^# Test Start: pipeline-debug-chat/m); assert.match(result.output, /bin\/lbs test plan pipeline-debug-chat/); - assert.match(result.output, /bin\/lbs test run pipeline-debug-chat --run-id .+ --output reports\/evidence\/.+pipeline-debug-chat/); - assert.match(result.output, /bin\/lbs test report pipeline-debug-chat --since ".+" --console-log reports\/evidence\/.+\/console\.log --evidence-dir reports\/evidence\/.+ --output reports\/.+pipeline-debug-chat\.md/); + assert.match( + result.output, + /bin\/lbs test run pipeline-debug-chat --run-id .+ --output reports\/evidence\/.+pipeline-debug-chat/, + ); + assert.match( + result.output, + /bin\/lbs test report pipeline-debug-chat --since ".+" --console-log reports\/evidence\/.+\/console\.log --evidence-dir reports\/evidence\/.+ --output reports\/.+pipeline-debug-chat\.md/, + ); assert.match(result.output, /Streaming completed/); }); test("test start JSON is parseable for agent orchestration", () => { - const result = capture(() => commandTestStart(ctx(["test", "start", "pipeline-debug-chat", "--json"]))); + const result = capture(() => + commandTestStart(ctx(["test", "start", "pipeline-debug-chat", "--json"])), + ); assert.equal(result.code, 0); const start = JSON.parse(result.output); assert.equal(start.case.id, "pipeline-debug-chat"); assert.match(start.run_id, /pipeline-debug-chat$/); assert.match(start.started_at_local, /\d{4}-\d{2}-\d{2}T/); assert.match(start.report_command, /--since/); - assert.match(start.result_command_template, /bin\/lbs test result pipeline-debug-chat/); - assert.match(start.automation.command, /bin\/lbs test run pipeline-debug-chat/); + assert.match( + start.result_command_template, + /bin\/lbs test result pipeline-debug-chat/, + ); + assert.match( + start.automation.command, + /bin\/lbs test run pipeline-debug-chat/, + ); assert.ok(start.success_patterns.includes("Streaming completed")); assert.ok(start.evidence_required.includes("backend_log")); }); @@ -1738,22 +2551,26 @@ test("test result writes a suite-readable result.json and enforces pass evidence const tmp = mkdtempSync(join(tmpdir(), "lbs-test-result-")); try { const evidenceDir = join(tmp, "pipeline-run"); - const ok = capture(() => commandTestResult(ctx([ - "test", - "result", - "pipeline-debug-chat", - "--result", - "pass", - "--reason", - "Debug Chat returned OK and logs were clean.", - "--evidence-dir", - evidenceDir, - "--started-at", - "2026-05-21T10:30:00.000+08:00", - "--evidence", - "ui,screenshot,console,backend_log", - "--json", - ]))); + const ok = capture(() => + commandTestResult( + ctx([ + "test", + "result", + "pipeline-debug-chat", + "--result", + "pass", + "--reason", + "Debug Chat returned OK and logs were clean.", + "--evidence-dir", + evidenceDir, + "--started-at", + "2026-05-21T10:30:00.000+08:00", + "--evidence", + "ui,screenshot,console,backend_log", + "--json", + ]), + ), + ); assert.equal(ok.code, 0); const record = JSON.parse(ok.output); @@ -1761,21 +2578,29 @@ test("test result writes a suite-readable result.json and enforces pass evidence assert.equal(record.status, "pass"); assert.equal(record.evidence_status, "complete"); assert.deepEqual(record.evidence_missing, []); - assert.equal(JSON.parse(readFileSync(join(evidenceDir, "result.json"), "utf8")).case_id, "pipeline-debug-chat"); - - const missing = captureAll(() => commandTestResult(ctx([ - "test", - "result", + assert.equal( + JSON.parse(readFileSync(join(evidenceDir, "result.json"), "utf8")) + .case_id, "pipeline-debug-chat", - "--result", - "pass", - "--reason", - "Missing backend evidence should not be accepted as pass.", - "--evidence-dir", - join(tmp, "missing-evidence"), - "--evidence", - "ui", - ]))); + ); + + const missing = captureAll(() => + commandTestResult( + ctx([ + "test", + "result", + "pipeline-debug-chat", + "--result", + "pass", + "--reason", + "Missing backend evidence should not be accepted as pass.", + "--evidence-dir", + join(tmp, "missing-evidence"), + "--evidence", + "ui", + ]), + ), + ); assert.equal(missing.code, 1); assert.match(missing.error, /missing required evidence/); } finally { @@ -1784,42 +2609,62 @@ test("test result writes a suite-readable result.json and enforces pass evidence }); test("test run dry-run exposes case automation script and evidence paths", () => { - const result = capture(() => commandTestRun(ctx([ - "test", - "run", - "pipeline-debug-chat", - "--run-id", - "run-123", - "--output", - "reports/evidence/run-123", - "--dry-run", - ]))); + const result = capture(() => + commandTestRun( + ctx([ + "test", + "run", + "pipeline-debug-chat", + "--run-id", + "run-123", + "--output", + "reports/evidence/run-123", + "--dry-run", + ]), + ), + ); assert.equal(result.code, 0); assert.match(result.output, /^# Test Automation: pipeline-debug-chat/m); assert.match(result.output, /scripts\/e2e\/pipeline-debug-chat\.mjs/); - assert.match(result.output, /console_log: reports\/evidence\/run-123\/console\.log/); - assert.match(result.output, /automation_result_json: reports\/evidence\/run-123\/automation-result\.json/); - assert.match(result.output, /result_json: reports\/evidence\/run-123\/result\.json/); + assert.match( + result.output, + /console_log: reports\/evidence\/run-123\/console\.log/, + ); + assert.match( + result.output, + /automation_result_json: reports\/evidence\/run-123\/automation-result\.json/, + ); + assert.match( + result.output, + /result_json: reports\/evidence\/run-123\/result\.json/, + ); assert.match(result.output, /LANGBOT_PIPELINE_URL/); }); test("test run dry-run JSON is parseable for automation orchestration", () => { - const result = capture(() => commandTestRun(ctx([ - "test", - "run", - "webui-login-state", - "--run-id", - "login-run", - "--dry-run", - "--json", - ]))); + const result = capture(() => + commandTestRun( + ctx([ + "test", + "run", + "webui-login-state", + "--run-id", + "login-run", + "--dry-run", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const run = JSON.parse(result.output); assert.equal(run.case.id, "webui-login-state"); assert.equal(run.run_id, "login-run"); assert.equal(run.automation.script, "scripts/e2e/webui-login-state.mjs"); assert.equal(run.automation.exists, true); - assert.match(run.automation.automation_result_json, /automation-result\.json$/); + assert.match( + run.automation.automation_result_json, + /automation-result\.json$/, + ); assert.match(run.automation.result_json, /result\.json$/); assert.match(run.automation.report_command, /--console-log/); }); @@ -1832,7 +2677,10 @@ test("test run JSON executes automation unless dry-run is explicit", () => { const scriptsDir = join(tmp, "scripts"); mkdirSync(casesDir, { recursive: true }); mkdirSync(scriptsDir, { recursive: true }); - writeFileSync(join(skillDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n"); + writeFileSync( + join(skillDir, "SKILL.md"), + "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", + ); writeFileSync(join(tmp, "skills", ".env"), ""); writeFileSync( join(casesDir, "json-exec.yaml"), @@ -1857,10 +2705,12 @@ test("test run JSON executes automation unless dry-run is explicit", () => { ].join("\n"), ); - const result = capture(() => commandTestRun({ - root: tmp, - args: ["test", "run", "json-exec", "--run-id", "json-run", "--json"], - })); + const result = capture(() => + commandTestRun({ + root: tmp, + args: ["test", "run", "json-exec", "--run-id", "json-run", "--json"], + }), + ); assert.equal(result.code, 0); assert.equal(readFileSync(join(tmp, "json-ran.txt"), "utf8"), "yes"); @@ -1881,7 +2731,10 @@ test("test run lets explicit environment override automation defaults", () => { const scriptsDir = join(tmp, "scripts"); mkdirSync(casesDir, { recursive: true }); mkdirSync(scriptsDir, { recursive: true }); - writeFileSync(join(skillDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n"); + writeFileSync( + join(skillDir, "SKILL.md"), + "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", + ); writeFileSync(join(tmp, "skills", ".env"), ""); writeFileSync( join(casesDir, "env-override.yaml"), @@ -1895,7 +2748,7 @@ test("test run lets explicit environment override automation defaults", () => { "risk: low", "ci_eligible: false", "automation: scripts/write-env.mjs", - "automation_runner_config_patch_json: '{\"source\":\"default\"}'", + 'automation_runner_config_patch_json: \'{"source":"default"}\'', ].join("\n"), ); writeFileSync( @@ -1910,16 +2763,21 @@ test("test run lets explicit environment override automation defaults", () => { ); process.env.LANGBOT_E2E_RUNNER_CONFIG_PATCH_JSON = '{"source":"explicit"}'; - const result = capture(() => commandTestRun({ - root: tmp, - args: ["test", "run", "env-override", "--run-id", "env-run"], - })); + const result = capture(() => + commandTestRun({ + root: tmp, + args: ["test", "run", "env-override", "--run-id", "env-run"], + }), + ); assert.equal(result.code, 0); - const observed = JSON.parse(readFileSync(join(tmp, "env-out.json"), "utf8")); + const observed = JSON.parse( + readFileSync(join(tmp, "env-out.json"), "utf8"), + ); assert.equal(observed.patch, '{"source":"explicit"}'); } finally { - if (originalPatch === undefined) delete process.env.LANGBOT_E2E_RUNNER_CONFIG_PATCH_JSON; + if (originalPatch === undefined) + delete process.env.LANGBOT_E2E_RUNNER_CONFIG_PATCH_JSON; else process.env.LANGBOT_E2E_RUNNER_CONFIG_PATCH_JSON = originalPatch; rmSync(tmp, { recursive: true, force: true }); } @@ -1933,8 +2791,14 @@ test("test run expands env references in automation defaults", () => { const scriptsDir = join(tmp, "scripts"); mkdirSync(casesDir, { recursive: true }); mkdirSync(scriptsDir, { recursive: true }); - writeFileSync(join(skillDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n"); - writeFileSync(join(tmp, "skills", ".env"), "QA_KB_UUID=kb-from-env\nQA_MODEL_UUID=model-from-env\n"); + writeFileSync( + join(skillDir, "SKILL.md"), + "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", + ); + writeFileSync( + join(tmp, "skills", ".env"), + "QA_KB_UUID=kb-from-env\nQA_MODEL_UUID=model-from-env\n", + ); writeFileSync( join(casesDir, "env-expand.yaml"), [ @@ -1947,7 +2811,7 @@ test("test run expands env references in automation defaults", () => { "risk: low", "ci_eligible: false", "automation: scripts/write-expanded-env.mjs", - "automation_runner_config_patch_json: '{\"knowledge-bases\":[\"${QA_KB_UUID}\"],\"model\":{\"primary\":\"${QA_MODEL_UUID}\"}}'", + 'automation_runner_config_patch_json: \'{"knowledge-bases":["${QA_KB_UUID}"],"model":{"primary":"${QA_MODEL_UUID}"}}\'', ].join("\n"), ); writeFileSync( @@ -1961,10 +2825,20 @@ test("test run expands env references in automation defaults", () => { ].join("\n"), ); - const dryRun = capture(() => commandTestRun({ - root: tmp, - args: ["test", "run", "env-expand", "--run-id", "env-expand-dry", "--dry-run", "--json"], - })); + const dryRun = capture(() => + commandTestRun({ + root: tmp, + args: [ + "test", + "run", + "env-expand", + "--run-id", + "env-expand-dry", + "--dry-run", + "--json", + ], + }), + ); assert.equal(dryRun.code, 0); const plan = JSON.parse(dryRun.output); assert.equal( @@ -1972,13 +2846,20 @@ test("test run expands env references in automation defaults", () => { '{"knowledge-bases":["kb-from-env"],"model":{"primary":"model-from-env"}}', ); - const run = capture(() => commandTestRun({ - root: tmp, - args: ["test", "run", "env-expand", "--run-id", "env-expand-run"], - })); + const run = capture(() => + commandTestRun({ + root: tmp, + args: ["test", "run", "env-expand", "--run-id", "env-expand-run"], + }), + ); assert.equal(run.code, 0); - const observed = JSON.parse(readFileSync(join(tmp, "expanded-env-out.json"), "utf8")); - assert.equal(observed.patch, '{"knowledge-bases":["kb-from-env"],"model":{"primary":"model-from-env"}}'); + const observed = JSON.parse( + readFileSync(join(tmp, "expanded-env-out.json"), "utf8"), + ); + assert.equal( + observed.patch, + '{"knowledge-bases":["kb-from-env"],"model":{"primary":"model-from-env"}}', + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -1992,7 +2873,10 @@ test("test run setup automation isolates evidence and reloads env", () => { const scriptsDir = join(tmp, "scripts"); mkdirSync(casesDir, { recursive: true }); mkdirSync(scriptsDir, { recursive: true }); - writeFileSync(join(skillDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n"); + writeFileSync( + join(skillDir, "SKILL.md"), + "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", + ); writeFileSync(join(tmp, "skills", ".env"), "SETUP_VALUE=\n"); writeFileSync( join(casesDir, "setup-main.yaml"), @@ -2008,7 +2892,7 @@ test("test run setup automation isolates evidence and reloads env", () => { "env:", " - SETUP_VALUE", "setup_automation:", - " - \"node:scripts/write-setup-env.mjs --write-env\"", + ' - "node:scripts/write-setup-env.mjs --write-env"', "setup_provides_env:", " - SETUP_VALUE", "automation: scripts/read-setup-env.mjs", @@ -2026,7 +2910,7 @@ test("test run setup automation isolates evidence and reloads env", () => { "risk: low", "ci_eligible: false", "setup_automation:", - " - \"node:scripts/write-setup-env-issue.mjs\"", + ' - "node:scripts/write-setup-env-issue.mjs"', "automation: scripts/read-setup-env.mjs", ].join("\n"), ); @@ -2042,7 +2926,7 @@ test("test run setup automation isolates evidence and reloads env", () => { "risk: low", "ci_eligible: false", "setup_automation:", - " - \"node:scripts/write-setup-pass-then-fail.mjs\"", + ' - "node:scripts/write-setup-pass-then-fail.mjs"', "automation: scripts/read-setup-env.mjs", ].join("\n"), ); @@ -2092,45 +2976,118 @@ test("test run setup automation isolates evidence and reloads env", () => { ].join("\n"), ); - const dryRun = capture(() => commandTestRun({ - root: tmp, - args: ["test", "run", "setup-main", "--run-id", "setup-run", "--output", join(tmp, "evidence"), "--dry-run", "--json"], - })); + const dryRun = capture(() => + commandTestRun({ + root: tmp, + args: [ + "test", + "run", + "setup-main", + "--run-id", + "setup-run", + "--output", + join(tmp, "evidence"), + "--dry-run", + "--json", + ], + }), + ); assert.equal(dryRun.code, 0); const plan = JSON.parse(dryRun.output); assert.equal(plan.setup_automation.length, 1); - assert.match(plan.setup_automation[0].evidence_dir, /setup\/01-write-setup-env$/); - assert.match(plan.setup_automation[0].command, /^node scripts\/write-setup-env\.mjs --write-env$/); + assert.match( + plan.setup_automation[0].evidence_dir, + /setup\/01-write-setup-env$/, + ); + assert.match( + plan.setup_automation[0].command, + /^node scripts\/write-setup-env\.mjs --write-env$/, + ); assert.equal(plan.setup_automation[0].dry_run_command, ""); assert.equal(existsSync(join(tmp, "skills", ".env.local")), false); - const run = capture(() => commandTestRun({ - root: tmp, - args: ["test", "run", "setup-main", "--run-id", "setup-run", "--output", join(tmp, "evidence")], - })); + const run = capture(() => + commandTestRun({ + root: tmp, + args: [ + "test", + "run", + "setup-main", + "--run-id", + "setup-run", + "--output", + join(tmp, "evidence"), + ], + }), + ); assert.equal(run.code, 0); - const observed = JSON.parse(readFileSync(join(tmp, "main-observed.json"), "utf8")); + const observed = JSON.parse( + readFileSync(join(tmp, "main-observed.json"), "utf8"), + ); assert.equal(observed.value, "from-setup"); - const setupResult = JSON.parse(readFileSync(join(tmp, "evidence", "setup", "01-write-setup-env", "automation-result.json"), "utf8")); - const mainResult = JSON.parse(readFileSync(join(tmp, "evidence", "automation-result.json"), "utf8")); + const setupResult = JSON.parse( + readFileSync( + join( + tmp, + "evidence", + "setup", + "01-write-setup-env", + "automation-result.json", + ), + "utf8", + ), + ); + const mainResult = JSON.parse( + readFileSync(join(tmp, "evidence", "automation-result.json"), "utf8"), + ); assert.equal(setupResult.stage, "setup"); assert.equal(mainResult.stage, "main"); - const envIssue = capture(() => commandTestRun({ - root: tmp, - args: ["test", "run", "setup-env-issue", "--run-id", "setup-env-issue-run", "--output", join(tmp, "evidence-env-issue")], - })); + const envIssue = capture(() => + commandTestRun({ + root: tmp, + args: [ + "test", + "run", + "setup-env-issue", + "--run-id", + "setup-env-issue-run", + "--output", + join(tmp, "evidence-env-issue"), + ], + }), + ); assert.equal(envIssue.code, 2); - const parentResult = JSON.parse(readFileSync(join(tmp, "evidence-env-issue", "automation-result.json"), "utf8")); + const parentResult = JSON.parse( + readFileSync( + join(tmp, "evidence-env-issue", "automation-result.json"), + "utf8", + ), + ); assert.equal(parentResult.status, "env_issue"); assert.equal(parentResult.reason, "setup env missing"); - const failAfterPass = capture(() => commandTestRun({ - root: tmp, - args: ["test", "run", "setup-fail-after-pass", "--run-id", "setup-fail-after-pass-run", "--output", join(tmp, "evidence-fail-after-pass")], - })); + const failAfterPass = capture(() => + commandTestRun({ + root: tmp, + args: [ + "test", + "run", + "setup-fail-after-pass", + "--run-id", + "setup-fail-after-pass-run", + "--output", + join(tmp, "evidence-fail-after-pass"), + ], + }), + ); assert.equal(failAfterPass.code, 1); - const failAfterPassResult = JSON.parse(readFileSync(join(tmp, "evidence-fail-after-pass", "automation-result.json"), "utf8")); + const failAfterPassResult = JSON.parse( + readFileSync( + join(tmp, "evidence-fail-after-pass", "automation-result.json"), + "utf8", + ), + ); assert.equal(failAfterPassResult.status, "fail"); assert.equal(failAfterPassResult.reason, "stale pass before crash"); } finally { @@ -2146,7 +3103,10 @@ test("test run setup automation can execute another case outside this source rep const scriptsDir = join(tmp, "scripts"); mkdirSync(casesDir, { recursive: true }); mkdirSync(scriptsDir, { recursive: true }); - writeFileSync(join(skillDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n"); + writeFileSync( + join(skillDir, "SKILL.md"), + "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", + ); writeFileSync(join(tmp, "skills", ".env"), "SETUP_VALUE=\n"); writeFileSync( join(casesDir, "setup-child.yaml"), @@ -2174,7 +3134,7 @@ test("test run setup automation can execute another case outside this source rep "risk: low", "ci_eligible: true", "setup_automation:", - " - \"case:setup-child\"", + ' - "case:setup-child"', "setup_provides_env:", " - SETUP_VALUE", "automation: scripts/read-child-env.mjs", @@ -2203,14 +3163,30 @@ test("test run setup automation can execute another case outside this source rep ].join("\n"), ); - const run = capture(() => commandTestRun({ - root: tmp, - args: ["test", "run", "setup-parent", "--run-id", "setup-parent-run", "--output", join(tmp, "evidence")], - })); + const run = capture(() => + commandTestRun({ + root: tmp, + args: [ + "test", + "run", + "setup-parent", + "--run-id", + "setup-parent-run", + "--output", + join(tmp, "evidence"), + ], + }), + ); assert.equal(run.code, 0); - assert.ok(existsSync(join(tmp, "evidence", "setup", "01-setup-child", "result.json"))); - const result = JSON.parse(readFileSync(join(tmp, "evidence", "automation-result.json"), "utf8")); + assert.ok( + existsSync( + join(tmp, "evidence", "setup", "01-setup-child", "result.json"), + ), + ); + const result = JSON.parse( + readFileSync(join(tmp, "evidence", "automation-result.json"), "utf8"), + ); assert.equal(result.value, "from-child"); } finally { rmSync(tmp, { recursive: true, force: true }); @@ -2225,7 +3201,10 @@ test("test run automation inherits parent process environment", () => { const scriptsDir = join(tmp, "scripts"); mkdirSync(casesDir, { recursive: true }); mkdirSync(scriptsDir, { recursive: true }); - writeFileSync(join(skillDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n"); + writeFileSync( + join(skillDir, "SKILL.md"), + "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", + ); writeFileSync(join(tmp, "skills", ".env"), ""); writeFileSync( join(casesDir, "env-inherit.yaml"), @@ -2252,13 +3231,25 @@ test("test run automation inherits parent process environment", () => { ].join("\n"), ); - const run = capture(() => commandTestRun({ - root: tmp, - args: ["test", "run", "env-inherit", "--run-id", "env-inherit-run", "--output", join(tmp, "evidence")], - })); + const run = capture(() => + commandTestRun({ + root: tmp, + args: [ + "test", + "run", + "env-inherit", + "--run-id", + "env-inherit-run", + "--output", + join(tmp, "evidence"), + ], + }), + ); assert.equal(run.code, 0); - const result = JSON.parse(readFileSync(join(tmp, "evidence", "automation-result.json"), "utf8")); + const result = JSON.parse( + readFileSync(join(tmp, "evidence", "automation-result.json"), "utf8"), + ); assert.equal(result.status, "pass"); } finally { rmSync(tmp, { recursive: true, force: true }); @@ -2273,7 +3264,10 @@ test("test run dry-run marks missing setup case targets", () => { const scriptsDir = join(tmp, "scripts"); mkdirSync(casesDir, { recursive: true }); mkdirSync(scriptsDir, { recursive: true }); - writeFileSync(join(skillDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n"); + writeFileSync( + join(skillDir, "SKILL.md"), + "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", + ); writeFileSync(join(tmp, "skills", ".env"), ""); writeFileSync( join(casesDir, "setup-parent.yaml"), @@ -2287,16 +3281,18 @@ test("test run dry-run marks missing setup case targets", () => { "risk: low", "ci_eligible: true", "setup_automation:", - " - \"case:missing-child\"", + ' - "case:missing-child"', "automation: scripts/pass.mjs", ].join("\n"), ); writeFileSync(join(scriptsDir, "pass.mjs"), "process.exit(0);\n"); - const result = capture(() => commandTestRun({ - root: tmp, - args: ["test", "run", "setup-parent", "--dry-run", "--json"], - })); + const result = capture(() => + commandTestRun({ + root: tmp, + args: ["test", "run", "setup-parent", "--dry-run", "--json"], + }), + ); assert.equal(result.code, 0); const run = JSON.parse(result.output); @@ -2310,159 +3306,352 @@ test("test run dry-run marks missing setup case targets", () => { }); test("local-agent effective prompt case has runnable automation defaults", () => { - const result = capture(() => commandTestRun(ctx([ - "test", - "run", - "local-agent-effective-prompt-debug-chat", - "--run-id", - "effective-run", - "--dry-run", - "--json", - ]))); + const result = capture(() => + commandTestRun( + ctx([ + "test", + "run", + "local-agent-effective-prompt-debug-chat", + "--run-id", + "effective-run", + "--dry-run", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const run = JSON.parse(result.output); assert.equal(run.automation.script, "scripts/e2e/pipeline-debug-chat.mjs"); - assert.equal(run.automation.env_defaults.LANGBOT_E2E_PROMPT, "qa-effective-prompt"); - assert.equal(run.automation.env_defaults.LANGBOT_E2E_EXPECTED_TEXT, "PROMPT_PREPROCESS_OK"); - assert.equal(run.automation.env_defaults.LANGBOT_E2E_RESPONSE_TIMEOUT_MS, "180000"); + assert.equal( + run.automation.env_defaults.LANGBOT_E2E_PROMPT, + "qa-effective-prompt", + ); + assert.equal( + run.automation.env_defaults.LANGBOT_E2E_EXPECTED_TEXT, + "PROMPT_PREPROCESS_OK", + ); + assert.equal( + run.automation.env_defaults.LANGBOT_E2E_RESPONSE_TIMEOUT_MS, + "180000", + ); assert.equal(run.automation.pipeline_env_required, true); - assert.ok(run.automation.env_aliases.some((alias: { target: string; source: string }) => ( - alias.target === "LANGBOT_E2E_PIPELINE_URL" && alias.source === "LANGBOT_LOCAL_AGENT_PIPELINE_URL" - ))); + assert.ok( + run.automation.env_aliases.some( + (alias: { target: string; source: string }) => + alias.target === "LANGBOT_E2E_PIPELINE_URL" && + alias.source === "LANGBOT_LOCAL_AGENT_PIPELINE_URL", + ), + ); }); test("local-agent basic case can setup the local-agent pipeline env", () => { - withEnv({ - LANGBOT_BROWSER_PROFILE: "/tmp/langbot-test-profile", - LANGBOT_CHROMIUM_EXECUTABLE: "/tmp/langbot-test-chromium", - }, () => { - const result = capture(() => commandTestRun(ctx([ - "test", - "run", - "local-agent-basic-debug-chat", - "--dry-run", - "--json", - ]))); - assert.equal(result.code, 0); - const run = JSON.parse(result.output); - assert.deepEqual(run.setup_automation.map((item: { entry: string }) => item.entry), [ - "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env", - ]); + withEnv( + { + LANGBOT_BROWSER_PROFILE: "/tmp/langbot-test-profile", + LANGBOT_CHROMIUM_EXECUTABLE: "/tmp/langbot-test-chromium", + }, + () => { + const result = capture(() => + commandTestRun( + ctx([ + "test", + "run", + "local-agent-basic-debug-chat", + "--dry-run", + "--json", + ]), + ), + ); + assert.equal(result.code, 0); + const run = JSON.parse(result.output); + assert.deepEqual( + run.setup_automation.map((item: { entry: string }) => item.entry), + ["node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env"], + ); - const planResult = capture(() => commandTestPlan(ctx(["test", "plan", "local-agent-basic-debug-chat", "--json"]))); - assert.equal(planResult.code, 0); - const plan = JSON.parse(planResult.output); - assert.deepEqual(plan.setup_provides_env, [ - "LANGBOT_LOCAL_AGENT_PIPELINE_URL", - "LANGBOT_LOCAL_AGENT_PIPELINE_NAME", - ]); - assert.equal(plan.automation_readiness.status, "ready"); - }); + const planResult = capture(() => + commandTestPlan( + ctx(["test", "plan", "local-agent-basic-debug-chat", "--json"]), + ), + ); + assert.equal(planResult.code, 0); + const plan = JSON.parse(planResult.output); + assert.deepEqual(plan.setup_provides_env, [ + "LANGBOT_LOCAL_AGENT_PIPELINE_URL", + "LANGBOT_LOCAL_AGENT_PIPELINE_NAME", + ]); + assert.equal(plan.automation_readiness.status, "ready"); + }, + ); }); test("local-agent nonstreaming case disables stream output through automation defaults", () => { - const result = capture(() => commandTestRun(ctx([ - "test", - "run", - "local-agent-nonstreaming-debug-chat", - "--dry-run", - "--json", - ]))); + const result = capture(() => + commandTestRun( + ctx([ + "test", + "run", + "local-agent-nonstreaming-debug-chat", + "--dry-run", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const run = JSON.parse(result.output); assert.equal(run.automation.script, "scripts/e2e/pipeline-debug-chat.mjs"); - assert.equal(run.automation.env_defaults.LANGBOT_E2E_PROMPT, "Reply only NONSTREAM_OK."); - assert.equal(run.automation.env_defaults.LANGBOT_E2E_EXPECTED_TEXT, "NONSTREAM_OK"); + assert.equal( + run.automation.env_defaults.LANGBOT_E2E_PROMPT, + "Reply only NONSTREAM_OK.", + ); + assert.equal( + run.automation.env_defaults.LANGBOT_E2E_EXPECTED_TEXT, + "NONSTREAM_OK", + ); assert.equal(run.automation.env_defaults.LANGBOT_E2E_STREAM_OUTPUT, "0"); assert.equal(run.automation.pipeline_env_required, true); }); test("local-agent multimodal case exposes image fixture automation defaults", () => { - const result = capture(() => commandTestRun(ctx([ - "test", - "run", - "local-agent-multimodal-debug-chat", - "--dry-run", - "--json", - ]))); + const result = capture(() => + commandTestRun( + ctx([ + "test", + "run", + "local-agent-multimodal-debug-chat", + "--dry-run", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const run = JSON.parse(result.output); assert.equal(run.automation.script, "scripts/e2e/pipeline-debug-chat.mjs"); - assert.equal(run.automation.env_defaults.LANGBOT_E2E_EXPECTED_TEXT, "IMAGE_OK"); - assert.match(run.automation.env_defaults.LANGBOT_E2E_IMAGE_BASE64_PATH, /red-square\.png\.base64$/); + assert.equal( + run.automation.env_defaults.LANGBOT_E2E_EXPECTED_TEXT, + "IMAGE_OK", + ); + assert.match( + run.automation.env_defaults.LANGBOT_E2E_IMAGE_BASE64_PATH, + /red-square\.png\.base64$/, + ); assert.equal(run.automation.pipeline_env_required, true); }); +test("fake provider returns IMAGE_OK only when image metadata is present", async () => { + const provider = await startFakeProviderForTest(); + try { + const request = async (content: string) => { + const json = await requestFakeProvider(provider, { + messages: [{ role: "user", content }], + }); + return fakeProviderMessage(json).content; + }; + + assert.equal( + await request( + "I attached an image. Reply only IMAGE_OK if you received the image.[Image]", + ), + "IMAGE_OK", + ); + assert.equal(await request("Say hello for a plain text request."), "OK"); + } finally { + await provider.stop(); + } +}); + +test("fake provider drives steering through qa_plugin_sleep and follow-up context", async () => { + const provider = await startFakeProviderForTest(); + try { + const initial = fakeProviderMessage( + await requestFakeProvider(provider, { + tools: [ + { + type: "function", + function: { + name: "qa_plugin_sleep", + parameters: { + type: "object", + properties: { + seconds: { type: "number" }, + text: { type: "string" }, + }, + }, + }, + }, + ], + messages: [ + { + role: "user", + content: + "First call qa_plugin_sleep with text=steering-e2e-anchor. If no follow-up was injected, reply only STEERING_NO_FOLLOWUP.", + }, + ], + }), + ); + assert.equal(initial.tool_calls?.[0]?.function?.name, "qa_plugin_sleep"); + assert.equal( + initial.tool_calls?.[0]?.function?.arguments, + JSON.stringify({ seconds: 8, text: "steering-e2e-anchor" }), + ); + + const noFollowup = fakeProviderMessage( + await requestFakeProvider(provider, { + messages: [ + { + role: "user", + content: + "First call qa_plugin_sleep with text=steering-e2e-anchor. If no follow-up was injected, reply only STEERING_NO_FOLLOWUP.", + }, + initial, + { + role: "tool", + tool_call_id: "call_qa_plugin_sleep_steering", + content: "qa-plugin-smoke:sleep:8:steering-e2e-anchor", + }, + ], + }), + ); + assert.equal(noFollowup.content, "STEERING_NO_FOLLOWUP"); + + const withFollowup = fakeProviderMessage( + await requestFakeProvider(provider, { + messages: [ + { + role: "user", + content: + "First call qa_plugin_sleep with text=steering-e2e-anchor. If no follow-up was injected, reply only STEERING_NO_FOLLOWUP.", + }, + initial, + { + role: "tool", + tool_call_id: "call_qa_plugin_sleep_steering", + content: "qa-plugin-smoke:sleep:8:steering-e2e-anchor", + }, + { + role: "user", + content: + "This is a steering follow-up. Return only qa_steering_sentinel_6194.", + }, + ], + }), + ); + assert.equal(withFollowup.content, "qa_steering_sentinel_6194"); + } finally { + await provider.stop(); + } +}); + test("MCP stdio case passes case-specific failure signals to automation defaults", () => { - const result = capture(() => commandTestRun(ctx([ - "test", - "run", - "mcp-stdio-tool-call", - "--dry-run", - "--json", - ]))); + const result = capture(() => + commandTestRun( + ctx(["test", "run", "mcp-stdio-tool-call", "--dry-run", "--json"]), + ), + ); assert.equal(result.code, 0); const run = JSON.parse(result.output); - assert.match(run.automation.env_defaults.LANGBOT_E2E_FAILURE_SIGNALS, /qa-plugin-smoke:mcp-ok-local-agent/); - assert.match(run.automation.env_defaults.LANGBOT_E2E_FAILURE_SIGNALS, /model_not_found/); + assert.match( + run.automation.env_defaults.LANGBOT_E2E_FAILURE_SIGNALS, + /qa-plugin-smoke:mcp-ok-local-agent/, + ); + assert.match( + run.automation.env_defaults.LANGBOT_E2E_FAILURE_SIGNALS, + /model_not_found/, + ); }); test("MCP stdio tool-call case setups pipeline and registered MCP server", () => { - const result = capture(() => commandTestRun(ctx([ - "test", - "run", - "mcp-stdio-tool-call", - "--dry-run", - "--json", - ]))); - assert.equal(result.code, 0); - const run = JSON.parse(result.output); - assert.deepEqual(run.setup_automation.map((item: { entry: string }) => item.entry), [ - "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env", - "case:mcp-stdio-register", - ]); + withEnv({ LANGBOT_MCP_QA_STDIO_SERVER_UUID: "mcp-server-uuid" }, () => { + const result = capture(() => + commandTestRun( + ctx(["test", "run", "mcp-stdio-tool-call", "--dry-run", "--json"]), + ), + ); + assert.equal(result.code, 0); + const run = JSON.parse(result.output); + assert.deepEqual( + run.setup_automation.map((item: { entry: string }) => item.entry), + [ + "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env", + "case:mcp-stdio-register", + ], + ); + assert.equal( + run.automation.env_defaults.LANGBOT_E2E_EXPECTED_RUNNER_ID, + "plugin:langbot/local-agent/default", + ); + assert.equal(run.automation.env_defaults.LANGBOT_E2E_RESET_DEBUG_CHAT, "1"); + assert.equal( + run.automation.env_defaults.LANGBOT_E2E_RESTORE_EXTENSIONS, + "1", + ); + assert.deepEqual( + JSON.parse(run.automation.env_defaults.LANGBOT_E2E_EXTENSIONS_PATCH_JSON), + { + enable_all_plugins: false, + bound_plugins: [{ author: "langbot", name: "local-agent" }], + enable_all_mcp_servers: false, + bound_mcp_servers: ["mcp-server-uuid"], + enable_all_skills: false, + bound_skills: [], + }, + ); - const planResult = capture(() => commandTestPlan(ctx(["test", "plan", "mcp-stdio-tool-call", "--json"]))); - assert.equal(planResult.code, 0); - const plan = JSON.parse(planResult.output); - assert.deepEqual(plan.setup_provides_env, [ - "LANGBOT_LOCAL_AGENT_PIPELINE_URL", - "LANGBOT_LOCAL_AGENT_PIPELINE_NAME", - ]); - assert.ok(!plan.preconditions.some((item: string) => item.includes("points to the local-agent pipeline"))); + const planResult = capture(() => + commandTestPlan(ctx(["test", "plan", "mcp-stdio-tool-call", "--json"])), + ); + assert.equal(planResult.code, 0); + const plan = JSON.parse(planResult.output); + assert.deepEqual(plan.setup_provides_env, [ + "LANGBOT_LOCAL_AGENT_PIPELINE_URL", + "LANGBOT_LOCAL_AGENT_PIPELINE_NAME", + "LANGBOT_MCP_QA_STDIO_SERVER_UUID", + ]); + assert.ok( + !plan.preconditions.some((item: string) => + item.includes("points to the local-agent pipeline"), + ), + ); + }); }); test("generic pipeline automation can still use the shared pipeline env", () => { - const result = capture(() => commandTestRun(ctx([ - "test", - "run", - "pipeline-debug-chat", - "--dry-run", - "--json", - ]))); + const result = capture(() => + commandTestRun( + ctx(["test", "run", "pipeline-debug-chat", "--dry-run", "--json"]), + ), + ); assert.equal(result.code, 0); const run = JSON.parse(result.output); assert.equal(run.automation.pipeline_env_required, false); assert.deepEqual(run.automation.env_aliases, []); - assert.ok(run.automation.required_env.includes("LANGBOT_PIPELINE_URL|LANGBOT_PIPELINE_NAME")); + assert.ok( + run.automation.required_env.includes( + "LANGBOT_PIPELINE_URL|LANGBOT_PIPELINE_NAME", + ), + ); }); test("AgentRunner live install case exposes package automation defaults", () => { - const result = capture(() => commandTestRun(ctx([ - "test", - "run", - "agent-runner-live-install", - "--dry-run", - "--json", - ]))); + const result = capture(() => + commandTestRun( + ctx(["test", "run", "agent-runner-live-install", "--dry-run", "--json"]), + ), + ); assert.equal(result.code, 0); const run = JSON.parse(result.output); assert.equal( run.automation.env_defaults.LANGBOT_E2E_PLUGIN_PACKAGE, "skills/langbot-testing/fixtures/plugins/qa-agent-runner/dist/qa-agent-runner-0.1.0.lbpkg", ); - assert.equal(run.automation.env_defaults.LANGBOT_E2E_EXPECTED_PLUGIN_ID, "qa/agent-runner"); - assert.equal(run.automation.env_defaults.LANGBOT_E2E_EXPECTED_RUNNER_ID, "plugin:qa/agent-runner/default"); + assert.equal( + run.automation.env_defaults.LANGBOT_E2E_EXPECTED_PLUGIN_ID, + "qa/agent-runner", + ); + assert.equal( + run.automation.env_defaults.LANGBOT_E2E_EXPECTED_RUNNER_ID, + "plugin:qa/agent-runner/default", + ); }); test("QA plugin live install checks the fixture package before installed state", () => { @@ -2493,18 +3682,19 @@ test("QA plugin live install checks the fixture package before installed state", }); test("AgentRunner QA Debug Chat case uses dedicated pipeline env", () => { - const result = capture(() => commandTestRun(ctx([ - "test", - "run", - "agent-runner-qa-debug-chat", - "--dry-run", - "--json", - ]))); + const result = capture(() => + commandTestRun( + ctx(["test", "run", "agent-runner-qa-debug-chat", "--dry-run", "--json"]), + ), + ); assert.equal(result.code, 0); const run = JSON.parse(result.output); assert.equal(run.automation.script, "scripts/e2e/pipeline-debug-chat.mjs"); assert.equal(run.automation.pipeline_env_required, true); - assert.equal(run.automation.env_defaults.LANGBOT_E2E_EXPECTED_RUNNER_ID, "plugin:qa/agent-runner/default"); + assert.equal( + run.automation.env_defaults.LANGBOT_E2E_EXPECTED_RUNNER_ID, + "plugin:qa/agent-runner/default", + ); assert.deepEqual( run.setup_automation.map((item: { entry: string }) => item.entry), [ @@ -2512,135 +3702,200 @@ test("AgentRunner QA Debug Chat case uses dedicated pipeline env", () => { "node:scripts/e2e/ensure-qa-agent-runner-pipeline.mjs --write-env", ], ); - assert.ok(run.automation.env_aliases.some((alias: { target: string; source: string }) => ( - alias.target === "LANGBOT_E2E_PIPELINE_URL" && alias.source === "LANGBOT_QA_AGENT_RUNNER_PIPELINE_URL" - ))); + assert.ok( + run.automation.env_aliases.some( + (alias: { target: string; source: string }) => + alias.target === "LANGBOT_E2E_PIPELINE_URL" && + alias.source === "LANGBOT_QA_AGENT_RUNNER_PIPELINE_URL", + ), + ); }); test("AgentRunner QA Debug Chat setup automation removes manual readiness", () => { - withEnv({ - LANGBOT_BROWSER_PROFILE: "/tmp/langbot-test-profile", - LANGBOT_CHROMIUM_EXECUTABLE: "/tmp/langbot-test-chromium", - }, () => { - const planResult = capture(() => commandTestPlan(ctx(["test", "plan", "agent-runner-qa-debug-chat", "--json"]))); - assert.equal(planResult.code, 0); - const plan = JSON.parse(planResult.output); - assert.equal(plan.manual_readiness.status, "not_required"); - assert.deepEqual(plan.setup_provides_env, [ - "LANGBOT_QA_AGENT_RUNNER_PIPELINE_URL", - "LANGBOT_QA_AGENT_RUNNER_PIPELINE_NAME", - ]); - assert.equal(plan.automation_readiness.status, "ready"); + withEnv( + { + LANGBOT_BROWSER_PROFILE: "/tmp/langbot-test-profile", + LANGBOT_CHROMIUM_EXECUTABLE: "/tmp/langbot-test-chromium", + }, + () => { + const planResult = capture(() => + commandTestPlan( + ctx(["test", "plan", "agent-runner-qa-debug-chat", "--json"]), + ), + ); + assert.equal(planResult.code, 0); + const plan = JSON.parse(planResult.output); + assert.equal(plan.manual_readiness.status, "not_required"); + assert.deepEqual(plan.setup_provides_env, [ + "LANGBOT_QA_AGENT_RUNNER_PIPELINE_URL", + "LANGBOT_QA_AGENT_RUNNER_PIPELINE_NAME", + ]); + assert.equal(plan.automation_readiness.status, "ready"); - const suiteResult = capture(() => commandSuitePlan(ctx(["suite", "plan", "agent-runner-release-gate", "--json"]))); - assert.equal(suiteResult.code, 0); - const suite = JSON.parse(suiteResult.output); - assert.ok(!suite.readiness.manual_check_cases.includes("agent-runner-qa-debug-chat")); - }); + const suiteResult = capture(() => + commandSuitePlan( + ctx(["suite", "plan", "agent-runner-release-gate", "--json"]), + ), + ); + assert.equal(suiteResult.code, 0); + const suite = JSON.parse(suiteResult.output); + assert.ok( + !suite.readiness.manual_check_cases.includes( + "agent-runner-qa-debug-chat", + ), + ); + }, + ); }); test("ACP AgentRunner Debug Chat case setups the ACP pipeline env", () => { - const result = capture(() => commandTestRun(ctx([ - "test", - "run", - "acp-agent-runner-debug-chat", - "--dry-run", - "--json", - ]))); + const result = capture(() => + commandTestRun( + ctx([ + "test", + "run", + "acp-agent-runner-debug-chat", + "--dry-run", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const run = JSON.parse(result.output); - assert.deepEqual(run.setup_automation.map((item: { entry: string }) => item.entry), [ - "node:scripts/e2e/ensure-acp-agent-runner-pipeline.mjs --write-env", - ]); - assert.ok(run.automation.env_aliases.some((alias: { target: string; source: string }) => ( - alias.target === "LANGBOT_E2E_PIPELINE_URL" && alias.source === "LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL" - ))); + assert.deepEqual( + run.setup_automation.map((item: { entry: string }) => item.entry), + ["node:scripts/e2e/ensure-acp-agent-runner-pipeline.mjs --write-env"], + ); + assert.ok( + run.automation.env_aliases.some( + (alias: { target: string; source: string }) => + alias.target === "LANGBOT_E2E_PIPELINE_URL" && + alias.source === "LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL", + ), + ); - const planResult = capture(() => commandTestPlan(ctx(["test", "plan", "acp-agent-runner-debug-chat", "--json"]))); + const planResult = capture(() => + commandTestPlan( + ctx(["test", "plan", "acp-agent-runner-debug-chat", "--json"]), + ), + ); assert.equal(planResult.code, 0); const plan = JSON.parse(planResult.output); assert.deepEqual(plan.setup_provides_env, [ "LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL", "LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME", ]); - assert.ok(!plan.preconditions.some((item: string) => item.includes("pipeline AI runner"))); + assert.ok( + !plan.preconditions.some((item: string) => + item.includes("pipeline AI runner"), + ), + ); }); test("local-agent plugin cases setup the QA plugin smoke fixture", () => { - const result = capture(() => commandTestRun(ctx([ - "test", - "run", - "local-agent-plugin-tool-call-debug-chat", - "--dry-run", - "--json", - ]))); + const result = capture(() => + commandTestRun( + ctx([ + "test", + "run", + "local-agent-plugin-tool-call-debug-chat", + "--dry-run", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const run = JSON.parse(result.output); - assert.deepEqual(run.setup_automation.map((item: { entry: string }) => item.entry), [ - "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env", - "case:qa-plugin-smoke-live-install", - ]); + assert.deepEqual( + run.setup_automation.map((item: { entry: string }) => item.entry), + [ + "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env", + "case:qa-plugin-smoke-live-install", + ], + ); }); test("local-agent RAG case only requires the KB fixture env", () => { - const result = capture(() => commandTestRun(ctx([ - "test", - "run", - "local-agent-rag-debug-chat", - "--dry-run", - "--json", - ]))); + const result = capture(() => + commandTestRun( + ctx(["test", "run", "local-agent-rag-debug-chat", "--dry-run", "--json"]), + ), + ); assert.equal(result.code, 0); const run = JSON.parse(result.output); - assert.ok(run.automation.required_env.includes("LANGBOT_LOCAL_AGENT_RAG_KB_UUID")); - assert.ok(!run.automation.required_env.includes("LANGBOT_LOCAL_AGENT_RAG_TEXT_MODEL_UUID")); + assert.ok( + run.automation.required_env.includes("LANGBOT_LOCAL_AGENT_RAG_KB_UUID"), + ); + assert.ok( + !run.automation.required_env.includes( + "LANGBOT_LOCAL_AGENT_RAG_TEXT_MODEL_UUID", + ), + ); assert.equal( run.automation.env_defaults.LANGBOT_E2E_RUNNER_CONFIG_PATCH_JSON, JSON.stringify({ - "knowledge-bases": [ - loadEnv(root).LANGBOT_LOCAL_AGENT_RAG_KB_UUID || "", - ], + "knowledge-bases": [loadEnv(root).LANGBOT_LOCAL_AGENT_RAG_KB_UUID || ""], }), ); }); test("LangRAG retrieve readiness requires a KB UUID alternative", () => { - const result = capture(() => commandTestPlan(ctx(["test", "plan", "langrag-kb-retrieve", "--json"]))); + const result = capture(() => + commandTestPlan(ctx(["test", "plan", "langrag-kb-retrieve", "--json"])), + ); assert.equal(result.code, 0); const plan = JSON.parse(result.output); - assert.ok(plan.automation_readiness.required.includes("LANGBOT_LOCAL_AGENT_RAG_KB_UUID|LANGBOT_RAG_KB_UUID")); + assert.ok( + plan.automation_readiness.required.includes( + "LANGBOT_LOCAL_AGENT_RAG_KB_UUID|LANGBOT_RAG_KB_UUID", + ), + ); }); test("local-agent RAG multimodal case setups the KB fixture env", () => { - const result = capture(() => commandTestRun(ctx([ - "test", - "run", - "local-agent-rag-multimodal-debug-chat", - "--dry-run", - "--json", - ]))); + const result = capture(() => + commandTestRun( + ctx([ + "test", + "run", + "local-agent-rag-multimodal-debug-chat", + "--dry-run", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const run = JSON.parse(result.output); - assert.ok(run.automation.required_env.includes("LANGBOT_LOCAL_AGENT_RAG_KB_UUID")); + assert.ok( + run.automation.required_env.includes("LANGBOT_LOCAL_AGENT_RAG_KB_UUID"), + ); assert.equal( run.automation.env_defaults.LANGBOT_E2E_RUNNER_CONFIG_PATCH_JSON, JSON.stringify({ - "knowledge-bases": [ - loadEnv(root).LANGBOT_LOCAL_AGENT_RAG_KB_UUID || "", - ], + "knowledge-bases": [loadEnv(root).LANGBOT_LOCAL_AGENT_RAG_KB_UUID || ""], }), ); - assert.deepEqual(run.setup_automation.map((item: { entry: string }) => item.entry), [ - "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env", - "node:scripts/e2e/ensure-langrag-sentinel-kb.mjs --write-env", - ]); + assert.deepEqual( + run.setup_automation.map((item: { entry: string }) => item.entry), + [ + "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env", + "node:scripts/e2e/ensure-langrag-sentinel-kb.mjs --write-env", + ], + ); }); test("test report renders a reusable evidence template", () => { - const result = capture(() => commandTestReport(ctx(["test", "report", "pipeline-debug-chat", "--no-auto-log"]))); + const result = capture(() => + commandTestReport( + ctx(["test", "report", "pipeline-debug-chat", "--no-auto-log"]), + ), + ); assert.equal(result.code, 0); assert.match(result.output, /^# Test Report: pipeline-debug-chat/m); - assert.match(result.output, /result: pass \| fail \| blocked \| env_issue \| flaky/); + assert.match( + result.output, + /result: pass \| fail \| blocked \| env_issue \| flaky/, + ); assert.match(result.output, /## Log Guard/); assert.match(result.output, /## Automation Result/); assert.match(result.output, /## Required Evidence/); @@ -2660,17 +3915,24 @@ test("test report promotes loaded automation evidence into result section", () = }), ); - const result = capture(() => commandTestReport(ctx([ - "test", - "report", - "langbot-live-backend-latency", - "--evidence-dir", - tmp, - "--no-auto-log", - ]))); + const result = capture(() => + commandTestReport( + ctx([ + "test", + "report", + "langbot-live-backend-latency", + "--evidence-dir", + tmp, + "--no-auto-log", + ]), + ), + ); assert.equal(result.code, 0); - assert.match(result.output, /## Result\n- result: pass\n- reason: latency thresholds passed/); + assert.match( + result.output, + /## Result\n- result: pass\n- reason: latency thresholds passed/, + ); assert.match(result.output, /- target_tested: http:\/\/127\.0\.0\.1:5300/); assert.doesNotMatch(result.output, /target_tested: TODO/); assert.match(result.output, /## Automation Result/); @@ -2691,11 +3953,22 @@ test("validate rejects dangling case references and missing automation scripts", mkdirSync(join(testingDir, "fixtures"), { recursive: true }); mkdirSync(join(testingDir, "suites"), { recursive: true }); mkdirSync(envSetupDir, { recursive: true }); - for (const schemaName of ["case.schema.json", "suite.schema.json", "troubleshooting.schema.json", "skill-index.schema.json"]) { + for (const schemaName of [ + "case.schema.json", + "suite.schema.json", + "troubleshooting.schema.json", + "skill-index.schema.json", + ]) { writeFileSync(join(schemasDir, schemaName), "{}"); } - writeFileSync(join(envSetupDir, "SKILL.md"), "---\nname: langbot-env-setup\ndescription: Env setup.\n---\n\n# Env\n"); - writeFileSync(join(testingDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n"); + writeFileSync( + join(envSetupDir, "SKILL.md"), + "---\nname: langbot-env-setup\ndescription: Env setup.\n---\n\n# Env\n", + ); + writeFileSync( + join(testingDir, "SKILL.md"), + "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", + ); writeFileSync( join(skillsDir, ".env"), [ @@ -2742,7 +4015,10 @@ test("validate rejects dangling case references and missing automation scripts", " - missing-trouble", ].join("\n"), ); - for (const [id, target] of [["cycle-a", "cycle-b"], ["cycle-b", "cycle-a"]]) { + for (const [id, target] of [ + ["cycle-a", "cycle-b"], + ["cycle-b", "cycle-a"], + ]) { writeFileSync( join(testingDir, "cases", `${id}.yaml`), [ @@ -2785,7 +4061,14 @@ test("validate rejects dangling case references and missing automation scripts", ); writeFileSync( join(testingDir, "fixtures", "fixtures.json"), - JSON.stringify([{ id: "bad-fixture", title: "Bad Fixture", path: "fixtures/missing.txt", related_cases: ["missing-case"] }]), + JSON.stringify([ + { + id: "bad-fixture", + title: "Bad Fixture", + path: "fixtures/missing.txt", + related_cases: ["missing-case"], + }, + ]), ); const result = captureAll(() => commandValidate(tmp)); @@ -2817,21 +4100,43 @@ test("test report JSON scans logs and redacts secrets", () => { ].join("\n"), ); - const result = capture(() => commandTestReport(ctx(["test", "report", "pipeline-debug-chat", "--backend-log", logPath, "--json"]))); + const result = capture(() => + commandTestReport( + ctx([ + "test", + "report", + "pipeline-debug-chat", + "--backend-log", + logPath, + "--json", + ]), + ), + ); assert.equal(result.code, 0); assert.doesNotMatch(result.output, /sk-test-secret/); const report = JSON.parse(result.output); assert.equal(report.log_guard.status, "fail"); - assert.ok(report.log_guard.findings.some((finding: { kind: string }) => ( - finding.kind === "case_failure_pattern" - ))); - assert.ok(report.log_guard.findings.some((finding: { troubleshooting_id?: string }) => ( - finding.troubleshooting_id === "plugin-runtime-timeout" - ))); - assert.ok(report.log_guard.findings.some((finding: { kind: string }) => finding.kind === "python_traceback")); + assert.ok( + report.log_guard.findings.some( + (finding: { kind: string }) => finding.kind === "case_failure_pattern", + ), + ); + assert.ok( + report.log_guard.findings.some( + (finding: { troubleshooting_id?: string }) => + finding.troubleshooting_id === "plugin-runtime-timeout", + ), + ); + assert.ok( + report.log_guard.findings.some( + (finding: { kind: string }) => finding.kind === "python_traceback", + ), + ); - const secretFinding = report.log_guard.findings.find((finding: { kind: string }) => finding.kind === "secret_leak"); + const secretFinding = report.log_guard.findings.find( + (finding: { kind: string }) => finding.kind === "secret_leak", + ); assert.ok(secretFinding); assert.match(secretFinding.excerpt, /\[redacted\]/); } finally { @@ -2848,15 +4153,33 @@ test("test report does not treat invalid api key wording as a secret leak", () = "RequesterError: 模型请求失败: 无效的 api-key: Error code: 401 - invalid api key\n", ); - const result = capture(() => commandTestReport(ctx(["test", "report", "mcp-stdio-tool-call", "--backend-log", logPath, "--json"]))); + const result = capture(() => + commandTestReport( + ctx([ + "test", + "report", + "mcp-stdio-tool-call", + "--backend-log", + logPath, + "--json", + ]), + ), + ); assert.equal(result.code, 0); assert.match(result.output, /api-key: Error code/); const report = JSON.parse(result.output); - assert.ok(!report.log_guard.findings.some((finding: { kind: string }) => finding.kind === "secret_leak")); - assert.ok(report.log_guard.findings.some((finding: { troubleshooting_id?: string }) => ( - finding.troubleshooting_id === "local-agent-model-route-unavailable" - ))); + assert.ok( + !report.log_guard.findings.some( + (finding: { kind: string }) => finding.kind === "secret_leak", + ), + ); + assert.ok( + report.log_guard.findings.some( + (finding: { troubleshooting_id?: string }) => + finding.troubleshooting_id === "local-agent-model-route-unavailable", + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -2874,21 +4197,28 @@ test("test report records declared success signals from logs", () => { ].join("\n"), ); - const result = capture(() => commandTestReport(ctx([ - "test", - "report", - "pipeline-debug-chat", - "--backend-log", - logPath, - "--json", - ]))); + const result = capture(() => + commandTestReport( + ctx([ + "test", + "report", + "pipeline-debug-chat", + "--backend-log", + logPath, + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); assert.equal(report.log_guard.status, "pass"); assert.equal(report.log_guard.success_signals.length, 2); - assert.ok(report.log_guard.success_signals.some((signal: { pattern: string }) => ( - signal.pattern === "Streaming completed" - ))); + assert.ok( + report.log_guard.success_signals.some( + (signal: { pattern: string }) => + signal.pattern === "Streaming completed", + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -2900,20 +4230,27 @@ test("test report warns when declared success signals are missing", () => { const logPath = join(tmp, "backend.log"); writeFileSync(logPath, "INFO request started\nINFO request ended\n"); - const result = capture(() => commandTestReport(ctx([ - "test", - "report", - "pipeline-debug-chat", - "--backend-log", - logPath, - "--json", - ]))); + const result = capture(() => + commandTestReport( + ctx([ + "test", + "report", + "pipeline-debug-chat", + "--backend-log", + logPath, + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); assert.equal(report.log_guard.status, "warning"); - assert.ok(report.log_guard.findings.some((finding: { kind: string }) => ( - finding.kind === "missing_success_signal" - ))); + assert.ok( + report.log_guard.findings.some( + (finding: { kind: string }) => + finding.kind === "missing_success_signal", + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -2933,28 +4270,39 @@ test("test report can limit log guard to tail lines", () => { ].join("\n"), ); - const result = capture(() => commandTestReport(ctx([ - "test", - "report", - "pipeline-debug-chat", - "--backend-log", - logPath, - "--tail-lines", - "2", - "--json", - ]))); + const result = capture(() => + commandTestReport( + ctx([ + "test", + "report", + "pipeline-debug-chat", + "--backend-log", + logPath, + "--tail-lines", + "2", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); assert.equal(report.log_guard.scan.mode, "tail-lines"); assert.equal(report.log_guard.scan.tail_lines, 2); assert.equal(report.log_guard.sources[0].line_count, 2); assert.equal(report.log_guard.sources[0].start_line, 3); - assert.ok(report.log_guard.findings.some((finding: { troubleshooting_id?: string }) => ( - finding.troubleshooting_id === "plugin-runtime-timeout" - ))); - assert.ok(!report.log_guard.findings.some((finding: { kind: string; excerpt?: string }) => ( - finding.kind === "error_log" && finding.excerpt?.includes("old failure") - ))); + assert.ok( + report.log_guard.findings.some( + (finding: { troubleshooting_id?: string }) => + finding.troubleshooting_id === "plugin-runtime-timeout", + ), + ); + assert.ok( + !report.log_guard.findings.some( + (finding: { kind: string; excerpt?: string }) => + finding.kind === "error_log" && + finding.excerpt?.includes("old failure"), + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -2974,26 +4322,38 @@ test("test report can limit log guard with since timestamp", () => { ].join("\n"), ); - const result = capture(() => commandTestReport(ctx([ - "test", - "report", - "pipeline-debug-chat", - "--backend-log", - logPath, - "--since", - "2026-05-21T10:30:00+08:00", - "--json", - ]))); + const result = capture(() => + commandTestReport( + ctx([ + "test", + "report", + "pipeline-debug-chat", + "--backend-log", + logPath, + "--since", + "2026-05-21T10:30:00+08:00", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); assert.equal(report.log_guard.scan.mode, "since"); assert.equal(report.log_guard.sources[0].line_count, 3); assert.equal(report.log_guard.sources[0].start_line, 2); assert.equal(report.log_guard.sources[0].timestamped_line_count, 3); - assert.ok(report.log_guard.findings.some((finding: { line?: number; troubleshooting_id?: string }) => ( - finding.line === 2 && finding.troubleshooting_id === "plugin-runtime-timeout" - ))); - assert.ok(!report.log_guard.findings.some((finding: { excerpt?: string }) => finding.excerpt?.includes("old failure"))); + assert.ok( + report.log_guard.findings.some( + (finding: { line?: number; troubleshooting_id?: string }) => + finding.line === 2 && + finding.troubleshooting_id === "plugin-runtime-timeout", + ), + ); + assert.ok( + !report.log_guard.findings.some((finding: { excerpt?: string }) => + finding.excerpt?.includes("old failure"), + ), + ); assert.doesNotMatch(result.output, /sk-since-secret/); } finally { rmSync(tmp, { recursive: true, force: true }); @@ -3014,18 +4374,22 @@ test("test report can limit log guard with since and until timestamps", () => { ].join("\n"), ); - const result = capture(() => commandTestReport(ctx([ - "test", - "report", - "pipeline-debug-chat", - "--backend-log", - logPath, - "--since", - "2026-05-21T10:30:00+08:00", - "--until", - "2026-05-21T10:32:00+08:00", - "--json", - ]))); + const result = capture(() => + commandTestReport( + ctx([ + "test", + "report", + "pipeline-debug-chat", + "--backend-log", + logPath, + "--since", + "2026-05-21T10:30:00+08:00", + "--until", + "2026-05-21T10:32:00+08:00", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); assert.equal(report.log_guard.scan.mode, "since+until"); @@ -3033,8 +4397,16 @@ test("test report can limit log guard with since and until timestamps", () => { assert.equal(report.log_guard.sources[0].start_line, 2); assert.equal(report.log_guard.sources[0].end_line, 3); assert.equal(report.log_guard.status, "pass"); - assert.ok(!report.log_guard.findings.some((finding: { excerpt?: string }) => finding.excerpt?.includes("old failure"))); - assert.ok(!report.log_guard.findings.some((finding: { excerpt?: string }) => finding.excerpt?.includes("later failure"))); + assert.ok( + !report.log_guard.findings.some((finding: { excerpt?: string }) => + finding.excerpt?.includes("old failure"), + ), + ); + assert.ok( + !report.log_guard.findings.some((finding: { excerpt?: string }) => + finding.excerpt?.includes("later failure"), + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -3049,20 +4421,28 @@ test("test report classifies model route failures as env_issue", () => { "[05-21 10:31:00.000] runner.py (2) - [ERROR] : runner.llm_error model_not_found no available channel for model gpt-test\n", ); - const result = capture(() => commandTestReport(ctx([ - "test", - "report", - "local-agent-plugin-tool-call-debug-chat", - "--backend-log", - logPath, - "--json", - ]))); + const result = capture(() => + commandTestReport( + ctx([ + "test", + "report", + "local-agent-plugin-tool-call-debug-chat", + "--backend-log", + logPath, + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); assert.equal(report.log_guard.status, "env_issue"); - assert.ok(report.log_guard.findings.some((finding: { severity?: string; troubleshooting_id?: string }) => ( - finding.severity === "env_issue" && finding.troubleshooting_id === "local-agent-model-route-unavailable" - ))); + assert.ok( + report.log_guard.findings.some( + (finding: { severity?: string; troubleshooting_id?: string }) => + finding.severity === "env_issue" && + finding.troubleshooting_id === "local-agent-model-route-unavailable", + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -3094,15 +4474,19 @@ test("test report infers scan window from automation result evidence", () => { }), ); - const result = capture(() => commandTestReport(ctx([ - "test", - "report", - "pipeline-debug-chat", - "--console-log", - consoleLog, - "--no-auto-log", - "--json", - ]))); + const result = capture(() => + commandTestReport( + ctx([ + "test", + "report", + "pipeline-debug-chat", + "--console-log", + consoleLog, + "--no-auto-log", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); assert.equal(report.log_guard.scan.mode, "since+until"); @@ -3113,8 +4497,16 @@ test("test report infers scan window from automation result evidence", () => { assert.equal(report.automation_result.status, "loaded"); assert.equal(report.automation_result.result, "pass"); assert.equal(report.automation_result.reason, "UI sentinel appeared."); - assert.ok(!report.log_guard.findings.some((finding: { excerpt?: string }) => finding.excerpt?.includes("old failure"))); - assert.ok(!report.log_guard.findings.some((finding: { excerpt?: string }) => finding.excerpt?.includes("later failure"))); + assert.ok( + !report.log_guard.findings.some((finding: { excerpt?: string }) => + finding.excerpt?.includes("old failure"), + ), + ); + assert.ok( + !report.log_guard.findings.some((finding: { excerpt?: string }) => + finding.excerpt?.includes("later failure"), + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -3126,7 +4518,10 @@ test("test report does not treat final result as automation evidence", () => { const evidenceDir = join(tmp, "evidence", "run-final"); mkdirSync(evidenceDir, { recursive: true }); const consoleLog = join(evidenceDir, "console.log"); - writeFileSync(consoleLog, "[05-21 10:31:00.000] ui.js (1) - [INFO] : opened\n"); + writeFileSync( + consoleLog, + "[05-21 10:31:00.000] ui.js (1) - [INFO] : opened\n", + ); writeFileSync( join(evidenceDir, "result.json"), JSON.stringify({ @@ -3139,20 +4534,27 @@ test("test report does not treat final result as automation evidence", () => { }), ); - const result = capture(() => commandTestReport(ctx([ - "test", - "report", - "webui-login-state", - "--console-log", - consoleLog, - "--no-auto-log", - "--json", - ]))); + const result = capture(() => + commandTestReport( + ctx([ + "test", + "report", + "webui-login-state", + "--console-log", + consoleLog, + "--no-auto-log", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); assert.equal(report.automation_result.status, "not_provided"); - assert.match(report.automation_result.reason, /only final result\.json is present/); + assert.match( + report.automation_result.reason, + /only final result\.json is present/, + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -3164,7 +4566,10 @@ test("test report still scans untimestamped explicit console evidence within an const evidenceDir = join(tmp, "evidence", "run-untimestamped"); mkdirSync(evidenceDir, { recursive: true }); const consoleLog = join(evidenceDir, "console.log"); - writeFileSync(consoleLog, "[error] Uncaught TypeError: Cannot read properties of undefined\n"); + writeFileSync( + consoleLog, + "[error] Uncaught TypeError: Cannot read properties of undefined\n", + ); writeFileSync( join(evidenceDir, "result.json"), JSON.stringify({ @@ -3175,15 +4580,19 @@ test("test report still scans untimestamped explicit console evidence within an }), ); - const result = capture(() => commandTestReport(ctx([ - "test", - "report", - "webui-login-state", - "--console-log", - consoleLog, - "--no-auto-log", - "--json", - ]))); + const result = capture(() => + commandTestReport( + ctx([ + "test", + "report", + "webui-login-state", + "--console-log", + consoleLog, + "--no-auto-log", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); @@ -3191,9 +4600,12 @@ test("test report still scans untimestamped explicit console evidence within an assert.equal(report.log_guard.sources[0].timestamped_line_count, 0); assert.ok(report.log_guard.sources[0].line_count >= 1); assert.equal(report.log_guard.status, "fail"); - assert.ok(report.log_guard.findings.some((finding: { kind: string }) => ( - finding.kind === "frontend_uncaught_error" - ))); + assert.ok( + report.log_guard.findings.some( + (finding: { kind: string }) => + finding.kind === "frontend_uncaught_error", + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -3203,10 +4615,17 @@ test("test report can write markdown to an output path", () => { const tmp = mkdtempSync(join(tmpdir(), "lbs-report-output-")); try { const output = join(tmp, "reports", "pipeline-debug-chat.md"); - const result = capture(() => commandTestReport(ctx(["test", "report", "pipeline-debug-chat", "--output", output]))); + const result = capture(() => + commandTestReport( + ctx(["test", "report", "pipeline-debug-chat", "--output", output]), + ), + ); assert.equal(result.code, 0); assert.match(result.output, /pipeline-debug-chat\.md$/); - assert.match(readFileSync(output, "utf8"), /^# Test Report: pipeline-debug-chat/m); + assert.match( + readFileSync(output, "utf8"), + /^# Test Report: pipeline-debug-chat/m, + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -3225,32 +4644,49 @@ test("log scan reuses case-aware log guard patterns", () => { ].join("\n"), ); - const result = capture(() => commandLogScan(ctx([ - "log", - "scan", - "--backend-log", - logPath, - "--case", - "pipeline-debug-chat", - "--json", - ]))); + const result = capture(() => + commandLogScan( + ctx([ + "log", + "scan", + "--backend-log", + logPath, + "--case", + "pipeline-debug-chat", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); assert.equal(report.status, "fail"); - assert.ok(report.success_signals.some((signal: { pattern: string }) => signal.pattern === "Streaming completed")); - assert.ok(report.findings.some((finding: { kind: string }) => finding.kind === "case_failure_pattern")); + assert.ok( + report.success_signals.some( + (signal: { pattern: string }) => + signal.pattern === "Streaming completed", + ), + ); + assert.ok( + report.findings.some( + (finding: { kind: string }) => finding.kind === "case_failure_pattern", + ), + ); - const strict = capture(() => commandLogScan(ctx([ - "log", - "scan", - "--backend-log", - logPath, - "--case", - "pipeline-debug-chat", - "--strict", - "--json", - ]))); + const strict = capture(() => + commandLogScan( + ctx([ + "log", + "scan", + "--backend-log", + logPath, + "--case", + "pipeline-debug-chat", + "--strict", + "--json", + ]), + ), + ); assert.equal(strict.code, 1); } finally { rmSync(tmp, { recursive: true, force: true }); @@ -3264,42 +4700,54 @@ test("log guard start and stop bound a QA log window", () => { const outputDir = join(tmp, "guards"); writeFileSync(logPath, "INFO before guard\n"); - const start = capture(() => commandLogGuard(ctx([ - "log", - "guard", - "start", - "--run-id", - "qa-run", - "--output-dir", - outputDir, - "--backend-log", - logPath, - "--case", - "pipeline-debug-chat", - "--json", - ]))); + const start = capture(() => + commandLogGuard( + ctx([ + "log", + "guard", + "start", + "--run-id", + "qa-run", + "--output-dir", + outputDir, + "--backend-log", + logPath, + "--case", + "pipeline-debug-chat", + "--json", + ]), + ), + ); assert.equal(start.code, 0); const session = JSON.parse(start.output); assert.equal(session.run_id, "qa-run"); assert.ok(existsSync(join(outputDir, "qa-run.json"))); appendFileSync(logPath, "Traceback (most recent call last):\n"); - const stop = capture(() => commandLogGuard(ctx([ - "log", - "guard", - "stop", - "--run-id", - "qa-run", - "--output-dir", - outputDir, - "--json", - ]))); + const stop = capture(() => + commandLogGuard( + ctx([ + "log", + "guard", + "stop", + "--run-id", + "qa-run", + "--output-dir", + outputDir, + "--json", + ]), + ), + ); assert.equal(stop.code, 1); const report = JSON.parse(stop.output); assert.equal(report.session.run_id, "qa-run"); assert.equal(report.result.status, "fail"); - assert.ok(report.result.findings.some((finding: { kind: string }) => finding.kind === "python_traceback")); + assert.ok( + report.result.findings.some( + (finding: { kind: string }) => finding.kind === "python_traceback", + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -3311,18 +4759,22 @@ test("log watch observes appended LangBot backend lines", async () => { const logPath = join(tmp, "backend.log"); writeFileSync(logPath, "INFO existing line\n"); - const watching = captureAsync(() => commandLogWatch(ctx([ - "log", - "watch", - "--backend-log", - logPath, - "--duration-ms", - "220", - "--interval-ms", - "20", - "--strict", - "--json", - ]))); + const watching = captureAsync(() => + commandLogWatch( + ctx([ + "log", + "watch", + "--backend-log", + logPath, + "--duration-ms", + "220", + "--interval-ms", + "20", + "--strict", + "--json", + ]), + ), + ); setTimeout(() => { appendFileSync(logPath, "Traceback (most recent call last):\n"); }, 50); @@ -3333,14 +4785,20 @@ test("log watch observes appended LangBot backend lines", async () => { assert.equal(summary.mode, "watch"); assert.equal(summary.status, "fail"); assert.ok(summary.bytes_read > 0); - assert.ok(summary.findings.some((finding: { kind: string }) => finding.kind === "python_traceback")); + assert.ok( + summary.findings.some( + (finding: { kind: string }) => finding.kind === "python_traceback", + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } }); test("trouble search finds structured troubleshooting entries", () => { - const result = capture(() => commandTroubleSearch(ctx(["trouble", "search", "proxy"]))); + const result = capture(() => + commandTroubleSearch(ctx(["trouble", "search", "proxy"])), + ); assert.equal(result.code, 0); assert.match(result.output, /proxy-env-mismatch/); }); @@ -3349,7 +4807,10 @@ test("env local overrides shared env defaults", () => { const tmp = mkdtempSync(join(tmpdir(), "lbs-env-")); try { mkdirSync(join(tmp, "skills")); - writeFileSync(join(tmp, "skills", ".env"), "LANGBOT_REPO=/shared\nLANGBOT_BACKEND_URL=http://127.0.0.1:5300\n"); + writeFileSync( + join(tmp, "skills", ".env"), + "LANGBOT_REPO=/shared\nLANGBOT_BACKEND_URL=http://127.0.0.1:5300\n", + ); writeFileSync(join(tmp, "skills", ".env.local"), "LANGBOT_REPO=/local\n"); assert.deepEqual(loadEnv(tmp), { diff --git a/src/langbot/pkg/api/http/controller/groups/pipelines/pipelines.py b/src/langbot/pkg/api/http/controller/groups/pipelines/pipelines.py index 2e45add77..c8b52127c 100644 --- a/src/langbot/pkg/api/http/controller/groups/pipelines/pipelines.py +++ b/src/langbot/pkg/api/http/controller/groups/pipelines/pipelines.py @@ -73,8 +73,13 @@ class PipelinesRouterGroup(group.RouterGroup): plugins = await self.ap.plugin_connector.list_plugins(component_kinds=pipeline_component_kinds) mcp_servers = await self.ap.mcp_service.get_mcp_servers(contain_runtime_info=True) - # Get available skills - available_skills = await self.ap.skill_service.list_skills() + # Skill listing depends on Box. Pipeline plugin/MCP binding + # must remain usable when Box is slow or unavailable. + try: + available_skills = await self.ap.skill_service.list_skills() + except Exception as exc: + self.ap.logger.warning('Unable to list skills for pipeline extensions: %s', exc) + available_skills = [] extensions_prefs = pipeline.get('extensions_preferences', {}) return self.success( diff --git a/src/langbot/pkg/api/http/service/skill.py b/src/langbot/pkg/api/http/service/skill.py index 94b926975..cbb739a48 100644 --- a/src/langbot/pkg/api/http/service/skill.py +++ b/src/langbot/pkg/api/http/service/skill.py @@ -2,6 +2,7 @@ from __future__ import annotations import io import inspect +import logging import os import posixpath import zipfile @@ -9,6 +10,8 @@ from typing import Optional from urllib.parse import quote, unquote, urlparse import httpx +from langbot_plugin.box.errors import BoxError +from langbot_plugin.entities.io.errors import ActionCallError, ActionCallTimeoutError from ....core import app from ....skill.utils import parse_frontmatter @@ -33,6 +36,8 @@ _GITHUB_ASSET_HOSTS = { 'codeload.github.com', } +logger = logging.getLogger(__name__) + class SkillService: """Filesystem-backed skill management service.""" @@ -81,12 +86,16 @@ class SkillService: async def list_skills(self) -> list[dict]: # When Box is unavailable, surface an empty list rather than raising — - # the skills page should render cleanly, and the UI separately renders - # a "Box disabled / unavailable" banner via useBoxStatus. + # the skills page and unrelated extension surfaces should render + # cleanly, and the UI separately renders Box status via useBoxStatus. box_service = self._box_service() if box_service is None: return [] - return [self._serialize_skill(skill) for skill in await box_service.list_skills()] + try: + return [self._serialize_skill(skill) for skill in await box_service.list_skills()] + except (ActionCallTimeoutError, ActionCallError, BoxError, TimeoutError, ConnectionError) as exc: + logger.warning('Box skill list unavailable; returning an empty skill list: %s', exc) + return [] async def get_skill(self, skill_name: str) -> Optional[dict]: box_service = self._box_service() diff --git a/tests/e2e/test_local_agent_runner_fake_provider.py b/tests/e2e/test_local_agent_runner_fake_provider.py new file mode 100644 index 000000000..910aeffe6 --- /dev/null +++ b/tests/e2e/test_local_agent_runner_fake_provider.py @@ -0,0 +1,885 @@ +"""E2E coverage for the official Local Agent runner with fake Host resources. + +These tests start the real LangBot application and the real SDK Plugin Runtime, +load the sibling ``langbot-local-agent`` plugin, and verify Local Agent paths +that must cross Host run-scoped APIs without calling any external provider. +""" + +from __future__ import annotations + +import asyncio +import json +import os +import shutil +import socket +import sqlite3 +import subprocess +import tempfile +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +from tests.e2e.utils.config_factory import create_minimal_config, create_test_directories +from tests.e2e.utils.process_manager import find_project_root + +pytestmark = pytest.mark.e2e + + +LOCAL_AGENT_RUNNER_ID = 'plugin:langbot/local-agent/default' +FAKE_PROVIDER_UUID = 'e2e-fake-provider' +FAKE_MODEL_UUID = 'e2e-fake-local-agent-model' +LOCAL_AGENT_PLUGIN_DIRNAME = 'langbot__local-agent' +E2E_TOOL_NAME = 'e2e_lookup' +E2E_KB_UUID = 'e2e-kb-local-agent' + + +def _free_port() -> int: + """Reserve a currently-free localhost TCP port for this E2E process.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(('127.0.0.1', 0)) + return int(sock.getsockname()[1]) + + +def _local_agent_repo() -> Path: + """Return the sibling local-agent repository used by this workspace E2E.""" + project_root = find_project_root() + return project_root.parent / 'langbot-local-agent' + + +def _copy_local_agent_plugin(tmpdir: Path) -> None: + """Copy the sibling Local Agent plugin into the temporary LangBot data dir.""" + local_agent_src = _local_agent_repo() + if not (local_agent_src / 'manifest.yaml').exists(): + pytest.skip(f'local-agent repository not found at {local_agent_src}') + + plugin_dst = tmpdir / 'data' / 'plugins' / LOCAL_AGENT_PLUGIN_DIRNAME + ignore = shutil.ignore_patterns( + '.git', + '.venv', + '__pycache__', + '.pytest_cache', + '.ruff_cache', + 'build', + 'dist', + ) + shutil.copytree(local_agent_src, plugin_dst, ignore=ignore) + + +def _content_text(content: Any) -> str: + """Flatten provider message content into text for assertions.""" + if content is None: + return '' + if isinstance(content, str): + return content + if isinstance(content, list): + parts: list[str] = [] + for item in content: + text = item.get('text') if isinstance(item, dict) else getattr(item, 'text', None) + if text: + parts.append(str(text)) + return ''.join(parts) + return str(content) + + +def _message_text(message: Any) -> str: + """Flatten a provider message into text for assertions.""" + return _content_text(getattr(message, 'content', None)) + + +def _invoke_payload_texts(fake_requester: Any) -> list[list[str]]: + """Return model-facing text for every fake LLM invocation.""" + payloads: list[list[str]] = [] + for payload in fake_requester._invoke_payloads: + payloads.append([_message_text(message) for message in payload['messages']]) + return payloads + + +def _event( + *, + event_id: str, + conversation_id: str, + text: str, + thread_id: str = 'e2e-local-agent-thread', +): + """Build an AgentRunner event envelope for Local Agent E2E probes.""" + from langbot.pkg.agent.runner.host_models import AgentEventEnvelope + from langbot_plugin.api.entities.builtin.agent_runner.delivery import DeliveryContext + from langbot_plugin.api.entities.builtin.agent_runner.event import ActorContext, SubjectContext + from langbot_plugin.api.entities.builtin.agent_runner.input import AgentInput + + return AgentEventEnvelope( + event_id=event_id, + event_type='message.received', + source='api', + conversation_id=conversation_id, + thread_id=thread_id, + actor=ActorContext(actor_type='user', actor_id='user-001', actor_name='E2E User'), + subject=SubjectContext(subject_type='chat', subject_id='chat-001'), + input=AgentInput(text=text), + delivery=DeliveryContext(surface='e2e', supports_streaming=False), + ) + + +def _binding( + *, + binding_id: str = 'e2e-local-agent-binding', + runner_config: dict[str, Any] | None = None, + allowed_tool_names: list[str] | None = None, + allowed_kb_uuids: list[str] | None = None, +): + """Build a Local Agent binding with fake model access.""" + from langbot.pkg.agent.runner.host_models import ( + AgentBinding, + BindingScope, + DeliveryPolicy, + ResourcePolicy, + StatePolicy, + ) + + config = { + 'model': {'primary': FAKE_MODEL_UUID, 'fallbacks': []}, + 'timeout': 60, + 'prompt': [{'role': 'system', 'content': 'You are a concise test assistant.'}], + 'knowledge-bases': [], + 'context-window-tokens': 8192, + 'context-reserve-tokens': 1024, + 'context-keep-recent-tokens': 1000, + 'context-summary-tokens': 500, + } + if runner_config: + config.update(runner_config) + + return AgentBinding( + binding_id=binding_id, + scope=BindingScope(scope_type='global'), + runner_id=LOCAL_AGENT_RUNNER_ID, + runner_config=config, + resource_policy=ResourcePolicy( + allowed_model_uuids=[FAKE_MODEL_UUID], + allowed_tool_names=allowed_tool_names, + allowed_kb_uuids=allowed_kb_uuids, + ), + state_policy=StatePolicy(enable_state=True, state_scopes=['conversation']), + delivery_policy=DeliveryPolicy(enable_streaming=False, enable_reply=True), + ) + + +class _FakeToolManager: + """Deterministic tool manager used behind the real Host CALL_TOOL action.""" + + def __init__(self): + self.calls: list[dict[str, Any]] = [] + + async def get_tool_schema(self, tool_name: str): + if tool_name != E2E_TOOL_NAME: + return None, None + return ( + 'Lookup a deterministic E2E value.', + { + 'type': 'object', + 'properties': { + 'query': {'type': 'string'}, + }, + 'required': ['query'], + }, + ) + + async def get_tool_detail(self, tool_name: str): + description, parameters = await self.get_tool_schema(tool_name) + if parameters is None: + return None + return {'name': tool_name, 'description': description, 'parameters': parameters} + + async def execute_func_call(self, name: str, parameters: dict[str, Any], query: Any = None): + del query + self.calls.append({'name': name, 'parameters': dict(parameters)}) + return { + 'value': f"tool-result:{parameters.get('query')}", + 'source': 'fake-tool-manager', + } + + +class _FakeKnowledgeBase: + """Minimal KB object used behind the real Host RETRIEVE_KNOWLEDGE action.""" + + def __init__(self): + self.knowledge_base_entity = SimpleNamespace(kb_type='fake') + self.retrieve_calls: list[dict[str, Any]] = [] + + def get_uuid(self) -> str: + return E2E_KB_UUID + + def get_name(self) -> str: + return 'E2E Fake KB' + + async def retrieve(self, query_text: str, settings: dict[str, Any]): + self.retrieve_calls.append({'query_text': query_text, 'settings': settings}) + return [ + SimpleNamespace( + content='RAG_SENTINEL Local Agent retrieved this deterministic chunk.', + metadata={'source': 'fake-kb'}, + id='fake-kb-chunk-1', + score=0.99, + model_dump=lambda mode='json': { + 'content': 'RAG_SENTINEL Local Agent retrieved this deterministic chunk.', + 'metadata': {'source': 'fake-kb'}, + 'id': 'fake-kb-chunk-1', + 'score': 0.99, + }, + ) + ] + + +class _FakeRagManager: + """Deterministic RAG manager used by resource builder and retrieval action.""" + + def __init__(self, kb: _FakeKnowledgeBase): + self.kb = kb + self.knowledge_bases = {E2E_KB_UUID: kb} + + async def get_knowledge_base_by_uuid(self, kb_uuid: str): + if kb_uuid == E2E_KB_UUID: + return self.kb + return None + + +@pytest.fixture(scope='session') +def local_agent_e2e_tmpdir(): + """Create temporary directory for Local Agent E2E testing.""" + tmpdir = Path(tempfile.mkdtemp(prefix='langbot_local_agent_e2e_')) + yield tmpdir + shutil.rmtree(tmpdir, ignore_errors=True) + + +@pytest.fixture(scope='session') +def local_agent_e2e_port() -> int: + """HTTP port for the real LangBot app used by this E2E.""" + return _free_port() + + +@pytest.fixture(scope='session') +def local_agent_runtime_ports() -> tuple[int, int]: + """Control/debug ports for the standalone plugin runtime.""" + control_port = _free_port() + debug_port = _free_port() + while debug_port == control_port: + debug_port = _free_port() + return control_port, debug_port + + +@pytest.fixture(scope='session') +def local_agent_e2e_config_path(local_agent_e2e_tmpdir, local_agent_e2e_port, local_agent_runtime_ports): + """Create a plugin-enabled config and install the Local Agent plugin fixture.""" + config_path = create_minimal_config(local_agent_e2e_tmpdir, port=local_agent_e2e_port) + create_test_directories(local_agent_e2e_tmpdir) + + import yaml + + with open(config_path, encoding='utf-8') as f: + config = yaml.safe_load(f) + runtime_control_port, _runtime_debug_port = local_agent_runtime_ports + config['api']['global_api_key'] = 'e2e-local-agent-key' + config['plugin']['enable'] = True + config['plugin']['runtime_ws_url'] = f'ws://127.0.0.1:{runtime_control_port}/control/ws' + config['plugin']['enable_marketplace'] = False + config['box']['enabled'] = False + config['system']['jwt']['secret'] = 'e2e-local-agent-secret-key' + with open(config_path, 'w', encoding='utf-8') as f: + yaml.safe_dump(config, f, default_flow_style=False) + + _copy_local_agent_plugin(local_agent_e2e_tmpdir) + return config_path + + +@pytest.fixture(scope='session') +def local_agent_runtime_process(local_agent_e2e_tmpdir, local_agent_runtime_ports, local_agent_e2e_config_path): + """Start the real SDK plugin runtime over WebSocket.""" + del local_agent_e2e_config_path + control_port, debug_port = local_agent_runtime_ports + stdout_path = local_agent_e2e_tmpdir / 'plugin-runtime.stdout.log' + stderr_path = local_agent_e2e_tmpdir / 'plugin-runtime.stderr.log' + stdout_file = open(stdout_path, 'wb') + stderr_file = open(stderr_path, 'wb') + proc = subprocess.Popen( + [ + str(find_project_root() / '.venv' / 'bin' / 'python'), + '-m', + 'langbot_plugin.cli.__init__', + 'rt', + '--ws-control-port', + str(control_port), + '--ws-debug-port', + str(debug_port), + ], + cwd=local_agent_e2e_tmpdir, + stdout=stdout_file, + stderr=stderr_file, + start_new_session=True, + ) + yield proc + proc.terminate() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + stdout_file.close() + stderr_file.close() + + +def _inject_fake_llm_model(ap) -> Any: + """Register a runtime-only fake model that supports count_tokens/invoke.""" + from langbot.pkg.entity.persistence import model as persistence_model + from langbot.pkg.provider.modelmgr import requester, token + from tests.unit_tests.provider.conftest import FakeProviderAPIRequester + + provider_entity = persistence_model.ModelProvider( + uuid=FAKE_PROVIDER_UUID, + name='E2E Fake Provider', + requester='fake-requester', + base_url='https://fake.invalid', + api_keys=['fake-key'], + ) + fake_requester = FakeProviderAPIRequester(ap, {'base_url': provider_entity.base_url}) + runtime_provider = requester.RuntimeProvider( + provider_entity=provider_entity, + token_mgr=token.TokenManager(name=provider_entity.uuid, tokens=provider_entity.api_keys), + requester=fake_requester, + ) + runtime_model = requester.RuntimeLLMModel( + model_entity=persistence_model.LLMModel( + uuid=FAKE_MODEL_UUID, + name=FAKE_MODEL_UUID, + provider_uuid=provider_entity.uuid, + abilities=['func_call'], + context_length=8192, + extra_args={}, + ), + provider=runtime_provider, + ) + ap.model_mgr.provider_dict[provider_entity.uuid] = runtime_provider + ap.model_mgr.llm_models.append(runtime_model) + return fake_requester + + +def _scripted_tool_call( + tool_name: str = E2E_TOOL_NAME, + *, + call_id: str = 'call-e2e-lookup', + query: str = 'alpha', +): + """Build an assistant message requesting a deterministic tool call.""" + from langbot_plugin.api.entities.builtin.provider import message as provider_message + + return provider_message.Message( + role='assistant', + content='', + tool_calls=[ + provider_message.ToolCall( + id=call_id, + type='function', + function=provider_message.FunctionCall( + name=tool_name, + arguments=json.dumps({'query': query}, ensure_ascii=False), + ), + ) + ], + ) + + +async def _boot_local_agent_app(tmpdir: Path): + """Boot LangBot and wait until the Local Agent runner is discoverable.""" + from langbot.pkg.core import boot + + ap = await boot.make_app(asyncio.get_running_loop()) + for _ in range(60): + handler = getattr(ap.plugin_connector, 'handler', None) + if handler is not None: + await handler.ping() + break + await asyncio.sleep(1) + else: + raise AssertionError(f'Plugin runtime did not connect; tmpdir={tmpdir}') + + for _ in range(60): + runners = await ap.agent_runner_registry.list_runners(use_cache=False) + if any(runner.id == LOCAL_AGENT_RUNNER_ID for runner in runners): + break + await asyncio.sleep(1) + else: + raise AssertionError(f'{LOCAL_AGENT_RUNNER_ID} was not discovered') + + return ap + + +def _run_local_agent_probe(tmpdir: Path, probe): + """Run one Local Agent probe inside the temporary LangBot app.""" + from langbot.pkg.utils import platform as platform_utils + + async def _run(): + previous_cwd = Path.cwd() + previous_standalone_runtime = platform_utils.standalone_runtime + os.chdir(tmpdir) + platform_utils.standalone_runtime = True + ap = None + try: + ap = await _boot_local_agent_app(tmpdir) + return await probe(ap) + finally: + if ap is not None: + ap.dispose() + platform_utils.standalone_runtime = previous_standalone_runtime + os.chdir(previous_cwd) + + return asyncio.run(_run()) + + +def test_local_agent_runner_uses_host_fake_provider_and_persists_ledger( + local_agent_e2e_tmpdir, + local_agent_e2e_config_path, + local_agent_runtime_process, +): + """Local Agent should execute through Host APIs with a token-free fake provider.""" + del local_agent_e2e_config_path, local_agent_runtime_process + + async def _run_probe(ap): + fake_requester = _inject_fake_llm_model(ap) + event = _event( + event_id='e2e-local-agent-event-001', + conversation_id='e2e-local-agent-conversation', + text='Say pong through the fake provider.', + ) + messages = [message async for message in ap.agent_run_orchestrator.run(event, _binding())] + return messages, list(fake_requester._count_tokens_payloads) + + messages, token_payloads = _run_local_agent_probe(local_agent_e2e_tmpdir, _run_probe) + + assert len(messages) == 1 + assert messages[0].role == 'assistant' + assert _content_text(messages[0].content) == 'Fake LLM response' + assert token_payloads + flattened_token_payloads = [item for payload in token_payloads for item in payload] + assert any(item.get('role') == 'system' for item in flattened_token_payloads) + assert any('Say pong through the fake provider.' in item.get('content', '') for item in flattened_token_payloads) + + db_path = local_agent_e2e_tmpdir / 'data' / 'langbot.db' + conn = sqlite3.connect(str(db_path)) + try: + run_row = conn.execute( + "SELECT run_id, status, runner_id, status_reason FROM agent_run WHERE event_id = ?", + ('e2e-local-agent-event-001',), + ).fetchone() + assert run_row is not None + run_id, status, runner_id, status_reason = run_row + assert status == 'completed' + assert runner_id == LOCAL_AGENT_RUNNER_ID + assert status_reason == 'stop' + + event_rows = conn.execute( + 'SELECT sequence, type, data_json FROM agent_run_event WHERE run_id = ? ORDER BY sequence', + (run_id,), + ).fetchall() + event_types = [row[1] for row in event_rows] + assert event_types == ['message.completed', 'run.completed'] + assert 'Fake LLM response' in event_rows[0][2] + + transcript_rows = conn.execute( + 'SELECT role, content, run_id, runner_id FROM transcript WHERE conversation_id = ? ORDER BY seq', + ('e2e-local-agent-conversation',), + ).fetchall() + assert [row[0] for row in transcript_rows] == ['user', 'assistant'] + assert transcript_rows[0][1] == 'Say pong through the fake provider.' + assert transcript_rows[1][1] == 'Fake LLM response' + assert transcript_rows[1][2] == run_id + assert transcript_rows[1][3] == LOCAL_AGENT_RUNNER_ID + finally: + conn.close() + + +def test_local_agent_runner_executes_authorized_tool_loop_through_host_action( + local_agent_e2e_tmpdir, + local_agent_e2e_config_path, + local_agent_runtime_process, +): + """Local Agent should execute a model tool call through Host CALL_TOOL and finish.""" + del local_agent_e2e_config_path, local_agent_runtime_process + + async def _run_probe(ap): + fake_requester = _inject_fake_llm_model(ap) + fake_requester.queue_llm_responses( + _scripted_tool_call(), + 'Tool loop final answer after tool-result:alpha', + ) + tool_mgr = _FakeToolManager() + ap.tool_mgr = tool_mgr + + event = _event( + event_id='e2e-local-agent-tool-event-001', + conversation_id='e2e-local-agent-tool-conversation', + text='Use the e2e lookup tool before answering.', + ) + binding = _binding( + binding_id='e2e-local-agent-tool-binding', + allowed_tool_names=[E2E_TOOL_NAME], + runner_config={ + 'max-tool-iterations': 2, + 'tool-execution-mode': 'serial', + }, + ) + messages = [message async for message in ap.agent_run_orchestrator.run(event, binding)] + return messages, tool_mgr.calls, _invoke_payload_texts(fake_requester) + + messages, tool_calls, invoke_payload_texts = _run_local_agent_probe(local_agent_e2e_tmpdir, _run_probe) + + assert len(messages) == 1 + assert _content_text(messages[0].content) == 'Tool loop final answer after tool-result:alpha' + assert tool_calls == [{'name': E2E_TOOL_NAME, 'parameters': {'query': 'alpha'}}] + assert any('tool-result:alpha' in text for text in invoke_payload_texts[-1]) + + db_path = local_agent_e2e_tmpdir / 'data' / 'langbot.db' + conn = sqlite3.connect(str(db_path)) + try: + run_row = conn.execute( + "SELECT run_id, status, status_reason FROM agent_run WHERE event_id = ?", + ('e2e-local-agent-tool-event-001',), + ).fetchone() + assert run_row is not None + run_id, status, status_reason = run_row + assert status == 'completed' + assert status_reason == 'stop' + + event_rows = conn.execute( + 'SELECT type, data_json FROM agent_run_event WHERE run_id = ? ORDER BY sequence', + (run_id,), + ).fetchall() + event_types = [row[0] for row in event_rows] + assert event_types == [ + 'tool.call.started', + 'tool.call.completed', + 'message.completed', + 'run.completed', + ] + assert E2E_TOOL_NAME in event_rows[0][1] + assert 'tool-result:alpha' in event_rows[1][1] + finally: + conn.close() + + +def test_local_agent_runner_retrieves_authorized_rag_context_through_host_action( + local_agent_e2e_tmpdir, + local_agent_e2e_config_path, + local_agent_runtime_process, +): + """Local Agent should retrieve configured KB context through Host RETRIEVE_KNOWLEDGE.""" + del local_agent_e2e_config_path, local_agent_runtime_process + + async def _run_probe(ap): + fake_requester = _inject_fake_llm_model(ap) + fake_requester.queue_llm_responses('RAG final answer with RAG_SENTINEL') + fake_kb = _FakeKnowledgeBase() + ap.rag_mgr = _FakeRagManager(fake_kb) + + event = _event( + event_id='e2e-local-agent-rag-event-001', + conversation_id='e2e-local-agent-rag-conversation', + text='Answer with the retrieved RAG sentinel.', + ) + binding = _binding( + binding_id='e2e-local-agent-rag-binding', + allowed_kb_uuids=[E2E_KB_UUID], + runner_config={ + 'knowledge-bases': [E2E_KB_UUID], + 'retrieval-top-k': 1, + }, + ) + messages = [message async for message in ap.agent_run_orchestrator.run(event, binding)] + return messages, fake_kb.retrieve_calls, _invoke_payload_texts(fake_requester) + + messages, retrieve_calls, invoke_payload_texts = _run_local_agent_probe(local_agent_e2e_tmpdir, _run_probe) + + assert len(messages) == 1 + assert _content_text(messages[0].content) == 'RAG final answer with RAG_SENTINEL' + assert retrieve_calls == [ + { + 'query_text': 'Answer with the retrieved RAG sentinel.', + 'settings': {'top_k': 1, 'filters': {}}, + } + ] + assert any( + 'RAG_SENTINEL Local Agent retrieved this deterministic chunk.' in text + for payload_texts in invoke_payload_texts + for text in payload_texts + ) + + db_path = local_agent_e2e_tmpdir / 'data' / 'langbot.db' + conn = sqlite3.connect(str(db_path)) + try: + run_row = conn.execute( + "SELECT run_id, status FROM agent_run WHERE event_id = ?", + ('e2e-local-agent-rag-event-001',), + ).fetchone() + assert run_row is not None + run_id, status = run_row + assert status == 'completed' + event_types = [ + row[0] + for row in conn.execute( + 'SELECT type FROM agent_run_event WHERE run_id = ? ORDER BY sequence', + (run_id,), + ) + ] + assert event_types == ['message.completed', 'run.completed'] + finally: + conn.close() + + +def test_local_agent_runner_compacts_history_and_persists_checkpoint( + local_agent_e2e_tmpdir, + local_agent_e2e_config_path, + local_agent_runtime_process, +): + """Local Agent should compact old Host history and write conversation checkpoint state.""" + del local_agent_e2e_config_path, local_agent_runtime_process + + async def _run_probe(ap): + from langbot.pkg.agent.runner.transcript_store import TranscriptStore + + fake_requester = _inject_fake_llm_model(ap) + fake_requester.queue_llm_responses( + 'SUMMARY_SENTINEL compacted older history including HIST_SENTINEL', + 'Compaction final answer', + ) + + store = TranscriptStore(ap.persistence_mgr.get_db_engine()) + for index in range(12): + await store.append_transcript( + transcript_id=None, + event_id=f'e2e-local-agent-history-{index}', + conversation_id='e2e-local-agent-compaction-conversation', + role='user' if index % 2 == 0 else 'assistant', + content=( + f'HIST_SENTINEL-{index} ' + 'This is intentionally long deterministic history for compaction. ' * 10 + ), + thread_id='e2e-local-agent-thread', + item_type='message', + ) + + event = _event( + event_id='e2e-local-agent-compaction-event-001', + conversation_id='e2e-local-agent-compaction-conversation', + text='Use compacted context and answer.', + ) + binding = _binding( + binding_id='e2e-local-agent-compaction-binding', + runner_config={ + 'context-window-tokens': 900, + 'context-reserve-tokens': 300, + 'context-keep-recent-tokens': 160, + 'context-summary-tokens': 240, + 'context-history-fetch-limit': 20, + }, + ) + messages = [message async for message in ap.agent_run_orchestrator.run(event, binding)] + return messages, _invoke_payload_texts(fake_requester), fake_requester._invoke_count + + messages, invoke_payload_texts, invoke_count = _run_local_agent_probe(local_agent_e2e_tmpdir, _run_probe) + + assert len(messages) == 1 + assert _content_text(messages[0].content) == 'Compaction final answer' + assert invoke_count == 2 + assert any('HIST_SENTINEL' in text for text in invoke_payload_texts[0]) + assert any('SUMMARY_SENTINEL compacted older history' in text for text in invoke_payload_texts[-1]) + + db_path = local_agent_e2e_tmpdir / 'data' / 'langbot.db' + conn = sqlite3.connect(str(db_path)) + try: + run_row = conn.execute( + "SELECT run_id, status FROM agent_run WHERE event_id = ?", + ('e2e-local-agent-compaction-event-001',), + ).fetchone() + assert run_row is not None + run_id, status = run_row + assert status == 'completed' + + event_types = [ + row[0] + for row in conn.execute( + 'SELECT type FROM agent_run_event WHERE run_id = ? ORDER BY sequence', + (run_id,), + ) + ] + assert event_types == ['message.completed', 'run.completed'] + + state_row = conn.execute( + "SELECT value_json FROM agent_runner_state WHERE state_key = 'runner.compaction.checkpoint'" + ).fetchone() + assert state_row is not None + checkpoint = json.loads(state_row[0]) + assert checkpoint['schema_version'] == 'langbot.local_agent.compaction_checkpoint.v1' + assert 'SUMMARY_SENTINEL compacted older history' in checkpoint['summary'] + assert checkpoint['conversation_id'] == 'e2e-local-agent-compaction-conversation' + assert checkpoint['covers_until'] + assert checkpoint['tokens_before'] > 600 + finally: + conn.close() + + +def test_local_agent_runner_combines_rag_compaction_and_multi_turn_tool_loop( + local_agent_e2e_tmpdir, + local_agent_e2e_config_path, + local_agent_runtime_process, +): + """Local Agent should preserve RAG, compressed history, and multi-turn tool results together.""" + del local_agent_e2e_config_path, local_agent_runtime_process + + async def _run_probe(ap): + from langbot.pkg.agent.runner.transcript_store import TranscriptStore + + fake_requester = _inject_fake_llm_model(ap) + + async def scripted_response(**kwargs): + messages = kwargs['messages'] + text = '\n'.join(_message_text(message) for message in messages) + if 'context summarization assistant' in text or '' in text: + return 'SUMMARY_COMBO compacted older history including HIST_COMBO_SENTINEL and RAG_TOOL_COMBO_GOAL' + if 'tool-result:alpha' not in text: + return _scripted_tool_call(call_id='call-combo-alpha', query='alpha') + if 'tool-result:beta' not in text: + return _scripted_tool_call(call_id='call-combo-beta', query='beta') + assert 'RAG_SENTINEL Local Agent retrieved this deterministic chunk.' in text + assert 'SUMMARY_COMBO compacted older history' in text + assert 'tool-result:alpha' in text + assert 'tool-result:beta' in text + assert 'current combo request must survive' in text + return 'COMBO_FINAL RAG_SENTINEL HIST_COMBO_SENTINEL tool-result:alpha tool-result:beta' + + fake_requester.queue_llm_responses(*(scripted_response for _ in range(20))) + tool_mgr = _FakeToolManager() + fake_kb = _FakeKnowledgeBase() + ap.tool_mgr = tool_mgr + ap.rag_mgr = _FakeRagManager(fake_kb) + + store = TranscriptStore(ap.persistence_mgr.get_db_engine()) + for index in range(16): + await store.append_transcript( + transcript_id=None, + event_id=f'e2e-local-agent-combo-history-{index}', + conversation_id='e2e-local-agent-combo-conversation', + role='user' if index % 2 == 0 else 'assistant', + content=( + f'HIST_COMBO_SENTINEL-{index} RAG_TOOL_COMBO_GOAL ' + 'This old message intentionally creates pressure for combo compaction. ' * 8 + ), + thread_id='e2e-local-agent-thread', + item_type='message', + ) + + event = _event( + event_id='e2e-local-agent-combo-event-001', + conversation_id='e2e-local-agent-combo-conversation', + text='current combo request must survive; use RAG and tools before answering.', + ) + binding = _binding( + binding_id='e2e-local-agent-combo-binding', + allowed_tool_names=[E2E_TOOL_NAME], + allowed_kb_uuids=[E2E_KB_UUID], + runner_config={ + 'knowledge-bases': [E2E_KB_UUID], + 'retrieval-top-k': 1, + 'max-tool-iterations': 4, + 'tool-execution-mode': 'serial', + 'context-window-tokens': 950, + 'context-reserve-tokens': 300, + 'context-keep-recent-tokens': 140, + 'context-summary-tokens': 260, + 'context-history-fetch-limit': 25, + }, + ) + messages = [message async for message in ap.agent_run_orchestrator.run(event, binding)] + return ( + messages, + tool_mgr.calls, + fake_kb.retrieve_calls, + _invoke_payload_texts(fake_requester), + fake_requester._invoke_count, + ) + + messages, tool_calls, retrieve_calls, invoke_payload_texts, invoke_count = _run_local_agent_probe( + local_agent_e2e_tmpdir, + _run_probe, + ) + + assert len(messages) == 1 + assert _content_text(messages[0].content) == ( + 'COMBO_FINAL RAG_SENTINEL HIST_COMBO_SENTINEL tool-result:alpha tool-result:beta' + ) + assert tool_calls == [ + {'name': E2E_TOOL_NAME, 'parameters': {'query': 'alpha'}}, + {'name': E2E_TOOL_NAME, 'parameters': {'query': 'beta'}}, + ] + assert retrieve_calls == [ + { + 'query_text': 'current combo request must survive; use RAG and tools before answering.', + 'settings': {'top_k': 1, 'filters': {}}, + } + ] + assert invoke_count >= 4 + assert any( + 'RAG_SENTINEL Local Agent retrieved this deterministic chunk.' in text + for payload_texts in invoke_payload_texts + for text in payload_texts + ) + assert any('SUMMARY_COMBO compacted older history' in text for text in invoke_payload_texts[-1]) + assert any('tool-result:alpha' in text for text in invoke_payload_texts[-1]) + assert any('tool-result:beta' in text for text in invoke_payload_texts[-1]) + assert any('current combo request must survive' in text for text in invoke_payload_texts[-1]) + + db_path = local_agent_e2e_tmpdir / 'data' / 'langbot.db' + conn = sqlite3.connect(str(db_path)) + try: + run_row = conn.execute( + "SELECT run_id, status, status_reason FROM agent_run WHERE event_id = ?", + ('e2e-local-agent-combo-event-001',), + ).fetchone() + assert run_row is not None + run_id, status, status_reason = run_row + assert status == 'completed' + assert status_reason == 'stop' + + event_rows = conn.execute( + 'SELECT type, data_json FROM agent_run_event WHERE run_id = ? ORDER BY sequence', + (run_id,), + ).fetchall() + event_types = [row[0] for row in event_rows] + assert event_types == [ + 'tool.call.started', + 'tool.call.completed', + 'tool.call.started', + 'tool.call.completed', + 'message.completed', + 'run.completed', + ] + assert 'tool-result:alpha' in event_rows[1][1] + assert 'tool-result:beta' in event_rows[3][1] + assert 'COMBO_FINAL' in event_rows[4][1] + + state_rows = conn.execute( + "SELECT value_json FROM agent_runner_state WHERE state_key = 'runner.compaction.checkpoint'" + ).fetchall() + checkpoints = [json.loads(row[0]) for row in state_rows] + checkpoint = next( + ( + item + for item in checkpoints + if item.get('conversation_id') == 'e2e-local-agent-combo-conversation' + ), + None, + ) + assert checkpoint is not None + assert checkpoint['conversation_id'] == 'e2e-local-agent-combo-conversation' + assert 'SUMMARY_COMBO compacted older history' in checkpoint['summary'] + finally: + conn.close() diff --git a/tests/integration/api/test_pipelines.py b/tests/integration/api/test_pipelines.py index 50ac37bc5..21441596f 100644 --- a/tests/integration/api/test_pipelines.py +++ b/tests/integration/api/test_pipelines.py @@ -278,3 +278,17 @@ class TestPipelineExtensionsEndpoint: assert response.status_code == 200 data = await response.get_json() assert data['code'] == 0 + + @pytest.mark.asyncio + async def test_get_extensions_tolerates_skill_list_failure(self, quart_test_client, fake_pipeline_app): + """Pipeline plugin/MCP binding UI should not fail when Box skill listing is unavailable.""" + fake_pipeline_app.skill_service.list_skills = AsyncMock(side_effect=TimeoutError('box_list_skills timeout')) + + response = await quart_test_client.get( + '/api/v1/pipelines/test-pipeline-uuid/extensions', headers={'Authorization': 'Bearer test_token'} + ) + + assert response.status_code == 200 + data = await response.get_json() + assert data['code'] == 0 + assert data['data']['available_skills'] == [] diff --git a/tests/unit_tests/provider/conftest.py b/tests/unit_tests/provider/conftest.py index db76aefe5..2332915ea 100644 --- a/tests/unit_tests/provider/conftest.py +++ b/tests/unit_tests/provider/conftest.py @@ -8,6 +8,8 @@ without calling real LLM APIs or network requests. from __future__ import annotations import json +import inspect +from typing import Any import pytest from unittest.mock import AsyncMock, Mock @@ -32,7 +34,11 @@ class FakeProviderAPIRequester(requester.ProviderAPIRequester): self._invoke_count = 0 self._last_messages = None self._last_model = None + self._last_funcs = None + self._invoke_payloads = [] self._last_count_tokens_payload = None + self._count_tokens_payloads = [] + self._scripted_llm_responses = [] @staticmethod def _content_to_text(content) -> str: @@ -52,6 +58,44 @@ class FakeProviderAPIRequester(requester.ProviderAPIRequester): return ''.join(parts) return str(content) + def queue_llm_responses(self, *responses: Any) -> None: + """Queue deterministic LLM responses for multi-turn tests.""" + self._scripted_llm_responses.extend(responses) + + async def _coerce_llm_response( + self, + response: Any, + *, + query: Any, + model: requester.RuntimeLLMModel, + messages: list, + funcs: list | None, + extra_args: dict, + remove_think: bool, + ): + """Convert scripted response values into provider Message objects.""" + import langbot_plugin.api.entities.builtin.provider.message as provider_message + + if callable(response): + response = response( + query=query, + model=model, + messages=messages, + funcs=funcs, + extra_args=extra_args, + remove_think=remove_think, + ) + if inspect.isawaitable(response): + response = await response + + if isinstance(response, provider_message.Message): + return response + if isinstance(response, dict): + return provider_message.Message.model_validate(response) + if isinstance(response, str): + return provider_message.Message(role='assistant', content=response) + return response + async def invoke_llm( self, query, @@ -65,10 +109,30 @@ class FakeProviderAPIRequester(requester.ProviderAPIRequester): self._invoke_count += 1 self._last_messages = messages self._last_model = model + self._last_funcs = funcs or [] + self._invoke_payloads.append( + { + 'messages': messages, + 'funcs': funcs or [], + 'extra_args': dict(extra_args or {}), + 'remove_think': remove_think, + } + ) # Import the message entity for response import langbot_plugin.api.entities.builtin.provider.message as provider_message + if self._scripted_llm_responses: + return await self._coerce_llm_response( + self._scripted_llm_responses.pop(0), + query=query, + model=model, + messages=messages, + funcs=funcs, + extra_args=extra_args, + remove_think=remove_think, + ) + return provider_message.Message( role='assistant', content=[provider_message.ContentElement(type='text', text='Fake LLM response')], @@ -106,6 +170,7 @@ class FakeProviderAPIRequester(requester.ProviderAPIRequester): 'role': getattr(message, 'role', ''), 'content': self._content_to_text(getattr(message, 'content', None)), 'tool_calls': getattr(message, 'tool_calls', None), + 'tool_call_id': getattr(message, 'tool_call_id', None), } ) @@ -119,6 +184,7 @@ class FakeProviderAPIRequester(requester.ProviderAPIRequester): ) self._last_count_tokens_payload = payload + self._count_tokens_payloads.append(payload) text = json.dumps(payload, ensure_ascii=False, sort_keys=True, default=str) return max(1, (len(text) + 3) // 4) diff --git a/tests/unit_tests/test_skill_service.py b/tests/unit_tests/test_skill_service.py index 6fd7d64f2..800ff8906 100644 --- a/tests/unit_tests/test_skill_service.py +++ b/tests/unit_tests/test_skill_service.py @@ -2,6 +2,7 @@ from types import SimpleNamespace from unittest.mock import AsyncMock import pytest +from langbot_plugin.entities.io.errors import ActionCallTimeoutError from langbot.pkg.api.http.service.skill import SkillService @@ -82,6 +83,17 @@ class TestRequireBoxForWrite: service = SkillService(self._ap_with_disabled_box()) assert await service.list_skills() == [] + @pytest.mark.asyncio + async def test_list_skills_returns_empty_when_box_action_times_out(self): + """A transient Box list failure should not break unrelated extension surfaces.""" + box_service = SimpleNamespace( + available=True, + list_skills=AsyncMock(side_effect=ActionCallTimeoutError('Action box_list_skills call timed out')), + ) + service = SkillService(SimpleNamespace(skill_mgr=SimpleNamespace(reload_skills=AsyncMock()), box_service=box_service)) + + assert await service.list_skills() == [] + @pytest.mark.asyncio async def test_read_skill_file_refused_when_box_unavailable(self): service = SkillService(self._ap_with_disabled_box())