xinyin025/LangBot
1
0
Fork 0
mirror of https://github.com/langbot-app/LangBot.git synced 2026-07-16 17:36:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(agent-runner): strengthen local agent e2e gate

Browse Source
This commit is contained in:
huanghuoguoguo
2026-07-01 20:20:43 +08:00
parent d0f8f080e9
commit 995888f6b2
44 changed files with 5678 additions and 1040 deletions
Download Patch File Download Diff File Expand all files Collapse all files
skills/scripts/e2e/ensure-langrag-sentinel-kb.mjs
+1 -1
View File
@@ -21,7 +21,7 @@ await ensureEvidence(paths);
const backendUrl = env.LANGBOT_BACKEND_URL || "";
const user = env.LANGBOT_E2E_LOGIN_USER || "";
const password = env.LANGBOT_E2E_LOGIN_PASSWORD || "LangBotE2ELocalPass!2026";
const expectedText = env.LANGBOT_E2E_EXPECTED_TEXT || "azalea-cobalt-7421";
const expectedText = env.LANGBOT_E2E_RAG_EXPECTED_TEXT || "azalea-cobalt-7421";
const query = env.LANGBOT_E2E_RETRIEVE_QUERY || "What is the local agent runner retrieval sentinel?";
const writeEnv = process.argv.includes("--write-env");
const checkOnly = process.argv.includes("--check-only");
skills/scripts/e2e/ensure-local-agent-pipeline.mjs
+423 -95
View File
@@ -18,12 +18,14 @@ import {
writeResult,
} from "./lib/langbot-e2e.mjs";
const RUNNER_ID = "local-agent";
const RUNNER_ID = "plugin:langbot/local-agent/default";
const SPACE_PROVIDER_UUID = "00000000-0000-0000-0000-000000000000";
const DEFAULT_PIPELINE_NAME = "Agent QA Local Agent Debug Chat";
const DEFAULT_LOCAL_PASSWORD = "LangBotE2ELocalPass!2026";
const DEFAULT_MODEL_TEST_LIMIT = 8;
const DEFAULT_MODEL_FALLBACK_COUNT = 3;
const DEFAULT_FAKE_MODEL_UUID = "langbot-e2e-fake-local-agent-model";
const DEFAULT_FAKE_PROVIDER_NAME = "LangBot E2E Fake OpenAI Provider";
const caseId = "ensure-local-agent-pipeline";
await loadEnvFiles();
@@ -31,7 +33,10 @@ const paths = evidencePaths(caseId);
await ensureEvidence(paths);
const writeEnv = process.argv.includes("--write-env");
const pipelineName = env.LANGBOT_E2E_CREATE_PIPELINE_NAME || env.LANGBOT_LOCAL_AGENT_PIPELINE_NAME || DEFAULT_PIPELINE_NAME;
const pipelineName =
env.LANGBOT_E2E_CREATE_PIPELINE_NAME ||
env.LANGBOT_LOCAL_AGENT_PIPELINE_NAME ||
DEFAULT_PIPELINE_NAME;
const frontendUrl = env.LANGBOT_FRONTEND_URL || "";
const backendUrl = env.LANGBOT_BACKEND_URL || "";
const envLocalPath = resolve("skills/.env.local");
@@ -51,6 +56,7 @@ const result = {
selected_model_id: "",
selected_model_name: "",
fallback_model_ids: [],
fake_provider: null,
model_count: 0,
space_model_count: 0,
scanned_space_model_count: 0,
@@ -83,7 +89,9 @@ try {
const password = env.LANGBOT_E2E_LOGIN_PASSWORD || DEFAULT_LOCAL_PASSWORD;
if (!user) {
result.status = "env_issue";
throw new Error("LANGBOT_E2E_LOGIN_USER is required so this setup can create/update the pipeline via backend API.");
throw new Error(
"LANGBOT_E2E_LOGIN_USER is required so this setup can create/update the pipeline via backend API.",
);
}
const auth = await resetAndAuthLocalUser({ backendUrl, user, password });
@@ -97,7 +105,9 @@ try {
result.wizard = wizard;
if (wizard.status !== "pass") {
result.status = "fail";
throw new Error(wizard.reason || "Failed to mark the local QA wizard as skipped.");
throw new Error(
wizard.reason || "Failed to mark the local QA wizard as skipped.",
);
}
const prepared = await ensureLocalAgentPipeline({
@@ -118,10 +128,12 @@ try {
LANGBOT_PIPELINE_NAME: result.pipeline_name || pipelineName,
LANGBOT_LOCAL_AGENT_PIPELINE_URL: result.pipeline_url,
LANGBOT_LOCAL_AGENT_PIPELINE_NAME: result.pipeline_name || pipelineName,
...(result.selected_model_id ? {
LANGBOT_LOCAL_AGENT_MODEL_UUID: result.selected_model_id,
LANGBOT_E2E_MODEL_UUID: result.selected_model_id,
} : {}),
...(result.selected_model_id
? {
LANGBOT_LOCAL_AGENT_MODEL_UUID: result.selected_model_id,
LANGBOT_E2E_MODEL_UUID: result.selected_model_id,
}
: {}),
});
result.wrote_env = true;
}
@@ -132,12 +144,21 @@ try {
const browserCheck = await verifyBrowserToken(page, backendUrl);
result.browser_token_check = browserCheck;
if (!browserCheck.authenticated) {
throw new Error(browserCheck.reason || "Browser token check failed after setup.");
throw new Error(
browserCheck.reason || "Browser token check failed after setup.",
);
}
await page.goto(result.pipeline_url || frontendUrl, { waitUntil: "domcontentloaded" });
await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {});
await page.goto(result.pipeline_url || frontendUrl, {
waitUntil: "domcontentloaded",
});
await page
.waitForLoadState("networkidle", { timeout: 10_000 })
.catch(() => {});
const text = await bodyText(page);
result.page_signal = ["Pipelines", "流水线", pipelineName].find((signal) => text.includes(signal)) || "";
result.page_signal =
["Pipelines", "流水线", pipelineName].find((signal) =>
text.includes(signal),
) || "";
} catch (error) {
result.status = result.status === "env_issue" ? "env_issue" : "fail";
result.reason = result.reason || error.message;
@@ -148,24 +169,51 @@ try {
console.log(JSON.stringify(result, null, 2));
}
process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1);
process.exit(
result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1,
);
async function skipWizard({ backendUrl, token }) {
const response = await apiJson(backendUrl, "/api/v1/system/wizard/completed", {
method: "POST",
token,
body: { status: "skipped" },
});
const response = await apiJson(
backendUrl,
"/api/v1/system/wizard/completed",
{
method: "POST",
token,
body: { status: "skipped" },
},
);
const ok = response.status < 400 && response.json.code === 0;
return {
status: ok ? "pass" : "fail",
http_status: response.status,
code: response.json.code ?? null,
reason: ok ? "Wizard marked skipped for local QA." : response.json.msg || "Wizard status update failed.",
reason: ok
? "Wizard marked skipped for local QA."
: response.json.msg || "Wizard status update failed.",
};
}
async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runnerId }) {
async function ensureLocalAgentPipeline({
backendUrl,
token,
pipelineName,
runnerId,
}) {
const fakeProviderBaseUrl =
env.LANGBOT_E2E_FAKE_PROVIDER_BASE_URL ||
env.LANGBOT_FAKE_PROVIDER_BASE_URL ||
"";
let fakeModel = null;
if (fakeProviderBaseUrl) {
fakeModel = await ensureFakeProviderModel({
backendUrl,
token,
baseUrl: fakeProviderBaseUrl,
});
if (fakeModel.status !== "pass") return fakeModel;
}
const [pipelineList, modelList] = await Promise.all([
apiJson(backendUrl, "/api/v1/pipelines", { token }),
apiJson(backendUrl, "/api/v1/provider/models/llm", { token }),
@@ -199,7 +247,9 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne
.map((item) => item.trim())
.filter(Boolean),
);
const spaceModels = models.filter((model) => isSpaceModel(model) && !skippedModelIds.has(model.uuid));
const spaceModels = models.filter(
(model) => isSpaceModel(model) && !skippedModelIds.has(model.uuid),
);
const pipelines = pipelineList.json.data?.pipelines || [];
let pipeline = pipelines.find((item) => item.name === pipelineName) || null;
let created = false;
@@ -210,7 +260,8 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne
token,
body: {
name: pipelineName,
description: "Local QA pipeline for AgentRunner Debug Chat smoke tests.",
description:
"Local QA pipeline for AgentRunner Debug Chat smoke tests.",
emoji: "QA",
},
});
@@ -224,7 +275,11 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne
};
}
const pipelineId = createdResponse.json.data?.uuid || "";
const loaded = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipelineId)}`, { token });
const loaded = await apiJson(
backendUrl,
`/api/v1/pipelines/${encodeURIComponent(pipelineId)}`,
{ token },
);
pipeline = loaded.json.data?.pipeline || null;
created = true;
}
@@ -238,7 +293,11 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne
};
}
const loaded = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, { token });
const loaded = await apiJson(
backendUrl,
`/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`,
{ token },
);
if (isApiFailure(loaded) || !loaded.json.data?.pipeline) {
return {
status: "fail",
@@ -251,32 +310,53 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne
}
pipeline = loaded.json.data.pipeline;
const config = pipeline.config && typeof pipeline.config === "object" ? pipeline.config : {};
const config =
pipeline.config && typeof pipeline.config === "object"
? pipeline.config
: {};
const ai = config.ai && typeof config.ai === "object" ? config.ai : {};
const rawExistingLocalAgentConfig = ai["local-agent"] && typeof ai["local-agent"] === "object"
? ai["local-agent"]
: {};
const runnerConfigs =
ai.runner_config && typeof ai.runner_config === "object"
? ai.runner_config
: {};
const rawExistingLocalAgentConfig =
runnerConfigs[runnerId] && typeof runnerConfigs[runnerId] === "object"
? runnerConfigs[runnerId]
: {};
const existingLocalAgentConfig = rawExistingLocalAgentConfig;
const existingModel = existingLocalAgentConfig.model && typeof existingLocalAgentConfig.model === "object"
? existingLocalAgentConfig.model
: {};
const requestedModelId = env.LANGBOT_LOCAL_AGENT_MODEL_UUID || env.LANGBOT_E2E_MODEL_UUID || "";
const selected = await selectWorkingSpaceModel({
backendUrl,
token,
models,
skippedModelIds,
skippedModelNames,
requestedModelId,
existingModelId: existingModel.primary || "",
});
const existingModel =
existingLocalAgentConfig.model &&
typeof existingLocalAgentConfig.model === "object"
? existingLocalAgentConfig.model
: {};
const requestedModelId =
env.LANGBOT_LOCAL_AGENT_MODEL_UUID || env.LANGBOT_E2E_MODEL_UUID || "";
const selected = fakeModel
? {
status: "pass",
reason: "",
selected_model_id: fakeModel.model_uuid,
selected_model_name: fakeModel.model_name,
fallback_model_ids: [],
scanned_space_model_count: 0,
tested_model_count: 0,
model_tests: [],
}
: await selectWorkingSpaceModel({
backendUrl,
token,
models,
skippedModelIds,
skippedModelNames,
requestedModelId,
existingModelId: existingModel.primary || "",
});
const selectedModelId = selected.selected_model_id || "";
const localAgentConfig = {
timeout: 300,
prompt: [{ role: "system", content: "You are a helpful assistant." }],
"remove-think": false,
"knowledge-bases": [],
"box-session-id-template": "{launcher_type}_{launcher_id}",
"retrieval-top-k": 5,
"rerank-model": "",
"rerank-top-k": 5,
@@ -288,7 +368,6 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne
"context-reserve-tokens": 16384,
"context-keep-recent-tokens": 20000,
"context-summary-tokens": 8000,
...existingLocalAgentConfig,
// Current backend truncation still reads this field directly.
"max-round": positiveInteger(existingLocalAgentConfig["max-round"], 10),
model: {
@@ -303,23 +382,30 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne
runner: {
...(ai.runner && typeof ai.runner === "object" ? ai.runner : {}),
id: runnerId,
runner: runnerId,
"expire-time": 0,
},
"local-agent": localAgentConfig,
runner_config: {
...runnerConfigs,
[runnerId]: localAgentConfig,
},
},
};
const updateResponse = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, {
method: "PUT",
token,
body: {
name: pipelineName,
description: "Local QA pipeline for AgentRunner Debug Chat smoke tests.",
emoji: "QA",
config: updatedConfig,
const updateResponse = await apiJson(
backendUrl,
`/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`,
{
method: "PUT",
token,
body: {
name: pipelineName,
description:
"Local QA pipeline for AgentRunner Debug Chat smoke tests.",
emoji: "QA",
config: updatedConfig,
},
},
});
);
if (isApiFailure(updateResponse)) {
return {
status: "fail",
@@ -341,7 +427,8 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne
status: selectedModelId ? "pass" : "env_issue",
reason: selectedModelId
? `Local-agent pipeline is configured for Debug Chat with Space model ${selected.selected_model_name || selectedModelId} and ${selected.fallback_model_ids.length} fallback(s).`
: selected.reason || "No working Space LLM model is configured in this LangBot instance.",
: selected.reason ||
"No working Space LLM model is configured in this LangBot instance.",
pipeline_id: pipeline.uuid,
pipeline_name: pipelineName,
model_count: models.length,
@@ -352,21 +439,214 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne
selected_model_id: selectedModelId,
selected_model_name: selected.selected_model_name,
fallback_model_ids: selected.fallback_model_ids,
fake_provider: fakeModel,
created,
updated: true,
};
}
async function ensureFakeProviderModel({ backendUrl, token, baseUrl }) {
const modelUuid = env.LANGBOT_E2E_FAKE_MODEL_UUID || DEFAULT_FAKE_MODEL_UUID;
const modelName =
env.LANGBOT_E2E_FAKE_MODEL_NAME ||
env.LANGBOT_FAKE_PROVIDER_MODEL ||
env.LANGBOT_FAKE_PROVIDER_MODEL_NAME ||
"langbot-e2e-fake-model";
const providerName =
env.LANGBOT_E2E_FAKE_PROVIDER_NAME || DEFAULT_FAKE_PROVIDER_NAME;
const providerRequester =
env.LANGBOT_E2E_FAKE_PROVIDER_REQUESTER || "openai-chat-completions";
const apiKey =
env.LANGBOT_E2E_FAKE_PROVIDER_API_KEY ||
env.LANGBOT_FAKE_PROVIDER_API_KEY ||
"fake-key";
const providersResponse = await apiJson(
backendUrl,
"/api/v1/provider/providers",
{ token },
);
if (isApiFailure(providersResponse)) {
return {
status: "fail",
reason:
providersResponse.json.msg ||
"Failed to list providers before creating fake provider.",
provider_status: providersResponse.status,
};
}
const normalizedBaseUrl = String(baseUrl || "").replace(/\/$/, "");
const providers = providersResponse.json.data?.providers || [];
let provider = providers.find(
(item) =>
item.name === providerName ||
(item.requester === providerRequester &&
String(item.base_url || "").replace(/\/$/, "") === normalizedBaseUrl),
);
const providerBody = {
name: providerName,
requester: providerRequester,
base_url: normalizedBaseUrl,
api_keys: [apiKey],
};
if (provider?.uuid) {
const updateResponse = await apiJson(
backendUrl,
`/api/v1/provider/providers/${encodeURIComponent(provider.uuid)}`,
{
method: "PUT",
token,
body: providerBody,
},
);
if (isApiFailure(updateResponse)) {
return {
status: "fail",
reason: updateResponse.json.msg || "Failed to update fake provider.",
provider_status: updateResponse.status,
};
}
} else {
const createProviderResponse = await apiJson(
backendUrl,
"/api/v1/provider/providers",
{
method: "POST",
token,
body: providerBody,
},
);
if (isApiFailure(createProviderResponse)) {
return {
status: "fail",
reason:
createProviderResponse.json.msg || "Failed to create fake provider.",
provider_status: createProviderResponse.status,
};
}
provider = { uuid: createProviderResponse.json.data?.uuid || "" };
}
if (!provider?.uuid) {
return {
status: "fail",
reason: "Fake provider did not return a provider uuid.",
};
}
let resolvedModelUuid = modelUuid;
let modelResponse = await apiJson(
backendUrl,
`/api/v1/provider/models/llm/${encodeURIComponent(modelUuid)}`,
{
token,
},
);
if (modelResponse.status === 404) {
const providerModelsResponse = await apiJson(
backendUrl,
`/api/v1/provider/models/llm?provider_uuid=${encodeURIComponent(provider.uuid)}`,
{ token },
);
if (!isApiFailure(providerModelsResponse)) {
const existingModel = (
providerModelsResponse.json.data?.models || []
).find((item) => item.name === modelName);
if (existingModel?.uuid) {
resolvedModelUuid = existingModel.uuid;
modelResponse = await apiJson(
backendUrl,
`/api/v1/provider/models/llm/${encodeURIComponent(resolvedModelUuid)}`,
{
token,
},
);
}
}
}
const modelBody = {
name: modelName,
provider_uuid: provider.uuid,
abilities: ["func_call", "vision"],
context_length: 8192,
extra_args: {},
prefered_ranking: 0,
};
if (modelResponse.status === 404) {
const createModelResponse = await apiJson(
backendUrl,
"/api/v1/provider/models/llm",
{
method: "POST",
token,
body: {
uuid: modelUuid,
...modelBody,
},
},
);
if (isApiFailure(createModelResponse)) {
return {
status: "fail",
reason: createModelResponse.json.msg || "Failed to create fake model.",
model_status: createModelResponse.status,
};
}
resolvedModelUuid = createModelResponse.json.data?.uuid || modelUuid;
} else if (isApiFailure(modelResponse)) {
return {
status: "fail",
reason: modelResponse.json.msg || "Failed to load fake model.",
model_status: modelResponse.status,
};
} else {
const updateModelResponse = await apiJson(
backendUrl,
`/api/v1/provider/models/llm/${encodeURIComponent(resolvedModelUuid)}`,
{
method: "PUT",
token,
body: modelBody,
},
);
if (isApiFailure(updateModelResponse)) {
return {
status: "fail",
reason: updateModelResponse.json.msg || "Failed to update fake model.",
model_status: updateModelResponse.status,
};
}
}
return {
status: "pass",
provider_uuid: provider.uuid,
provider_requester: providerRequester,
base_url: normalizedBaseUrl,
model_uuid: resolvedModelUuid,
model_name: modelName,
};
}
function isApiFailure(response) {
return response.status >= 400 || (response.json.code !== undefined && response.json.code !== 0);
return (
response.status >= 400 ||
(response.json.code !== undefined && response.json.code !== 0)
);
}
function isSpaceModel(model) {
const provider = model?.provider && typeof model.provider === "object" ? model.provider : {};
return model?.provider_uuid === SPACE_PROVIDER_UUID
|| provider.uuid === SPACE_PROVIDER_UUID
|| provider.requester === "space-chat-completions"
|| provider.name === "LangBot Models";
const provider =
model?.provider && typeof model.provider === "object" ? model.provider : {};
return (
model?.provider_uuid === SPACE_PROVIDER_UUID ||
provider.uuid === SPACE_PROVIDER_UUID ||
provider.requester === "space-chat-completions" ||
provider.name === "LangBot Models"
);
}
async function selectWorkingSpaceModel({
@@ -379,15 +659,24 @@ async function selectWorkingSpaceModel({
existingModelId,
}) {
const modelTests = [];
const testLimit = positiveInteger(env.LANGBOT_E2E_MODEL_TEST_LIMIT, DEFAULT_MODEL_TEST_LIMIT);
const fallbackCount = positiveInteger(env.LANGBOT_E2E_MODEL_FALLBACK_COUNT, DEFAULT_MODEL_FALLBACK_COUNT);
const testLimit = positiveInteger(
env.LANGBOT_E2E_MODEL_TEST_LIMIT,
DEFAULT_MODEL_TEST_LIMIT,
);
const fallbackCount = positiveInteger(
env.LANGBOT_E2E_MODEL_FALLBACK_COUNT,
DEFAULT_MODEL_FALLBACK_COUNT,
);
const workingModels = [];
const spaceModels = rankModels(models.filter((model) => (
model.uuid
&& isSpaceModel(model)
&& !skippedModelIds.has(model.uuid)
&& !skippedModelNames.has(model.name)
)));
const spaceModels = rankModels(
models.filter(
(model) =>
model.uuid &&
isSpaceModel(model) &&
!skippedModelIds.has(model.uuid) &&
!skippedModelNames.has(model.name),
),
);
const requestedModel = requestedModelId
? spaceModels.find((model) => model.uuid === requestedModelId) || null
: null;
@@ -396,7 +685,9 @@ async function selectWorkingSpaceModel({
: null;
const candidates = uniqueCandidates([
...(requestedModel ? [existingCandidate(requestedModel, "requested")] : []),
...(existingModel ? [existingCandidate(existingModel, "existing-pipeline")] : []),
...(existingModel
? [existingCandidate(existingModel, "existing-pipeline")]
: []),
...spaceModels.map((model) => existingCandidate(model, "configured-space")),
]);
@@ -405,9 +696,16 @@ async function selectWorkingSpaceModel({
scanResult = await scanSpaceModels({ backendUrl, token });
if (scanResult.status === "pass") {
const knownNames = new Set(spaceModels.map((model) => model.name));
candidates.push(...scanResult.models
.filter((model) => model.name && !knownNames.has(model.name) && !skippedModelNames.has(model.name))
.map((model) => scannedCandidate(model)));
candidates.push(
...scanResult.models
.filter(
(model) =>
model.name &&
!knownNames.has(model.name) &&
!skippedModelNames.has(model.name),
)
.map((model) => scannedCandidate(model)),
);
}
}
@@ -435,14 +733,16 @@ async function selectWorkingSpaceModel({
};
}
const baseReason = unique.length === 0
? scanResult.reason || "No Space LLM model candidates are available."
: `No working Space LLM model found after testing ${modelTests.length} candidate(s).`;
const baseReason =
unique.length === 0
? scanResult.reason || "No Space LLM model candidates are available."
: `No working Space LLM model found after testing ${modelTests.length} candidate(s).`;
return {
status: "env_issue",
reason: requestedModelId && !requestedModel
? `Requested Space LLM model ${requestedModelId} is missing or skipped; ${baseReason}`
: baseReason,
reason:
requestedModelId && !requestedModel
? `Requested Space LLM model ${requestedModelId} is missing or skipped; ${baseReason}`
: baseReason,
selected_model_id: "",
selected_model_name: "",
fallback_model_ids: [],
@@ -462,7 +762,11 @@ async function scanSpaceModels({ backendUrl, token }) {
return {
status: "env_issue",
models: [],
reason: safeReason(response.json.msg || response.json.message || "Failed to scan Space LLM models."),
reason: safeReason(
response.json.msg ||
response.json.message ||
"Failed to scan Space LLM models.",
),
};
}
return {
@@ -492,28 +796,42 @@ async function ensureAndTestModel({ backendUrl, token, candidate }) {
if (isApiFailure(create) || !modelUuid) {
return modelTestResult(candidate, {
status: "fail",
reason: safeReason(create.json.msg || "Failed to create scanned Space model."),
reason: safeReason(
create.json.msg || "Failed to create scanned Space model.",
),
http_status: create.status,
});
}
created = true;
}
const test = await apiJson(backendUrl, `/api/v1/provider/models/llm/${encodeURIComponent(modelUuid)}/test`, {
method: "POST",
token,
body: { extra_args: {} },
});
const test = await apiJson(
backendUrl,
`/api/v1/provider/models/llm/${encodeURIComponent(modelUuid)}/test`,
{
method: "POST",
token,
body: { extra_args: {} },
},
);
const passed = !isApiFailure(test);
if (!passed && created) {
await apiJson(backendUrl, `/api/v1/provider/models/llm/${encodeURIComponent(modelUuid)}`, {
method: "DELETE",
token,
}).catch(() => {});
await apiJson(
backendUrl,
`/api/v1/provider/models/llm/${encodeURIComponent(modelUuid)}`,
{
method: "DELETE",
token,
},
).catch(() => {});
}
return modelTestResult(candidate, {
status: passed ? "pass" : "fail",
reason: passed ? "" : safeReason(test.json.msg || test.json.message || "Space model test failed."),
reason: passed
? ""
: safeReason(
test.json.msg || test.json.message || "Space model test failed.",
),
http_status: test.status,
model_uuid: modelUuid,
created,
@@ -558,7 +876,9 @@ function uniqueCandidates(candidates) {
const seen = new Set();
const result = [];
for (const candidate of candidates) {
const key = candidate.uuid ? `uuid:${candidate.uuid}` : `name:${candidate.name}`;
const key = candidate.uuid
? `uuid:${candidate.uuid}`
: `name:${candidate.name}`;
if (!candidate.name || seen.has(key)) continue;
seen.add(key);
result.push(candidate);
@@ -568,8 +888,12 @@ function uniqueCandidates(candidates) {
function rankModels(models) {
return [...models].sort((left, right) => {
const leftRank = Number.isFinite(Number(left.prefered_ranking)) ? Number(left.prefered_ranking) : 9999;
const rightRank = Number.isFinite(Number(right.prefered_ranking)) ? Number(right.prefered_ranking) : 9999;
const leftRank = Number.isFinite(Number(left.prefered_ranking))
? Number(left.prefered_ranking)
: 9999;
const rightRank = Number.isFinite(Number(right.prefered_ranking))
? Number(right.prefered_ranking)
: 9999;
if (leftRank !== rightRank) return leftRank - rightRank;
return String(left.name || "").localeCompare(String(right.name || ""));
});
@@ -605,5 +929,9 @@ async function upsertEnvLocal(path, updates) {
for (const [key, value] of Object.entries(updates)) {
if (!seen.has(key)) next.push(`${key}=${value}`);
}
await writeFile(path, `${next.filter((line, index) => line !== "" || index < next.length - 1).join("\n")}\n`, "utf8");
await writeFile(
path,
`${next.filter((line, index) => line !== "" || index < next.length - 1).join("\n")}\n`,
"utf8",
);
}
skills/scripts/e2e/fake-openai-provider.mjs
+350 -14
View File
@@ -9,7 +9,43 @@ const args = parseArgs(process.argv.slice(2));
const host = args.host || env.LANGBOT_FAKE_PROVIDER_HOST || "127.0.0.1";
const port = integer(args.port ?? env.LANGBOT_FAKE_PROVIDER_PORT, 0);
const stateFile = args["state-file"] || env.LANGBOT_FAKE_PROVIDER_STATE_FILE || "";
const modelName = env.LANGBOT_FAKE_PROVIDER_MODEL_NAME || "gpt-4o-mini";
const modelName = args.model || env.LANGBOT_FAKE_PROVIDER_MODEL_NAME || env.LANGBOT_FAKE_PROVIDER_MODEL || "gpt-4o-mini";
const COMPACTION_SENTINEL = "qa_compaction_sentinel_7391";
const COMBO_SENTINEL = "qa_combo_compaction_sentinel_2406";
const RAG_SENTINEL = "azalea-cobalt-7421";
const COMBO_TOOL_INPUT = "combo-tool-ok-local-agent";
const COMBO_TOOL_RESULT = `qa-plugin-smoke:${COMBO_TOOL_INPUT}`;
const COMBO_FINAL = `COMBO_FINAL ${COMBO_SENTINEL} ${RAG_SENTINEL} ${COMBO_TOOL_RESULT}`;
const MULTITOOL_SENTINEL = "qa_multitool_compaction_sentinel_6718";
const MULTITOOL_TOOL_A_INPUT = "multi-tool-a-local-agent";
const MULTITOOL_TOOL_B_INPUT = "multi-tool-b-local-agent";
const MULTITOOL_TOOL_A_RESULT = `qa-plugin-smoke:${MULTITOOL_TOOL_A_INPUT}`;
const MULTITOOL_TOOL_B_RESULT = `qa-plugin-smoke:${MULTITOOL_TOOL_B_INPUT}`;
const MULTITOOL_FINAL = [
"MULTITOOL_COMBO_FINAL",
MULTITOOL_SENTINEL,
RAG_SENTINEL,
MULTITOOL_TOOL_A_RESULT,
MULTITOOL_TOOL_B_RESULT,
].join(" ");
const PARALLEL_SENTINEL = "qa_parallel_compaction_sentinel_8142";
const PARALLEL_TOOL_A_INPUT = "parallel-tool-a-local-agent";
const PARALLEL_TOOL_B_INPUT = "parallel-tool-b-local-agent";
const PARALLEL_TOOL_A_RESULT = `qa-plugin-smoke:${PARALLEL_TOOL_A_INPUT}`;
const PARALLEL_TOOL_B_RESULT = `qa-plugin-smoke:${PARALLEL_TOOL_B_INPUT}`;
const PARALLEL_FINAL = [
"PARALLEL_COMBO_FINAL",
PARALLEL_SENTINEL,
RAG_SENTINEL,
PARALLEL_TOOL_A_RESULT,
PARALLEL_TOOL_B_RESULT,
].join(" ");
const LOOP_LIMIT_INPUT = "loop-limit-repeat-local-agent";
const TOOL_ERROR_INPUT = "tool-error-recovery-local-agent";
const TOOL_ERROR_FINAL = "TOOL_ERROR_RECOVERY_FINAL qa-plugin-smoke forced failure observed";
const STEERING_FOLLOWUP_SENTINEL = "qa_steering_sentinel_6194";
const STEERING_SLEEP_INPUT = "steering-e2e-anchor";
const STEERING_SLEEP_RESULT = `qa-plugin-smoke:sleep:8:${STEERING_SLEEP_INPUT}`;
const config = {
response_text: env.LANGBOT_FAKE_PROVIDER_RESPONSE_TEXT || "OK",
first_token_delay_ms: integer(env.LANGBOT_FAKE_PROVIDER_FIRST_TOKEN_DELAY_MS, 25),
@@ -88,6 +124,7 @@ const server = createServer(async (request, response) => {
created: 1,
owned_by: "langbot-qa",
type: "llm",
context_length: 8192,
},
],
});
@@ -100,7 +137,8 @@ const server = createServer(async (request, response) => {
const requestId = `chatcmpl-langbot-fake-${requestCount}`;
const shouldFail = requestCount <= config.fail_first_n
|| (config.fail_every_n > 0 && requestCount % config.fail_every_n === 0);
const replyText = responseTextForBody(body);
const replyMessage = buildResponse(body);
const replyText = replyMessage.content || "";
requestRecord = recordRequest({
id: requestId,
request_number: requestCount,
@@ -140,7 +178,7 @@ const server = createServer(async (request, response) => {
await streamCompletion(response, {
requestId,
model: body.model || modelName,
content: replyText,
message: replyMessage,
failAfterFirstChunk: config.fail_after_first_chunk,
requestRecord,
startedPerf,
@@ -150,7 +188,7 @@ const server = createServer(async (request, response) => {
sendJson(response, 200, completionPayload({
requestId,
model: body.model || modelName,
content: replyText,
message: replyMessage,
}));
markRequestTiming(requestRecord, "first_chunk", startedPerf);
markRequestTiming(requestRecord, "first_content_chunk", startedPerf);
@@ -270,8 +308,8 @@ function sendJson(response, status, payload) {
response.end(text);
}
function completionPayload({ requestId, model, content }) {
const completionTokens = tokenEstimate(content);
function completionPayload({ requestId, model, message }) {
const completionTokens = tokenEstimate(message.content || JSON.stringify(message.tool_calls || []));
return {
id: requestId,
object: "chat.completion",
@@ -280,11 +318,8 @@ function completionPayload({ requestId, model, content }) {
choices: [
{
index: 0,
message: {
role: "assistant",
content,
},
finish_reason: "stop",
message,
finish_reason: message.tool_calls?.length ? "tool_calls" : "stop",
},
],
usage: {
@@ -298,7 +333,7 @@ function completionPayload({ requestId, model, content }) {
async function streamCompletion(response, {
requestId,
model,
content,
message,
failAfterFirstChunk: failMidStream,
requestRecord,
startedPerf,
@@ -319,7 +354,56 @@ async function streamCompletion(response, {
choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }],
});
const chunks = splitContent(content);
const chunks = message.tool_calls?.length ? [] : splitContent(message.content || "");
if (message.tool_calls?.length) {
await sleep(config.chunk_delay_ms);
markRequestTiming(requestRecord, "first_content_chunk", startedPerf);
requestRecord.content_chunk_count = (requestRecord.content_chunk_count || 0) + 1;
writeSse(response, {
id: requestId,
object: "chat.completion.chunk",
created: Math.floor(Date.now() / 1000),
model,
choices: [
{
index: 0,
delta: {
tool_calls: message.tool_calls.map((call, index) => ({
index,
id: call.id,
type: call.type,
function: {
name: call.function.name,
arguments: call.function.arguments,
},
})),
},
finish_reason: null,
},
],
});
await sleep(config.chunk_delay_ms);
writeSse(response, {
id: requestId,
object: "chat.completion.chunk",
created: Math.floor(Date.now() / 1000),
model,
choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }],
usage: {
prompt_tokens: 8,
completion_tokens: tokenEstimate(JSON.stringify(message.tool_calls)),
total_tokens: 8 + tokenEstimate(JSON.stringify(message.tool_calls)),
},
});
response.write("data: [DONE]\n\n");
response.end();
finishRequestRecord(requestRecord, startedPerf, {
status: "ok",
http_status: 200,
});
return;
}
for (let index = 0; index < chunks.length; index += 1) {
await sleep(config.chunk_delay_ms);
if (index === 0) markRequestTiming(requestRecord, "first_content_chunk", startedPerf);
@@ -342,7 +426,7 @@ async function streamCompletion(response, {
}
await sleep(config.chunk_delay_ms);
const completionTokens = tokenEstimate(content);
const completionTokens = tokenEstimate(message.content || "");
writeSse(response, {
id: requestId,
object: "chat.completion.chunk",
@@ -399,6 +483,258 @@ function responseTextForBody(body) {
return config.response_text;
}
function messageText(messages = []) {
return messages
.map((message) => {
const parts = [contentText(message?.content)];
if (Array.isArray(message?.tool_calls)) {
parts.push(
message.tool_calls
.map((call) => `${call?.function?.name || call?.name || ""} ${call?.function?.arguments || ""}`)
.join("\n"),
);
}
return parts.filter(Boolean).join("\n");
})
.join("\n");
}
function contentText(content) {
if (typeof content === "string") return content;
if (Array.isArray(content)) return content.map((part) => contentText(part)).join("");
if (content && typeof content === "object") {
for (const key of ["text", "content", "message", "error", "value"]) {
if (content[key] !== undefined && content[key] !== null) {
return contentText(content[key]);
}
}
return JSON.stringify(content);
}
return "";
}
function latestUserText(messages = []) {
const latest = [...messages].reverse().find((message) => message?.role === "user");
return contentText(latest?.content);
}
function toolNames(tools = []) {
return tools
.map((tool) => tool?.function?.name || tool?.name || "")
.filter(Boolean);
}
function firstToolName(tools = [], candidates = []) {
const names = toolNames(tools);
return candidates.find((candidate) => names.includes(candidate)) || "";
}
function isSummarizationRequest(messages = []) {
const text = messageText(messages);
return /context summarization assistant|conversation to summarize|<previous-summary>/i.test(text);
}
function buildSummaryResponse(text) {
const references = [];
if (text.includes(COMPACTION_SENTINEL)) references.push(COMPACTION_SENTINEL);
if (text.includes(COMBO_SENTINEL)) references.push(COMBO_SENTINEL);
if (text.includes(MULTITOOL_SENTINEL)) references.push(MULTITOOL_SENTINEL);
if (text.includes(PARALLEL_SENTINEL)) references.push(PARALLEL_SENTINEL);
if (/RAG_TOOL_COMBO_GOAL/i.test(text)) references.push("RAG_TOOL_COMBO_GOAL");
if (/MULTITOOL_RAG_GOAL/i.test(text)) references.push("MULTITOOL_RAG_GOAL");
if (/PARALLEL_RAG_GOAL/i.test(text)) references.push("PARALLEL_RAG_GOAL");
const critical = references.length
? references.map((reference) => `- ${reference}`).join("\n")
: "- (none)";
return {
role: "assistant",
content: [
"## Goal",
"Preserve deterministic LangBot E2E context across compaction.",
"",
"## Constraints & Preferences",
"- Keep exact sentinel values verbatim.",
"",
"## Progress",
"### Done",
"- [x] Earlier conversation was compacted by the fake provider.",
"",
"### In Progress",
"- [ ] Continue the current Debug Chat run.",
"",
"### Blocked",
"- (none)",
"",
"## Key Decisions",
"- **Deterministic QA**: Return only known sentinel values from compacted input.",
"",
"## Next Steps",
"1. Answer the current user request using preserved context.",
"",
"## Critical Context",
critical,
].join("\n"),
};
}
function buildResponse(payload) {
const text = messageText(payload.messages || []);
const current = latestUserText(payload.messages || []);
const tools = payload.tools || [];
if (isSummarizationRequest(payload.messages || [])) {
return buildSummaryResponse(text);
}
const pluginTool = firstToolName(tools, ["qa_plugin_echo"]);
const pluginFailTool = firstToolName(tools, ["qa_plugin_fail"]);
const pluginSleepTool = firstToolName(tools, ["qa_plugin_sleep"]);
const mcpTool = firstToolName(tools, ["qa_mcp_echo"]);
if (/STEERING_NO_FOLLOWUP|qa_plugin_sleep|steering-e2e-anchor|qa_steering_sentinel_6194/i.test(current || text)) {
if (text.includes(STEERING_FOLLOWUP_SENTINEL) && text.includes(STEERING_SLEEP_RESULT)) {
return { role: "assistant", content: STEERING_FOLLOWUP_SENTINEL };
}
if (text.includes(STEERING_SLEEP_RESULT) && !text.includes(STEERING_FOLLOWUP_SENTINEL)) {
return { role: "assistant", content: "STEERING_NO_FOLLOWUP" };
}
if (pluginSleepTool && !text.includes(STEERING_SLEEP_RESULT)) {
return toolCall("call_qa_plugin_sleep_steering", pluginSleepTool, { seconds: 8, text: STEERING_SLEEP_INPUT });
}
}
if (/测试暗号是什么|original sentinel|first.*sentinel/i.test(current)) {
return {
role: "assistant",
content: text.includes(COMPACTION_SENTINEL) ? COMPACTION_SENTINEL : "COMPACTION_SENTINEL_MISSING",
};
}
if ((current.includes(COMPACTION_SENTINEL)
|| current.includes(COMBO_SENTINEL)
|| current.includes(MULTITOOL_SENTINEL)
|| current.includes(PARALLEL_SENTINEL))
&& /请只回复 MEMORY_SET|only reply MEMORY_SET/i.test(current)) {
return { role: "assistant", content: "MEMORY_SET" };
}
if (/PARALLEL_CONTEXT_PRESSURE_READY/.test(current)) return { role: "assistant", content: "PARALLEL_CONTEXT_PRESSURE_READY" };
if (/MULTITOOL_CONTEXT_PRESSURE_READY/.test(current)) return { role: "assistant", content: "MULTITOOL_CONTEXT_PRESSURE_READY" };
if (/COMBO_CONTEXT_PRESSURE_READY/.test(current)) return { role: "assistant", content: "COMBO_CONTEXT_PRESSURE_READY" };
if (/CONTEXT_PRESSURE_READY/.test(current)) return { role: "assistant", content: "CONTEXT_PRESSURE_READY" };
if (/qa_mcp_echo:mcp-ok-local-agent/.test(text)) return { role: "assistant", content: "qa_mcp_echo:mcp-ok-local-agent" };
if (/qa_mcp_echo|mcp-ok-local-agent/i.test(current || text) && mcpTool && !/qa_mcp_echo:mcp-ok-local-agent/.test(text)) {
return toolCall("call_qa_mcp_echo", mcpTool, { text: "mcp-ok-local-agent" });
}
if (/LOOP_LIMIT|loop-limit-repeat-local-agent|iteration limit/i.test(current || text) && pluginTool) {
return toolCall(`call_qa_plugin_echo_loop_limit_${Date.now()}`, pluginTool, { text: LOOP_LIMIT_INPUT });
}
if (/TOOL_ERROR_RECOVERY|tool-error-recovery-local-agent|qa_plugin_fail/i.test(current || text)) {
if (/(?:Error:|Tool execution failed:|ActionCallError:|RuntimeError:)/i.test(text)
&& /(?:qa-plugin-smoke forced failure|qa_plugin_fail|tool-error-recovery-local-agent)/i.test(text)) {
return { role: "assistant", content: TOOL_ERROR_FINAL };
}
if (pluginFailTool) {
return toolCall("call_qa_plugin_fail_recovery", pluginFailTool, { text: TOOL_ERROR_INPUT });
}
}
const isParallelComboRequest = /PARALLEL_COMBO|parallel-tool-a-local-agent|parallel-tool-b-local-agent|PARALLEL_RAG_GOAL/i.test(current || text);
if (isParallelComboRequest && pluginTool && !text.includes(PARALLEL_TOOL_A_RESULT) && !text.includes(PARALLEL_TOOL_B_RESULT)) {
return toolCalls([
["call_qa_plugin_echo_parallel_a", pluginTool, { text: PARALLEL_TOOL_A_INPUT }],
["call_qa_plugin_echo_parallel_b", pluginTool, { text: PARALLEL_TOOL_B_INPUT }],
]);
}
if (isParallelComboRequest && (text.includes(PARALLEL_TOOL_A_RESULT) || text.includes(PARALLEL_TOOL_B_RESULT))) {
return missingOrFinal(text, [
[PARALLEL_SENTINEL, "parallel-memory"],
[RAG_SENTINEL, "rag"],
[PARALLEL_TOOL_A_RESULT, "tool-a"],
[PARALLEL_TOOL_B_RESULT, "tool-b"],
], "PARALLEL_COMBO_MISSING", PARALLEL_FINAL);
}
const isMultiToolComboRequest = /MULTITOOL_COMBO|multi-tool-a-local-agent|multi-tool-b-local-agent|MULTITOOL_RAG_GOAL/i.test(current || text);
if (isMultiToolComboRequest && pluginTool && !text.includes(MULTITOOL_TOOL_A_RESULT)) {
return toolCall("call_qa_plugin_echo_multi_a", pluginTool, { text: MULTITOOL_TOOL_A_INPUT });
}
if (isMultiToolComboRequest && pluginTool && text.includes(MULTITOOL_TOOL_A_RESULT) && !text.includes(MULTITOOL_TOOL_B_RESULT)) {
return toolCall("call_qa_plugin_echo_multi_b", pluginTool, { text: MULTITOOL_TOOL_B_INPUT });
}
if (isMultiToolComboRequest && text.includes(MULTITOOL_TOOL_A_RESULT) && text.includes(MULTITOOL_TOOL_B_RESULT)) {
return missingOrFinal(text, [
[MULTITOOL_SENTINEL, "multi-memory"],
[RAG_SENTINEL, "rag"],
[MULTITOOL_TOOL_A_RESULT, "tool-a"],
[MULTITOOL_TOOL_B_RESULT, "tool-b"],
], "MULTITOOL_COMBO_MISSING", MULTITOOL_FINAL);
}
const isComboRequest = /qa_combo|\bCOMBO_FINAL\b|combo-tool-ok-local-agent|RAG_TOOL_COMBO_GOAL/i.test(current || text);
if (isComboRequest && pluginTool && !text.includes(COMBO_TOOL_RESULT)) {
return toolCall("call_qa_plugin_echo_combo", pluginTool, { text: COMBO_TOOL_INPUT });
}
if (isComboRequest && text.includes(COMBO_TOOL_RESULT)) {
return missingOrFinal(text, [
[COMBO_SENTINEL, "combo-memory"],
[RAG_SENTINEL, "rag"],
[COMBO_TOOL_RESULT, "tool-result"],
], "COMBO_MISSING", COMBO_FINAL);
}
if (/qa-plugin-smoke:plugin-tool-ok-local-agent/.test(text)) {
return { role: "assistant", content: "qa-plugin-smoke:plugin-tool-ok-local-agent" };
}
if (/qa_plugin_echo|plugin-tool-ok-local-agent/i.test(current || text) && pluginTool && !/qa-plugin-smoke:plugin-tool-ok-local-agent/.test(text)) {
return toolCall("call_qa_plugin_echo", pluginTool, { text: "plugin-tool-ok-local-agent" });
}
const e2eTool = firstToolName(tools, ["e2e_lookup"]);
if (/Use the e2e lookup tool|e2e lookup tool|tool loop/i.test(current || text) && e2eTool && !/tool-result:alpha/.test(text)) {
return toolCall("call_e2e_lookup", e2eTool, { query: "alpha" });
}
if (/tool-result:alpha/.test(text)) return { role: "assistant", content: "Tool loop final answer after tool-result:alpha" };
if (/RAG_SENTINEL/.test(text)) return { role: "assistant", content: "RAG final answer with RAG_SENTINEL" };
if (/NONSTREAM_OK/i.test(current || text)) return { role: "assistant", content: "NONSTREAM_OK" };
if (/IMAGE_OK/i.test(current || text)
&& /(?:\[Image\]|langbot_input_attachments|data:image\/|\"type\":\s*\"image\")/i.test(current || text)) {
return { role: "assistant", content: "IMAGE_OK" };
}
if (/azalea-cobalt-7421/.test(text)) return { role: "assistant", content: "azalea-cobalt-7421" };
return { role: "assistant", content: responseTextForBody(payload) };
}
function toolCall(id, name, args) {
return toolCalls([[id, name, args]]);
}
function toolCalls(calls) {
return {
role: "assistant",
content: "",
tool_calls: calls.map(([id, name, args]) => ({
id,
type: "function",
function: {
name,
arguments: JSON.stringify(args),
},
})),
};
}
function missingOrFinal(text, requirements, prefix, finalText) {
const missing = requirements
.filter(([needle]) => !text.includes(needle))
.map(([, label]) => label);
return {
role: "assistant",
content: missing.length ? `${prefix}_${missing.join("_")}` : finalText,
};
}
function flattenContent(content) {
if (typeof content === "string") return content;
if (Array.isArray(content)) {
skills/scripts/e2e/install-qa-plugin-smoke.mjs
+62 -18
View File
@@ -26,7 +26,12 @@ const packagePath = resolve(
|| "skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/dist/qa-plugin-smoke-0.1.0.lbpkg",
);
const expectedPluginId = env.LANGBOT_E2E_EXPECTED_PLUGIN_ID || "qa/plugin-smoke";
const expectedTool = env.LANGBOT_E2E_EXPECTED_TOOL || (expectedPluginId === "qa/plugin-smoke" ? "qa_plugin_echo" : "");
const expectedTools = (env.LANGBOT_E2E_EXPECTED_TOOLS || env.LANGBOT_E2E_EXPECTED_TOOL || (
expectedPluginId === "qa/plugin-smoke" ? "qa_plugin_echo" : ""
))
.split(",")
.map((item) => item.trim())
.filter(Boolean);
const expectedRunnerId = env.LANGBOT_E2E_EXPECTED_RUNNER_ID || "";
const result = {
@@ -40,6 +45,7 @@ const result = {
package_preview: null,
task_id: null,
task: null,
reinstall_reason: "",
plugin_present_before: false,
plugin_present_after: false,
tool_names: [],
@@ -64,21 +70,23 @@ try {
}
result.plugin_present_before = await hasPlugin(backendUrl, auth.token);
if (!result.plugin_present_before) {
const form = new FormData();
form.set("file", new Blob([bytes]), packagePath.split("/").pop());
const response = await fetch(`${backendUrl.replace(/\/$/, "")}/api/v1/plugins/install/local`, {
method: "POST",
headers: { Authorization: `Bearer ${auth.token}` },
body: form,
});
const json = await response.json().catch(() => ({}));
if (response.status >= 400 || json.code !== 0) {
throw new Error(json.msg || `Plugin install request failed with HTTP ${response.status}.`);
if (expectedTools.length > 0) {
result.tool_names = await listToolNames(backendUrl, auth.token);
}
const missingToolsBefore = expectedTools.filter((tool) => !result.tool_names.includes(tool));
if (result.plugin_present_before && missingToolsBefore.length > 0) {
result.reinstall_reason = `Installed plugin is missing expected tools: ${missingToolsBefore.join(", ")}`;
const removeTask = await removePlugin(backendUrl, auth.token);
if (!isTaskComplete(removeTask)) {
throw new Error(`Plugin reinstall cleanup did not complete successfully: ${JSON.stringify(removeTask)}`);
}
result.task_id = json.data?.task_id ?? null;
if (!result.task_id) throw new Error("Plugin install response did not include task_id.");
result.task = await waitForTask(backendUrl, auth.token, result.task_id);
result.plugin_present_before = false;
await sleep(1000);
}
if (!result.plugin_present_before) {
result.task = await installPlugin(backendUrl, auth.token, bytes, packagePath);
result.task_id = result.task?.id || result.task_id;
if (!isTaskComplete(result.task)) {
throw new Error(`Plugin install task did not complete successfully: ${JSON.stringify(result.task)}`);
}
@@ -87,10 +95,11 @@ try {
await sleep(1000);
result.plugin_present_after = await hasPlugin(backendUrl, auth.token);
if (!result.plugin_present_after) throw new Error(`${expectedPluginId} is not listed by /api/v1/plugins after install.`);
if (expectedTool) {
if (expectedTools.length > 0) {
result.tool_names = await listToolNames(backendUrl, auth.token);
if (!result.tool_names.includes(expectedTool)) {
throw new Error(`${expectedTool} is not listed by /api/v1/tools after install.`);
const missingTools = expectedTools.filter((tool) => !result.tool_names.includes(tool));
if (missingTools.length > 0) {
throw new Error(`${missingTools.join(", ")} is not listed by /api/v1/tools after install.`);
}
}
if (expectedRunnerId) {
@@ -121,6 +130,41 @@ async function hasPlugin(backendUrl, token) {
});
}
async function installPlugin(backendUrl, token, bytes, packagePath) {
const form = new FormData();
form.set("file", new Blob([bytes]), packagePath.split("/").pop());
const response = await fetch(`${backendUrl.replace(/\/$/, "")}/api/v1/plugins/install/local`, {
method: "POST",
headers: { Authorization: `Bearer ${token}` },
body: form,
});
const json = await response.json().catch(() => ({}));
if (response.status >= 400 || json.code !== 0) {
throw new Error(json.msg || `Plugin install request failed with HTTP ${response.status}.`);
}
const taskId = json.data?.task_id ?? null;
if (!taskId) throw new Error("Plugin install response did not include task_id.");
const task = await waitForTask(backendUrl, token, taskId);
return { id: taskId, ...task };
}
async function removePlugin(backendUrl, token) {
const [author, pluginName] = expectedPluginId.split("/");
if (!author || !pluginName) throw new Error(`Invalid expected plugin id: ${expectedPluginId}`);
const response = await apiJson(
backendUrl,
`/api/v1/plugins/${encodeURIComponent(author)}/${encodeURIComponent(pluginName)}?delete_data=false`,
{ method: "DELETE", token },
);
if (response.status >= 400 || response.json.code !== 0) {
throw new Error(response.json.msg || `Plugin delete request failed with HTTP ${response.status}.`);
}
const taskId = response.json.data?.task_id ?? null;
if (!taskId) throw new Error("Plugin delete response did not include task_id.");
const task = await waitForTask(backendUrl, token, taskId);
return { id: taskId, ...task };
}
async function previewPackage(backendUrl, token, bytes, packagePath) {
const form = new FormData();
form.set("file", new Blob([bytes]), packagePath.split("/").pop());
skills/scripts/e2e/lib/langbot-e2e.mjs
+159 -3
View File
@@ -1,5 +1,5 @@
import { appendFile, mkdir, readFile, stat, writeFile } from "node:fs/promises";
import { join, resolve } from "node:path";
import { basename, join, resolve } from "node:path";
import { env } from "node:process";
const secretRe = /(?:authorization|bearer|token|secret|password|api[_-]?key|jwt|oauth)\s*[:=]\s*["']?[^"',\s]+/gi;
@@ -71,6 +71,86 @@ export async function writeResult(paths, result) {
}
}
function browserDiagnosticFindings(source, text) {
const findings = [];
const lines = String(text || "").split(/\r?\n/);
for (let index = 0; index < lines.length; index += 1) {
const line = lines[index];
if (!line) continue;
const lineNumber = index + 1;
if (source === "console") {
const checks = [
["pageerror", /\[pageerror\]/i],
["frontend_uncaught_error", /\[error\].*(?:\bUncaught\b|Unhandled(?: promise rejection|Rejection)|TypeError|ReferenceError)/i],
["http_5xx", /Failed to load resource: the server responded with a status of 5\d\d/i],
["api_server_error", /\[error\].*Server error:/i],
["plugin_runtime_timeout", /\[error\].*Action [A-Za-z0-9_]+ call timed out/i],
];
for (const [kind, regex] of checks) {
if (!regex.test(line)) continue;
findings.push({
source,
severity: "fail",
kind,
line: lineNumber,
excerpt: redact(line.trim()),
});
break;
}
continue;
}
if (source === "network") {
if (/\[response\]\s+5\d\d\b/i.test(line)) {
findings.push({
source,
severity: "fail",
kind: "http_5xx",
line: lineNumber,
excerpt: redact(line.trim()),
});
continue;
}
if (/\[requestfailed\]/i.test(line) && !/net::ERR_ABORTED/i.test(line)) {
findings.push({
source,
severity: "warning",
kind: "request_failed",
line: lineNumber,
excerpt: redact(line.trim()),
});
}
}
}
return findings;
}
export async function scanBrowserDiagnostics(paths) {
const sources = [
["console", paths.consoleLog],
["network", paths.networkLog],
];
const findings = [];
for (const [source, path] of sources) {
let text = "";
try {
text = await readFile(path, "utf8");
} catch {
continue;
}
findings.push(...browserDiagnosticFindings(source, text));
}
const hasFailure = findings.some((finding) => finding.severity === "fail");
return {
status: hasFailure ? "fail" : "pass",
findings,
reason: hasFailure
? `Browser diagnostics found ${findings.filter((finding) => finding.severity === "fail").length} failing signal(s).`
: "No failing browser diagnostics found.",
};
}
export async function loadEnvFiles(paths = ["skills/.env", "skills/.env.local"]) {
const processEnvKeys = new Set(Object.keys(env));
for (const path of paths) {
@@ -92,8 +172,28 @@ export async function loadEnvFiles(paths = ["skills/.env", "skills/.env.local"])
}
}
export async function readRecoveryKey(repo = env.LANGBOT_REPO || "../LangBot") {
const configPath = resolve(repo, "data/config.yaml");
export async function resolveLangBotRepo(repo = env.LANGBOT_REPO || "", cwd = process.cwd()) {
if (repo) return resolve(repo);
const candidates = [
resolve(cwd),
basename(cwd) === "skills" ? resolve(cwd, "..") : "",
resolve(cwd, "../LangBot"),
resolve(cwd, "LangBot"),
].filter(Boolean);
const seen = new Set();
for (const candidate of candidates) {
if (seen.has(candidate)) continue;
seen.add(candidate);
if (await pathExists(resolve(candidate, "data/config.yaml"))) return candidate;
}
return resolve(cwd, "../LangBot");
}
export async function readRecoveryKey(repo = env.LANGBOT_REPO || "") {
const configPath = resolve(await resolveLangBotRepo(repo), "data/config.yaml");
const config = await readFile(configPath, "utf8");
const match = config.match(/^\s*recovery_key:\s*['"]?([^'"\s#]+)['"]?\s*$/m);
return match?.[1] || "";
@@ -202,6 +302,62 @@ export async function verifyBrowserToken(page, backendUrl) {
}, backendUrl);
}
export async function ensureAuthenticatedBrowser(page, {
frontendUrl = env.LANGBOT_FRONTEND_URL || "",
backendUrl = env.LANGBOT_BACKEND_URL || "",
user = env.LANGBOT_E2E_LOGIN_USER || "",
password = env.LANGBOT_E2E_LOGIN_PASSWORD || "LangBotE2ELocalPass!2026",
recoveryKey = "",
} = {}) {
if (!frontendUrl) return { status: "env_issue", reason: "LANGBOT_FRONTEND_URL is not configured." };
if (!backendUrl) return { status: "env_issue", reason: "LANGBOT_BACKEND_URL is not configured." };
const current = await verifyBrowserToken(page, backendUrl).catch((error) => ({
authenticated: false,
reason: error.message,
}));
if (current.authenticated) {
return {
status: "pass",
reason: "Existing browser token is valid.",
backend_token_check: null,
browser_token_check: current,
injected: false,
};
}
if (!user) {
return {
status: "blocked",
reason: "Browser profile is not authenticated for LANGBOT_FRONTEND_URL, and LANGBOT_E2E_LOGIN_USER is not configured for automatic local login.",
backend_token_check: null,
browser_token_check: current,
injected: false,
};
}
const auth = await resetAndAuthLocalUser({ backendUrl, user, password, recoveryKey });
await setBrowserToken(page, frontendUrl, auth.token);
const browserCheck = await verifyBrowserToken(page, backendUrl);
if (!browserCheck.authenticated) {
return {
status: "blocked",
reason: browserCheck.reason || "Browser token check failed after automatic local login.",
backend_token_check: auth.check,
browser_token_check: browserCheck,
injected: true,
};
}
return {
status: "pass",
reason: "Browser token injected from local recovery login.",
backend_token_check: auth.check,
browser_token_check: browserCheck,
injected: true,
};
}
export function exitCode(status) {
if (status === "pass") return 0;
if (status === "blocked" || status === "env_issue") return 2;
skills/scripts/e2e/local-agent-steering-debug-chat.mjs
+14
View File
@@ -11,6 +11,7 @@ import {
} from "./lib/debug-chat.mjs";
import {
createBrowser,
ensureAuthenticatedBrowser,
ensureEvidence,
evidencePaths,
exitCode,
@@ -74,6 +75,7 @@ const result = {
pipeline_config: null,
debug_chat_reset: null,
tool_diagnostic: null,
browser_auth: null,
steering: null,
evidence: {
console_log: paths.consoleLog,
@@ -95,6 +97,18 @@ try {
browser = await createBrowser(paths);
const { page } = browser;
const authDiagnostic = await ensureAuthenticatedBrowser(page, {
frontendUrl: env.LANGBOT_FRONTEND_URL || "",
backendUrl,
});
result.browser_auth = authDiagnostic;
if (!result.evidence_collected.includes("api_diagnostic")) result.evidence_collected.push("api_diagnostic");
if (authDiagnostic.status === "env_issue" || authDiagnostic.status === "blocked" || authDiagnostic.status === "fail") {
result.status = authDiagnostic.status;
result.reason = authDiagnostic.reason || "Browser authentication failed.";
throw new Error(result.reason);
}
const openResult = await openPipelineDebugChat(page, {
pipelineUrl,
pipelineName,
skills/scripts/e2e/mcp-stdio-register.mjs
+44 -2
View File
@@ -1,7 +1,7 @@
#!/usr/bin/env node
import { existsSync, readFileSync } from "node:fs";
import { writeFile } from "node:fs/promises";
import { readFile, writeFile } from "node:fs/promises";
import { resolve } from "node:path";
import { env } from "node:process";
import {
@@ -46,6 +46,8 @@ const startupTimeoutSec = Number(env.LANGBOT_MCP_STARTUP_TIMEOUT_SEC || "300");
const readyTimeoutMs = Number(env.LANGBOT_MCP_READY_TIMEOUT_MS || "360000");
const backendUrl = env.LANGBOT_BACKEND_URL || "";
const apiDiagnosticPath = resolve(paths.evidenceDir, "api-diagnostic.json");
const envLocalPath = resolve("skills/.env.local");
const serverUuidEnvKey = env.LANGBOT_MCP_SERVER_UUID_ENV_KEY || "LANGBOT_MCP_QA_STDIO_SERVER_UUID";
let browser;
const result = {
@@ -70,6 +72,7 @@ const result = {
result_json: paths.resultJson,
},
evidence_collected: ["api_diagnostic"],
wrote_env: false,
};
async function run() {
@@ -159,6 +162,7 @@ async function run() {
const deadline = Date.now() + readyTimeoutMs;
let lastTools = [];
let lastRuntime = null;
let lastServer = null;
while (Date.now() < deadline) {
await new Promise((resolveReady) => setTimeout(resolveReady, 500));
const tools = await getJson("/api/v1/tools");
@@ -167,7 +171,8 @@ async function run() {
.map((tool) => tool.name || tool.tool_name || tool.function?.name || "")
.filter(Boolean)
.sort();
lastRuntime = server.json.data?.server?.runtime_info || null;
lastServer = server.json.data?.server || null;
lastRuntime = lastServer?.runtime_info || null;
if (lastTools.includes(expectedTool)) break;
}
@@ -177,6 +182,7 @@ async function run() {
save_status: save.status,
save_code: save.json.code ?? null,
save_msg: save.json.msg || "",
server_uuid: lastServer?.uuid || save.json.data?.uuid || "",
tool_names: lastTools,
has_expected_tool: lastTools.includes(expectedTool),
runtime_status: lastRuntime?.status || null,
@@ -212,11 +218,47 @@ async function run() {
result.reason = `MCP server ${serverName} did not expose ${expectedTool}. See ${apiDiagnosticPath}.`;
return;
}
if (!diagnostic.server_uuid) {
result.status = "fail";
result.reason = `MCP server ${serverName} exposed ${expectedTool}, but the server UUID was not returned. See ${apiDiagnosticPath}.`;
return;
}
await upsertEnvLocal(envLocalPath, {
[serverUuidEnvKey]: diagnostic.server_uuid,
});
result.wrote_env = true;
result.server_uuid = diagnostic.server_uuid;
result.server_uuid_env_key = serverUuidEnvKey;
result.status = "pass";
result.reason = `MCP server ${serverName} is connected and exposes ${expectedTool} through LangBot /api/v1/tools.`;
}
async function upsertEnvLocal(path, updates) {
let text = "";
try {
text = await readFile(path, "utf8");
} catch {
text = "";
}
const lines = text ? text.split(/\r?\n/) : [];
const seen = new Set();
const updated = lines.map((line) => {
const match = line.match(/^([A-Z][A-Z0-9_]*)=/);
if (!match || !Object.prototype.hasOwnProperty.call(updates, match[1])) return line;
seen.add(match[1]);
return `${match[1]}=${updates[match[1]]}`;
});
for (const [key, value] of Object.entries(updates)) {
if (!seen.has(key)) updated.push(`${key}=${value}`);
}
await writeFile(path, `${updated.filter((line, index) => line || index < updated.length - 1).join("\n")}\n`, "utf8");
}
try {
await run();
} catch (error) {
skills/scripts/e2e/pipeline-debug-chat.mjs
+308 -15
View File
@@ -11,12 +11,14 @@ import {
} from "./lib/debug-chat.mjs";
import {
createBrowser,
ensureAuthenticatedBrowser,
ensureEvidence,
evidencePaths,
exitCode,
localIsoWithOffset,
pathExists,
safeScreenshot,
scanBrowserDiagnostics,
writeResult,
} from "./lib/langbot-e2e.mjs";
@@ -50,15 +52,19 @@ const pipelineName = pipelineRequired
const expectedRunnerId = env.LANGBOT_E2E_EXPECTED_RUNNER_ID || "";
const resetDebugChat = boolFromEnv(env.LANGBOT_E2E_RESET_DEBUG_CHAT, false);
const restoreRunnerConfig = boolFromEnv(env.LANGBOT_E2E_RESTORE_RUNNER_CONFIG, true);
const restoreExtensions = boolFromEnv(env.LANGBOT_E2E_RESTORE_EXTENSIONS, true);
const debugChatSessionType = env.LANGBOT_E2E_DEBUG_CHAT_SESSION_TYPE || "person";
const pipelineConfigDiagnosticPath = resolve(paths.evidenceDir, "pipeline-config-diagnostic.json");
const pipelineExtensionsDiagnosticPath = resolve(paths.evidenceDir, "pipeline-extensions-diagnostic.json");
const debugChatResetDiagnosticPath = resolve(paths.evidenceDir, "debug-chat-reset-diagnostic.json");
const pipelineConfigRestoreDiagnosticPath = resolve(paths.evidenceDir, "pipeline-config-restore-diagnostic.json");
const pipelineExtensionsRestoreDiagnosticPath = resolve(paths.evidenceDir, "pipeline-extensions-restore-diagnostic.json");
const metricsPath = resolve(paths.evidenceDir, "metrics.json");
const startedAt = new Date();
let browser;
let restorePlan = null;
let restoreConfigPlan = null;
let restoreExtensionsPlan = null;
let result = {
source: "automation",
case_id: caseId,
@@ -317,6 +323,11 @@ function sanitizePipelineDiagnostic(diagnostic) {
return safe;
}
function sanitizePipelineExtensionsDiagnostic(diagnostic) {
const { restore_extensions: _restoreExtensions, ...safe } = diagnostic || {};
return safe;
}
async function prepareImageFixture(paths) {
if (imagePathEnv) return resolve(imagePathEnv);
if (!imageBase64Path) return "";
@@ -451,6 +462,31 @@ async function inspectAndPatchPipelineConfig(page, {
? runnerConfigs[runnerId]
: {};
const patchKeys = Object.keys(runnerConfigPatch || {});
const sensitiveConfigKeyRe = /(?:api[_-]?key|authorization|bearer|credential|jwt|oauth|password|secret)/i;
const sanitizeConfigValue = (key, value, depth = 0) => {
if (sensitiveConfigKeyRe.test(String(key))) return "[redacted]";
if (typeof value === "string") {
return value.length > 1200 ? `${value.slice(0, 1200)}...[truncated ${value.length - 1200} chars]` : value;
}
if (Array.isArray(value)) {
const items = value.slice(0, 50).map((item) => sanitizeConfigValue(key, item, depth + 1));
if (value.length > 50) items.push(`[truncated ${value.length - 50} items]`);
return items;
}
if (value && typeof value === "object") {
if (depth >= 3) return "[object truncated]";
return Object.fromEntries(
Object.entries(value).map(([childKey, childValue]) => [
childKey,
sanitizeConfigValue(childKey, childValue, depth + 1),
]),
);
}
return value;
};
const pickPatchValues = (config) => Object.fromEntries(
patchKeys.map((key) => [key, sanitizeConfigValue(key, config?.[key])]),
);
const baseDiagnostic = {
status: "ready",
authenticated: true,
@@ -462,6 +498,7 @@ async function inspectAndPatchPipelineConfig(page, {
expected_runner_id: expectedRunnerId || "",
patch_keys: patchKeys,
runner_config_before_keys: Object.keys(currentRunnerConfig),
runner_config_patch_before: pickPatchValues(currentRunnerConfig),
patched: patchKeys.length > 0,
};
@@ -506,6 +543,7 @@ async function inspectAndPatchPipelineConfig(page, {
put_status: update.status,
put_code: update.json.code ?? null,
runner_config_after_keys: Object.keys(updatedRunnerConfig),
runner_config_patch_after: pickPatchValues(updatedRunnerConfig),
restore_config: config,
};
}, {
@@ -548,6 +586,201 @@ async function restorePipelineConfig(page, { backendUrl, pipelineId, config }) {
}, { backendUrl, pipelineId, config });
}
async function inspectAndPatchPipelineExtensions(page, {
backendUrl,
pipelineId,
extensionsPatch,
}) {
return await page.evaluate(async ({
backendUrl,
pipelineId,
extensionsPatch,
}) => {
const token = localStorage.getItem("token");
if (!token) {
return {
status: "blocked",
authenticated: false,
pipeline_id: pipelineId,
reason: "Browser profile has no localStorage token.",
};
}
const headers = {
Authorization: `Bearer ${token}`,
"Content-Type": "application/json",
};
const getJson = async (path) => {
const response = await fetch(`${backendUrl}${path}`, { headers });
return {
status: response.status,
json: await response.json().catch(() => ({})),
};
};
const putJson = async (path, body) => {
const response = await fetch(`${backendUrl}${path}`, {
method: "PUT",
headers,
body: JSON.stringify(body),
});
return {
status: response.status,
json: await response.json().catch(() => ({})),
};
};
if (!pipelineId) {
return {
status: "blocked",
authenticated: true,
pipeline_resolved: false,
reason: "Pipeline id is required before patching extensions.",
};
}
const before = await getJson(`/api/v1/pipelines/${encodeURIComponent(pipelineId)}/extensions`);
let extensions = before.json.data || {};
let getStatus = before.status;
let getCode = before.json.code ?? null;
let fallbackReason = "";
if (before.status >= 400 || before.json.code !== 0) {
const pipelineResponse = await getJson(`/api/v1/pipelines/${encodeURIComponent(pipelineId)}`);
const pipeline = pipelineResponse.json.data?.pipeline || {};
const prefs = pipeline.extensions_preferences || {};
if (pipelineResponse.status >= 400 || pipelineResponse.json.code !== 0 || !pipeline.uuid) {
return {
status: "fail",
authenticated: true,
pipeline_id: pipelineId,
get_status: before.status,
get_code: before.json.code ?? null,
fallback_pipeline_status: pipelineResponse.status,
fallback_pipeline_code: pipelineResponse.json.code ?? null,
reason: before.json.msg || "Could not load pipeline extensions.",
};
}
fallbackReason = before.json.msg || "Could not load pipeline extensions; restored from pipeline preferences.";
extensions = {
enable_all_plugins: prefs.enable_all_plugins ?? true,
enable_all_mcp_servers: prefs.enable_all_mcp_servers ?? true,
enable_all_skills: prefs.enable_all_skills ?? true,
bound_plugins: prefs.plugins || [],
bound_mcp_servers: prefs.mcp_servers || [],
bound_skills: prefs.skills || [],
};
}
const patchKeys = Object.keys(extensionsPatch || {});
const restoreExtensions = {
enable_all_plugins: extensions.enable_all_plugins ?? true,
enable_all_mcp_servers: extensions.enable_all_mcp_servers ?? true,
enable_all_skills: extensions.enable_all_skills ?? true,
bound_plugins: extensions.bound_plugins || [],
bound_mcp_servers: extensions.bound_mcp_servers || [],
bound_skills: extensions.bound_skills || [],
};
const baseDiagnostic = {
status: "ready",
authenticated: true,
pipeline_id: pipelineId,
patch_keys: patchKeys,
patched: patchKeys.length > 0,
get_status: getStatus,
get_code: getCode,
fallback_reason: fallbackReason,
extensions_before: {
enable_all_plugins: restoreExtensions.enable_all_plugins,
enable_all_mcp_servers: restoreExtensions.enable_all_mcp_servers,
enable_all_skills: restoreExtensions.enable_all_skills,
bound_plugins: restoreExtensions.bound_plugins,
bound_mcp_servers: restoreExtensions.bound_mcp_servers,
bound_skills: restoreExtensions.bound_skills,
},
};
if (patchKeys.length === 0) {
return baseDiagnostic;
}
const updateBody = {
enable_all_plugins: Object.prototype.hasOwnProperty.call(extensionsPatch, "enable_all_plugins")
? Boolean(extensionsPatch.enable_all_plugins)
: restoreExtensions.enable_all_plugins,
enable_all_mcp_servers: Object.prototype.hasOwnProperty.call(extensionsPatch, "enable_all_mcp_servers")
? Boolean(extensionsPatch.enable_all_mcp_servers)
: restoreExtensions.enable_all_mcp_servers,
enable_all_skills: Object.prototype.hasOwnProperty.call(extensionsPatch, "enable_all_skills")
? Boolean(extensionsPatch.enable_all_skills)
: restoreExtensions.enable_all_skills,
bound_plugins: Object.prototype.hasOwnProperty.call(extensionsPatch, "bound_plugins")
? extensionsPatch.bound_plugins
: restoreExtensions.bound_plugins,
bound_mcp_servers: Object.prototype.hasOwnProperty.call(extensionsPatch, "bound_mcp_servers")
? extensionsPatch.bound_mcp_servers
: restoreExtensions.bound_mcp_servers,
bound_skills: Object.prototype.hasOwnProperty.call(extensionsPatch, "bound_skills")
? extensionsPatch.bound_skills
: restoreExtensions.bound_skills,
};
const update = await putJson(`/api/v1/pipelines/${encodeURIComponent(pipelineId)}/extensions`, updateBody);
if (update.status >= 400 || update.json.code !== 0) {
return {
...baseDiagnostic,
status: "fail",
put_status: update.status,
put_code: update.json.code ?? null,
reason: update.json.msg || "Pipeline extensions update failed.",
};
}
return {
...baseDiagnostic,
put_status: update.status,
put_code: update.json.code ?? null,
extensions_after: updateBody,
restore_extensions: restoreExtensions,
};
}, {
backendUrl,
pipelineId,
extensionsPatch,
});
}
async function restorePipelineExtensions(page, { backendUrl, pipelineId, extensions }) {
return await page.evaluate(async ({ backendUrl, pipelineId, extensions }) => {
const token = localStorage.getItem("token");
if (!token) {
return {
status: "blocked",
authenticated: false,
pipeline_id: pipelineId,
reason: "Browser profile has no localStorage token.",
};
}
const response = await fetch(`${backendUrl}/api/v1/pipelines/${encodeURIComponent(pipelineId)}/extensions`, {
method: "PUT",
headers: {
Authorization: `Bearer ${token}`,
"Content-Type": "application/json",
},
body: JSON.stringify(extensions),
});
const json = await response.json().catch(() => ({}));
return {
status: response.status >= 400 || json.code !== 0 ? "fail" : "ready",
authenticated: true,
pipeline_id: pipelineId,
put_status: response.status,
put_code: json.code ?? null,
reason: response.status >= 400 || json.code !== 0
? json.msg || "Pipeline extensions restore failed."
: "Pipeline extensions restored.",
};
}, { backendUrl, pipelineId, extensions });
}
async function resetPipelineDebugChat(page, { backendUrl, pipelineId, sessionType }) {
return await page.evaluate(async ({ backendUrl, pipelineId, sessionType }) => {
const token = localStorage.getItem("token");
@@ -590,11 +823,13 @@ try {
const promptSteps = promptStepsFromEnv();
const filesystemChecks = parseJsonEnv("LANGBOT_E2E_FILESYSTEM_CHECKS_JSON", []);
const runnerConfigPatch = parseJsonEnv("LANGBOT_E2E_RUNNER_CONFIG_PATCH_JSON", {});
const extensionsPatch = parseJsonEnv("LANGBOT_E2E_EXTENSIONS_PATCH_JSON", {});
const runnerPatchKeys = Object.keys(runnerConfigPatch);
if (runnerPatchKeys.length > 0 || resetDebugChat || expectedRunnerId) {
const extensionsPatchKeys = Object.keys(extensionsPatch);
if (runnerPatchKeys.length > 0 || extensionsPatchKeys.length > 0 || resetDebugChat || expectedRunnerId) {
if (!backendUrl) {
result.status = "env_issue";
result.reason = "LANGBOT_BACKEND_URL is required for runner config patch, runner assertion, or Debug Chat reset.";
result.reason = "LANGBOT_BACKEND_URL is required for runner config patch, extensions patch, runner assertion, or Debug Chat reset.";
throw new Error(result.reason);
}
}
@@ -602,13 +837,30 @@ try {
result.prompt = promptSteps.length === 1 ? promptSteps[0].prompt : `${promptSteps.length} prompts`;
result.expected_text = promptSteps.at(-1)?.expectedText || expectedText;
const openResult = await openPipelineDebugChat(page, {
pipelineUrl,
pipelineName,
envHint: pipelineRequired
? "case-specific pipeline env mapped to LANGBOT_E2E_PIPELINE_URL or LANGBOT_E2E_PIPELINE_NAME"
: "LANGBOT_PIPELINE_URL or LANGBOT_PIPELINE_NAME",
const authDiagnostic = await ensureAuthenticatedBrowser(page, {
frontendUrl: env.LANGBOT_FRONTEND_URL || "",
backendUrl,
});
result.browser_auth = authDiagnostic;
if (!result.evidence_collected.includes("api_diagnostic")) result.evidence_collected.push("api_diagnostic");
const authFailed = authDiagnostic.status === "env_issue" || authDiagnostic.status === "blocked" || authDiagnostic.status === "fail";
if (authFailed) {
result.status = authDiagnostic.status;
result.reason = authDiagnostic.reason || "Browser authentication failed.";
} else {
result.status = "running";
result.reason = "";
}
const openResult = authFailed
? { opened: false, status: result.status, reason: result.reason }
: await openPipelineDebugChat(page, {
pipelineUrl,
pipelineName,
envHint: pipelineRequired
? "case-specific pipeline env mapped to LANGBOT_E2E_PIPELINE_URL or LANGBOT_E2E_PIPELINE_NAME"
: "LANGBOT_PIPELINE_URL or LANGBOT_PIPELINE_NAME",
});
result.url = page.url();
if (!openResult.opened) {
@@ -617,7 +869,7 @@ try {
} else {
result.status = "running";
result.reason = "";
if (runnerPatchKeys.length > 0 || resetDebugChat || expectedRunnerId) {
if (runnerPatchKeys.length > 0 || extensionsPatchKeys.length > 0 || resetDebugChat || expectedRunnerId) {
const pipelineDiagnostic = await inspectAndPatchPipelineConfig(page, {
backendUrl,
pipelineUrl,
@@ -636,13 +888,35 @@ try {
result.reason = pipelineDiagnostic.reason || "Pipeline config preparation failed.";
} else {
if (pipelineDiagnostic.restore_config && restoreRunnerConfig) {
restorePlan = {
restoreConfigPlan = {
backendUrl,
pipelineId: pipelineDiagnostic.pipeline_id,
config: pipelineDiagnostic.restore_config,
};
}
if (resetDebugChat) {
if (extensionsPatchKeys.length > 0) {
const extensionsDiagnostic = await inspectAndPatchPipelineExtensions(page, {
backendUrl,
pipelineId: pipelineDiagnostic.pipeline_id,
extensionsPatch,
});
const safeExtensionsDiagnostic = sanitizePipelineExtensionsDiagnostic(extensionsDiagnostic);
await writeFile(pipelineExtensionsDiagnosticPath, `${JSON.stringify(safeExtensionsDiagnostic, null, 2)}\n`, "utf8");
result.evidence.pipeline_extensions_diagnostic_json = pipelineExtensionsDiagnosticPath;
result.pipeline_extensions = safeExtensionsDiagnostic;
if (!result.evidence_collected.includes("api_diagnostic")) result.evidence_collected.push("api_diagnostic");
if (extensionsDiagnostic.status === "fail" || extensionsDiagnostic.status === "blocked") {
result.status = extensionsDiagnostic.status;
result.reason = extensionsDiagnostic.reason || "Pipeline extensions preparation failed.";
} else if (extensionsDiagnostic.restore_extensions && restoreExtensions) {
restoreExtensionsPlan = {
backendUrl,
pipelineId: extensionsDiagnostic.pipeline_id,
extensions: extensionsDiagnostic.restore_extensions,
};
}
}
if (!["fail", "blocked", "env_issue"].includes(result.status) && resetDebugChat) {
const resetDiagnostic = await resetPipelineDebugChat(page, {
backendUrl,
pipelineId: pipelineDiagnostic.pipeline_id,
@@ -720,6 +994,15 @@ try {
result.reason = filesystemResult.reason || "Filesystem checks failed.";
}
}
if (result.status === "pass" && !boolFromEnv(env.LANGBOT_E2E_ALLOW_BROWSER_ERRORS, false)) {
const browserDiagnostics = await scanBrowserDiagnostics(paths);
result.browser_diagnostics = browserDiagnostics;
if (browserDiagnostics.status === "fail") {
result.status = "fail";
result.reason = browserDiagnostics.reason;
}
}
}
} catch (error) {
if (!["env_issue", "blocked", "fail", "pass"].includes(result.status) || !result.reason) {
@@ -728,10 +1011,20 @@ try {
result.reason = result.reason || error.message;
} finally {
if (browser?.page) await safeScreenshot(browser.page, paths.screenshot);
if (browser?.page && restorePlan) {
const restoreDiagnostic = await restorePipelineConfig(browser.page, restorePlan).catch((error) => ({
if (browser?.page && restoreExtensionsPlan) {
const restoreDiagnostic = await restorePipelineExtensions(browser.page, restoreExtensionsPlan).catch((error) => ({
status: "fail",
pipeline_id: restorePlan.pipelineId,
pipeline_id: restoreExtensionsPlan.pipelineId,
reason: error.message,
}));
await writeFile(pipelineExtensionsRestoreDiagnosticPath, `${JSON.stringify(restoreDiagnostic, null, 2)}\n`, "utf8");
result.evidence.pipeline_extensions_restore_diagnostic_json = pipelineExtensionsRestoreDiagnosticPath;
result.pipeline_extensions_restore = restoreDiagnostic;
}
if (browser?.page && restoreConfigPlan) {
const restoreDiagnostic = await restorePipelineConfig(browser.page, restoreConfigPlan).catch((error) => ({
status: "fail",
pipeline_id: restoreConfigPlan.pipelineId,
reason: error.message,
}));
await writeFile(pipelineConfigRestoreDiagnosticPath, `${JSON.stringify(restoreDiagnostic, null, 2)}\n`, "utf8");
skills/skills.index.json
+209 -4
View File
@@ -166,14 +166,19 @@
"langrag-parser-golden-e2e",
"langrag-sentinel-kb-discover",
"local-agent-basic-debug-chat",
"local-agent-combo-rag-compaction-tool-debug-chat",
"local-agent-context-compaction-debug-chat",
"local-agent-effective-prompt-debug-chat",
"local-agent-multimodal-debug-chat",
"local-agent-multitool-rag-compaction-debug-chat",
"local-agent-nonstreaming-debug-chat",
"local-agent-parallel-tools-rag-compaction-debug-chat",
"local-agent-plugin-tool-call-debug-chat",
"local-agent-rag-debug-chat",
"local-agent-rag-multimodal-debug-chat",
"local-agent-steering-debug-chat",
"local-agent-tool-error-recovery-debug-chat",
"local-agent-tool-loop-limit-debug-chat",
"mcp-stdio-register",
"mcp-stdio-tool-call",
"pipeline-debug-chat",
@@ -911,6 +916,43 @@
"backend_log"
]
},
{
"id": "local-agent-combo-rag-compaction-tool-debug-chat",
"title": "Local Agent preserves RAG, compacted history, and plugin tool result in one Debug Chat run",
"mode": "agent-browser",
"area": "pipeline",
"type": "regression",
"priority": "p1",
"risk": "high",
"ci_eligible": false,
"tags": [
"local-agent",
"langrag",
"context",
"compaction",
"plugin",
"tools",
"pipeline"
],
"automation": "scripts/e2e/pipeline-debug-chat.mjs",
"setup_automation": [
"node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env",
"node:scripts/e2e/ensure-langrag-sentinel-kb.mjs --write-env",
"case:qa-plugin-smoke-live-install"
],
"setup_provides_env": [
"LANGBOT_LOCAL_AGENT_PIPELINE_URL",
"LANGBOT_LOCAL_AGENT_PIPELINE_NAME",
"LANGBOT_LOCAL_AGENT_RAG_KB_UUID"
],
"evidence_required": [
"ui",
"screenshot",
"console",
"backend_log",
"api_diagnostic"
]
},
{
"id": "local-agent-context-compaction-debug-chat",
"title": "Local Agent compacts long Debug Chat history and preserves older facts",
@@ -1002,9 +1044,47 @@
"backend_log"
]
},
{
"id": "local-agent-multitool-rag-compaction-debug-chat",
"title": "Local Agent preserves RAG, compacted history, and two-step plugin tool loop",
"mode": "agent-browser",
"area": "pipeline",
"type": "regression",
"priority": "p1",
"risk": "high",
"ci_eligible": false,
"tags": [
"local-agent",
"langrag",
"context",
"compaction",
"plugin",
"tools",
"tool-loop",
"pipeline"
],
"automation": "scripts/e2e/pipeline-debug-chat.mjs",
"setup_automation": [
"node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env",
"node:scripts/e2e/ensure-langrag-sentinel-kb.mjs --write-env",
"case:qa-plugin-smoke-live-install"
],
"setup_provides_env": [
"LANGBOT_LOCAL_AGENT_PIPELINE_URL",
"LANGBOT_LOCAL_AGENT_PIPELINE_NAME",
"LANGBOT_LOCAL_AGENT_RAG_KB_UUID"
],
"evidence_required": [
"ui",
"screenshot",
"console",
"backend_log",
"api_diagnostic"
]
},
{
"id": "local-agent-nonstreaming-debug-chat",
"title": "Local Agent Debug Chat returns a deterministic non-streaming response",
"title": "Local Agent Debug Chat returns a deterministic response with UI streaming disabled",
"mode": "agent-browser",
"area": "pipeline",
"type": "regression",
@@ -1030,6 +1110,44 @@
"backend_log"
]
},
{
"id": "local-agent-parallel-tools-rag-compaction-debug-chat",
"title": "Local Agent preserves RAG, compacted history, and parallel plugin tools",
"mode": "agent-browser",
"area": "pipeline",
"type": "regression",
"priority": "p1",
"risk": "high",
"ci_eligible": false,
"tags": [
"local-agent",
"langrag",
"context",
"compaction",
"plugin",
"tools",
"parallel-tools",
"pipeline"
],
"automation": "scripts/e2e/pipeline-debug-chat.mjs",
"setup_automation": [
"node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env",
"node:scripts/e2e/ensure-langrag-sentinel-kb.mjs --write-env",
"case:qa-plugin-smoke-live-install"
],
"setup_provides_env": [
"LANGBOT_LOCAL_AGENT_PIPELINE_URL",
"LANGBOT_LOCAL_AGENT_PIPELINE_NAME",
"LANGBOT_LOCAL_AGENT_RAG_KB_UUID"
],
"evidence_required": [
"ui",
"screenshot",
"console",
"backend_log",
"api_diagnostic"
]
},
{
"id": "local-agent-plugin-tool-call-debug-chat",
"title": "Local Agent can call a plugin-provided tool",
@@ -1157,6 +1275,73 @@
"api_diagnostic"
]
},
{
"id": "local-agent-tool-error-recovery-debug-chat",
"title": "Local Agent feeds plugin tool errors back to the model",
"mode": "agent-browser",
"area": "pipeline",
"type": "regression",
"priority": "p1",
"risk": "high",
"ci_eligible": false,
"tags": [
"local-agent",
"plugin",
"tools",
"tool-error",
"pipeline"
],
"automation": "scripts/e2e/pipeline-debug-chat.mjs",
"setup_automation": [
"node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env",
"case:qa-plugin-smoke-live-install"
],
"setup_provides_env": [
"LANGBOT_LOCAL_AGENT_PIPELINE_URL",
"LANGBOT_LOCAL_AGENT_PIPELINE_NAME"
],
"evidence_required": [
"ui",
"screenshot",
"console",
"backend_log",
"api_diagnostic"
]
},
{
"id": "local-agent-tool-loop-limit-debug-chat",
"title": "Local Agent stops repeated tool calls at max-tool-iterations",
"mode": "agent-browser",
"area": "pipeline",
"type": "regression",
"priority": "p1",
"risk": "high",
"ci_eligible": false,
"tags": [
"local-agent",
"plugin",
"tools",
"tool-loop",
"limit",
"pipeline"
],
"automation": "scripts/e2e/pipeline-debug-chat.mjs",
"setup_automation": [
"node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env",
"case:qa-plugin-smoke-live-install"
],
"setup_provides_env": [
"LANGBOT_LOCAL_AGENT_PIPELINE_URL",
"LANGBOT_LOCAL_AGENT_PIPELINE_NAME"
],
"evidence_required": [
"ui",
"screenshot",
"console",
"backend_log",
"api_diagnostic"
]
},
{
"id": "mcp-stdio-register",
"title": "MCP stdio fixture is registered and exposes qa_mcp_echo",
@@ -1203,7 +1388,8 @@
],
"setup_provides_env": [
"LANGBOT_LOCAL_AGENT_PIPELINE_URL",
"LANGBOT_LOCAL_AGENT_PIPELINE_NAME"
"LANGBOT_LOCAL_AGENT_PIPELINE_NAME",
"LANGBOT_MCP_QA_STDIO_SERVER_UUID"
],
"evidence_required": [
"ui",
@@ -1483,6 +1669,11 @@
"local-agent-context-compaction-debug-chat",
"local-agent-rag-debug-chat",
"local-agent-plugin-tool-call-debug-chat",
"local-agent-tool-error-recovery-debug-chat",
"local-agent-tool-loop-limit-debug-chat",
"local-agent-combo-rag-compaction-tool-debug-chat",
"local-agent-multitool-rag-compaction-debug-chat",
"local-agent-parallel-tools-rag-compaction-debug-chat",
"mcp-stdio-register",
"mcp-stdio-tool-call",
"local-agent-nonstreaming-debug-chat",
@@ -1633,6 +1824,11 @@
"local-agent-context-compaction-debug-chat",
"local-agent-rag-debug-chat",
"local-agent-plugin-tool-call-debug-chat",
"local-agent-tool-error-recovery-debug-chat",
"local-agent-tool-loop-limit-debug-chat",
"local-agent-combo-rag-compaction-tool-debug-chat",
"local-agent-multitool-rag-compaction-debug-chat",
"local-agent-parallel-tools-rag-compaction-debug-chat",
"local-agent-steering-debug-chat",
"mcp-stdio-tool-call",
"local-agent-nonstreaming-debug-chat",
@@ -1692,7 +1888,10 @@
"path": "fixtures/rag/sentinel-doc.txt",
"related_cases": [
"langrag-kb-retrieve",
"local-agent-rag-debug-chat"
"local-agent-rag-debug-chat",
"local-agent-combo-rag-compaction-tool-debug-chat",
"local-agent-multitool-rag-compaction-debug-chat",
"local-agent-parallel-tools-rag-compaction-debug-chat"
]
},
{
@@ -1724,6 +1923,11 @@
"plugin-e2e-smoke",
"local-agent-effective-prompt-debug-chat",
"local-agent-plugin-tool-call-debug-chat",
"local-agent-tool-error-recovery-debug-chat",
"local-agent-tool-loop-limit-debug-chat",
"local-agent-combo-rag-compaction-tool-debug-chat",
"local-agent-multitool-rag-compaction-debug-chat",
"local-agent-parallel-tools-rag-compaction-debug-chat",
"local-agent-steering-debug-chat"
]
},
@@ -1734,7 +1938,8 @@
"path": "fixtures/plugins/qa-plugin-smoke/dist/qa-plugin-smoke-0.1.0.lbpkg",
"related_cases": [
"qa-plugin-smoke-live-install",
"plugin-e2e-smoke"
"plugin-e2e-smoke",
"local-agent-tool-error-recovery-debug-chat"
]
}
],
skills/skills/langbot-testing/cases/local-agent-combo-rag-compaction-tool-debug-chat.yaml
+103
View File
@@ -0,0 +1,103 @@
id: local-agent-combo-rag-compaction-tool-debug-chat
title: "Local Agent preserves RAG, compacted history, and plugin tool result in one Debug Chat run"
mode: agent-browser
area: pipeline
type: regression
priority: p1
risk: high
ci_eligible: false
tags:
- local-agent
- langrag
- context
- compaction
- plugin
- tools
- pipeline
skills:
- langbot-env-setup
- langbot-testing
env:
- LANGBOT_FRONTEND_URL
- LANGBOT_BACKEND_URL
- LANGBOT_LOCAL_AGENT_PIPELINE_URL
- LANGBOT_LOCAL_AGENT_PIPELINE_NAME
env_optional:
- LANGBOT_E2E_FAKE_PROVIDER_BASE_URL
automation: scripts/e2e/pipeline-debug-chat.mjs
automation_env:
- LANGBOT_FRONTEND_URL
- LANGBOT_BACKEND_URL
- LANGBOT_BROWSER_PROFILE
- LANGBOT_CHROMIUM_EXECUTABLE
- LANGBOT_LOCAL_AGENT_PIPELINE_URL
- LANGBOT_LOCAL_AGENT_PIPELINE_NAME
- LANGBOT_LOCAL_AGENT_RAG_KB_UUID
automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL
automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME
automation_expected_runner_id: "plugin:langbot/local-agent/default"
automation_runner_config_patch_json: '{"knowledge-bases":["${LANGBOT_LOCAL_AGENT_RAG_KB_UUID}"],"retrieval-top-k":1,"context-window-tokens":650,"context-reserve-tokens":180,"context-keep-recent-tokens":120,"context-summary-tokens":220,"max-tool-iterations":4,"tool-execution-mode":"serial"}'
automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot","name":"local-agent"},{"author":"qa","name":"plugin-smoke"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}'
automation_restore_runner_config: "1"
automation_restore_extensions: "1"
automation_reset_debug_chat: "1"
automation_debug_chat_session_type: "person"
automation_expected_text: "COMBO_FINAL qa_combo_compaction_sentinel_2406 azalea-cobalt-7421 qa-plugin-smoke:combo-tool-ok-local-agent"
automation_response_timeout_ms: "180000"
automation_prompts_json: '[{"prompt":"请记住这个用于 local-agent 组合回归测试的暗号：qa_combo_compaction_sentinel_2406。请只回复 MEMORY_SET。","expected_text":"MEMORY_SET","response_timeout_ms":"180000"},{"prompt":"下面这轮只用于制造长历史压力，目标标记 RAG_TOOL_COMBO_GOAL 必须被压缩保留。请忽略填充内容，最后只回复 COMBO_CONTEXT_PRESSURE_READY。填充片段 C001 combo context padding. C002 combo context padding. C003 combo context padding. C004 combo context padding. C005 combo context padding. C006 combo context padding. C007 combo context padding. C008 combo context padding. C009 combo context padding. C010 combo context padding. C011 combo context padding. C012 combo context padding. C013 combo context padding. C014 combo context padding. C015 combo context padding. C016 combo context padding. C017 combo context padding. C018 combo context padding. C019 combo context padding. C020 combo context padding. C021 combo context padding. C022 combo context padding. C023 combo context padding. C024 combo context padding. C025 combo context padding. C026 combo context padding. C027 combo context padding. C028 combo context padding. C029 combo context padding. C030 combo context padding. C031 combo context padding. C032 combo context padding. C033 combo context padding. C034 combo context padding. C035 combo context padding. C036 combo context padding. C037 combo context padding. C038 combo context padding. C039 combo context padding. C040 combo context padding.","expected_text":"COMBO_CONTEXT_PRESSURE_READY","response_timeout_ms":"180000"},{"prompt":"qa_combo final check: using the knowledge base, the compacted earlier passcode, and qa_plugin_echo with exactly combo-tool-ok-local-agent, return only COMBO_FINAL plus the passcode, the RAG sentinel, and the plugin tool result.","expected_text":"COMBO_FINAL qa_combo_compaction_sentinel_2406 azalea-cobalt-7421 qa-plugin-smoke:combo-tool-ok-local-agent","response_timeout_ms":"180000"}]'
setup_automation:
- "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env"
- "node:scripts/e2e/ensure-langrag-sentinel-kb.mjs --write-env"
- "case:qa-plugin-smoke-live-install"
setup_provides_env:
- LANGBOT_LOCAL_AGENT_PIPELINE_URL
- LANGBOT_LOCAL_AGENT_PIPELINE_NAME
- LANGBOT_LOCAL_AGENT_RAG_KB_UUID
preconditions:
- "The selected model route supports function/tool calling and deterministic multi-turn answers, or LANGBOT_E2E_FAKE_PROVIDER_BASE_URL points to scripts/e2e/fake-openai-provider.mjs."
- "The LangRAG sentinel knowledge base contains azalea-cobalt-7421 and qa-plugin-smoke exposes qa_plugin_echo."
steps:
- "Ensure the local-agent pipeline, LangRAG sentinel KB, and qa-plugin-smoke fixture plugin are available."
- "Temporarily bind the LangRAG KB, shrink the runner context budget, and allow serial plugin tool execution."
- "Temporarily bind only langbot/local-agent and qa/plugin-smoke; disable MCP servers and skills to isolate the tool surface."
- "Reset the person Debug Chat session for the target pipeline."
- "Send the passcode memory prompt and wait for MEMORY_SET."
- "Send the long pressure prompt and wait for COMBO_CONTEXT_PRESSURE_READY."
- "Ask the runner to combine compacted passcode, RAG sentinel, and qa_plugin_echo result in one final answer."
- "Restore the original runner config and pipeline extension bindings."
checks:
- "UI: All three user messages appear in Debug Chat."
- "UI: The final Bot message contains COMBO_FINAL qa_combo_compaction_sentinel_2406 azalea-cobalt-7421 qa-plugin-smoke:combo-tool-ok-local-agent."
- "API diagnostic: pipeline-config-diagnostic.json shows knowledge-bases, context token settings, and tool-loop settings were patched."
- "API diagnostic: pipeline-extensions-diagnostic.json shows only langbot/local-agent and qa/plugin-smoke were bound."
- "API diagnostic: restore diagnostics show the original runner config and extension bindings were restored."
- "Logs: Backend completes RAG retrieval, plugin tool call, and Debug Chat response without runner timeout or model setup failure."
- "Console: No unexpected frontend runtime errors appear during the run."
evidence_required:
- ui
- screenshot
- console
- backend_log
- api_diagnostic
diagnostics:
- "For token-free deterministic UI coverage, start scripts/e2e/fake-openai-provider.mjs and pass LANGBOT_E2E_FAKE_PROVIDER_BASE_URL to this case."
- "The ensure-langrag-sentinel-kb setup script verifies LANGBOT_E2E_RAG_EXPECTED_TEXT, defaulting to azalea-cobalt-7421, while this case's Debug Chat expected text is the full COMBO_FINAL string."
- "The fake provider intentionally returns COMBO_MISSING_* if the final model input lacks compacted memory, RAG context, or plugin tool result."
- "If the final answer lacks azalea-cobalt-7421, first rerun ensure-langrag-sentinel-kb and local-agent-rag-debug-chat."
- "If the final answer lacks qa-plugin-smoke:combo-tool-ok-local-agent, inspect /api/v1/tools and the pipeline extensions diagnostic."
success_patterns:
- "Processing request from person_websocket"
- "Streaming completed"
failure_patterns:
- "Action invoke_llm_stream call timed out"
- "All models failed during streaming setup"
- "Task exception was never retrieved"
- "COMBO_MISSING_"
- "survey widget blocks debug chat"
troubleshooting:
- local-agent-model-route-unavailable
- tool-name-collision-between-mcp-and-plugin
- plugin-runtime-timeout
- proxy-env-mismatch
- survey-widget-blocks-debug-chat
- debug-chat-history-contaminates-automation
skills/skills/langbot-testing/cases/local-agent-context-compaction-debug-chat.yaml
+10 -2
View File
@@ -19,6 +19,8 @@ env:
- LANGBOT_BACKEND_URL
- LANGBOT_LOCAL_AGENT_PIPELINE_URL
- LANGBOT_LOCAL_AGENT_PIPELINE_NAME
env_optional:
- LANGBOT_E2E_FAKE_PROVIDER_BASE_URL
automation: scripts/e2e/pipeline-debug-chat.mjs
automation_env:
- LANGBOT_FRONTEND_URL
@@ -31,7 +33,9 @@ automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL
automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME
automation_expected_runner_id: "plugin:langbot/local-agent/default"
automation_runner_config_patch_json: '{"context-window-tokens":225,"context-reserve-tokens":50,"context-keep-recent-tokens":30,"context-summary-tokens":105,"knowledge-bases":[]}'
automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot","name":"local-agent"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}'
automation_restore_runner_config: "1"
automation_restore_extensions: "1"
automation_reset_debug_chat: "1"
automation_debug_chat_session_type: "person"
automation_expected_text: "qa_compaction_sentinel_7391"
@@ -43,12 +47,13 @@ setup_provides_env:
- LANGBOT_LOCAL_AGENT_PIPELINE_URL
- LANGBOT_LOCAL_AGENT_PIPELINE_NAME
preconditions:
- "The selected model route can follow short deterministic instructions across multiple Debug Chat turns."
- "The selected model route can follow short deterministic instructions across multiple Debug Chat turns, or LANGBOT_E2E_FAKE_PROVIDER_BASE_URL points to scripts/e2e/fake-openai-provider.mjs."
steps:
- "Open the target local-agent pipeline through LANGBOT_FRONTEND_URL."
- "Use the authenticated browser token only inside automation to GET and PUT /api/v1/pipelines/{uuid}."
- "Assert the saved runner is plugin:langbot/local-agent/default."
- "Temporarily set context-window-tokens, context-reserve-tokens, context-keep-recent-tokens, and context-summary-tokens to force compaction, and clear knowledge-bases so RAG does not answer the memory question."
- "Temporarily bind only the local-agent plugin, disable MCP servers, and disable skills so unrelated visible tools cannot answer or distract the memory question."
- "Reset the person Debug Chat session for the target pipeline."
- "Send the sentinel memory prompt and wait for MEMORY_SET."
- "Send the long padding prompt and wait for CONTEXT_PRESSURE_READY."
@@ -59,6 +64,8 @@ checks:
- "UI: The final Bot message contains qa_compaction_sentinel_7391."
- "API diagnostic: pipeline-config-diagnostic.json shows patched=true and patch_keys include the four token context compaction fields plus knowledge-bases."
- "API diagnostic: pipeline-config-restore-diagnostic.json shows the original runner config was restored."
- "API diagnostic: pipeline-extensions-diagnostic.json shows enable_all_plugins=false, bound_plugins contains langbot/local-agent, enable_all_mcp_servers=false, and enable_all_skills=false."
- "API diagnostic: pipeline-extensions-restore-diagnostic.json shows the original extension bindings were restored."
- "Logs: Backend completes the multi-turn Debug Chat path without runner timeout or model setup failure."
- "Console: No unexpected frontend runtime errors appear during the run."
evidence_required:
@@ -68,7 +75,8 @@ evidence_required:
- backend_log
- api_diagnostic
diagnostics:
- "If the final sentinel is missing, inspect whether pipeline-config-diagnostic.json targeted ai.runner_config[runnerId], cleared knowledge-bases, and whether the backend log shows the local-agent runner loading the small context settings."
- "For token-free deterministic UI coverage, start scripts/e2e/fake-openai-provider.mjs and pass LANGBOT_E2E_FAKE_PROVIDER_BASE_URL to this case; setup will bind the local-agent pipeline to that fake OpenAI-compatible model."
- "If the final sentinel is missing, inspect whether pipeline-config-diagnostic.json targeted ai.runner_config[runnerId], cleared knowledge-bases, whether pipeline-extensions-diagnostic.json isolated unrelated tools, and whether the backend log shows the local-agent runner loading the small context settings."
- "If the model ignores deterministic replies, rerun with a known-good model route before diagnosing ContextAssembler."
- "If restore fails, use pipeline-config-restore-diagnostic.json and GET /api/v1/pipelines/{uuid} to confirm the current saved config before retrying."
success_patterns:
skills/skills/langbot-testing/cases/local-agent-multitool-rag-compaction-debug-chat.yaml
+102
View File
@@ -0,0 +1,102 @@
id: local-agent-multitool-rag-compaction-debug-chat
title: "Local Agent preserves RAG, compacted history, and two-step plugin tool loop"
mode: agent-browser
area: pipeline
type: regression
priority: p1
risk: high
ci_eligible: false
tags:
- local-agent
- langrag
- context
- compaction
- plugin
- tools
- tool-loop
- pipeline
skills:
- langbot-env-setup
- langbot-testing
env:
- LANGBOT_FRONTEND_URL
- LANGBOT_BACKEND_URL
- LANGBOT_LOCAL_AGENT_PIPELINE_URL
- LANGBOT_LOCAL_AGENT_PIPELINE_NAME
env_optional:
- LANGBOT_E2E_FAKE_PROVIDER_BASE_URL
automation: scripts/e2e/pipeline-debug-chat.mjs
automation_env:
- LANGBOT_FRONTEND_URL
- LANGBOT_BACKEND_URL
- LANGBOT_BROWSER_PROFILE
- LANGBOT_CHROMIUM_EXECUTABLE
- LANGBOT_LOCAL_AGENT_PIPELINE_URL
- LANGBOT_LOCAL_AGENT_PIPELINE_NAME
- LANGBOT_LOCAL_AGENT_RAG_KB_UUID
automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL
automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME
automation_expected_runner_id: "plugin:langbot/local-agent/default"
automation_runner_config_patch_json: '{"knowledge-bases":["${LANGBOT_LOCAL_AGENT_RAG_KB_UUID}"],"retrieval-top-k":1,"context-window-tokens":900,"context-reserve-tokens":220,"context-keep-recent-tokens":160,"context-summary-tokens":260,"max-tool-iterations":5,"tool-execution-mode":"serial"}'
automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot","name":"local-agent"},{"author":"qa","name":"plugin-smoke"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}'
automation_restore_runner_config: "1"
automation_restore_extensions: "1"
automation_reset_debug_chat: "1"
automation_debug_chat_session_type: "person"
automation_expected_text: "MULTITOOL_COMBO_FINAL qa_multitool_compaction_sentinel_6718 azalea-cobalt-7421 qa-plugin-smoke:multi-tool-a-local-agent qa-plugin-smoke:multi-tool-b-local-agent"
automation_response_timeout_ms: "180000"
automation_prompts_json: '[{"prompt":"请记住这个用于 local-agent 多工具组合回归测试的暗号：qa_multitool_compaction_sentinel_6718。请只回复 MEMORY_SET。","expected_text":"MEMORY_SET","response_timeout_ms":"180000"},{"prompt":"下面这轮只用于制造长历史压力，目标标记 MULTITOOL_RAG_GOAL 必须被压缩保留。请忽略填充内容，最后只回复 MULTITOOL_CONTEXT_PRESSURE_READY。填充片段 M001 multitool context padding. M002 multitool context padding. M003 multitool context padding. M004 multitool context padding. M005 multitool context padding. M006 multitool context padding. M007 multitool context padding. M008 multitool context padding. M009 multitool context padding. M010 multitool context padding. M011 multitool context padding. M012 multitool context padding. M013 multitool context padding. M014 multitool context padding. M015 multitool context padding. M016 multitool context padding. M017 multitool context padding. M018 multitool context padding. M019 multitool context padding. M020 multitool context padding. M021 multitool context padding. M022 multitool context padding. M023 multitool context padding. M024 multitool context padding. M025 multitool context padding. M026 multitool context padding. M027 multitool context padding. M028 multitool context padding. M029 multitool context padding. M030 multitool context padding. M031 multitool context padding. M032 multitool context padding.","expected_text":"MULTITOOL_CONTEXT_PRESSURE_READY","response_timeout_ms":"180000"},{"prompt":"MULTITOOL_COMBO final check: using the knowledge base, the compacted earlier passcode, and qa_plugin_echo twice with exactly multi-tool-a-local-agent then multi-tool-b-local-agent, return only MULTITOOL_COMBO_FINAL plus the passcode, the RAG sentinel, and both plugin tool results.","expected_text":"MULTITOOL_COMBO_FINAL qa_multitool_compaction_sentinel_6718 azalea-cobalt-7421 qa-plugin-smoke:multi-tool-a-local-agent qa-plugin-smoke:multi-tool-b-local-agent","response_timeout_ms":"180000"}]'
setup_automation:
- "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env"
- "node:scripts/e2e/ensure-langrag-sentinel-kb.mjs --write-env"
- "case:qa-plugin-smoke-live-install"
setup_provides_env:
- LANGBOT_LOCAL_AGENT_PIPELINE_URL
- LANGBOT_LOCAL_AGENT_PIPELINE_NAME
- LANGBOT_LOCAL_AGENT_RAG_KB_UUID
preconditions:
- "The selected model route supports function/tool calling and repeated tool loop turns, or LANGBOT_E2E_FAKE_PROVIDER_BASE_URL points to scripts/e2e/fake-openai-provider.mjs."
- "The LangRAG sentinel knowledge base contains azalea-cobalt-7421 and qa-plugin-smoke exposes qa_plugin_echo."
steps:
- "Ensure the local-agent pipeline, LangRAG sentinel KB, and qa-plugin-smoke fixture plugin are available."
- "Temporarily bind the LangRAG KB, shrink the runner context budget, and set max-tool-iterations high enough for two serial plugin tool calls."
- "Temporarily bind only langbot/local-agent and qa/plugin-smoke; disable MCP servers and skills to isolate the tool surface."
- "Reset the person Debug Chat session for the target pipeline."
- "Send the passcode memory prompt and wait for MEMORY_SET."
- "Send the long pressure prompt and wait for MULTITOOL_CONTEXT_PRESSURE_READY."
- "Ask the runner to combine compacted passcode, RAG sentinel, and two qa_plugin_echo results in one final answer."
- "Restore the original runner config and pipeline extension bindings."
checks:
- "UI: All three user messages appear in Debug Chat."
- "UI: The final Bot message contains MULTITOOL_COMBO_FINAL with the compacted sentinel, azalea-cobalt-7421, and both qa-plugin-smoke tool results."
- "API diagnostic: pipeline-config-diagnostic.json shows knowledge-bases, context token settings, and max-tool-iterations were patched."
- "API diagnostic: pipeline-extensions-diagnostic.json shows only langbot/local-agent and qa/plugin-smoke were bound."
- "Logs: Backend completes RAG retrieval, two plugin tool calls, and Debug Chat response without runner timeout or model setup failure."
- "Console: No unexpected frontend runtime errors appear during the run."
evidence_required:
- ui
- screenshot
- console
- backend_log
- api_diagnostic
diagnostics:
- "For token-free deterministic UI coverage, start scripts/e2e/fake-openai-provider.mjs and pass LANGBOT_E2E_FAKE_PROVIDER_BASE_URL to this case."
- "The fake provider intentionally returns MULTITOOL_COMBO_MISSING_* if the final model input lacks compacted memory, RAG context, or either plugin tool result."
- "If only one tool result is present, inspect max-tool-iterations and the runner tool-loop events."
- "If the final answer lacks azalea-cobalt-7421, first rerun ensure-langrag-sentinel-kb and local-agent-rag-debug-chat."
success_patterns:
- "Processing request from person_websocket"
- "Streaming completed"
failure_patterns:
- "Action invoke_llm_stream call timed out"
- "All models failed during streaming setup"
- "Task exception was never retrieved"
- "MULTITOOL_COMBO_MISSING_"
- "survey widget blocks debug chat"
troubleshooting:
- local-agent-model-route-unavailable
- tool-name-collision-between-mcp-and-plugin
- plugin-runtime-timeout
- proxy-env-mismatch
- survey-widget-blocks-debug-chat
- debug-chat-history-contaminates-automation
skills/skills/langbot-testing/cases/local-agent-nonstreaming-debug-chat.yaml
+5 -3
View File
@@ -1,5 +1,5 @@
id: local-agent-nonstreaming-debug-chat
title: "Local Agent Debug Chat returns a deterministic non-streaming response"
title: "Local Agent Debug Chat returns a deterministic response with UI streaming disabled"
mode: agent-browser
area: pipeline
type: regression
@@ -48,14 +48,16 @@ steps:
checks:
- "UI: The user message appears in Debug Chat."
- "UI: A Bot message appears and contains NONSTREAM_OK."
- "Logs: Backend completes the request as a normal response rather than only relying on the streaming-completed path."
- "UI: The Debug Chat stream switch is disabled for this send path."
- "Logs: Backend completes the request without plugin/runtime timeout or frontend streaming UI errors."
- "Console: No unexpected frontend runtime errors appear during the send/receive path."
evidence_required:
- ui
- console
- backend_log
diagnostics:
- "If the UI still streams after the switch is disabled, inspect the adapter streaming capability and runner config before diagnosing the model."
- "This browser case validates the Debug Chat non-streaming UI delivery path. The runner-internal invoke_llm path is covered by langbot-local-agent component tests using runtime_metadata.streaming_supported=false."
- "If the UI still streams after the switch is disabled, inspect the Debug Chat WebSocket stream flag before diagnosing the model."
- "Use GET /api/v1/pipelines/{uuid} only to confirm the saved runner and model config."
troubleshooting:
- local-agent-model-route-unavailable
skills/skills/langbot-testing/cases/local-agent-parallel-tools-rag-compaction-debug-chat.yaml
+103
View File
@@ -0,0 +1,103 @@
id: local-agent-parallel-tools-rag-compaction-debug-chat
title: "Local Agent preserves RAG, compacted history, and parallel plugin tools"
mode: agent-browser
area: pipeline
type: regression
priority: p1
risk: high
ci_eligible: false
tags:
- local-agent
- langrag
- context
- compaction
- plugin
- tools
- parallel-tools
- pipeline
skills:
- langbot-env-setup
- langbot-testing
env:
- LANGBOT_FRONTEND_URL
- LANGBOT_BACKEND_URL
- LANGBOT_LOCAL_AGENT_PIPELINE_URL
- LANGBOT_LOCAL_AGENT_PIPELINE_NAME
env_optional:
- LANGBOT_E2E_FAKE_PROVIDER_BASE_URL
automation: scripts/e2e/pipeline-debug-chat.mjs
automation_env:
- LANGBOT_FRONTEND_URL
- LANGBOT_BACKEND_URL
- LANGBOT_BROWSER_PROFILE
- LANGBOT_CHROMIUM_EXECUTABLE
- LANGBOT_LOCAL_AGENT_PIPELINE_URL
- LANGBOT_LOCAL_AGENT_PIPELINE_NAME
- LANGBOT_LOCAL_AGENT_RAG_KB_UUID
automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL
automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME
automation_expected_runner_id: "plugin:langbot/local-agent/default"
automation_runner_config_patch_json: '{"knowledge-bases":["${LANGBOT_LOCAL_AGENT_RAG_KB_UUID}"],"retrieval-top-k":1,"context-window-tokens":900,"context-reserve-tokens":220,"context-keep-recent-tokens":160,"context-summary-tokens":260,"max-tool-iterations":3,"tool-execution-mode":"parallel"}'
automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot","name":"local-agent"},{"author":"qa","name":"plugin-smoke"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}'
automation_restore_runner_config: "1"
automation_restore_extensions: "1"
automation_reset_debug_chat: "1"
automation_debug_chat_session_type: "person"
automation_expected_text: "PARALLEL_COMBO_FINAL qa_parallel_compaction_sentinel_8142 azalea-cobalt-7421 qa-plugin-smoke:parallel-tool-a-local-agent qa-plugin-smoke:parallel-tool-b-local-agent"
automation_response_timeout_ms: "180000"
automation_prompts_json: '[{"prompt":"请记住这个用于 local-agent 并行工具组合回归测试的暗号：qa_parallel_compaction_sentinel_8142。请只回复 MEMORY_SET。","expected_text":"MEMORY_SET","response_timeout_ms":"180000"},{"prompt":"下面这轮只用于制造长历史压力，目标标记 PARALLEL_RAG_GOAL 必须被压缩保留。请忽略填充内容，最后只回复 PARALLEL_CONTEXT_PRESSURE_READY。填充片段 P001 parallel context padding. P002 parallel context padding. P003 parallel context padding. P004 parallel context padding. P005 parallel context padding. P006 parallel context padding. P007 parallel context padding. P008 parallel context padding. P009 parallel context padding. P010 parallel context padding. P011 parallel context padding. P012 parallel context padding. P013 parallel context padding. P014 parallel context padding. P015 parallel context padding. P016 parallel context padding. P017 parallel context padding. P018 parallel context padding. P019 parallel context padding. P020 parallel context padding. P021 parallel context padding. P022 parallel context padding. P023 parallel context padding. P024 parallel context padding. P025 parallel context padding. P026 parallel context padding. P027 parallel context padding. P028 parallel context padding. P029 parallel context padding. P030 parallel context padding. P031 parallel context padding. P032 parallel context padding.","expected_text":"PARALLEL_CONTEXT_PRESSURE_READY","response_timeout_ms":"180000"},{"prompt":"PARALLEL_COMBO final check: using the knowledge base, the compacted earlier passcode, and one model turn that calls qa_plugin_echo for both parallel-tool-a-local-agent and parallel-tool-b-local-agent, return only PARALLEL_COMBO_FINAL plus the passcode, the RAG sentinel, and both plugin tool results.","expected_text":"PARALLEL_COMBO_FINAL qa_parallel_compaction_sentinel_8142 azalea-cobalt-7421 qa-plugin-smoke:parallel-tool-a-local-agent qa-plugin-smoke:parallel-tool-b-local-agent","response_timeout_ms":"180000"}]'
setup_automation:
- "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env"
- "node:scripts/e2e/ensure-langrag-sentinel-kb.mjs --write-env"
- "case:qa-plugin-smoke-live-install"
setup_provides_env:
- LANGBOT_LOCAL_AGENT_PIPELINE_URL
- LANGBOT_LOCAL_AGENT_PIPELINE_NAME
- LANGBOT_LOCAL_AGENT_RAG_KB_UUID
preconditions:
- "The selected model route supports function/tool calling with multiple tool calls in one turn, or LANGBOT_E2E_FAKE_PROVIDER_BASE_URL points to scripts/e2e/fake-openai-provider.mjs."
- "The LangRAG sentinel knowledge base contains azalea-cobalt-7421 and qa-plugin-smoke exposes qa_plugin_echo."
steps:
- "Ensure the local-agent pipeline, LangRAG sentinel KB, and qa-plugin-smoke fixture plugin are available."
- "Temporarily bind the LangRAG KB, shrink the runner context budget, and set tool-execution-mode to parallel."
- "Temporarily bind only langbot/local-agent and qa/plugin-smoke; disable MCP servers and skills to isolate the tool surface."
- "Reset the person Debug Chat session for the target pipeline."
- "Send the passcode memory prompt and wait for MEMORY_SET."
- "Send the long pressure prompt and wait for PARALLEL_CONTEXT_PRESSURE_READY."
- "Ask the runner to combine compacted passcode, RAG sentinel, and two same-turn qa_plugin_echo results in one final answer."
- "Restore the original runner config and pipeline extension bindings."
checks:
- "UI: All three user messages appear in Debug Chat."
- "UI: The final Bot message contains PARALLEL_COMBO_FINAL with the compacted sentinel, azalea-cobalt-7421, and both qa-plugin-smoke tool results."
- "API diagnostic: pipeline-config-diagnostic.json shows knowledge-bases, context token settings, max-tool-iterations, and tool-execution-mode=parallel were patched."
- "API diagnostic: pipeline-extensions-diagnostic.json shows only langbot/local-agent and qa/plugin-smoke were bound."
- "Logs: Backend completes RAG retrieval, both plugin tool calls, and Debug Chat response without runner timeout or model setup failure."
- "Console: No unexpected frontend runtime errors appear during the run."
evidence_required:
- ui
- screenshot
- console
- backend_log
- api_diagnostic
diagnostics:
- "For token-free deterministic UI coverage, start scripts/e2e/fake-openai-provider.mjs and pass LANGBOT_E2E_FAKE_PROVIDER_BASE_URL to this case."
- "The fake provider intentionally returns two qa_plugin_echo tool_calls in the same streamed model turn for this prompt."
- "The fake provider intentionally returns PARALLEL_COMBO_MISSING_* if the final model input lacks compacted memory, RAG context, or either plugin tool result."
- "If only one tool result is present, inspect the provider trace and runner parallel tool batch handling."
- "If the final answer lacks azalea-cobalt-7421, first rerun ensure-langrag-sentinel-kb and local-agent-rag-debug-chat."
success_patterns:
- "Processing request from person_websocket"
- "Streaming completed"
failure_patterns:
- "Action invoke_llm_stream call timed out"
- "All models failed during streaming setup"
- "Task exception was never retrieved"
- "PARALLEL_COMBO_MISSING_"
- "survey widget blocks debug chat"
troubleshooting:
- local-agent-model-route-unavailable
- tool-name-collision-between-mcp-and-plugin
- plugin-runtime-timeout
- proxy-env-mismatch
- survey-widget-blocks-debug-chat
- debug-chat-history-contaminates-automation
skills/skills/langbot-testing/cases/local-agent-plugin-tool-call-debug-chat.yaml
+4 -1
View File
@@ -19,6 +19,8 @@ env:
- LANGBOT_BACKEND_URL
- LANGBOT_LOCAL_AGENT_PIPELINE_URL
- LANGBOT_LOCAL_AGENT_PIPELINE_NAME
env_optional:
- LANGBOT_E2E_FAKE_PROVIDER_BASE_URL
automation: scripts/e2e/pipeline-debug-chat.mjs
automation_env:
- LANGBOT_FRONTEND_URL
@@ -38,7 +40,7 @@ setup_provides_env:
- LANGBOT_LOCAL_AGENT_PIPELINE_URL
- LANGBOT_LOCAL_AGENT_PIPELINE_NAME
preconditions:
- "The selected model route supports function/tool calling."
- "The selected model route supports function/tool calling, or LANGBOT_E2E_FAKE_PROVIDER_BASE_URL points to scripts/e2e/fake-openai-provider.mjs."
steps:
- "Install or enable the bundled qa-plugin-smoke fixture plugin."
- "Confirm /api/v1/tools or the plugin detail shows qa_plugin_echo."
@@ -58,6 +60,7 @@ evidence_required:
- backend_log
- api_diagnostic
diagnostics:
- "For token-free deterministic UI coverage, start scripts/e2e/fake-openai-provider.mjs and pass LANGBOT_E2E_FAKE_PROVIDER_BASE_URL to this case; setup will bind the local-agent pipeline to that fake OpenAI-compatible model."
- "If qa_plugin_echo is not listed, rebuild and reinstall the qa-plugin-smoke fixture plugin."
- "If the selected model returns model_not_found or no available channel only when tools are provided, switch to a known-good function-calling model before diagnosing plugin tools or local-agent."
troubleshooting:
skills/skills/langbot-testing/cases/local-agent-rag-debug-chat.yaml
+5 -1
View File
@@ -18,6 +18,8 @@ env:
- LANGBOT_BACKEND_URL
- LANGBOT_LOCAL_AGENT_PIPELINE_URL
- LANGBOT_LOCAL_AGENT_PIPELINE_NAME
env_optional:
- LANGBOT_E2E_FAKE_PROVIDER_BASE_URL
automation: scripts/e2e/pipeline-debug-chat.mjs
automation_env:
- LANGBOT_FRONTEND_URL
@@ -43,7 +45,7 @@ setup_provides_env:
- LANGBOT_LOCAL_AGENT_PIPELINE_NAME
- LANGBOT_LOCAL_AGENT_RAG_KB_UUID
preconditions:
- "The target pipeline already has a text-capable model route that is available for this run."
- "The target pipeline already has a text-capable model route that is available for this run, or LANGBOT_E2E_FAKE_PROVIDER_BASE_URL points to scripts/e2e/fake-openai-provider.mjs."
steps:
- "Ensure case langrag-kb-retrieve has produced a knowledge base containing sentinel azalea-cobalt-7421."
- "Open LANGBOT_FRONTEND_URL."
@@ -68,6 +70,8 @@ evidence_required:
- backend_log
- api_diagnostic
diagnostics:
- "For token-free deterministic UI coverage, start scripts/e2e/fake-openai-provider.mjs and pass LANGBOT_E2E_FAKE_PROVIDER_BASE_URL to this case; setup will bind the local-agent pipeline to that fake OpenAI-compatible model."
- "The ensure-langrag-sentinel-kb setup script verifies LANGBOT_E2E_RAG_EXPECTED_TEXT, defaulting to azalea-cobalt-7421, so the Debug Chat expected text does not leak into KB setup."
- "Use GET /api/v1/pipelines/{uuid} only to confirm the saved runner_config contains the knowledge base uuid."
- "If the bot ignores the knowledge base, rerun Retrieve Test before debugging the runner."
troubleshooting:
skills/skills/langbot-testing/cases/local-agent-steering-debug-chat.yaml
-1
View File
@@ -74,7 +74,6 @@ success_patterns:
- "Steering"
- "Streaming completed"
failure_patterns:
- "STEERING_NO_FOLLOWUP"
- "Action invoke_llm_stream call timed out"
- "All models failed during streaming setup"
- "Task exception was never retrieved"
skills/skills/langbot-testing/cases/local-agent-tool-error-recovery-debug-chat.yaml
+94
View File
@@ -0,0 +1,94 @@
id: local-agent-tool-error-recovery-debug-chat
title: "Local Agent feeds plugin tool errors back to the model"
mode: agent-browser
area: pipeline
type: regression
priority: p1
risk: high
ci_eligible: false
tags:
- local-agent
- plugin
- tools
- tool-error
- pipeline
skills:
- langbot-env-setup
- langbot-testing
env:
- LANGBOT_FRONTEND_URL
- LANGBOT_BACKEND_URL
- LANGBOT_LOCAL_AGENT_PIPELINE_URL
- LANGBOT_LOCAL_AGENT_PIPELINE_NAME
env_optional:
- LANGBOT_E2E_FAKE_PROVIDER_BASE_URL
automation: scripts/e2e/pipeline-debug-chat.mjs
automation_env:
- LANGBOT_FRONTEND_URL
- LANGBOT_BACKEND_URL
- LANGBOT_BROWSER_PROFILE
- LANGBOT_CHROMIUM_EXECUTABLE
- LANGBOT_LOCAL_AGENT_PIPELINE_URL
- LANGBOT_LOCAL_AGENT_PIPELINE_NAME
automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL
automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME
automation_expected_runner_id: "plugin:langbot/local-agent/default"
automation_runner_config_patch_json: '{"knowledge-bases":[],"max-tool-iterations":3,"tool-execution-mode":"serial"}'
automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot","name":"local-agent"},{"author":"qa","name":"plugin-smoke"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}'
automation_restore_runner_config: "1"
automation_restore_extensions: "1"
automation_reset_debug_chat: "1"
automation_debug_chat_session_type: "person"
automation_prompt: "TOOL_ERROR_RECOVERY regression: call qa_plugin_fail with exactly tool-error-recovery-local-agent. After observing the tool error, return only TOOL_ERROR_RECOVERY_FINAL qa-plugin-smoke forced failure observed."
automation_expected_text: "TOOL_ERROR_RECOVERY_FINAL qa-plugin-smoke forced failure observed"
automation_response_timeout_ms: "180000"
setup_automation:
- "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env"
- "case:qa-plugin-smoke-live-install"
setup_provides_env:
- LANGBOT_LOCAL_AGENT_PIPELINE_URL
- LANGBOT_LOCAL_AGENT_PIPELINE_NAME
preconditions:
- "The selected model route supports function/tool calling, or LANGBOT_E2E_FAKE_PROVIDER_BASE_URL points to scripts/e2e/fake-openai-provider.mjs."
- "qa-plugin-smoke exposes qa_plugin_fail; the setup case reinstalls the fixture if an older installed package is missing it."
steps:
- "Ensure the local-agent pipeline and qa-plugin-smoke fixture plugin are available."
- "Temporarily bind only langbot/local-agent plus qa/plugin-smoke and clear RAG knowledge bases."
- "Reset the person Debug Chat session for the target pipeline."
- "Send the TOOL_ERROR_RECOVERY prompt."
- "Verify the failing plugin tool result is fed back into the model and the final answer contains the recovery sentinel."
- "Restore the original runner config and pipeline extension bindings."
checks:
- "UI: The final Bot message contains TOOL_ERROR_RECOVERY_FINAL qa-plugin-smoke forced failure observed."
- "API diagnostic: pipeline-config-diagnostic.json shows max-tool-iterations=3 and knowledge-bases cleared."
- "API diagnostic: pipeline-extensions-diagnostic.json shows only langbot/local-agent and qa/plugin-smoke were bound."
- "Logs: Backend shows the request completed without runner.timeout, runner.tool_error run failure, or model setup failure."
- "Console: No unexpected frontend runtime errors appear during Debug Chat."
evidence_required:
- ui
- screenshot
- console
- backend_log
- api_diagnostic
diagnostics:
- "For token-free deterministic UI coverage, start scripts/e2e/fake-openai-provider.mjs and pass LANGBOT_E2E_FAKE_PROVIDER_BASE_URL to this case."
- "The fake provider calls qa_plugin_fail once, then only returns the final sentinel after it sees an Error: tool result."
- "If the run returns a generic answer, inspect whether qa_plugin_fail was exposed by /api/v1/tools and whether the installed fixture plugin was refreshed."
success_patterns:
- "TOOL_ERROR_RECOVERY_FINAL"
- "Streaming completed"
failure_patterns:
- "Action invoke_llm_stream call timed out"
- "All models failed during streaming setup"
- "Task exception was never retrieved"
- "runner.timeout"
- "runner.tool_error"
- "Tool call iteration limit reached"
- "survey widget blocks debug chat"
troubleshooting:
- local-agent-model-route-unavailable
- tool-name-collision-between-mcp-and-plugin
- plugin-runtime-timeout
- proxy-env-mismatch
- survey-widget-blocks-debug-chat
- debug-chat-history-contaminates-automation
skills/skills/langbot-testing/cases/local-agent-tool-loop-limit-debug-chat.yaml
+92
View File
@@ -0,0 +1,92 @@
id: local-agent-tool-loop-limit-debug-chat
title: "Local Agent stops repeated tool calls at max-tool-iterations"
mode: agent-browser
area: pipeline
type: regression
priority: p1
risk: high
ci_eligible: false
tags:
- local-agent
- plugin
- tools
- tool-loop
- limit
- pipeline
skills:
- langbot-env-setup
- langbot-testing
env:
- LANGBOT_FRONTEND_URL
- LANGBOT_BACKEND_URL
- LANGBOT_LOCAL_AGENT_PIPELINE_URL
- LANGBOT_LOCAL_AGENT_PIPELINE_NAME
env_optional:
- LANGBOT_E2E_FAKE_PROVIDER_BASE_URL
automation: scripts/e2e/pipeline-debug-chat.mjs
automation_env:
- LANGBOT_FRONTEND_URL
- LANGBOT_BACKEND_URL
- LANGBOT_BROWSER_PROFILE
- LANGBOT_CHROMIUM_EXECUTABLE
- LANGBOT_LOCAL_AGENT_PIPELINE_URL
- LANGBOT_LOCAL_AGENT_PIPELINE_NAME
automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL
automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME
automation_expected_runner_id: "plugin:langbot/local-agent/default"
automation_runner_config_patch_json: '{"knowledge-bases":[],"max-tool-iterations":2,"tool-execution-mode":"serial"}'
automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot","name":"local-agent"},{"author":"qa","name":"plugin-smoke"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}'
automation_restore_runner_config: "1"
automation_restore_extensions: "1"
automation_reset_debug_chat: "1"
automation_debug_chat_session_type: "person"
automation_prompt: "LOOP_LIMIT regression: keep requesting qa_plugin_echo with exactly loop-limit-repeat-local-agent until the runner stops you. The correct final visible answer is the runner limit message."
automation_expected_text: "Tool call iteration limit reached. I stopped before calling more tools."
automation_response_timeout_ms: "180000"
setup_automation:
- "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env"
- "case:qa-plugin-smoke-live-install"
setup_provides_env:
- LANGBOT_LOCAL_AGENT_PIPELINE_URL
- LANGBOT_LOCAL_AGENT_PIPELINE_NAME
preconditions:
- "The selected model route supports function/tool calling, or LANGBOT_E2E_FAKE_PROVIDER_BASE_URL points to scripts/e2e/fake-openai-provider.mjs."
- "qa-plugin-smoke exposes qa_plugin_echo."
steps:
- "Ensure the local-agent pipeline and qa-plugin-smoke fixture plugin are available."
- "Temporarily set max-tool-iterations to 2 and bind only langbot/local-agent plus qa/plugin-smoke."
- "Reset the person Debug Chat session for the target pipeline."
- "Send the LOOP_LIMIT prompt and wait for the runner limit message."
- "Restore the original runner config and pipeline extension bindings."
checks:
- "UI: The final Bot message contains Tool call iteration limit reached. I stopped before calling more tools."
- "API diagnostic: pipeline-config-diagnostic.json shows max-tool-iterations was patched to 2."
- "API diagnostic: pipeline-extensions-diagnostic.json shows only langbot/local-agent and qa/plugin-smoke were bound."
- "Logs: Backend shows the request completed without infinite loop, runner timeout, or model setup failure."
- "Console: No unexpected frontend runtime errors appear during Debug Chat."
evidence_required:
- ui
- screenshot
- console
- backend_log
- api_diagnostic
diagnostics:
- "For token-free deterministic UI coverage, start scripts/e2e/fake-openai-provider.mjs and pass LANGBOT_E2E_FAKE_PROVIDER_BASE_URL to this case."
- "The fake provider intentionally keeps requesting qa_plugin_echo; this case passes only when the runner stops at max-tool-iterations."
- "If the run times out, inspect whether max-tool-iterations was patched or whether the runner ignored tool loop limits."
success_patterns:
- "Tool call iteration limit reached"
- "Streaming completed"
failure_patterns:
- "Action invoke_llm_stream call timed out"
- "All models failed during streaming setup"
- "Task exception was never retrieved"
- "runner.timeout"
- "survey widget blocks debug chat"
troubleshooting:
- local-agent-model-route-unavailable
- tool-name-collision-between-mcp-and-plugin
- plugin-runtime-timeout
- proxy-env-mismatch
- survey-widget-blocks-debug-chat
- debug-chat-history-contaminates-automation
skills/skills/langbot-testing/cases/mcp-stdio-register.yaml
+2
View File
@@ -34,8 +34,10 @@ steps:
- "Poll /api/v1/tools and /api/v1/mcp/servers/qa-local-stdio until qa_mcp_echo is visible."
checks:
- "API diagnostic: qa-local-stdio runtime_status is connected."
- "API diagnostic: qa-local-stdio server_uuid is present for pipeline extension binding."
- "API diagnostic: runtime_tool_names includes qa_mcp_echo."
- "API diagnostic: /api/v1/tools includes qa_mcp_echo."
- "Automation writes LANGBOT_MCP_QA_STDIO_SERVER_UUID into skills/.env.local for downstream tool-call cases."
- "Console and network logs contain no unexpected frontend/runtime failures."
evidence_required:
- screenshot
skills/skills/langbot-testing/cases/mcp-stdio-tool-call.yaml
+12 -1
View File
@@ -18,15 +18,23 @@ env:
- LANGBOT_BACKEND_URL
- LANGBOT_LOCAL_AGENT_PIPELINE_URL
- LANGBOT_LOCAL_AGENT_PIPELINE_NAME
env_optional:
- LANGBOT_E2E_FAKE_PROVIDER_BASE_URL
automation: scripts/e2e/pipeline-debug-chat.mjs
automation_env:
- LANGBOT_FRONTEND_URL
- LANGBOT_BACKEND_URL
- LANGBOT_BROWSER_PROFILE
- LANGBOT_CHROMIUM_EXECUTABLE
- LANGBOT_LOCAL_AGENT_PIPELINE_URL
- LANGBOT_LOCAL_AGENT_PIPELINE_NAME
- LANGBOT_MCP_QA_STDIO_SERVER_UUID
automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL
automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME
automation_expected_runner_id: "plugin:langbot/local-agent/default"
automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot","name":"local-agent"}],"enable_all_mcp_servers":false,"bound_mcp_servers":["${LANGBOT_MCP_QA_STDIO_SERVER_UUID}"],"enable_all_skills":false,"bound_skills":[]}'
automation_restore_extensions: "1"
automation_reset_debug_chat: "1"
automation_prompt: "Call the qa_mcp_echo MCP tool with exactly this text: mcp-ok-local-agent. Return only the tool result."
automation_expected_text: "qa_mcp_echo:mcp-ok-local-agent"
automation_response_timeout_ms: "180000"
@@ -36,6 +44,7 @@ setup_automation:
setup_provides_env:
- LANGBOT_LOCAL_AGENT_PIPELINE_URL
- LANGBOT_LOCAL_AGENT_PIPELINE_NAME
- LANGBOT_MCP_QA_STDIO_SERVER_UUID
failure_patterns:
- "qa-plugin-smoke:mcp-ok-local-agent"
- "qa_echo:mcp-ok-local-agent"
@@ -45,7 +54,7 @@ failure_patterns:
- "no available channel for model"
preconditions:
- "box.local.allowed_mount_roots includes the bundled MCP fixture directory when LangBot runs stdio MCP servers through Box."
- "The selected model route supports function/tool calling."
- "The selected model route supports function/tool calling, or LANGBOT_E2E_FAKE_PROVIDER_BASE_URL points to scripts/e2e/fake-openai-provider.mjs."
steps:
- "Open LANGBOT_FRONTEND_URL."
- "Navigate to MCP Servers."
@@ -66,6 +75,7 @@ checks:
- "API diagnostic: /api/v1/tools contains qa_mcp_echo."
- "UI: Debug Chat bot response contains qa_mcp_echo:mcp-ok-local-agent."
- "Logs: Backend logs show the MCP tool call was executed, not only listed."
- "API diagnostic: pipeline-extensions-diagnostic.json shows the qa-local-stdio server UUID is the only bound MCP server for the run."
- "Console: No unexpected frontend errors appear during MCP form use or Debug Chat."
evidence_required:
- ui
@@ -73,6 +83,7 @@ evidence_required:
- backend_log
- api_diagnostic
diagnostics:
- "For token-free deterministic UI coverage, start scripts/e2e/fake-openai-provider.mjs and pass LANGBOT_E2E_FAKE_PROVIDER_BASE_URL to this case; setup will bind the local-agent pipeline to that fake OpenAI-compatible model."
- "Run node scripts/e2e/mcp-stdio-fixture.mjs to verify the bundled stdio fixture can list and call qa_mcp_echo without involving a model provider."
- "Run node scripts/e2e/mcp-stdio-register.mjs to upsert qa-local-stdio in LangBot and verify /api/v1/tools exposes qa_mcp_echo."
- "If backend logs show host_path is outside allowed_mount_roots, add the fixture directory to box.local.allowed_mount_roots in the local LangBot data config."
skills/skills/langbot-testing/cases/qa-plugin-smoke-live-install.yaml
+3 -2
View File
@@ -20,14 +20,14 @@ env:
automation: scripts/e2e/install-qa-plugin-smoke.mjs
automation_plugin_package: "skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/dist/qa-plugin-smoke-0.1.0.lbpkg"
automation_expected_plugin_id: "qa/plugin-smoke"
automation_expected_tool: "qa_plugin_echo"
automation_expected_tool: "qa_plugin_echo,qa_plugin_fail"
steps:
- "Run `rtk bin/lbs test run qa-plugin-smoke-live-install --dry-run` first; remove `--dry-run` only after readiness points at a local test LangBot instance."
- "Automation authenticates the local test user, uploads the QA plugin smoke .lbpkg package when missing, waits for install, and checks tool exposure."
checks:
- "automation-result.json status is pass."
- "/api/v1/plugins lists qa/plugin-smoke after install."
- "/api/v1/tools lists qa_plugin_echo after install."
- "/api/v1/tools lists qa_plugin_echo and qa_plugin_fail after install."
evidence_required:
- api_diagnostic
- filesystem
@@ -39,6 +39,7 @@ success_patterns:
failure_patterns:
- "Plugin install task did not complete successfully"
- "qa_plugin_echo is not listed"
- "qa_plugin_fail is not listed"
troubleshooting:
- plugin-runtime-timeout
- plugin-dependency-install-offline
skills/skills/langbot-testing/fixtures/fixtures.json
+13 -2
View File
@@ -49,7 +49,13 @@
"title": "LangRAG sentinel text document",
"kind": "text",
"path": "fixtures/rag/sentinel-doc.txt",
"related_cases": ["langrag-kb-retrieve", "local-agent-rag-debug-chat"],
"related_cases": [
"langrag-kb-retrieve",
"local-agent-rag-debug-chat",
"local-agent-combo-rag-compaction-tool-debug-chat",
"local-agent-multitool-rag-compaction-debug-chat",
"local-agent-parallel-tools-rag-compaction-debug-chat"
],
"checks": ["exists"]
},
{
@@ -78,6 +84,11 @@
"plugin-e2e-smoke",
"local-agent-effective-prompt-debug-chat",
"local-agent-plugin-tool-call-debug-chat",
"local-agent-tool-error-recovery-debug-chat",
"local-agent-tool-loop-limit-debug-chat",
"local-agent-combo-rag-compaction-tool-debug-chat",
"local-agent-multitool-rag-compaction-debug-chat",
"local-agent-parallel-tools-rag-compaction-debug-chat",
"local-agent-steering-debug-chat"
],
"checks": ["exists"]
@@ -87,7 +98,7 @@
"title": "QA plugin smoke prebuilt package",
"kind": "plugin_package",
"path": "fixtures/plugins/qa-plugin-smoke/dist/qa-plugin-smoke-0.1.0.lbpkg",
"related_cases": ["qa-plugin-smoke-live-install", "plugin-e2e-smoke"],
"related_cases": ["qa-plugin-smoke-live-install", "plugin-e2e-smoke", "local-agent-tool-error-recovery-debug-chat"],
"checks": ["exists", "zip_package"]
}
]
skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/README.md
+1
View File
@@ -7,3 +7,4 @@ Tools:
- `qa_echo(text)` returns `qa-plugin-smoke:<text>`.
- `qa_plugin_echo(text)` returns `qa-plugin-smoke:<text>`.
- `qa_plugin_sleep(seconds, text)` waits up to 15 seconds and returns `qa-plugin-smoke:sleep:<seconds>:<text>`.
- `qa_plugin_fail(text)` raises a deterministic error containing `text`.
skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/components/tools/qa_plugin_fail.py
+17
View File
@@ -0,0 +1,17 @@
from __future__ import annotations
from typing import Any
from langbot_plugin.api.definition.components.tool.tool import Tool
from langbot_plugin.api.entities.builtin.provider import session as provider_session
class QAPluginFailTool(Tool):
async def call(
self,
params: dict[str, Any],
session: provider_session.Session,
query_id: int,
) -> str:
text = str(params.get("text", "qa-plugin-fail"))
raise RuntimeError(f"qa-plugin-smoke forced failure: {text}")
skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/components/tools/qa_plugin_fail.yaml
+24
View File
@@ -0,0 +1,24 @@
apiVersion: v1
kind: Tool
metadata:
name: qa_plugin_fail
label:
en_US: QA Plugin Fail
zh_Hans: QA 插件失败工具
description:
en_US: Raises a deterministic error for local-agent tool error recovery tests.
zh_Hans: 为 local-agent 工具错误恢复测试抛出确定性错误。
spec:
parameters:
type: object
properties:
text:
type: string
description: Text included in the deterministic failure message.
required:
- text
llm_prompt: Always fail with a deterministic qa-plugin-smoke error for tool error recovery tests.
execution:
python:
path: qa_plugin_fail.py
attr: QAPluginFailTool
skills/skills/langbot-testing/references/agent-runner-release-gate.md
+3 -1
View File
@@ -111,13 +111,15 @@ Each probe writes `automation-result.json` and probe logs under
| Deterministic QA runner install | `agent-runner-live-install` | A local `.lbpkg` AgentRunner package can install and register a runner. |
| Deterministic QA runner Debug Chat | `agent-runner-qa-debug-chat` | The installed QA runner executes through WebUI Debug Chat without a model provider. |
| Required runner plugins | `agent-runner-release-preflight` | `langbot/local-agent` and `langbot/acp-agent-runner` are visible to the host. |
| Required QA plugin tool | `plugin-e2e-smoke`, `agent-runner-release-preflight` | The deterministic `qa_plugin_echo` tool is exposed before tool-loop cases start. |
| Required QA plugin tools | `plugin-e2e-smoke`, `agent-runner-release-preflight`, `qa-plugin-smoke-live-install` | The deterministic `qa_plugin_echo` and `qa_plugin_fail` tools are exposed before tool-loop and tool-error cases start. |
| Knowledge base fixture | `langrag-kb-retrieve`, `local-agent-rag-debug-chat` | LangRAG data is queryable and the runner inserts retrieved context. |
| Effective prompt bridge | `local-agent-effective-prompt-debug-chat` | Host prompt preprocessing reaches the runner. |
| History and compaction | `local-agent-context-compaction-debug-chat` | Runner-owned history budgeting keeps recoverable older context. |
| Streaming LLM | `local-agent-basic-debug-chat` | The default streaming path returns a visible answer. |
| Non-streaming LLM | `local-agent-nonstreaming-debug-chat` | The non-streaming adapter path returns a visible answer. |
| Plugin tool loop | `local-agent-plugin-tool-call-debug-chat` | Function-call capable models can call host plugin tools through authorization. |
| Plugin tool error recovery | `local-agent-tool-error-recovery-debug-chat` | Tool execution errors are serialized into model-facing tool results and the model can produce a final answer instead of failing the run. |
| Parallel plugin tool batch | `local-agent-parallel-tools-rag-compaction-debug-chat` | Local-agent executes multiple same-turn plugin tool calls and returns both results with RAG and compacted history. |
| MCP registration | `mcp-stdio-register` | The deterministic stdio MCP server is registered and exposes `qa_mcp_echo`. |
| MCP tool loop | `mcp-stdio-tool-call` | Local-agent can call the registered MCP tool through the same tool loop. |
| Multimodal input | `local-agent-multimodal-debug-chat` | Image upload and structured input reach the runner. |
skills/skills/langbot-testing/references/local-agent-runner-coverage.md
+12 -3
View File
@@ -29,10 +29,15 @@ These browser cases are the minimum gate for a local-agent migration check:
| `local-agent-rag-debug-chat` | Knowledge-base authorization, retrieval, and RAG prompt insertion | Bot returns the KB sentinel, not a generic answer. |
| `mcp-stdio-tool-call` | MCP stdio discovery, tool detail, model function calling, and tool execution | Bot returns `qa_mcp_echo:<input>` and backend logs the MCP tool call. |
| `local-agent-plugin-tool-call-debug-chat` | Plugin tool discovery, tool detail, model function calling, and tool execution | Bot returns `qa-plugin-smoke:<input>` and backend logs the plugin tool call. |
| `local-agent-tool-error-recovery-debug-chat` | Plugin tool execution failure, model-facing error tool result, and final recovery turn | Bot returns `TOOL_ERROR_RECOVERY_FINAL...` only after the model sees the tool error result. |
| `local-agent-tool-loop-limit-debug-chat` | Repeated plugin tool requests and max tool iteration enforcement | Bot returns the runner limit message instead of looping indefinitely. |
| `local-agent-combo-rag-compaction-tool-debug-chat` | RAG, compacted history, and plugin tool execution in the same Debug Chat run | Bot returns `COMBO_FINAL` with the compacted sentinel, KB sentinel, and plugin tool result together. |
| `local-agent-multitool-rag-compaction-debug-chat` | RAG, compacted history, and repeated serial plugin tool loop turns in the same Debug Chat run | Bot returns `MULTITOOL_COMBO_FINAL` with the compacted sentinel, KB sentinel, and both plugin tool results together. |
| `local-agent-parallel-tools-rag-compaction-debug-chat` | RAG, compacted history, and two plugin tool calls emitted in the same model turn | Bot returns `PARALLEL_COMBO_FINAL` with the compacted sentinel, KB sentinel, and both plugin tool results together. |
| `local-agent-steering-debug-chat` | Host steering claim, runner pull at turn boundary, and follow-up injection during an active tool loop | Two user messages produce one assistant response containing the steering sentinel. |
| `local-agent-multimodal-debug-chat` | Image upload, structured input contents, and multimodal runner consumption | UI shows uploaded image and bot returns `IMAGE_OK`; backend receives an image input. |
| `local-agent-rag-multimodal-debug-chat` | RAG insertion while structured image input is present | UI shows uploaded image, bot returns the KB sentinel, and backend logs the same request with `[Image]`. |
| `local-agent-nonstreaming-debug-chat` | Host non-streaming adapter path and runner non-streaming invocation | Bot returns `NONSTREAM_OK`; backend completes without the streaming-completed path. |
| `local-agent-nonstreaming-debug-chat` | Debug Chat non-streaming UI delivery path | Bot returns `NONSTREAM_OK` with the Debug Chat stream switch disabled; runner-internal `invoke_llm` mode is covered by local-agent component tests using `runtime_metadata.streaming_supported=false`. |
## Full Coverage Matrix
@@ -44,14 +49,18 @@ These browser cases are the minimum gate for a local-agent migration check:
| Multimodal plus RAG | Run `local-agent-rag-multimodal-debug-chat`. | RAG sentinel is still retrievable and the image is not dropped from the user message; exact image-preservation inside the model message is covered by unit tests. |
| History and context compaction | Run `local-agent-context-compaction-debug-chat` with a small temporary `context-window-tokens` budget. | The runner compacts older history into `<conversation_summary>` and the final answer still recovers the older sentinel from the compacted context. |
| Streaming model invocation | Enable Debug Chat streaming and ask for `OK`. | UI receives incremental bot output and backend logs streaming completion. |
| Non-streaming model invocation | Disable Debug Chat streaming or use a non-streaming adapter path. | UI receives a final bot message and backend logs a normal response completion. |
| Non-streaming UI delivery | Disable Debug Chat streaming. | UI receives a final bot message without frontend streaming errors. |
| Non-streaming model invocation | Use local-agent component tests with `runtime_metadata.streaming_supported=false` or a host adapter that does not support streaming. | Runner calls `invoke_llm` instead of `invoke_llm_stream` and emits `message.completed`. |
| Model fallback before first chunk | Configure a failing primary and working fallback, preferably with a controlled test provider. | First model failure does not fail the run; fallback model produces the final answer. |
| Failure after streaming commit | Use a controlled provider that emits one chunk and then fails. | Runner reports a terminal run failure and does not fallback after partial output. |
| No authorized model | Clear model config or configure a model not in run resources. | Runner returns `runner.no_model` instead of calling an unauthorized model. |
| MCP tool call | Use `qa-local-stdio` and `qa_mcp_echo`. | Bot returns the exact `qa_mcp_echo:<input>` result; `/api/v1/tools` contains `qa_mcp_echo`. |
| Plugin tool call | Install a fixture plugin exposing a deterministic tool and bind it to the pipeline. | Runner lists the plugin tool and can call it through the same tool loop as MCP tools. |
| Repeated tool loop | Run `local-agent-multitool-rag-compaction-debug-chat` with `max-tool-iterations` high enough for two serial calls. | Runner calls `qa_plugin_echo` twice and the final answer contains both distinct tool results with RAG and compacted history. |
| Parallel tool batch | Run `local-agent-parallel-tools-rag-compaction-debug-chat` with `tool-execution-mode: parallel`. | Runner executes two same-turn `qa_plugin_echo` calls and the final answer contains both results with RAG and compacted history. |
| Tool iteration limit | Run `local-agent-tool-loop-limit-debug-chat` with `max-tool-iterations: 2` and a fake provider that keeps requesting tools. | Runner stops and returns `Tool call iteration limit reached...`; it does not keep invoking tools indefinitely. |
| Run steering | Use `local-agent-steering-debug-chat` with the fixture `qa_plugin_sleep` tool. | A follow-up sent while the sleep tool keeps the run active is claimed into the same run: two user messages, one assistant response, sentinel included. |
| Tool errors | Make the model request an unauthorized tool or invalid arguments in a controlled unit/component test. | Tool result contains an error message and the run does not bypass authorization. |
| Tool errors | Run `local-agent-tool-error-recovery-debug-chat` for plugin tool execution errors; keep unauthorized-tool branches in unit/component tests. | The model receives an `Error:` tool result and returns the recovery sentinel; unauthorized calls remain blocked in component tests. |
| Tool iteration limit | Use a controlled model/tool fixture that repeatedly requests more tool calls. | Runner stops with `runner.tool_loop_limit` at the configured limit. |
| Knowledge retrieval | Bind a KB containing a unique sentinel. | Bot returns the sentinel and backend logs LangRAG retrieval. |
| Legacy `knowledge-base` config | Load a pipeline config using the old single-KB field. | Runner still retrieves from the KB. |
skills/skills/langbot-testing/references/local-agent-runner.md
+7 -2
View File
@@ -36,7 +36,7 @@ If the direct MCP fixture passes but `/api/v1/tools` still shows the old MCP nam
For a multimodal check, upload a small image and ask for a deterministic acknowledgement. Prefer the bundled 64x64 red-square fixture over a 1x1 image because some model providers reject tiny images before the runner path is exercised.
For a non-streaming check, disable the Debug Chat stream switch before sending the prompt.
For a Debug Chat non-streaming delivery check, disable the Debug Chat stream switch before sending the prompt. This validates the UI/adapter delivery path. Runner-internal non-streaming model invocation is covered by component tests that set `runtime_metadata.streaming_supported=false`.
## Timeout And Tool Regression Checks
@@ -53,9 +53,14 @@ Run these cases before saying the pluginized local-agent behavior is healthy:
- `local-agent-rag-debug-chat`: LangRAG retrieval reaches the runner and affects the answer.
- `mcp-stdio-tool-call`: MCP tool discovery and local-agent tool loop.
- `local-agent-plugin-tool-call-debug-chat`: plugin tool discovery and local-agent tool loop.
- `local-agent-tool-error-recovery-debug-chat`: plugin tool execution errors are returned to the model as tool results and can produce a final recovery answer.
- `local-agent-tool-loop-limit-debug-chat`: repeated plugin tool requests stop at the configured max tool iteration limit.
- `local-agent-combo-rag-compaction-tool-debug-chat`: one Debug Chat run combines compacted history, LangRAG context, and a plugin tool result.
- `local-agent-multitool-rag-compaction-debug-chat`: one Debug Chat run combines compacted history, LangRAG context, and two serial plugin tool calls.
- `local-agent-parallel-tools-rag-compaction-debug-chat`: one Debug Chat run combines compacted history, LangRAG context, and two same-turn parallel plugin tool calls.
- `local-agent-multimodal-debug-chat`: uploaded image reaches `ctx.input.contents`.
- `local-agent-rag-multimodal-debug-chat`: RAG retrieval still works when the same user message carries an image.
- `local-agent-nonstreaming-debug-chat`: runner works when the host adapter cannot or should not stream.
- `local-agent-nonstreaming-debug-chat`: Debug Chat still returns a complete visible response when UI streaming is disabled.
## Pass Criteria
skills/skills/langbot-testing/references/mcp-stdio-testing.md
+3 -1
View File
@@ -79,7 +79,9 @@ node scripts/e2e/mcp-stdio-register.mjs
It upserts `qa-local-stdio` through the authenticated WebUI session, points it at
the bundled `qa_mcp_echo_server.py`, then checks `/api/v1/tools` and the MCP
runtime info. A pass proves LangBot has refreshed the saved server and exposes
`qa_mcp_echo` before any model provider is involved.
`qa_mcp_echo` before any model provider is involved. It also writes the resolved
server UUID to `LANGBOT_MCP_QA_STDIO_SERVER_UUID` in `skills/.env.local`; pipeline
extension binding uses this UUID, not the human-readable server name.
## Local-Agent Tool Call Check
skills/skills/langbot-testing/suites/agent-runner-release-gate.yaml
+5
View File
@@ -30,6 +30,11 @@ cases:
- local-agent-context-compaction-debug-chat
- local-agent-rag-debug-chat
- local-agent-plugin-tool-call-debug-chat
- local-agent-tool-error-recovery-debug-chat
- local-agent-tool-loop-limit-debug-chat
- local-agent-combo-rag-compaction-tool-debug-chat
- local-agent-multitool-rag-compaction-debug-chat
- local-agent-parallel-tools-rag-compaction-debug-chat
- mcp-stdio-register
- mcp-stdio-tool-call
- local-agent-nonstreaming-debug-chat
skills/skills/langbot-testing/suites/local-agent-gate.yaml
+5
View File
@@ -14,6 +14,11 @@ cases:
- local-agent-context-compaction-debug-chat
- local-agent-rag-debug-chat
- local-agent-plugin-tool-call-debug-chat
- local-agent-tool-error-recovery-debug-chat
- local-agent-tool-loop-limit-debug-chat
- local-agent-combo-rag-compaction-tool-debug-chat
- local-agent-multitool-rag-compaction-debug-chat
- local-agent-parallel-tools-rag-compaction-debug-chat
- local-agent-steering-debug-chat
- mcp-stdio-tool-call
- local-agent-nonstreaming-debug-chat
skills/src/commands/log.ts
+1 -1
View File
@@ -117,7 +117,7 @@ function patternContextFromOptions(root: string, options: Record<string, string
function latestOrExplicitBackendLog(root: string, options: Record<string, string | boolean>): string {
const explicit = optionString(options, "backend-log");
if (explicit) return resolve(explicit);
const auto = latestLangBotLogPath(loadEnv(root));
const auto = latestLangBotLogPath(loadEnv(root), root);
return auto ? resolve(auto) : "";
}
skills/src/commands/test.ts
+77 -3
View File
@@ -343,6 +343,78 @@ function manualEvidenceTemplate(mode: string): ManualEvidenceTemplate {
};
}
function existingEvidencePath(evidence: Record<string, string> | undefined, key: string): string {
const path = evidence?.[key];
return path && existsSync(path) ? path : "";
}
function renderAutomationEvidence(
mode: string,
automation: AutomationResultEvidence,
logGuard: LogGuardResult,
): ManualEvidenceTemplate {
if (automation.status !== "loaded") return manualEvidenceTemplate(mode);
const isProbe = isProbeMode(mode);
const screenshot = existingEvidencePath(automation.evidence, "screenshot");
const consoleLog = existingEvidencePath(automation.evidence, "console_log");
const networkLog = existingEvidencePath(automation.evidence, "network_log");
const apiDiagnostics = Object.entries(automation.evidence ?? {})
.filter(([key, value]) => key.endsWith("_diagnostic_json") && value && existsSync(value))
.map(([, value]) => value);
const successLines = logGuard.success_signals
.map((signal) => signal.excerpt || `${signal.source}:${signal.line ?? ""} ${signal.pattern}`.trim())
.slice(0, 3);
const findings = logGuard.findings
.map((finding) => `${finding.severity}/${finding.kind}${finding.excerpt ? `: ${finding.excerpt}` : ""}`)
.slice(0, 3);
const chatSummary = automation.chat_results?.length
? automation.chat_results
.map((item) => `#${(item.index ?? 0) + 1} ${item.status ?? "unknown"} ${item.expected_text ?? ""}`.trim())
.join("; ")
: automation.reason || "";
const diagnostics = [
automation.browser_diagnostics?.reason,
apiDiagnostics.length ? `api diagnostics: ${apiDiagnostics.join(", ")}` : "",
].filter(Boolean).join("; ") || "No extra diagnostics recorded.";
if (isProbe) {
return {
result: automation.result || "unknown",
target_tested: String(automation.url || automation.path || "automation target"),
execution_path: "automation script",
probe_result: automation.reason || chatSummary || "Automation completed.",
logs_or_artifacts: [
consoleLog ? `console=${consoleLog}` : "",
networkLog ? `network=${networkLog}` : "",
automation.path ? `automation=${automation.path}` : "",
].filter(Boolean).join(", ") || "No log artifacts recorded.",
diagnostics,
matched_troubleshooting: findings.length ? findings.join(" | ") : "None.",
assets_to_update: automation.result === "pass" ? "None." : "Review case/troubleshooting assets if this failure is expected to recur.",
};
}
return {
result: automation.result || "unknown",
url_tested: automation.url || "Not recorded.",
browser_path: "direct Playwright automation",
ui_result: chatSummary || automation.reason || "Automation completed.",
console_errors: automation.browser_diagnostics?.status === "fail"
? automation.browser_diagnostics.reason || "Browser diagnostics failed."
: "None found by browser diagnostics.",
network_symptoms: networkLog ? `Captured in ${networkLog}` : "No network log artifact recorded.",
backend_logs: successLines.length
? successLines.join(" | ")
: `Log Guard status: ${logGuard.status}${findings.length ? `; ${findings.join(" | ")}` : ""}`,
frontend_logs: consoleLog ? `Captured in ${consoleLog}` : "No frontend console log artifact recorded.",
screenshots: screenshot || "No screenshot artifact recorded.",
diagnostics,
matched_troubleshooting: findings.length ? findings.join(" | ") : "None.",
assets_to_update: automation.result === "pass" ? "None." : "Review case/troubleshooting assets if this failure is expected to recur.",
};
}
function envSummary(item: StructuredItem, env: Record<string, string>): Record<string, string> {
const keys = [
...listValue(item.fields, "env"),
@@ -1321,6 +1393,8 @@ export function commandTestRun(ctx: CommandContext): number {
function buildReport(root: string, item: StructuredItem, options: Record<string, string | boolean>): TestReport {
const env = loadEnv(root);
const mode = caseMode(item);
const automationResult = readAutomationResultEvidence(options);
const logGuard = scanStructuredLogSources(root, item, options);
const related = relatedTroubleshooting(root, item).map((entry) => ({
id: scalar(entry.fields, "id"),
title: scalar(entry.fields, "title"),
@@ -1332,8 +1406,8 @@ function buildReport(root: string, item: StructuredItem, options: Record<string,
generated_at: new Date().toISOString(),
case: caseSummary(item),
result_options: ["pass", "fail", "blocked", "env_issue", "flaky"],
automation_result: readAutomationResultEvidence(options),
manual_evidence: manualEvidenceTemplate(mode),
automation_result: automationResult,
manual_evidence: renderAutomationEvidence(mode, automationResult, logGuard),
environment: envSummary(item, env),
required_skills: listValue(item.fields, "skills"),
steps: listValue(item.fields, "steps"),
@@ -1344,7 +1418,7 @@ function buildReport(root: string, item: StructuredItem, options: Record<string,
failure_patterns: listValue(item.fields, "failure_patterns"),
expected_failures: listValue(item.fields, "expected_failures"),
troubleshooting: related,
log_guard: scanStructuredLogSources(root, item, options),
log_guard: logGuard,
};
}
skills/src/commands/validate.ts
+2
View File
@@ -125,6 +125,8 @@ function validateCaseItem(root: string, item: StructuredItem, skillNames: Set<st
...validateEnvKeyList(item, "setup_provides_env"),
...validateEnvKeyScalar(item, "automation_pipeline_url_env"),
...validateEnvKeyScalar(item, "automation_pipeline_name_env"),
...validateJsonScalar(item, "automation_runner_config_patch_json"),
...validateJsonScalar(item, "automation_extensions_patch_json"),
...validateJsonScalar(item, "automation_filesystem_checks_json"),
...validateJsonScalar(item, "metrics_thresholds_json"),
...validateJsonScalar(item, "load_profile_json"),
skills/src/log-guard.ts
+92 -7
View File
@@ -1,5 +1,5 @@
import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
import { dirname, join, resolve } from "node:path";
import { basename, dirname, join, resolve } from "node:path";
import type { StructuredItem } from "./types.ts";
import { listValue, loadEnv, loadStructuredItems, scalar } from "./fs.ts";
@@ -99,6 +99,22 @@ export type AutomationResultEvidence = {
url?: string;
prompt?: string;
expected_text?: string;
stream_output?: string;
prompt_count?: number;
image_fixture?: string;
evidence?: Record<string, string>;
evidence_collected?: string[];
browser_diagnostics?: {
status?: string;
reason?: string;
finding_count?: number;
};
chat_results?: Array<{
index?: number;
expected_text?: string;
status?: string;
reason?: string;
}>;
metrics_summary?: Record<string, unknown>;
thresholds_summary?: Record<string, unknown>;
artifacts?: Record<string, unknown>;
@@ -216,13 +232,13 @@ export function scanLogSources(
for (const configured of configuredSources) {
const explicitPath = options[configured.option];
const autoPath = configured.source === "backend" && options["no-auto-log"] !== true
? latestLangBotLogPath(env)
? latestLangBotLogPath(env, root)
: null;
const rawPath = typeof explicitPath === "string" ? explicitPath : autoPath;
const autoDetected = typeof explicitPath !== "string" && rawPath === autoPath;
if (!rawPath) {
if (configured.source === "backend" && options["no-auto-log"] !== true) {
const logsDir = env.LANGBOT_REPO ? join(env.LANGBOT_REPO, "data", "logs") : "LANGBOT_REPO/data/logs";
const logsDir = langBotLogsDir(env, root) ?? "LANGBOT_REPO/data/logs";
sources.push({ source: "backend", path: join(logsDir, "langbot-*.log"), status: "auto_not_found", line_count: 0, auto_detected: true });
}
continue;
@@ -610,6 +626,47 @@ function objectField(data: Record<string, unknown>, key: string): Record<string,
: undefined;
}
function stringMapField(data: Record<string, unknown>, key: string): Record<string, string> | undefined {
const value = data[key];
if (!value || typeof value !== "object" || Array.isArray(value)) return undefined;
const entries = Object.entries(value as Record<string, unknown>)
.filter((entry): entry is [string, string] => typeof entry[1] === "string" && entry[1].trim().length > 0);
return entries.length ? Object.fromEntries(entries) : undefined;
}
function stringListField(data: Record<string, unknown>, key: string): string[] | undefined {
const value = data[key];
if (!Array.isArray(value)) return undefined;
const items = value.map((item) => typeof item === "string" ? item : "").filter(Boolean);
return items.length ? items : undefined;
}
function chatResultsField(data: Record<string, unknown>): AutomationResultEvidence["chat_results"] {
const value = data.chat_results;
if (!Array.isArray(value)) return undefined;
const results = value
.filter((item): item is Record<string, unknown> => Boolean(item) && typeof item === "object" && !Array.isArray(item))
.map((item) => ({
index: numberField(item, "index"),
expected_text: stringField(item, "expected_text"),
status: stringField(item, "status"),
reason: stringField(item, "reason"),
}));
return results.length ? results : undefined;
}
function browserDiagnosticsField(data: Record<string, unknown>): AutomationResultEvidence["browser_diagnostics"] {
const value = data.browser_diagnostics;
if (!value || typeof value !== "object" || Array.isArray(value)) return undefined;
const diagnostics = value as Record<string, unknown>;
const findings = Array.isArray(diagnostics.findings) ? diagnostics.findings.length : undefined;
return {
status: stringField(diagnostics, "status"),
reason: stringField(diagnostics, "reason"),
finding_count: findings,
};
}
function evidenceDirFromOptions(options: Record<string, string | boolean>): string | undefined {
const explicit = typeof options["evidence-dir"] === "string" ? options["evidence-dir"] : undefined;
if (explicit) return resolve(explicit);
@@ -652,6 +709,13 @@ export function readAutomationResultEvidence(options: Record<string, string | bo
url: stringField(result, "url"),
prompt: redactSecrets(stringField(result, "prompt") ?? ""),
expected_text: stringField(result, "expected_text"),
stream_output: typeof result.stream_output === "boolean" ? String(result.stream_output) : stringField(result, "stream_output"),
prompt_count: numberField(result, "prompt_count"),
image_fixture: stringField(result, "image_fixture"),
evidence: stringMapField(result, "evidence"),
evidence_collected: stringListField(result, "evidence_collected"),
browser_diagnostics: browserDiagnosticsField(result),
chat_results: chatResultsField(result),
metrics_summary: objectField(result, "metrics_summary"),
thresholds_summary: objectField(result, "thresholds_summary"),
artifacts: objectField(result, "artifacts"),
@@ -661,10 +725,31 @@ export function readAutomationResultEvidence(options: Record<string, string | bo
}
}
export function latestLangBotLogPath(env: Record<string, string>): string | null {
const repo = env.LANGBOT_REPO;
if (!repo) return null;
const logsDir = join(repo, "data", "logs");
function langBotRepoFromRoot(root: string): string | null {
const candidates = [
resolve(root),
basename(resolve(root)) === "skills" ? resolve(root, "..") : "",
resolve(root, "../LangBot"),
resolve(root, "LangBot"),
].filter(Boolean);
const seen = new Set<string>();
for (const candidate of candidates) {
if (seen.has(candidate)) continue;
seen.add(candidate);
if (existsSync(join(candidate, "data", "config.yaml"))) return candidate;
}
return null;
}
function langBotLogsDir(env: Record<string, string>, root: string): string | null {
const repo = env.LANGBOT_REPO ? resolve(env.LANGBOT_REPO) : langBotRepoFromRoot(root);
return repo ? join(repo, "data", "logs") : null;
}
export function latestLangBotLogPath(env: Record<string, string>, root = process.cwd()): string | null {
const logsDir = langBotLogsDir(env, root);
if (!logsDir) return null;
if (!existsSync(logsDir)) return null;
const candidates = readdirSync(logsDir)
skills/src/readiness.ts
+2
View File
@@ -111,6 +111,8 @@ export function automationEnvDefaults(item: StructuredItem, env: EnvSource = pro
["automation_image_base64_fixture", "LANGBOT_E2E_IMAGE_BASE64_PATH"],
["automation_runner_config_patch_json", "LANGBOT_E2E_RUNNER_CONFIG_PATCH_JSON"],
["automation_restore_runner_config", "LANGBOT_E2E_RESTORE_RUNNER_CONFIG"],
["automation_extensions_patch_json", "LANGBOT_E2E_EXTENSIONS_PATCH_JSON"],
["automation_restore_extensions", "LANGBOT_E2E_RESTORE_EXTENSIONS"],
["automation_expected_runner_id", "LANGBOT_E2E_EXPECTED_RUNNER_ID"],
["automation_reset_debug_chat", "LANGBOT_E2E_RESET_DEBUG_CHAT"],
["automation_debug_chat_session_type", "LANGBOT_E2E_DEBUG_CHAT_SESSION_TYPE"],
skills/test/lbs-cli.test.ts
+2313 -852
View File
File diff suppressed because it is too large Load Diff
src/langbot/pkg/api/http/controller/groups/pipelines/pipelines.py
+7 -2
View File
@@ -73,8 +73,13 @@ class PipelinesRouterGroup(group.RouterGroup):
plugins = await self.ap.plugin_connector.list_plugins(component_kinds=pipeline_component_kinds)
mcp_servers = await self.ap.mcp_service.get_mcp_servers(contain_runtime_info=True)
# Get available skills
available_skills = await self.ap.skill_service.list_skills()
# Skill listing depends on Box. Pipeline plugin/MCP binding
# must remain usable when Box is slow or unavailable.
try:
available_skills = await self.ap.skill_service.list_skills()
except Exception as exc:
self.ap.logger.warning('Unable to list skills for pipeline extensions: %s', exc)
available_skills = []
extensions_prefs = pipeline.get('extensions_preferences', {})
return self.success(
src/langbot/pkg/api/http/service/skill.py
+12 -3
View File
@@ -2,6 +2,7 @@ from __future__ import annotations
import io
import inspect
import logging
import os
import posixpath
import zipfile
@@ -9,6 +10,8 @@ from typing import Optional
from urllib.parse import quote, unquote, urlparse
import httpx
from langbot_plugin.box.errors import BoxError
from langbot_plugin.entities.io.errors import ActionCallError, ActionCallTimeoutError
from ....core import app
from ....skill.utils import parse_frontmatter
@@ -33,6 +36,8 @@ _GITHUB_ASSET_HOSTS = {
'codeload.github.com',
}
logger = logging.getLogger(__name__)
class SkillService:
"""Filesystem-backed skill management service."""
@@ -81,12 +86,16 @@ class SkillService:
async def list_skills(self) -> list[dict]:
# When Box is unavailable, surface an empty list rather than raising —
# the skills page should render cleanly, and the UI separately renders
# a "Box disabled / unavailable" banner via useBoxStatus.
# the skills page and unrelated extension surfaces should render
# cleanly, and the UI separately renders Box status via useBoxStatus.
box_service = self._box_service()
if box_service is None:
return []
return [self._serialize_skill(skill) for skill in await box_service.list_skills()]
try:
return [self._serialize_skill(skill) for skill in await box_service.list_skills()]
except (ActionCallTimeoutError, ActionCallError, BoxError, TimeoutError, ConnectionError) as exc:
logger.warning('Box skill list unavailable; returning an empty skill list: %s', exc)
return []
async def get_skill(self, skill_name: str) -> Optional[dict]:
box_service = self._box_service()
tests/e2e/test_local_agent_runner_fake_provider.py
+885
View File
@@ -0,0 +1,885 @@
"""E2E coverage for the official Local Agent runner with fake Host resources.
These tests start the real LangBot application and the real SDK Plugin Runtime,
load the sibling ``langbot-local-agent`` plugin, and verify Local Agent paths
that must cross Host run-scoped APIs without calling any external provider.
"""
from __future__ import annotations
import asyncio
import json
import os
import shutil
import socket
import sqlite3
import subprocess
import tempfile
from pathlib import Path
from types import SimpleNamespace
from typing import Any
import pytest
from tests.e2e.utils.config_factory import create_minimal_config, create_test_directories
from tests.e2e.utils.process_manager import find_project_root
pytestmark = pytest.mark.e2e
LOCAL_AGENT_RUNNER_ID = 'plugin:langbot/local-agent/default'
FAKE_PROVIDER_UUID = 'e2e-fake-provider'
FAKE_MODEL_UUID = 'e2e-fake-local-agent-model'
LOCAL_AGENT_PLUGIN_DIRNAME = 'langbot__local-agent'
E2E_TOOL_NAME = 'e2e_lookup'
E2E_KB_UUID = 'e2e-kb-local-agent'
def _free_port() -> int:
"""Reserve a currently-free localhost TCP port for this E2E process."""
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.bind(('127.0.0.1', 0))
return int(sock.getsockname()[1])
def _local_agent_repo() -> Path:
"""Return the sibling local-agent repository used by this workspace E2E."""
project_root = find_project_root()
return project_root.parent / 'langbot-local-agent'
def _copy_local_agent_plugin(tmpdir: Path) -> None:
"""Copy the sibling Local Agent plugin into the temporary LangBot data dir."""
local_agent_src = _local_agent_repo()
if not (local_agent_src / 'manifest.yaml').exists():
pytest.skip(f'local-agent repository not found at {local_agent_src}')
plugin_dst = tmpdir / 'data' / 'plugins' / LOCAL_AGENT_PLUGIN_DIRNAME
ignore = shutil.ignore_patterns(
'.git',
'.venv',
'__pycache__',
'.pytest_cache',
'.ruff_cache',
'build',
'dist',
)
shutil.copytree(local_agent_src, plugin_dst, ignore=ignore)
def _content_text(content: Any) -> str:
"""Flatten provider message content into text for assertions."""
if content is None:
return ''
if isinstance(content, str):
return content
if isinstance(content, list):
parts: list[str] = []
for item in content:
text = item.get('text') if isinstance(item, dict) else getattr(item, 'text', None)
if text:
parts.append(str(text))
return ''.join(parts)
return str(content)
def _message_text(message: Any) -> str:
"""Flatten a provider message into text for assertions."""
return _content_text(getattr(message, 'content', None))
def _invoke_payload_texts(fake_requester: Any) -> list[list[str]]:
"""Return model-facing text for every fake LLM invocation."""
payloads: list[list[str]] = []
for payload in fake_requester._invoke_payloads:
payloads.append([_message_text(message) for message in payload['messages']])
return payloads
def _event(
*,
event_id: str,
conversation_id: str,
text: str,
thread_id: str = 'e2e-local-agent-thread',
):
"""Build an AgentRunner event envelope for Local Agent E2E probes."""
from langbot.pkg.agent.runner.host_models import AgentEventEnvelope
from langbot_plugin.api.entities.builtin.agent_runner.delivery import DeliveryContext
from langbot_plugin.api.entities.builtin.agent_runner.event import ActorContext, SubjectContext
from langbot_plugin.api.entities.builtin.agent_runner.input import AgentInput
return AgentEventEnvelope(
event_id=event_id,
event_type='message.received',
source='api',
conversation_id=conversation_id,
thread_id=thread_id,
actor=ActorContext(actor_type='user', actor_id='user-001', actor_name='E2E User'),
subject=SubjectContext(subject_type='chat', subject_id='chat-001'),
input=AgentInput(text=text),
delivery=DeliveryContext(surface='e2e', supports_streaming=False),
)
def _binding(
*,
binding_id: str = 'e2e-local-agent-binding',
runner_config: dict[str, Any] | None = None,
allowed_tool_names: list[str] | None = None,
allowed_kb_uuids: list[str] | None = None,
):
"""Build a Local Agent binding with fake model access."""
from langbot.pkg.agent.runner.host_models import (
AgentBinding,
BindingScope,
DeliveryPolicy,
ResourcePolicy,
StatePolicy,
)
config = {
'model': {'primary': FAKE_MODEL_UUID, 'fallbacks': []},
'timeout': 60,
'prompt': [{'role': 'system', 'content': 'You are a concise test assistant.'}],
'knowledge-bases': [],
'context-window-tokens': 8192,
'context-reserve-tokens': 1024,
'context-keep-recent-tokens': 1000,
'context-summary-tokens': 500,
}
if runner_config:
config.update(runner_config)
return AgentBinding(
binding_id=binding_id,
scope=BindingScope(scope_type='global'),
runner_id=LOCAL_AGENT_RUNNER_ID,
runner_config=config,
resource_policy=ResourcePolicy(
allowed_model_uuids=[FAKE_MODEL_UUID],
allowed_tool_names=allowed_tool_names,
allowed_kb_uuids=allowed_kb_uuids,
),
state_policy=StatePolicy(enable_state=True, state_scopes=['conversation']),
delivery_policy=DeliveryPolicy(enable_streaming=False, enable_reply=True),
)
class _FakeToolManager:
"""Deterministic tool manager used behind the real Host CALL_TOOL action."""
def __init__(self):
self.calls: list[dict[str, Any]] = []
async def get_tool_schema(self, tool_name: str):
if tool_name != E2E_TOOL_NAME:
return None, None
return (
'Lookup a deterministic E2E value.',
{
'type': 'object',
'properties': {
'query': {'type': 'string'},
},
'required': ['query'],
},
)
async def get_tool_detail(self, tool_name: str):
description, parameters = await self.get_tool_schema(tool_name)
if parameters is None:
return None
return {'name': tool_name, 'description': description, 'parameters': parameters}
async def execute_func_call(self, name: str, parameters: dict[str, Any], query: Any = None):
del query
self.calls.append({'name': name, 'parameters': dict(parameters)})
return {
'value': f"tool-result:{parameters.get('query')}",
'source': 'fake-tool-manager',
}
class _FakeKnowledgeBase:
"""Minimal KB object used behind the real Host RETRIEVE_KNOWLEDGE action."""
def __init__(self):
self.knowledge_base_entity = SimpleNamespace(kb_type='fake')
self.retrieve_calls: list[dict[str, Any]] = []
def get_uuid(self) -> str:
return E2E_KB_UUID
def get_name(self) -> str:
return 'E2E Fake KB'
async def retrieve(self, query_text: str, settings: dict[str, Any]):
self.retrieve_calls.append({'query_text': query_text, 'settings': settings})
return [
SimpleNamespace(
content='RAG_SENTINEL Local Agent retrieved this deterministic chunk.',
metadata={'source': 'fake-kb'},
id='fake-kb-chunk-1',
score=0.99,
model_dump=lambda mode='json': {
'content': 'RAG_SENTINEL Local Agent retrieved this deterministic chunk.',
'metadata': {'source': 'fake-kb'},
'id': 'fake-kb-chunk-1',
'score': 0.99,
},
)
]
class _FakeRagManager:
"""Deterministic RAG manager used by resource builder and retrieval action."""
def __init__(self, kb: _FakeKnowledgeBase):
self.kb = kb
self.knowledge_bases = {E2E_KB_UUID: kb}
async def get_knowledge_base_by_uuid(self, kb_uuid: str):
if kb_uuid == E2E_KB_UUID:
return self.kb
return None
@pytest.fixture(scope='session')
def local_agent_e2e_tmpdir():
"""Create temporary directory for Local Agent E2E testing."""
tmpdir = Path(tempfile.mkdtemp(prefix='langbot_local_agent_e2e_'))
yield tmpdir
shutil.rmtree(tmpdir, ignore_errors=True)
@pytest.fixture(scope='session')
def local_agent_e2e_port() -> int:
"""HTTP port for the real LangBot app used by this E2E."""
return _free_port()
@pytest.fixture(scope='session')
def local_agent_runtime_ports() -> tuple[int, int]:
"""Control/debug ports for the standalone plugin runtime."""
control_port = _free_port()
debug_port = _free_port()
while debug_port == control_port:
debug_port = _free_port()
return control_port, debug_port
@pytest.fixture(scope='session')
def local_agent_e2e_config_path(local_agent_e2e_tmpdir, local_agent_e2e_port, local_agent_runtime_ports):
"""Create a plugin-enabled config and install the Local Agent plugin fixture."""
config_path = create_minimal_config(local_agent_e2e_tmpdir, port=local_agent_e2e_port)
create_test_directories(local_agent_e2e_tmpdir)
import yaml
with open(config_path, encoding='utf-8') as f:
config = yaml.safe_load(f)
runtime_control_port, _runtime_debug_port = local_agent_runtime_ports
config['api']['global_api_key'] = 'e2e-local-agent-key'
config['plugin']['enable'] = True
config['plugin']['runtime_ws_url'] = f'ws://127.0.0.1:{runtime_control_port}/control/ws'
config['plugin']['enable_marketplace'] = False
config['box']['enabled'] = False
config['system']['jwt']['secret'] = 'e2e-local-agent-secret-key'
with open(config_path, 'w', encoding='utf-8') as f:
yaml.safe_dump(config, f, default_flow_style=False)
_copy_local_agent_plugin(local_agent_e2e_tmpdir)
return config_path
@pytest.fixture(scope='session')
def local_agent_runtime_process(local_agent_e2e_tmpdir, local_agent_runtime_ports, local_agent_e2e_config_path):
"""Start the real SDK plugin runtime over WebSocket."""
del local_agent_e2e_config_path
control_port, debug_port = local_agent_runtime_ports
stdout_path = local_agent_e2e_tmpdir / 'plugin-runtime.stdout.log'
stderr_path = local_agent_e2e_tmpdir / 'plugin-runtime.stderr.log'
stdout_file = open(stdout_path, 'wb')
stderr_file = open(stderr_path, 'wb')
proc = subprocess.Popen(
[
str(find_project_root() / '.venv' / 'bin' / 'python'),
'-m',
'langbot_plugin.cli.__init__',
'rt',
'--ws-control-port',
str(control_port),
'--ws-debug-port',
str(debug_port),
],
cwd=local_agent_e2e_tmpdir,
stdout=stdout_file,
stderr=stderr_file,
start_new_session=True,
)
yield proc
proc.terminate()
try:
proc.wait(timeout=5)
except subprocess.TimeoutExpired:
proc.kill()
proc.wait()
stdout_file.close()
stderr_file.close()
def _inject_fake_llm_model(ap) -> Any:
"""Register a runtime-only fake model that supports count_tokens/invoke."""
from langbot.pkg.entity.persistence import model as persistence_model
from langbot.pkg.provider.modelmgr import requester, token
from tests.unit_tests.provider.conftest import FakeProviderAPIRequester
provider_entity = persistence_model.ModelProvider(
uuid=FAKE_PROVIDER_UUID,
name='E2E Fake Provider',
requester='fake-requester',
base_url='https://fake.invalid',
api_keys=['fake-key'],
)
fake_requester = FakeProviderAPIRequester(ap, {'base_url': provider_entity.base_url})
runtime_provider = requester.RuntimeProvider(
provider_entity=provider_entity,
token_mgr=token.TokenManager(name=provider_entity.uuid, tokens=provider_entity.api_keys),
requester=fake_requester,
)
runtime_model = requester.RuntimeLLMModel(
model_entity=persistence_model.LLMModel(
uuid=FAKE_MODEL_UUID,
name=FAKE_MODEL_UUID,
provider_uuid=provider_entity.uuid,
abilities=['func_call'],
context_length=8192,
extra_args={},
),
provider=runtime_provider,
)
ap.model_mgr.provider_dict[provider_entity.uuid] = runtime_provider
ap.model_mgr.llm_models.append(runtime_model)
return fake_requester
def _scripted_tool_call(
tool_name: str = E2E_TOOL_NAME,
*,
call_id: str = 'call-e2e-lookup',
query: str = 'alpha',
):
"""Build an assistant message requesting a deterministic tool call."""
from langbot_plugin.api.entities.builtin.provider import message as provider_message
return provider_message.Message(
role='assistant',
content='',
tool_calls=[
provider_message.ToolCall(
id=call_id,
type='function',
function=provider_message.FunctionCall(
name=tool_name,
arguments=json.dumps({'query': query}, ensure_ascii=False),
),
)
],
)
async def _boot_local_agent_app(tmpdir: Path):
"""Boot LangBot and wait until the Local Agent runner is discoverable."""
from langbot.pkg.core import boot
ap = await boot.make_app(asyncio.get_running_loop())
for _ in range(60):
handler = getattr(ap.plugin_connector, 'handler', None)
if handler is not None:
await handler.ping()
break
await asyncio.sleep(1)
else:
raise AssertionError(f'Plugin runtime did not connect; tmpdir={tmpdir}')
for _ in range(60):
runners = await ap.agent_runner_registry.list_runners(use_cache=False)
if any(runner.id == LOCAL_AGENT_RUNNER_ID for runner in runners):
break
await asyncio.sleep(1)
else:
raise AssertionError(f'{LOCAL_AGENT_RUNNER_ID} was not discovered')
return ap
def _run_local_agent_probe(tmpdir: Path, probe):
"""Run one Local Agent probe inside the temporary LangBot app."""
from langbot.pkg.utils import platform as platform_utils
async def _run():
previous_cwd = Path.cwd()
previous_standalone_runtime = platform_utils.standalone_runtime
os.chdir(tmpdir)
platform_utils.standalone_runtime = True
ap = None
try:
ap = await _boot_local_agent_app(tmpdir)
return await probe(ap)
finally:
if ap is not None:
ap.dispose()
platform_utils.standalone_runtime = previous_standalone_runtime
os.chdir(previous_cwd)
return asyncio.run(_run())
def test_local_agent_runner_uses_host_fake_provider_and_persists_ledger(
local_agent_e2e_tmpdir,
local_agent_e2e_config_path,
local_agent_runtime_process,
):
"""Local Agent should execute through Host APIs with a token-free fake provider."""
del local_agent_e2e_config_path, local_agent_runtime_process
async def _run_probe(ap):
fake_requester = _inject_fake_llm_model(ap)
event = _event(
event_id='e2e-local-agent-event-001',
conversation_id='e2e-local-agent-conversation',
text='Say pong through the fake provider.',
)
messages = [message async for message in ap.agent_run_orchestrator.run(event, _binding())]
return messages, list(fake_requester._count_tokens_payloads)
messages, token_payloads = _run_local_agent_probe(local_agent_e2e_tmpdir, _run_probe)
assert len(messages) == 1
assert messages[0].role == 'assistant'
assert _content_text(messages[0].content) == 'Fake LLM response'
assert token_payloads
flattened_token_payloads = [item for payload in token_payloads for item in payload]
assert any(item.get('role') == 'system' for item in flattened_token_payloads)
assert any('Say pong through the fake provider.' in item.get('content', '') for item in flattened_token_payloads)
db_path = local_agent_e2e_tmpdir / 'data' / 'langbot.db'
conn = sqlite3.connect(str(db_path))
try:
run_row = conn.execute(
"SELECT run_id, status, runner_id, status_reason FROM agent_run WHERE event_id = ?",
('e2e-local-agent-event-001',),
).fetchone()
assert run_row is not None
run_id, status, runner_id, status_reason = run_row
assert status == 'completed'
assert runner_id == LOCAL_AGENT_RUNNER_ID
assert status_reason == 'stop'
event_rows = conn.execute(
'SELECT sequence, type, data_json FROM agent_run_event WHERE run_id = ? ORDER BY sequence',
(run_id,),
).fetchall()
event_types = [row[1] for row in event_rows]
assert event_types == ['message.completed', 'run.completed']
assert 'Fake LLM response' in event_rows[0][2]
transcript_rows = conn.execute(
'SELECT role, content, run_id, runner_id FROM transcript WHERE conversation_id = ? ORDER BY seq',
('e2e-local-agent-conversation',),
).fetchall()
assert [row[0] for row in transcript_rows] == ['user', 'assistant']
assert transcript_rows[0][1] == 'Say pong through the fake provider.'
assert transcript_rows[1][1] == 'Fake LLM response'
assert transcript_rows[1][2] == run_id
assert transcript_rows[1][3] == LOCAL_AGENT_RUNNER_ID
finally:
conn.close()
def test_local_agent_runner_executes_authorized_tool_loop_through_host_action(
local_agent_e2e_tmpdir,
local_agent_e2e_config_path,
local_agent_runtime_process,
):
"""Local Agent should execute a model tool call through Host CALL_TOOL and finish."""
del local_agent_e2e_config_path, local_agent_runtime_process
async def _run_probe(ap):
fake_requester = _inject_fake_llm_model(ap)
fake_requester.queue_llm_responses(
_scripted_tool_call(),
'Tool loop final answer after tool-result:alpha',
)
tool_mgr = _FakeToolManager()
ap.tool_mgr = tool_mgr
event = _event(
event_id='e2e-local-agent-tool-event-001',
conversation_id='e2e-local-agent-tool-conversation',
text='Use the e2e lookup tool before answering.',
)
binding = _binding(
binding_id='e2e-local-agent-tool-binding',
allowed_tool_names=[E2E_TOOL_NAME],
runner_config={
'max-tool-iterations': 2,
'tool-execution-mode': 'serial',
},
)
messages = [message async for message in ap.agent_run_orchestrator.run(event, binding)]
return messages, tool_mgr.calls, _invoke_payload_texts(fake_requester)
messages, tool_calls, invoke_payload_texts = _run_local_agent_probe(local_agent_e2e_tmpdir, _run_probe)
assert len(messages) == 1
assert _content_text(messages[0].content) == 'Tool loop final answer after tool-result:alpha'
assert tool_calls == [{'name': E2E_TOOL_NAME, 'parameters': {'query': 'alpha'}}]
assert any('tool-result:alpha' in text for text in invoke_payload_texts[-1])
db_path = local_agent_e2e_tmpdir / 'data' / 'langbot.db'
conn = sqlite3.connect(str(db_path))
try:
run_row = conn.execute(
"SELECT run_id, status, status_reason FROM agent_run WHERE event_id = ?",
('e2e-local-agent-tool-event-001',),
).fetchone()
assert run_row is not None
run_id, status, status_reason = run_row
assert status == 'completed'
assert status_reason == 'stop'
event_rows = conn.execute(
'SELECT type, data_json FROM agent_run_event WHERE run_id = ? ORDER BY sequence',
(run_id,),
).fetchall()
event_types = [row[0] for row in event_rows]
assert event_types == [
'tool.call.started',
'tool.call.completed',
'message.completed',
'run.completed',
]
assert E2E_TOOL_NAME in event_rows[0][1]
assert 'tool-result:alpha' in event_rows[1][1]
finally:
conn.close()
def test_local_agent_runner_retrieves_authorized_rag_context_through_host_action(
local_agent_e2e_tmpdir,
local_agent_e2e_config_path,
local_agent_runtime_process,
):
"""Local Agent should retrieve configured KB context through Host RETRIEVE_KNOWLEDGE."""
del local_agent_e2e_config_path, local_agent_runtime_process
async def _run_probe(ap):
fake_requester = _inject_fake_llm_model(ap)
fake_requester.queue_llm_responses('RAG final answer with RAG_SENTINEL')
fake_kb = _FakeKnowledgeBase()
ap.rag_mgr = _FakeRagManager(fake_kb)
event = _event(
event_id='e2e-local-agent-rag-event-001',
conversation_id='e2e-local-agent-rag-conversation',
text='Answer with the retrieved RAG sentinel.',
)
binding = _binding(
binding_id='e2e-local-agent-rag-binding',
allowed_kb_uuids=[E2E_KB_UUID],
runner_config={
'knowledge-bases': [E2E_KB_UUID],
'retrieval-top-k': 1,
},
)
messages = [message async for message in ap.agent_run_orchestrator.run(event, binding)]
return messages, fake_kb.retrieve_calls, _invoke_payload_texts(fake_requester)
messages, retrieve_calls, invoke_payload_texts = _run_local_agent_probe(local_agent_e2e_tmpdir, _run_probe)
assert len(messages) == 1
assert _content_text(messages[0].content) == 'RAG final answer with RAG_SENTINEL'
assert retrieve_calls == [
{
'query_text': 'Answer with the retrieved RAG sentinel.',
'settings': {'top_k': 1, 'filters': {}},
}
]
assert any(
'RAG_SENTINEL Local Agent retrieved this deterministic chunk.' in text
for payload_texts in invoke_payload_texts
for text in payload_texts
)
db_path = local_agent_e2e_tmpdir / 'data' / 'langbot.db'
conn = sqlite3.connect(str(db_path))
try:
run_row = conn.execute(
"SELECT run_id, status FROM agent_run WHERE event_id = ?",
('e2e-local-agent-rag-event-001',),
).fetchone()
assert run_row is not None
run_id, status = run_row
assert status == 'completed'
event_types = [
row[0]
for row in conn.execute(
'SELECT type FROM agent_run_event WHERE run_id = ? ORDER BY sequence',
(run_id,),
)
]
assert event_types == ['message.completed', 'run.completed']
finally:
conn.close()
def test_local_agent_runner_compacts_history_and_persists_checkpoint(
local_agent_e2e_tmpdir,
local_agent_e2e_config_path,
local_agent_runtime_process,
):
"""Local Agent should compact old Host history and write conversation checkpoint state."""
del local_agent_e2e_config_path, local_agent_runtime_process
async def _run_probe(ap):
from langbot.pkg.agent.runner.transcript_store import TranscriptStore
fake_requester = _inject_fake_llm_model(ap)
fake_requester.queue_llm_responses(
'SUMMARY_SENTINEL compacted older history including HIST_SENTINEL',
'Compaction final answer',
)
store = TranscriptStore(ap.persistence_mgr.get_db_engine())
for index in range(12):
await store.append_transcript(
transcript_id=None,
event_id=f'e2e-local-agent-history-{index}',
conversation_id='e2e-local-agent-compaction-conversation',
role='user' if index % 2 == 0 else 'assistant',
content=(
f'HIST_SENTINEL-{index} '
'This is intentionally long deterministic history for compaction. ' * 10
),
thread_id='e2e-local-agent-thread',
item_type='message',
)
event = _event(
event_id='e2e-local-agent-compaction-event-001',
conversation_id='e2e-local-agent-compaction-conversation',
text='Use compacted context and answer.',
)
binding = _binding(
binding_id='e2e-local-agent-compaction-binding',
runner_config={
'context-window-tokens': 900,
'context-reserve-tokens': 300,
'context-keep-recent-tokens': 160,
'context-summary-tokens': 240,
'context-history-fetch-limit': 20,
},
)
messages = [message async for message in ap.agent_run_orchestrator.run(event, binding)]
return messages, _invoke_payload_texts(fake_requester), fake_requester._invoke_count
messages, invoke_payload_texts, invoke_count = _run_local_agent_probe(local_agent_e2e_tmpdir, _run_probe)
assert len(messages) == 1
assert _content_text(messages[0].content) == 'Compaction final answer'
assert invoke_count == 2
assert any('HIST_SENTINEL' in text for text in invoke_payload_texts[0])
assert any('SUMMARY_SENTINEL compacted older history' in text for text in invoke_payload_texts[-1])
db_path = local_agent_e2e_tmpdir / 'data' / 'langbot.db'
conn = sqlite3.connect(str(db_path))
try:
run_row = conn.execute(
"SELECT run_id, status FROM agent_run WHERE event_id = ?",
('e2e-local-agent-compaction-event-001',),
).fetchone()
assert run_row is not None
run_id, status = run_row
assert status == 'completed'
event_types = [
row[0]
for row in conn.execute(
'SELECT type FROM agent_run_event WHERE run_id = ? ORDER BY sequence',
(run_id,),
)
]
assert event_types == ['message.completed', 'run.completed']
state_row = conn.execute(
"SELECT value_json FROM agent_runner_state WHERE state_key = 'runner.compaction.checkpoint'"
).fetchone()
assert state_row is not None
checkpoint = json.loads(state_row[0])
assert checkpoint['schema_version'] == 'langbot.local_agent.compaction_checkpoint.v1'
assert 'SUMMARY_SENTINEL compacted older history' in checkpoint['summary']
assert checkpoint['conversation_id'] == 'e2e-local-agent-compaction-conversation'
assert checkpoint['covers_until']
assert checkpoint['tokens_before'] > 600
finally:
conn.close()
def test_local_agent_runner_combines_rag_compaction_and_multi_turn_tool_loop(
local_agent_e2e_tmpdir,
local_agent_e2e_config_path,
local_agent_runtime_process,
):
"""Local Agent should preserve RAG, compressed history, and multi-turn tool results together."""
del local_agent_e2e_config_path, local_agent_runtime_process
async def _run_probe(ap):
from langbot.pkg.agent.runner.transcript_store import TranscriptStore
fake_requester = _inject_fake_llm_model(ap)
async def scripted_response(**kwargs):
messages = kwargs['messages']
text = '\n'.join(_message_text(message) for message in messages)
if 'context summarization assistant' in text or '<conversation>' in text:
return 'SUMMARY_COMBO compacted older history including HIST_COMBO_SENTINEL and RAG_TOOL_COMBO_GOAL'
if 'tool-result:alpha' not in text:
return _scripted_tool_call(call_id='call-combo-alpha', query='alpha')
if 'tool-result:beta' not in text:
return _scripted_tool_call(call_id='call-combo-beta', query='beta')
assert 'RAG_SENTINEL Local Agent retrieved this deterministic chunk.' in text
assert 'SUMMARY_COMBO compacted older history' in text
assert 'tool-result:alpha' in text
assert 'tool-result:beta' in text
assert 'current combo request must survive' in text
return 'COMBO_FINAL RAG_SENTINEL HIST_COMBO_SENTINEL tool-result:alpha tool-result:beta'
fake_requester.queue_llm_responses(*(scripted_response for _ in range(20)))
tool_mgr = _FakeToolManager()
fake_kb = _FakeKnowledgeBase()
ap.tool_mgr = tool_mgr
ap.rag_mgr = _FakeRagManager(fake_kb)
store = TranscriptStore(ap.persistence_mgr.get_db_engine())
for index in range(16):
await store.append_transcript(
transcript_id=None,
event_id=f'e2e-local-agent-combo-history-{index}',
conversation_id='e2e-local-agent-combo-conversation',
role='user' if index % 2 == 0 else 'assistant',
content=(
f'HIST_COMBO_SENTINEL-{index} RAG_TOOL_COMBO_GOAL '
'This old message intentionally creates pressure for combo compaction. ' * 8
),
thread_id='e2e-local-agent-thread',
item_type='message',
)
event = _event(
event_id='e2e-local-agent-combo-event-001',
conversation_id='e2e-local-agent-combo-conversation',
text='current combo request must survive; use RAG and tools before answering.',
)
binding = _binding(
binding_id='e2e-local-agent-combo-binding',
allowed_tool_names=[E2E_TOOL_NAME],
allowed_kb_uuids=[E2E_KB_UUID],
runner_config={
'knowledge-bases': [E2E_KB_UUID],
'retrieval-top-k': 1,
'max-tool-iterations': 4,
'tool-execution-mode': 'serial',
'context-window-tokens': 950,
'context-reserve-tokens': 300,
'context-keep-recent-tokens': 140,
'context-summary-tokens': 260,
'context-history-fetch-limit': 25,
},
)
messages = [message async for message in ap.agent_run_orchestrator.run(event, binding)]
return (
messages,
tool_mgr.calls,
fake_kb.retrieve_calls,
_invoke_payload_texts(fake_requester),
fake_requester._invoke_count,
)
messages, tool_calls, retrieve_calls, invoke_payload_texts, invoke_count = _run_local_agent_probe(
local_agent_e2e_tmpdir,
_run_probe,
)
assert len(messages) == 1
assert _content_text(messages[0].content) == (
'COMBO_FINAL RAG_SENTINEL HIST_COMBO_SENTINEL tool-result:alpha tool-result:beta'
)
assert tool_calls == [
{'name': E2E_TOOL_NAME, 'parameters': {'query': 'alpha'}},
{'name': E2E_TOOL_NAME, 'parameters': {'query': 'beta'}},
]
assert retrieve_calls == [
{
'query_text': 'current combo request must survive; use RAG and tools before answering.',
'settings': {'top_k': 1, 'filters': {}},
}
]
assert invoke_count >= 4
assert any(
'RAG_SENTINEL Local Agent retrieved this deterministic chunk.' in text
for payload_texts in invoke_payload_texts
for text in payload_texts
)
assert any('SUMMARY_COMBO compacted older history' in text for text in invoke_payload_texts[-1])
assert any('tool-result:alpha' in text for text in invoke_payload_texts[-1])
assert any('tool-result:beta' in text for text in invoke_payload_texts[-1])
assert any('current combo request must survive' in text for text in invoke_payload_texts[-1])
db_path = local_agent_e2e_tmpdir / 'data' / 'langbot.db'
conn = sqlite3.connect(str(db_path))
try:
run_row = conn.execute(
"SELECT run_id, status, status_reason FROM agent_run WHERE event_id = ?",
('e2e-local-agent-combo-event-001',),
).fetchone()
assert run_row is not None
run_id, status, status_reason = run_row
assert status == 'completed'
assert status_reason == 'stop'
event_rows = conn.execute(
'SELECT type, data_json FROM agent_run_event WHERE run_id = ? ORDER BY sequence',
(run_id,),
).fetchall()
event_types = [row[0] for row in event_rows]
assert event_types == [
'tool.call.started',
'tool.call.completed',
'tool.call.started',
'tool.call.completed',
'message.completed',
'run.completed',
]
assert 'tool-result:alpha' in event_rows[1][1]
assert 'tool-result:beta' in event_rows[3][1]
assert 'COMBO_FINAL' in event_rows[4][1]
state_rows = conn.execute(
"SELECT value_json FROM agent_runner_state WHERE state_key = 'runner.compaction.checkpoint'"
).fetchall()
checkpoints = [json.loads(row[0]) for row in state_rows]
checkpoint = next(
(
item
for item in checkpoints
if item.get('conversation_id') == 'e2e-local-agent-combo-conversation'
),
None,
)
assert checkpoint is not None
assert checkpoint['conversation_id'] == 'e2e-local-agent-combo-conversation'
assert 'SUMMARY_COMBO compacted older history' in checkpoint['summary']
finally:
conn.close()
tests/integration/api/test_pipelines.py
+14
View File
@@ -278,3 +278,17 @@ class TestPipelineExtensionsEndpoint:
assert response.status_code == 200
data = await response.get_json()
assert data['code'] == 0
@pytest.mark.asyncio
async def test_get_extensions_tolerates_skill_list_failure(self, quart_test_client, fake_pipeline_app):
"""Pipeline plugin/MCP binding UI should not fail when Box skill listing is unavailable."""
fake_pipeline_app.skill_service.list_skills = AsyncMock(side_effect=TimeoutError('box_list_skills timeout'))
response = await quart_test_client.get(
'/api/v1/pipelines/test-pipeline-uuid/extensions', headers={'Authorization': 'Bearer test_token'}
)
assert response.status_code == 200
data = await response.get_json()
assert data['code'] == 0
assert data['data']['available_skills'] == []
tests/unit_tests/provider/conftest.py
+66
View File
@@ -8,6 +8,8 @@ without calling real LLM APIs or network requests.
from __future__ import annotations
import json
import inspect
from typing import Any
import pytest
from unittest.mock import AsyncMock, Mock
@@ -32,7 +34,11 @@ class FakeProviderAPIRequester(requester.ProviderAPIRequester):
self._invoke_count = 0
self._last_messages = None
self._last_model = None
self._last_funcs = None
self._invoke_payloads = []
self._last_count_tokens_payload = None
self._count_tokens_payloads = []
self._scripted_llm_responses = []
@staticmethod
def _content_to_text(content) -> str:
@@ -52,6 +58,44 @@ class FakeProviderAPIRequester(requester.ProviderAPIRequester):
return ''.join(parts)
return str(content)
def queue_llm_responses(self, *responses: Any) -> None:
"""Queue deterministic LLM responses for multi-turn tests."""
self._scripted_llm_responses.extend(responses)
async def _coerce_llm_response(
self,
response: Any,
*,
query: Any,
model: requester.RuntimeLLMModel,
messages: list,
funcs: list | None,
extra_args: dict,
remove_think: bool,
):
"""Convert scripted response values into provider Message objects."""
import langbot_plugin.api.entities.builtin.provider.message as provider_message
if callable(response):
response = response(
query=query,
model=model,
messages=messages,
funcs=funcs,
extra_args=extra_args,
remove_think=remove_think,
)
if inspect.isawaitable(response):
response = await response
if isinstance(response, provider_message.Message):
return response
if isinstance(response, dict):
return provider_message.Message.model_validate(response)
if isinstance(response, str):
return provider_message.Message(role='assistant', content=response)
return response
async def invoke_llm(
self,
query,
@@ -65,10 +109,30 @@ class FakeProviderAPIRequester(requester.ProviderAPIRequester):
self._invoke_count += 1
self._last_messages = messages
self._last_model = model
self._last_funcs = funcs or []
self._invoke_payloads.append(
{
'messages': messages,
'funcs': funcs or [],
'extra_args': dict(extra_args or {}),
'remove_think': remove_think,
}
)
# Import the message entity for response
import langbot_plugin.api.entities.builtin.provider.message as provider_message
if self._scripted_llm_responses:
return await self._coerce_llm_response(
self._scripted_llm_responses.pop(0),
query=query,
model=model,
messages=messages,
funcs=funcs,
extra_args=extra_args,
remove_think=remove_think,
)
return provider_message.Message(
role='assistant',
content=[provider_message.ContentElement(type='text', text='Fake LLM response')],
@@ -106,6 +170,7 @@ class FakeProviderAPIRequester(requester.ProviderAPIRequester):
'role': getattr(message, 'role', ''),
'content': self._content_to_text(getattr(message, 'content', None)),
'tool_calls': getattr(message, 'tool_calls', None),
'tool_call_id': getattr(message, 'tool_call_id', None),
}
)
@@ -119,6 +184,7 @@ class FakeProviderAPIRequester(requester.ProviderAPIRequester):
)
self._last_count_tokens_payload = payload
self._count_tokens_payloads.append(payload)
text = json.dumps(payload, ensure_ascii=False, sort_keys=True, default=str)
return max(1, (len(text) + 3) // 4)
tests/unit_tests/test_skill_service.py
+12
View File
@@ -2,6 +2,7 @@ from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
from langbot_plugin.entities.io.errors import ActionCallTimeoutError
from langbot.pkg.api.http.service.skill import SkillService
@@ -82,6 +83,17 @@ class TestRequireBoxForWrite:
service = SkillService(self._ap_with_disabled_box())
assert await service.list_skills() == []
@pytest.mark.asyncio
async def test_list_skills_returns_empty_when_box_action_times_out(self):
"""A transient Box list failure should not break unrelated extension surfaces."""
box_service = SimpleNamespace(
available=True,
list_skills=AsyncMock(side_effect=ActionCallTimeoutError('Action box_list_skills call timed out')),
)
service = SkillService(SimpleNamespace(skill_mgr=SimpleNamespace(reload_skills=AsyncMock()), box_service=box_service))
assert await service.list_skills() == []
@pytest.mark.asyncio
async def test_read_skill_file_refused_when_box_unavailable(self):
service = SkillService(self._ap_with_disabled_box())