feat(agent-runner): finalize 4.x processor integration

This commit is contained in:
huanghuoguoguo
2026-07-12 15:44:05 +08:00
parent 29689962f3
commit e6384aae5d
109 changed files with 2200 additions and 1204 deletions
@@ -0,0 +1,265 @@
#!/usr/bin/env node
import {
apiJson,
createBrowser,
ensureAuthenticatedBrowser,
ensureEvidence,
evidencePaths,
exitCode,
loadEnvFiles,
localIsoWithOffset,
safeScreenshot,
scanBrowserDiagnostics,
writeResult,
} from "./lib/langbot-e2e.mjs";
const caseId = "agent-runner-health-visibility";
await loadEnvFiles();
const paths = evidencePaths(caseId);
await ensureEvidence(paths);
const readyScreenshot = paths.screenshot.replace(/\.png$/, "-ready.png");
const unavailableScreenshot = paths.screenshot.replace(
/\.png$/,
"-unavailable.png",
);
const startedAt = new Date();
const frontendUrl = process.env.LANGBOT_FRONTEND_URL || "";
const backendUrl = process.env.LANGBOT_BACKEND_URL || "";
const missingRunnerId = "plugin:qa/missing-runner/default";
let browser;
let token = "";
let agentId = "";
const result = {
source: "automation",
case_id: caseId,
run_id: paths.runId,
started_at: startedAt.toISOString(),
started_at_local: localIsoWithOffset(startedAt),
finished_at: "",
finished_at_local: "",
status: "fail",
reason: "",
url: "",
visible_signals: [],
api: {},
diagnostics: null,
cleanup: null,
evidence: {
console_log: paths.consoleLog,
network_log: paths.networkLog,
screenshot: paths.screenshot,
ready_screenshot: readyScreenshot,
unavailable_screenshot: unavailableScreenshot,
automation_result_json: paths.automationResultJson,
result_json: paths.resultJson,
},
evidence_collected: ["ui", "screenshot", "console", "api_diagnostic"],
};
function schemaDefaults(items = []) {
return Object.fromEntries(
items
.filter((item) => item.name && Object.hasOwn(item, "default"))
.map((item) => [item.name, item.default]),
);
}
async function openRunnerSettings(page, agentUrl) {
await page.goto(agentUrl, { waitUntil: "domcontentloaded" });
await page
.getByRole("button", { name: /Runner|运行器|ランナー/ })
.first()
.waitFor({ timeout: 15_000 });
await page
.getByRole("button", { name: /Runner|运行器|ランナー/ })
.first()
.click();
}
try {
if (!frontendUrl) throw new Error("LANGBOT_FRONTEND_URL is not configured.");
if (!backendUrl) throw new Error("LANGBOT_BACKEND_URL is not configured.");
browser = await createBrowser(paths);
const { page } = browser;
await page.goto(frontendUrl, { waitUntil: "domcontentloaded" });
const auth = await ensureAuthenticatedBrowser(page, {
frontendUrl,
backendUrl,
});
if (auth.status !== "pass") {
result.status = auth.status;
throw new Error(auth.reason);
}
token = await page.evaluate(() => localStorage.getItem("token") || "");
if (!token) {
result.status = "blocked";
throw new Error("Authenticated browser has no reusable local token.");
}
const pluginStatus = await apiJson(
backendUrl,
"/api/v1/system/status/plugin-system",
{ token },
);
const pluginData = pluginStatus.json.data || {};
result.api.plugin_status = {
http_status: pluginStatus.status,
code: pluginStatus.json.code ?? null,
is_enable: pluginData.is_enable ?? null,
is_connected: pluginData.is_connected ?? null,
};
if (pluginStatus.status >= 400 || pluginStatus.json.code !== 0) {
result.status = "env_issue";
throw new Error(pluginStatus.json.msg || "Plugin status request failed.");
}
if (!pluginData.is_enable || !pluginData.is_connected) {
result.status = "env_issue";
throw new Error("The plugin runtime is not enabled and connected.");
}
const metadata = await apiJson(backendUrl, "/api/v1/agents/_/metadata", {
token,
});
const runnerTab = metadata.json.data?.runner_config;
const runnerStage = runnerTab?.stages?.find(
(stage) => stage.name === "runner",
);
const runnerOptions =
runnerStage?.config?.find((item) => item.name === "id")?.options || [];
const runner = runnerOptions[0];
result.api.agent_metadata = {
http_status: metadata.status,
code: metadata.json.code ?? null,
runner_count: runnerOptions.length,
selected_runner: runner?.name || null,
};
if (metadata.status >= 400 || metadata.json.code !== 0) {
throw new Error(metadata.json.msg || "Agent metadata request failed.");
}
if (!runner?.name) {
result.status = "blocked";
throw new Error("No registered AgentRunner is available for the UI check.");
}
const runnerConfigStage = runnerTab.stages.find(
(stage) => stage.name === runner.name,
);
const create = await apiJson(backendUrl, "/api/v1/agents", {
method: "POST",
token,
body: {
kind: "agent",
name: `Runner Health ${paths.runId.slice(-40)}`,
description: "Temporary AgentRunner health visibility fixture",
emoji: "H",
component_ref: runner.name,
config: {
runner: { id: runner.name, "expire-time": 0 },
runner_config: {
[runner.name]: schemaDefaults(runnerConfigStage?.config),
},
},
enabled: true,
supported_event_patterns: ["message.*"],
},
});
agentId = create.json.data?.uuid || "";
result.api.create_agent = {
http_status: create.status,
code: create.json.code ?? null,
};
if (create.status >= 400 || create.json.code !== 0 || !agentId) {
throw new Error(create.json.msg || "Failed to create the temporary Agent.");
}
const agentUrl = `${frontendUrl.replace(/\/$/, "")}/home/agents?id=${encodeURIComponent(agentId)}`;
result.url = agentUrl;
await openRunnerSettings(page, agentUrl);
await page
.getByText(/Runner ready|运行器已就绪|Runner の準備完了/, { exact: true })
.waitFor({ timeout: 15_000 });
result.visible_signals.push("registered-runner-ready");
await safeScreenshot(page, readyScreenshot);
const staleUpdate = await apiJson(
backendUrl,
`/api/v1/agents/${encodeURIComponent(agentId)}`,
{
method: "PUT",
token,
body: {
component_ref: missingRunnerId,
config: {
runner: { id: missingRunnerId, "expire-time": 0 },
runner_config: { [missingRunnerId]: {} },
},
},
},
);
result.api.set_stale_runner = {
http_status: staleUpdate.status,
code: staleUpdate.json.code ?? null,
};
if (staleUpdate.status >= 400 || staleUpdate.json.code !== 0) {
throw new Error(
staleUpdate.json.msg || "Failed to set the stale runner fixture.",
);
}
await openRunnerSettings(page, agentUrl);
await page
.getByText(
/Selected runner is unavailable|所选运行器不可用|選択した Runner は利用できません/,
{ exact: true },
)
.waitFor({ timeout: 15_000 });
await page.getByRole("link", { name: /Extensions|扩展|拡張機能/ }).waitFor();
result.visible_signals.push("stale-runner-unavailable", "recovery-action");
await safeScreenshot(page, unavailableScreenshot);
await safeScreenshot(page, paths.screenshot);
result.diagnostics = await scanBrowserDiagnostics(paths);
if (result.diagnostics.status !== "pass") {
throw new Error(result.diagnostics.reason);
}
result.status = "pass";
result.reason =
"Agent Runner settings visibly distinguished a registered runner from a stale binding.";
} catch (error) {
if (!["blocked", "env_issue"].includes(result.status)) result.status = "fail";
result.reason = result.reason || error.message;
if (browser?.page) await safeScreenshot(browser.page, paths.screenshot);
} finally {
const cleanup = {};
if (agentId && token && backendUrl) {
const deletedAgent = await apiJson(
backendUrl,
`/api/v1/agents/${encodeURIComponent(agentId)}`,
{ method: "DELETE", token },
).catch((error) => ({
status: 0,
json: { code: null, msg: error.message },
}));
cleanup.agent_deleted =
deletedAgent.status < 400 && deletedAgent.json.code === 0;
cleanup.agent_http_status = deletedAgent.status;
}
result.cleanup = cleanup;
if (agentId && !cleanup.agent_deleted && result.status === "pass") {
result.status = "fail";
result.reason = "The temporary runner health Agent was not deleted.";
}
if (browser) await browser.close().catch(() => {});
const finishedAt = new Date();
result.finished_at = finishedAt.toISOString();
result.finished_at_local = localIsoWithOffset(finishedAt);
await writeResult(paths, result);
console.log(JSON.stringify(result, null, 2));
}
process.exit(exitCode(result.status));
@@ -70,7 +70,7 @@ const startedAt = new Date();
const targets = [
{
id: "local-agent",
expected_runner_id: "plugin:langbot/local-agent/default",
expected_runner_id: "plugin:langbot-team/LocalAgent/default",
pipeline_url: firstEnv("LANGBOT_LOCAL_AGENT_PIPELINE_URL"),
pipeline_name: firstEnv("LANGBOT_LOCAL_AGENT_PIPELINE_NAME"),
require_func_call_model: true,
@@ -79,7 +79,7 @@ const targets = [
},
{
id: "acp-agent-runner",
expected_runner_id: "plugin:langbot/acp-agent-runner/default",
expected_runner_id: "plugin:langbot-team/ACPAgentRunner/default",
pipeline_url: firstEnv("LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL", "LANGBOT_AGENT_RUNNER_PIPELINE_URL"),
pipeline_name: firstEnv("LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME", "LANGBOT_AGENT_RUNNER_PIPELINE_NAME"),
require_func_call_model: false,
@@ -219,7 +219,7 @@ async function run() {
return metadata.author && metadata.name ? `${metadata.author}/${metadata.name}` : "";
})
.filter(Boolean);
const requiredPlugins = ["langbot/local-agent", "langbot/acp-agent-runner", "qa/plugin-smoke"];
const requiredPlugins = ["langbot-team/LocalAgent", "langbot-team/ACPAgentRunner", "qa/plugin-smoke"];
const pluginPresence = Object.fromEntries(requiredPlugins.map((id) => [id, installedPluginIds.includes(id)]));
for (const [id, present] of Object.entries(pluginPresence)) {
addCheck(`plugin:${id}`, present ? "pass" : "blocked", { plugin_id: id, reason: present ? "" : "Required plugin is not listed by /api/v1/plugins." });
@@ -309,7 +309,7 @@ async function run() {
const config = pipeline.config || {};
const aiConfig = config.ai && typeof config.ai === "object" ? config.ai : {};
const runner = aiConfig.runner && typeof aiConfig.runner === "object" ? aiConfig.runner : {};
const runnerId = runner.id || runner.runner || "";
const runnerId = runner.id || "";
const runnerConfigs = aiConfig.runner_config && typeof aiConfig.runner_config === "object" ? aiConfig.runner_config : {};
const runnerConfig = runnerConfigs[runnerId] && typeof runnerConfigs[runnerId] === "object" ? runnerConfigs[runnerId] : {};
const pipelineSummary = {
@@ -12,7 +12,7 @@ import {
writeResult,
} from "./lib/langbot-e2e.mjs";
const RUNNER_ID = "plugin:langbot/acp-agent-runner/default";
const RUNNER_ID = "plugin:langbot-team/ACPAgentRunner/default";
const DEFAULT_PIPELINE_NAME = "Agent QA ACP Claude Debug Chat";
const DEFAULT_LOCAL_PASSWORD = "LangBotE2ELocalPass!2026";
const caseId = "ensure-acp-agent-runner-pipeline";
@@ -14,7 +14,7 @@ import {
writeResult,
} from "./lib/langbot-e2e.mjs";
const RUNNER_ID = "local-agent";
const RUNNER_ID = "plugin:langbot-team/LocalAgent/default";
const DEFAULT_LOCAL_PASSWORD = "LangBotE2ELocalPass!2026";
const DEFAULT_PIPELINE_NAME = "LangBot QA Fake Provider Debug Chat";
const DEFAULT_PROVIDER_NAME = "LangBot QA Fake OpenAI Provider";
@@ -511,8 +511,11 @@ async function ensurePipeline({ backendUrl, token, name, modelUuid }) {
const config = pipeline.config && typeof pipeline.config === "object" ? pipeline.config : {};
const ai = config.ai && typeof config.ai === "object" ? config.ai : {};
const existingLocalAgentConfig = ai["local-agent"] && typeof ai["local-agent"] === "object"
? ai["local-agent"]
const runnerConfigs = ai.runner_config && typeof ai.runner_config === "object"
? ai.runner_config
: {};
const existingLocalAgentConfig = runnerConfigs[RUNNER_ID] && typeof runnerConfigs[RUNNER_ID] === "object"
? runnerConfigs[RUNNER_ID]
: {};
const localAgentConfig = {
timeout: 60,
@@ -546,10 +549,12 @@ async function ensurePipeline({ backendUrl, token, name, modelUuid }) {
runner: {
...(ai.runner && typeof ai.runner === "object" ? ai.runner : {}),
id: RUNNER_ID,
runner: RUNNER_ID,
"expire-time": 0,
},
"local-agent": localAgentConfig,
runner_config: {
...runnerConfigs,
[RUNNER_ID]: localAgentConfig,
},
},
};
@@ -18,7 +18,7 @@ import {
writeResult,
} from "./lib/langbot-e2e.mjs";
const RUNNER_ID = "plugin:langbot/local-agent/default";
const RUNNER_ID = "plugin:langbot-team/LocalAgent/default";
const SPACE_PROVIDER_UUID = "00000000-0000-0000-0000-000000000000";
const DEFAULT_PIPELINE_NAME = "Agent QA Local Agent Debug Chat";
const DEFAULT_LOCAL_PASSWORD = "LangBotE2ELocalPass!2026";
@@ -31,7 +31,7 @@ await ensureEvidence(paths);
const backendUrl = (env.LANGBOT_BACKEND_URL || "").replace(/\/$/, "");
const pipelineUrl = env.LANGBOT_E2E_PIPELINE_URL || env.LANGBOT_LOCAL_AGENT_PIPELINE_URL || env.LANGBOT_PIPELINE_URL || "";
const pipelineName = env.LANGBOT_E2E_PIPELINE_NAME || env.LANGBOT_LOCAL_AGENT_PIPELINE_NAME || env.LANGBOT_PIPELINE_NAME || "";
const expectedRunnerId = env.LANGBOT_E2E_EXPECTED_RUNNER_ID || "plugin:langbot/local-agent/default";
const expectedRunnerId = env.LANGBOT_E2E_EXPECTED_RUNNER_ID || "plugin:langbot-team/LocalAgent/default";
const expectedText = env.LANGBOT_E2E_EXPECTED_TEXT || "qa_steering_sentinel_6194";
const responseTimeoutMs = positiveInt(env.LANGBOT_E2E_RESPONSE_TIMEOUT_MS, 240000);
const followupDelayMs = 1000;
@@ -446,7 +446,7 @@ async function inspectPipeline(page, { backendUrl, pipelineUrl, pipelineName, ex
}
const config = pipeline.config || {};
const runner = config.ai?.runner || {};
const runnerId = runner.id || runner.runner || "";
const runnerId = runner.id || "";
if (!runnerId) {
return {
status: "blocked",
@@ -455,7 +455,7 @@ async function inspectPipeline(page, { backendUrl, pipelineUrl, pipelineName, ex
pipeline_id: pipelineId,
pipeline_name: pipeline.name,
matched_by: matchedBy,
reason: "Pipeline has no ai.runner.id or legacy ai.runner.runner.",
reason: "Pipeline has no ai.runner.id.",
};
}
if (expectedRunnerId && runnerId !== expectedRunnerId) {
+2 -2
View File
@@ -429,7 +429,7 @@ async function inspectAndPatchPipelineConfig(page, {
const config = JSON.parse(JSON.stringify(pipeline.config || {}));
const aiConfig = config.ai && typeof config.ai === "object" ? config.ai : {};
const runner = aiConfig.runner && typeof aiConfig.runner === "object" ? aiConfig.runner : {};
const runnerId = runner.id || runner.runner || "";
const runnerId = runner.id || "";
if (!runnerId) {
return {
status: "blocked",
@@ -438,7 +438,7 @@ async function inspectAndPatchPipelineConfig(page, {
pipeline_id: pipelineId,
pipeline_name: pipeline.name,
matched_by: matchedBy,
reason: "Pipeline has no ai.runner.id or legacy ai.runner.runner.",
reason: "Pipeline has no ai.runner.id.",
};
}
if (expectedRunnerId && runnerId !== expectedRunnerId) {
@@ -0,0 +1,280 @@
#!/usr/bin/env node
import {
apiJson,
createBrowser,
ensureAuthenticatedBrowser,
ensureEvidence,
evidencePaths,
exitCode,
loadEnvFiles,
localIsoWithOffset,
safeScreenshot,
scanBrowserDiagnostics,
writeResult,
} from "./lib/langbot-e2e.mjs";
const caseId = "wizard-runner-marketplace-catalog";
await loadEnvFiles();
const paths = evidencePaths(caseId);
await ensureEvidence(paths);
const mobileScreenshot = paths.screenshot.replace(/\.png$/, "-mobile.png");
const startedAt = new Date();
const frontendUrl = process.env.LANGBOT_FRONTEND_URL || "";
const backendUrl = process.env.LANGBOT_BACKEND_URL || "";
let browser;
let token = "";
let botId = "";
const result = {
source: "automation",
case_id: caseId,
run_id: paths.runId,
started_at: startedAt.toISOString(),
started_at_local: localIsoWithOffset(startedAt),
finished_at: "",
finished_at_local: "",
status: "fail",
reason: "",
url: "",
visible_signals: [],
api: {},
marketplace_request: null,
diagnostics: null,
cleanup: {},
evidence: {
screenshot: paths.screenshot,
mobile_screenshot: mobileScreenshot,
console_log: paths.consoleLog,
network_log: paths.networkLog,
automation_result_json: paths.automationResultJson,
result_json: paths.resultJson,
},
evidence_collected: ["ui", "screenshot", "console", "api_diagnostic"],
};
try {
if (!frontendUrl) throw new Error("LANGBOT_FRONTEND_URL is not configured.");
if (!backendUrl) throw new Error("LANGBOT_BACKEND_URL is not configured.");
browser = await createBrowser(paths);
const { page } = browser;
page.on("request", (request) => {
if (
!/\/api\/v1\/marketplace\/(extensions|plugins)\/search$/.test(
new URL(request.url()).pathname,
)
) {
return;
}
try {
const payload = request.postDataJSON();
if (payload?.component_filter === "AgentRunner") {
result.marketplace_request = {
endpoint: new URL(request.url()).pathname,
component_filter: payload.component_filter,
type_filter: payload.type_filter || null,
page_size: payload.page_size || null,
};
}
} catch {
// The assertion below reports a missing or malformed catalog request.
}
});
await page.goto(frontendUrl, { waitUntil: "domcontentloaded" });
const auth = await ensureAuthenticatedBrowser(page, {
frontendUrl,
backendUrl,
});
if (auth.status !== "pass") {
result.status = auth.status;
throw new Error(auth.reason);
}
token = await page.evaluate(() => localStorage.getItem("token") || "");
if (!token) {
result.status = "blocked";
throw new Error("Authenticated browser token is unavailable.");
}
const [plugins, metadata, system] = await Promise.all([
apiJson(backendUrl, "/api/v1/plugins", { token }),
apiJson(backendUrl, "/api/v1/pipelines/_/metadata", { token }),
apiJson(backendUrl, "/api/v1/system/info", { token }),
]);
const runnerStage = metadata.json.data?.configs
?.find((config) => config.name === "ai")
?.stages?.find((stage) => stage.name === "runner");
const runnerOptions =
runnerStage?.config?.find((item) => item.name === "id")?.options || [];
const installedPlugins = plugins.json.data?.plugins || [];
result.api.clean_state = {
plugin_count: installedPlugins.length,
runner_count: runnerOptions.length,
wizard_status: system.json.data?.wizard_status || null,
};
if (installedPlugins.length !== 0 || runnerOptions.length !== 0) {
result.status = "blocked";
throw new Error(
`This case requires zero plugins and zero runners; found ${installedPlugins.length} plugins and ${runnerOptions.length} runners.`,
);
}
if (system.json.data?.wizard_status !== "none") {
result.status = "blocked";
throw new Error(
`This case requires a first-run instance; wizard status is ${system.json.data?.wizard_status}.`,
);
}
const suffix = paths.runId.slice(-40);
const bot = await apiJson(backendUrl, "/api/v1/platform/bots", {
method: "POST",
token,
body: {
name: `Runner Catalog Wizard ${suffix}`,
description: "Temporary clean-runner catalog fixture",
adapter: "aiocqhttp",
adapter_config: {
host: "127.0.0.1",
port: 2280,
"access-token": "",
},
enable: false,
event_bindings: [],
},
});
botId = bot.json.data?.uuid || "";
if (bot.status >= 400 || bot.json.code !== 0 || !botId) {
throw new Error(bot.json.msg || "Failed to create the temporary Bot.");
}
const progress = await apiJson(backendUrl, "/api/v1/system/wizard/progress", {
method: "PUT",
token,
body: {
step: 2,
selected_scenario: "message_reply",
selected_adapter: "aiocqhttp",
created_bot_uuid: botId,
bot_saved: true,
selected_runner: null,
},
});
if (progress.status >= 400 || progress.json.code !== 0) {
throw new Error(progress.json.msg || "Failed to prepare Wizard progress.");
}
await page.goto(`${frontendUrl.replace(/\/$/, "")}/wizard`, {
waitUntil: "domcontentloaded",
});
result.url = page.url();
await page
.getByText(/Select an AI Engine|选择 AI 引擎|AIエンジンを選択/, {
exact: true,
})
.waitFor({ timeout: 15_000 });
await page
.getByText(
/No AgentRunner extensions are published yet|市场暂未发布 AgentRunner 扩展|AgentRunner 拡張機能はまだ公開されていません/,
{ exact: true },
)
.waitFor({ timeout: 15_000 });
const browseLink = page.getByRole("link", {
name: /Browse Runner Extensions|浏览运行器扩展|Runner 拡張機能を見る/,
});
await browseLink.waitFor();
const href = await browseLink.getAttribute("href");
if (href !== "/home/extensions?type=plugin&component=AgentRunner") {
throw new Error(`Unexpected Runner marketplace URL: ${href}`);
}
const nextButton = page.getByRole("button", {
name: /Create & Deploy|创建并部署|作成&デプロイ/,
});
if (!(await nextButton.isDisabled())) {
throw new Error("Wizard allowed continuing without an installed Runner.");
}
if (!result.marketplace_request) {
throw new Error(
"Wizard did not request the AgentRunner Marketplace catalog.",
);
}
result.visible_signals.push(
"first-run-ai-engine-step",
"empty-runner-catalog-state",
"runner-marketplace-link",
"next-disabled-without-runner",
);
await safeScreenshot(page, paths.screenshot);
await page.setViewportSize({ width: 390, height: 844 });
await page.waitForTimeout(250);
const overflow = await page.evaluate(
() => document.documentElement.scrollWidth - window.innerWidth,
);
if (overflow > 1) {
throw new Error(`Runner catalog Wizard overflows mobile by ${overflow}px.`);
}
await browseLink.scrollIntoViewIfNeeded();
await safeScreenshot(page, mobileScreenshot);
result.visible_signals.push("mobile-layout");
result.diagnostics = await scanBrowserDiagnostics(paths);
if (result.diagnostics.status !== "pass") {
throw new Error(result.diagnostics.reason);
}
result.status = "pass";
result.reason =
"A clean first-run instance requested the AgentRunner catalog, showed a safe empty state, and prevented continuing without a Runner.";
} catch (error) {
if (!["blocked", "env_issue"].includes(result.status)) result.status = "fail";
result.reason = result.reason || error.message;
if (browser?.page) await safeScreenshot(browser.page, paths.screenshot);
} finally {
const cleanup = {};
if (token && backendUrl) {
const resetProgress = await apiJson(
backendUrl,
"/api/v1/system/wizard/progress",
{
method: "PUT",
token,
body: {
step: 0,
selected_scenario: null,
selected_adapter: null,
created_bot_uuid: null,
bot_saved: false,
selected_runner: null,
},
},
).catch(() => ({ status: 0, json: {} }));
cleanup.progress_reset =
resetProgress.status < 400 && resetProgress.json.code === 0;
}
if (botId && token && backendUrl) {
const deletedBot = await apiJson(
backendUrl,
`/api/v1/platform/bots/${encodeURIComponent(botId)}`,
{ method: "DELETE", token },
).catch(() => ({ status: 0, json: {} }));
cleanup.bot_deleted = deletedBot.status < 400 && deletedBot.json.code === 0;
}
result.cleanup = cleanup;
if (
result.status === "pass" &&
(!cleanup.progress_reset || (botId && !cleanup.bot_deleted))
) {
result.status = "fail";
result.reason = "The clean Wizard fixtures were not fully reset.";
}
if (browser) await browser.close().catch(() => {});
const finishedAt = new Date();
result.finished_at = finishedAt.toISOString();
result.finished_at_local = localIsoWithOffset(finishedAt);
await writeResult(paths, result);
console.log(JSON.stringify(result, null, 2));
}
process.exit(exitCode(result.status));
+55
View File
@@ -143,6 +143,7 @@
"agent-runner-async-db-readiness",
"agent-runner-behavior-matrix",
"agent-runner-fixture-contract",
"agent-runner-health-visibility",
"agent-runner-ledger-concurrency",
"agent-runner-ledger-contention",
"agent-runner-ledger-invariants",
@@ -192,6 +193,7 @@
"skill-discovery-via-mcp-gateway",
"webui-login-state",
"wizard-onebot-agent-runtime",
"wizard-runner-marketplace-catalog",
"wizard-scenario-routing"
],
"case_summaries": [
@@ -291,6 +293,31 @@
"filesystem"
]
},
{
"id": "agent-runner-health-visibility",
"title": "Agent configuration shows whether its selected runner is usable",
"mode": "agent-browser",
"area": "agent",
"type": "feature",
"priority": "p0",
"risk": "medium",
"ci_eligible": false,
"tags": [
"agent-runner",
"agent",
"productization",
"release-gate"
],
"automation": "scripts/e2e/agent-runner-health-visibility.mjs",
"setup_automation": [],
"setup_provides_env": [],
"evidence_required": [
"ui",
"screenshot",
"console",
"api_diagnostic"
]
},
{
"id": "agent-runner-ledger-concurrency",
"title": "AgentRunner run ledger concurrency and auth pytest probe",
@@ -1676,6 +1703,32 @@
"api_diagnostic"
]
},
{
"id": "wizard-runner-marketplace-catalog",
"title": "Quick Start discovers AgentRunner extensions on a clean instance",
"mode": "agent-browser",
"area": "wizard",
"type": "feature",
"priority": "p0",
"risk": "high",
"ci_eligible": false,
"tags": [
"wizard",
"agent-runner",
"marketplace",
"clean-install",
"productization"
],
"automation": "scripts/e2e/wizard-runner-marketplace-catalog.mjs",
"setup_automation": [],
"setup_provides_env": [],
"evidence_required": [
"ui",
"screenshot",
"console",
"api_diagnostic"
]
},
{
"id": "wizard-scenario-routing",
"title": "Quick Start filters channels by the selected bot scenario",
@@ -1793,6 +1846,8 @@
],
"cases": [
"wizard-scenario-routing",
"wizard-runner-marketplace-catalog",
"agent-runner-health-visibility",
"bot-event-routing-product-flow",
"wizard-onebot-agent-runtime"
]
@@ -24,7 +24,7 @@ Healthy startup includes:
```text
Running on http://0.0.0.0:<backend-port>
Connected to plugin runtime.
Plugin langbot/local-agent initialized
Plugin langbot-team/LocalAgent initialized
```
Quick check:
+1 -1
View File
@@ -59,7 +59,7 @@ The tools wrap the LangBot service layer. Current tools (v1):
| `get_system_info` | Version, edition, instance id |
| `list_bots` / `get_bot` / `create_bot` / `update_bot` / `delete_bot` | Manage messaging-platform bots (secrets redacted on read) |
| `list_bot_event_route_statuses` / `test_bot_event_route` | Inspect bot event-route runtime status and dispatch a synthetic test event through saved routes without sending real outbound platform messages |
| `list_agents` / `get_agent` / `create_agent` / `update_agent` / `delete_agent` | Manage the Agent product surface, including Agent orchestrations and Pipelines |
| `list_processors` / `get_processor` / `create_processor` / `update_processor` / `delete_processor` | Manage the peer Agent and Pipeline processor types |
| `list_pipelines` / `get_pipeline` / `create_pipeline` / `update_pipeline` / `delete_pipeline` | Manage pipelines |
| `list_llm_models` / `get_llm_model` / `list_embedding_models` / `list_model_providers` | Inspect models & providers |
| `list_knowledge_bases` / `get_knowledge_base` / `retrieve_knowledge_base` | RAG knowledge bases (incl. semantic search) |
+1 -1
View File
@@ -422,7 +422,7 @@ When a plugin doesn't work:
3. **Verify plugin loaded**: `GET /api/v1/plugins` — should list your plugin
4. **Test person mode first**: `session_type=person` always triggers pipeline, isolating trigger rule issues
5. **Check trigger rules**: Group mode requires @bot, prefix match, or random% to enter pipeline
6. **Verify model configured**: Pipeline's `config.ai.local-agent.model.primary` must point to a valid model UUID with working API keys
6. **Verify model configured**: Pipeline's `config.ai.runner_config[config.ai.runner.id].model.primary` must point to a valid model UUID with working API keys
## Publishing Plugins
@@ -34,7 +34,7 @@ automation_env:
- LANGBOT_E2E_RESPONSE_TIMEOUT_MS
automation_pipeline_url_env: LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL
automation_pipeline_name_env: LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME
automation_expected_runner_id: "plugin:langbot/acp-agent-runner/default"
automation_expected_runner_id: "plugin:langbot-team/ACPAgentRunner/default"
automation_prompt: "Use the injected LangBot MCP server tool langbot_get_current_event once. If the MCP call succeeds, reply with exactly ACP_AGENT_RUNNER_E2E_OK."
automation_expected_text: "ACP_AGENT_RUNNER_E2E_OK"
automation_response_timeout_ms: "300000"
@@ -49,7 +49,7 @@ preconditions:
steps:
- "Open LANGBOT_FRONTEND_URL."
- "Open the ACP AgentRunner QA pipeline."
- "Confirm the pipeline AI runner is plugin:langbot/acp-agent-runner/default."
- "Confirm the pipeline AI runner is plugin:langbot-team/ACPAgentRunner/default."
- "Open Debug Chat."
- "Ask the real remote Claude ACP agent to call langbot_get_current_event and return ACP_AGENT_RUNNER_E2E_OK exactly."
checks:
@@ -72,7 +72,7 @@ success_patterns:
failure_patterns:
- "acp.command_not_found"
- "acp.process_exited"
- "Agent runner plugin:langbot/acp-agent-runner/default execution failed"
- "Agent runner plugin:langbot-team/ACPAgentRunner/default execution failed"
troubleshooting:
- backend-not-listening
- plugin-runtime-timeout
@@ -0,0 +1,53 @@
id: agent-runner-health-visibility
title: "Agent configuration shows whether its selected runner is usable"
mode: agent-browser
area: agent
type: feature
priority: p0
risk: medium
ci_eligible: false
tags:
- agent-runner
- agent
- productization
- release-gate
skills:
- langbot-env-setup
- langbot-testing
env:
- LANGBOT_FRONTEND_URL
- LANGBOT_BACKEND_URL
- LANGBOT_BROWSER_PROFILE
automation: scripts/e2e/agent-runner-health-visibility.mjs
automation_env:
- LANGBOT_FRONTEND_URL
- LANGBOT_BACKEND_URL
- LANGBOT_BROWSER_PROFILE
- LANGBOT_CHROMIUM_EXECUTABLE
preconditions:
- "The plugin runtime is enabled and connected, with at least one AgentRunner registered."
- "The target is a local test instance where a temporary Agent may be created and deleted."
steps:
- "Read the live plugin runtime status and select a registered runner from Agent metadata."
- "Create a temporary Agent using that runner and open its Runner settings in the WebUI."
- "Confirm the UI reports that the runner is ready."
- "Replace the temporary Agent binding with a syntactically valid but unregistered runner id."
- "Reload Runner settings and confirm the UI reports that the selected runner is unavailable."
checks:
- "UI: A registered runner and connected plugin runtime produce a visible ready status."
- "UI: A stale runner binding produces a visible unavailable status with recovery actions."
- "API: Plugin runtime status is connected and Agent metadata contains the registered runner."
- "Console: No unexpected frontend errors appear during the flow."
- "Cleanup: The temporary Agent is deleted after evidence is collected."
evidence_required:
- ui
- screenshot
- console
- api_diagnostic
diagnostics:
- "A disconnected plugin runtime is env_issue, not a passing unavailable-runner product check."
- "A ready status inferred only from API metadata without opening the Runner UI is not a pass."
troubleshooting:
- backend-not-listening
- plugin-runtime-timeout
- proxy-env-mismatch
@@ -45,7 +45,7 @@ steps:
- "Run the local-agent primary model test endpoint unless LANGBOT_PREFLIGHT_TEST_MODELS=0."
checks:
- "API diagnostic: api-diagnostic.json has no blockers and no env_issues."
- "API diagnostic: required pipelines resolve to plugin:langbot/local-agent/default and plugin:langbot/acp-agent-runner/default."
- "API diagnostic: required pipelines resolve to plugin:langbot-team/LocalAgent/default and plugin:langbot-team/ACPAgentRunner/default."
- "API diagnostic: qa_plugin_echo is exposed by /api/v1/tools."
- "API diagnostic: local-agent model check catches invalid credentials or missing func_call/vision before release E2E starts."
- "Secret safety: token values, api keys, and provider secrets are not printed."
@@ -38,7 +38,7 @@ steps:
- "Open LANGBOT_FRONTEND_URL."
- "Navigate to Pipelines and open the target local-agent pipeline."
- "Open Configuration > AI."
- "Use runner Default or the pluginized langbot/local-agent runner."
- "Use runner Default or the pluginized langbot-team/LocalAgent runner."
- "Select a model that is known to answer Debug Chat in the current environment."
- "Clear Knowledge Bases unless this run intentionally combines RAG with the smoke path."
- "Save the pipeline."
@@ -35,7 +35,7 @@ automation_env:
- LANGBOT_LOCAL_AGENT_RAG_KB_UUID
automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL
automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME
automation_expected_runner_id: "plugin:langbot/local-agent/default"
automation_expected_runner_id: "plugin:langbot-team/LocalAgent/default"
automation_runner_config_patch_json: '{"knowledge-bases":["${LANGBOT_LOCAL_AGENT_RAG_KB_UUID}"],"retrieval-top-k":1,"context-window-tokens":650,"context-reserve-tokens":180,"context-keep-recent-tokens":120,"context-summary-tokens":220,"max-tool-iterations":4,"tool-execution-mode":"serial"}'
automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot","name":"local-agent"},{"author":"qa","name":"plugin-smoke"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}'
automation_restore_runner_config: "1"
@@ -59,7 +59,7 @@ preconditions:
steps:
- "Ensure the local-agent pipeline, LangRAG sentinel KB, and qa-plugin-smoke fixture plugin are available."
- "Temporarily bind the LangRAG KB, shrink the runner context budget, and allow serial plugin tool execution."
- "Temporarily bind only langbot/local-agent and qa/plugin-smoke; disable MCP servers and skills to isolate the tool surface."
- "Temporarily bind only langbot-team/LocalAgent and qa/plugin-smoke; disable MCP servers and skills to isolate the tool surface."
- "Reset the person Debug Chat session for the target pipeline."
- "Send the passcode memory prompt and wait for MEMORY_SET."
- "Send the long pressure prompt and wait for COMBO_CONTEXT_PRESSURE_READY."
@@ -69,7 +69,7 @@ checks:
- "UI: All three user messages appear in Debug Chat."
- "UI: The final Bot message contains COMBO_FINAL qa_combo_compaction_sentinel_2406 azalea-cobalt-7421 qa-plugin-smoke:combo-tool-ok-local-agent."
- "API diagnostic: pipeline-config-diagnostic.json shows knowledge-bases, context token settings, and tool-loop settings were patched."
- "API diagnostic: pipeline-extensions-diagnostic.json shows only langbot/local-agent and qa/plugin-smoke were bound."
- "API diagnostic: pipeline-extensions-diagnostic.json shows only langbot-team/LocalAgent and qa/plugin-smoke were bound."
- "API diagnostic: restore diagnostics show the original runner config and extension bindings were restored."
- "Logs: Backend completes RAG retrieval, plugin tool call, and Debug Chat response without runner timeout or model setup failure."
- "Console: No unexpected frontend runtime errors appear during the run."
@@ -31,7 +31,7 @@ automation_env:
- LANGBOT_LOCAL_AGENT_PIPELINE_NAME
automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL
automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME
automation_expected_runner_id: "plugin:langbot/local-agent/default"
automation_expected_runner_id: "plugin:langbot-team/LocalAgent/default"
automation_runner_config_patch_json: '{"context-window-tokens":225,"context-reserve-tokens":50,"context-keep-recent-tokens":30,"context-summary-tokens":105,"knowledge-bases":[]}'
automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot","name":"local-agent"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}'
automation_restore_runner_config: "1"
@@ -51,7 +51,7 @@ preconditions:
steps:
- "Open the target local-agent pipeline through LANGBOT_FRONTEND_URL."
- "Use the authenticated browser token only inside automation to GET and PUT /api/v1/pipelines/{uuid}."
- "Assert the saved runner is plugin:langbot/local-agent/default."
- "Assert the saved runner is plugin:langbot-team/LocalAgent/default."
- "Temporarily set context-window-tokens, context-reserve-tokens, context-keep-recent-tokens, and context-summary-tokens to force compaction, and clear knowledge-bases so RAG does not answer the memory question."
- "Temporarily bind only the local-agent plugin, disable MCP servers, and disable skills so unrelated visible tools cannot answer or distract the memory question."
- "Reset the person Debug Chat session for the target pipeline."
@@ -64,7 +64,7 @@ checks:
- "UI: The final Bot message contains qa_compaction_sentinel_7391."
- "API diagnostic: pipeline-config-diagnostic.json shows patched=true and patch_keys include the four token context compaction fields plus knowledge-bases."
- "API diagnostic: pipeline-config-restore-diagnostic.json shows the original runner config was restored."
- "API diagnostic: pipeline-extensions-diagnostic.json shows enable_all_plugins=false, bound_plugins contains langbot/local-agent, enable_all_mcp_servers=false, and enable_all_skills=false."
- "API diagnostic: pipeline-extensions-diagnostic.json shows enable_all_plugins=false, bound_plugins contains langbot-team/LocalAgent, enable_all_mcp_servers=false, and enable_all_skills=false."
- "API diagnostic: pipeline-extensions-restore-diagnostic.json shows the original extension bindings were restored."
- "Logs: Backend completes the multi-turn Debug Chat path without runner timeout or model setup failure."
- "Console: No unexpected frontend runtime errors appear during the run."
@@ -44,7 +44,7 @@ steps:
- "Confirm the fixture plugin is bound to the target pipeline through Extensions."
- "Open the target local-agent pipeline."
- "Open Configuration > AI."
- "Use runner Default or the pluginized langbot/local-agent runner."
- "Use runner Default or the pluginized langbot-team/LocalAgent runner."
- "Select a model that is known to follow system prompts in Debug Chat."
- "Save the pipeline."
- "Open Debug Chat."
@@ -42,7 +42,7 @@ steps:
- "Open LANGBOT_FRONTEND_URL."
- "Navigate to Pipelines and open the target local-agent pipeline."
- "Open Configuration > AI."
- "Use runner Default or the pluginized langbot/local-agent runner."
- "Use runner Default or the pluginized langbot-team/LocalAgent runner."
- "Select a model that supports image input in the current environment, or use a known model that at least accepts the uploaded image payload."
- "Save the pipeline."
- "Open Debug Chat."
@@ -36,7 +36,7 @@ automation_env:
- LANGBOT_LOCAL_AGENT_RAG_KB_UUID
automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL
automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME
automation_expected_runner_id: "plugin:langbot/local-agent/default"
automation_expected_runner_id: "plugin:langbot-team/LocalAgent/default"
automation_runner_config_patch_json: '{"knowledge-bases":["${LANGBOT_LOCAL_AGENT_RAG_KB_UUID}"],"retrieval-top-k":1,"context-window-tokens":900,"context-reserve-tokens":220,"context-keep-recent-tokens":160,"context-summary-tokens":260,"max-tool-iterations":5,"tool-execution-mode":"serial"}'
automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot","name":"local-agent"},{"author":"qa","name":"plugin-smoke"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}'
automation_restore_runner_config: "1"
@@ -60,7 +60,7 @@ preconditions:
steps:
- "Ensure the local-agent pipeline, LangRAG sentinel KB, and qa-plugin-smoke fixture plugin are available."
- "Temporarily bind the LangRAG KB, shrink the runner context budget, and set max-tool-iterations high enough for two serial plugin tool calls."
- "Temporarily bind only langbot/local-agent and qa/plugin-smoke; disable MCP servers and skills to isolate the tool surface."
- "Temporarily bind only langbot-team/LocalAgent and qa/plugin-smoke; disable MCP servers and skills to isolate the tool surface."
- "Reset the person Debug Chat session for the target pipeline."
- "Send the passcode memory prompt and wait for MEMORY_SET."
- "Send the long pressure prompt and wait for MULTITOOL_CONTEXT_PRESSURE_READY."
@@ -70,7 +70,7 @@ checks:
- "UI: All three user messages appear in Debug Chat."
- "UI: The final Bot message contains MULTITOOL_COMBO_FINAL with the compacted sentinel, azalea-cobalt-7421, and both qa-plugin-smoke tool results."
- "API diagnostic: pipeline-config-diagnostic.json shows knowledge-bases, context token settings, and max-tool-iterations were patched."
- "API diagnostic: pipeline-extensions-diagnostic.json shows only langbot/local-agent and qa/plugin-smoke were bound."
- "API diagnostic: pipeline-extensions-diagnostic.json shows only langbot-team/LocalAgent and qa/plugin-smoke were bound."
- "Logs: Backend completes RAG retrieval, two plugin tool calls, and Debug Chat response without runner timeout or model setup failure."
- "Console: No unexpected frontend runtime errors appear during the run."
evidence_required:
@@ -39,7 +39,7 @@ steps:
- "Open LANGBOT_FRONTEND_URL."
- "Navigate to Pipelines and open the target local-agent pipeline."
- "Open Configuration > AI."
- "Use runner Default or the pluginized langbot/local-agent runner."
- "Use runner Default or the pluginized langbot-team/LocalAgent runner."
- "Select a model that is known to answer Debug Chat in the current environment."
- "Save the pipeline."
- "Open Debug Chat."
@@ -36,7 +36,7 @@ automation_env:
- LANGBOT_LOCAL_AGENT_RAG_KB_UUID
automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL
automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME
automation_expected_runner_id: "plugin:langbot/local-agent/default"
automation_expected_runner_id: "plugin:langbot-team/LocalAgent/default"
automation_runner_config_patch_json: '{"knowledge-bases":["${LANGBOT_LOCAL_AGENT_RAG_KB_UUID}"],"retrieval-top-k":1,"context-window-tokens":900,"context-reserve-tokens":220,"context-keep-recent-tokens":160,"context-summary-tokens":260,"max-tool-iterations":3,"tool-execution-mode":"parallel"}'
automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot","name":"local-agent"},{"author":"qa","name":"plugin-smoke"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}'
automation_restore_runner_config: "1"
@@ -60,7 +60,7 @@ preconditions:
steps:
- "Ensure the local-agent pipeline, LangRAG sentinel KB, and qa-plugin-smoke fixture plugin are available."
- "Temporarily bind the LangRAG KB, shrink the runner context budget, and set tool-execution-mode to parallel."
- "Temporarily bind only langbot/local-agent and qa/plugin-smoke; disable MCP servers and skills to isolate the tool surface."
- "Temporarily bind only langbot-team/LocalAgent and qa/plugin-smoke; disable MCP servers and skills to isolate the tool surface."
- "Reset the person Debug Chat session for the target pipeline."
- "Send the passcode memory prompt and wait for MEMORY_SET."
- "Send the long pressure prompt and wait for PARALLEL_CONTEXT_PRESSURE_READY."
@@ -70,7 +70,7 @@ checks:
- "UI: All three user messages appear in Debug Chat."
- "UI: The final Bot message contains PARALLEL_COMBO_FINAL with the compacted sentinel, azalea-cobalt-7421, and both qa-plugin-smoke tool results."
- "API diagnostic: pipeline-config-diagnostic.json shows knowledge-bases, context token settings, max-tool-iterations, and tool-execution-mode=parallel were patched."
- "API diagnostic: pipeline-extensions-diagnostic.json shows only langbot/local-agent and qa/plugin-smoke were bound."
- "API diagnostic: pipeline-extensions-diagnostic.json shows only langbot-team/LocalAgent and qa/plugin-smoke were bound."
- "Logs: Backend completes RAG retrieval, both plugin tool calls, and Debug Chat response without runner timeout or model setup failure."
- "Console: No unexpected frontend runtime errors appear during the run."
evidence_required:
@@ -46,7 +46,7 @@ steps:
- "Confirm /api/v1/tools or the plugin detail shows qa_plugin_echo."
- "Confirm the fixture plugin is bound to the target pipeline through Extensions, or that all plugins are enabled."
- "Open the target local-agent pipeline."
- "Use runner Default or the pluginized langbot/local-agent runner."
- "Use runner Default or the pluginized langbot-team/LocalAgent runner."
- "Select a model with function-calling ability that is known to work with tools in the current environment."
- "Open Debug Chat."
- "Send: Call the qa_plugin_echo tool with exactly this text: plugin-tool-ok-local-agent. Return only the tool result."
@@ -31,7 +31,7 @@ automation_env:
- LANGBOT_LOCAL_AGENT_RAG_KB_UUID
automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL
automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME
automation_expected_runner_id: "plugin:langbot/local-agent/default"
automation_expected_runner_id: "plugin:langbot-team/LocalAgent/default"
automation_runner_config_patch_json: '{"knowledge-bases":["${LANGBOT_LOCAL_AGENT_RAG_KB_UUID}"]}'
automation_restore_runner_config: "1"
automation_reset_debug_chat: "1"
@@ -29,7 +29,7 @@ automation_env:
- LANGBOT_LOCAL_AGENT_PIPELINE_NAME
automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL
automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME
automation_expected_runner_id: "plugin:langbot/local-agent/default"
automation_expected_runner_id: "plugin:langbot-team/LocalAgent/default"
automation_reset_debug_chat: "1"
automation_stream_output: "1"
automation_prompt: "You are running the LangBot steering E2E test. First call the qa_plugin_sleep tool with seconds=8 and text=steering-e2e-anchor. Do not answer before the tool result is available. After the tool returns, answer the latest user follow-up. If no follow-up was injected, reply only STEERING_NO_FOLLOWUP."
@@ -45,7 +45,7 @@ preconditions:
- "The selected model route supports function/tool calling and can call qa_plugin_sleep."
steps:
- "Open the target local-agent pipeline."
- "Assert the saved runner is plugin:langbot/local-agent/default."
- "Assert the saved runner is plugin:langbot-team/LocalAgent/default."
- "Reset the person Debug Chat session for the target pipeline."
- "Assert /api/v1/tools contains qa_plugin_sleep."
- "Send the first prompt that instructs the model to call qa_plugin_sleep for 8 seconds."
@@ -32,7 +32,7 @@ automation_env:
- LANGBOT_LOCAL_AGENT_PIPELINE_NAME
automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL
automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME
automation_expected_runner_id: "plugin:langbot/local-agent/default"
automation_expected_runner_id: "plugin:langbot-team/LocalAgent/default"
automation_runner_config_patch_json: '{"knowledge-bases":[],"max-tool-iterations":3,"tool-execution-mode":"serial"}'
automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot","name":"local-agent"},{"author":"qa","name":"plugin-smoke"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}'
automation_restore_runner_config: "1"
@@ -53,7 +53,7 @@ preconditions:
- "qa-plugin-smoke exposes qa_plugin_fail; the setup case reinstalls the fixture if an older installed package is missing it."
steps:
- "Ensure the local-agent pipeline and qa-plugin-smoke fixture plugin are available."
- "Temporarily bind only langbot/local-agent plus qa/plugin-smoke and clear RAG knowledge bases."
- "Temporarily bind only langbot-team/LocalAgent plus qa/plugin-smoke and clear RAG knowledge bases."
- "Reset the person Debug Chat session for the target pipeline."
- "Send the TOOL_ERROR_RECOVERY prompt."
- "Verify the failing plugin tool result is fed back into the model and the final answer contains the recovery sentinel."
@@ -61,7 +61,7 @@ steps:
checks:
- "UI: The final Bot message contains TOOL_ERROR_RECOVERY_FINAL qa-plugin-smoke forced failure observed."
- "API diagnostic: pipeline-config-diagnostic.json shows max-tool-iterations=3 and knowledge-bases cleared."
- "API diagnostic: pipeline-extensions-diagnostic.json shows only langbot/local-agent and qa/plugin-smoke were bound."
- "API diagnostic: pipeline-extensions-diagnostic.json shows only langbot-team/LocalAgent and qa/plugin-smoke were bound."
- "Logs: Backend shows the request completed without runner.timeout, runner.tool_error run failure, or model setup failure."
- "Console: No unexpected frontend runtime errors appear during Debug Chat."
evidence_required:
@@ -33,7 +33,7 @@ automation_env:
- LANGBOT_LOCAL_AGENT_PIPELINE_NAME
automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL
automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME
automation_expected_runner_id: "plugin:langbot/local-agent/default"
automation_expected_runner_id: "plugin:langbot-team/LocalAgent/default"
automation_runner_config_patch_json: '{"knowledge-bases":[],"max-tool-iterations":2,"tool-execution-mode":"serial"}'
automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot","name":"local-agent"},{"author":"qa","name":"plugin-smoke"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}'
automation_restore_runner_config: "1"
@@ -54,14 +54,14 @@ preconditions:
- "qa-plugin-smoke exposes qa_plugin_echo."
steps:
- "Ensure the local-agent pipeline and qa-plugin-smoke fixture plugin are available."
- "Temporarily set max-tool-iterations to 2 and bind only langbot/local-agent plus qa/plugin-smoke."
- "Temporarily set max-tool-iterations to 2 and bind only langbot-team/LocalAgent plus qa/plugin-smoke."
- "Reset the person Debug Chat session for the target pipeline."
- "Send the LOOP_LIMIT prompt and wait for the runner limit message."
- "Restore the original runner config and pipeline extension bindings."
checks:
- "UI: The final Bot message contains Tool call iteration limit reached. I stopped before calling more tools."
- "API diagnostic: pipeline-config-diagnostic.json shows max-tool-iterations was patched to 2."
- "API diagnostic: pipeline-extensions-diagnostic.json shows only langbot/local-agent and qa/plugin-smoke were bound."
- "API diagnostic: pipeline-extensions-diagnostic.json shows only langbot-team/LocalAgent and qa/plugin-smoke were bound."
- "Logs: Backend shows the request completed without infinite loop, runner timeout, or model setup failure."
- "Console: No unexpected frontend runtime errors appear during Debug Chat."
evidence_required:
@@ -31,7 +31,7 @@ automation_env:
- LANGBOT_MCP_QA_STDIO_SERVER_UUID
automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL
automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME
automation_expected_runner_id: "plugin:langbot/local-agent/default"
automation_expected_runner_id: "plugin:langbot-team/LocalAgent/default"
automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot","name":"local-agent"}],"enable_all_mcp_servers":false,"bound_mcp_servers":["${LANGBOT_MCP_QA_STDIO_SERVER_UUID}"],"enable_all_skills":false,"bound_skills":[]}'
automation_restore_extensions: "1"
automation_reset_debug_chat: "1"
@@ -65,7 +65,7 @@ steps:
- "Submit the server."
- "Confirm the server detail page shows Tools: 1 and qa_mcp_echo."
- "Open the target local-agent pipeline."
- "Use runner Default or the pluginized langbot/local-agent runner."
- "Use runner Default or the pluginized langbot-team/LocalAgent runner."
- "Select a model with function-calling ability that is known to work with tools in the current environment."
- "Open Debug Chat."
- "Send: Call the qa_mcp_echo tool with exactly this text: mcp-ok-local-agent. Return only the tool result."
@@ -0,0 +1,59 @@
id: wizard-runner-marketplace-catalog
title: "Quick Start discovers AgentRunner extensions on a clean instance"
mode: agent-browser
area: wizard
type: feature
priority: p0
risk: high
ci_eligible: false
tags:
- wizard
- agent-runner
- marketplace
- clean-install
- productization
skills:
- langbot-env-setup
- langbot-testing
env:
- LANGBOT_FRONTEND_URL
- LANGBOT_BACKEND_URL
- LANGBOT_E2E_LOGIN_USER
automation: scripts/e2e/wizard-runner-marketplace-catalog.mjs
automation_env:
- LANGBOT_FRONTEND_URL
- LANGBOT_BACKEND_URL
- LANGBOT_E2E_LOGIN_USER
- LANGBOT_BROWSER_PROFILE
- LANGBOT_CHROMIUM_EXECUTABLE
preconditions:
- "The target is a newly initialized local instance with wizard_status=none."
- "No plugins or AgentRunner components are installed."
steps:
- "Confirm the backend reports zero installed plugins and zero registered runners."
- "Resume Quick Start at the AI Engine step with a temporary disabled Bot."
- "Confirm the browser requests Marketplace plugins with component_filter=AgentRunner."
- "When Space has not published Runner plugins, confirm the safe empty state and Runner Extensions link."
- "Confirm Create & Deploy remains disabled until a Runner is installed and selected."
- "Verify the layout at desktop and mobile widths."
checks:
- "API: The instance has zero installed plugins and zero registered runners."
- "API: The instance wizard status is none."
- "Network: Marketplace search uses component_filter=AgentRunner and type_filter=plugin."
- "UI: The AI Engine step remains usable when the Runner catalog is empty."
- "UI: Browse Runner Extensions links to the AgentRunner-filtered market."
- "UI: Create & Deploy remains disabled without a selected Runner."
- "Console: No unexpected frontend errors appear during the flow."
- "Cleanup: Wizard progress and the temporary Bot are removed."
evidence_required:
- ui
- screenshot
- console
- api_diagnostic
diagnostics:
- "This case validates the clean-instance discovery and empty state. Run the install continuation after Space publishes at least one AgentRunner plugin."
- "A Marketplace API response alone is not a pass; the empty or installable Runner UI must be visible."
troubleshooting:
- backend-not-listening
- plugin-runtime-timeout
- proxy-env-mismatch
@@ -110,7 +110,7 @@ Each probe writes `automation-result.json` and probe logs under
| Generic Pipeline Debug Chat | `pipeline-debug-chat` | The WebUI Debug Chat path itself works before runner-specific failures are diagnosed. |
| Deterministic QA runner install | `agent-runner-live-install` | A local `.lbpkg` AgentRunner package can install and register a runner. |
| Deterministic QA runner Debug Chat | `agent-runner-qa-debug-chat` | The installed QA runner executes through WebUI Debug Chat without a model provider. |
| Required runner plugins | `agent-runner-release-preflight` | `langbot/local-agent` and `langbot/acp-agent-runner` are visible to the host. |
| Required runner plugins | `agent-runner-release-preflight` | `langbot-team/LocalAgent` and `langbot-team/ACPAgentRunner` are visible to the host. |
| Required QA plugin tools | `plugin-e2e-smoke`, `agent-runner-release-preflight`, `qa-plugin-smoke-live-install` | The deterministic `qa_plugin_echo` and `qa_plugin_fail` tools are exposed before tool-loop and tool-error cases start. |
| Knowledge base fixture | `langrag-kb-retrieve`, `local-agent-rag-debug-chat` | LangRAG data is queryable and the runner inserts retrieved context. |
| Effective prompt bridge | `local-agent-effective-prompt-debug-chat` | Host prompt preprocessing reaches the runner. |
@@ -1,6 +1,6 @@
# Dify AgentRunner
Use this reference when validating `langbot/dify-agent` through LangBot WebUI.
Use this reference when validating `langbot-team/DifyAgent` through LangBot WebUI.
## Prepare Dify
@@ -38,7 +38,7 @@ Pass only when:
## Diagnostics
- `GET /api/v1/pipelines/{uuid}` can confirm the saved runner id is `plugin:langbot/dify-agent/default` and runner config contains `app-type`, `base-url`, and `api-key`.
- `GET /api/v1/pipelines/{uuid}` can confirm the saved runner id is `plugin:langbot-team/DifyAgent/default` and runner config contains `app-type`, `base-url`, and `api-key`.
- Direct Dify streaming API calls are useful only to distinguish invalid Dify credentials from LangBot runner issues.
- If Debug Chat returns `Agent runner execution failed`, inspect backend logs before changing UI settings.
@@ -1,6 +1,6 @@
# Local Agent Runner Coverage
Use this matrix when judging whether the external `langbot/local-agent` plugin still behaves like the old built-in local-agent runner.
Use this matrix when judging whether the external `langbot-team/LocalAgent` plugin still behaves like the old built-in local-agent runner.
The QA target is end-to-end behavior. UI cases prove the host, SDK, plugin runtime, and WebUI work together. Unit or component tests are still needed for negative branches that are hard to trigger reliably through a live provider.
@@ -1,6 +1,6 @@
# Local Agent Runner
Use this reference when validating the pluginized `langbot/local-agent` runner through the WebUI.
Use this reference when validating the pluginized `langbot-team/LocalAgent` runner through the WebUI.
The goal is behavior parity with the old built-in local-agent runner. The code does not need to be identical, but the visible behavior should match: effective prompt, current input, history, model selection and fallback, tool calling, knowledge retrieval, multimodal input, streaming and non-streaming output all have to reach the runner through the host and SDK.
@@ -87,7 +87,7 @@ extension binding uses this UUID, not the human-readable server name.
1. Open the target pipeline.
2. Confirm `Extensions` allows the MCP server, or that all MCP servers are enabled.
3. Use runner `Default` or the pluginized `langbot/local-agent` runner.
3. Use runner `Default` or the pluginized `langbot-team/LocalAgent` runner.
4. Select a model with function-calling ability that is known to work with tools in the current environment.
5. Open `Debug Chat`.
6. Ask:
@@ -20,7 +20,7 @@ An old `langbot_plugin` runtime process survived a backend restart, or multiple
### Fix
Stop the LangBot backend and any orphaned `langbot_plugin.cli` runtime processes, confirm the configured backend URL is free/reachable as appropriate, then start LangBot again. A healthy startup logs `Connected to plugin runtime`, mounts `langbot/local-agent`, and initializes the default agent runner.
Stop the LangBot backend and any orphaned `langbot_plugin.cli` runtime processes, confirm the configured backend URL is free/reachable as appropriate, then start LangBot again. A healthy startup logs `Connected to plugin runtime`, mounts `langbot-team/LocalAgent`, and initializes the default agent runner.
### Verification
@@ -10,5 +10,7 @@ tags:
- release-gate
cases:
- wizard-scenario-routing
- wizard-runner-marketplace-catalog
- agent-runner-health-visibility
- bot-event-routing-product-flow
- wizard-onebot-agent-runtime
@@ -16,7 +16,7 @@ fix_steps:
- "Change metadata.label to the provider or runner family name, such as Dify, Local Agent, Coze, DashScope, Langflow, n8n, or Tbox."
- "Restart LangBot or plugin runtime to refresh runner metadata cache."
- "Check /api/v1/pipelines/_/metadata and the WebUI runner selector."
verification: "Runner options are distinguishable by visible label while their ids remain stable, such as plugin:langbot/dify-agent/default."
verification: "Runner options are distinguishable by visible label while their ids remain stable, such as plugin:langbot-team/DifyAgent/default."
related_cases:
- dify-agent-debug-chat
- local-agent-rag-debug-chat
@@ -22,7 +22,7 @@ fix_steps:
- "Stop the LangBot backend."
- "Stop orphaned langbot_plugin.cli runtime processes."
- "Confirm the configured backend URL is free or reachable as appropriate."
- "Start LangBot again and wait for Connected to plugin runtime plus langbot/local-agent initialization."
- "Start LangBot again and wait for Connected to plugin runtime plus langbot-team/LocalAgent initialization."
- "For disposable test data only, move suspected fixture plugins out of data/plugins and restart to isolate plugin discovery failures."
verification: "Open Installed Plugins and confirm the plugin list loads, then run pipeline-debug-chat and confirm the UI shows a Bot response while backend logs include Streaming completed."
related_cases:
+1 -1
View File
@@ -3578,7 +3578,7 @@ test("MCP stdio tool-call case setups pipeline and registered MCP server", () =>
);
assert.equal(
run.automation.env_defaults.LANGBOT_E2E_EXPECTED_RUNNER_ID,
"plugin:langbot/local-agent/default",
"plugin:langbot-team/LocalAgent/default",
);
assert.equal(run.automation.env_defaults.LANGBOT_E2E_RESET_DEBUG_CHAT, "1");
assert.equal(