mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-09-16 23:07:14 +00:00
fix(agent-debug): stream execution traces with platform mocks and coverage
This commit is contained in:
@@ -549,7 +549,7 @@
|
||||
},
|
||||
{
|
||||
"id": "bot-event-routing-product-flow",
|
||||
"title": "Bot event routing can be configured and tested from the WebUI",
|
||||
"title": "Bot event routing and adapter input can be inspected from the WebUI",
|
||||
"mode": "agent-browser",
|
||||
"area": "bot",
|
||||
"type": "feature",
|
||||
|
||||
@@ -66,6 +66,7 @@ The tools wrap the LangBot service layer. Current tools (v1):
|
||||
| `list_bots` / `get_bot` / `create_bot` / `update_bot` / `delete_bot` | Manage messaging-platform bots (secrets redacted on read) |
|
||||
| `list_bot_event_route_statuses` | Inspect bot event-route runtime status |
|
||||
| `list_processors` / `get_processor` / `create_processor` / `update_processor` / `delete_processor` | Manage the peer Agent and Pipeline processor types |
|
||||
| `debug_agent` | Execute a synthetic Agent event (`processor_uuid`, `payload`); requires `runtime.operate`. Returns final text and up to 1000 execution events (thinking, text, tool arguments/results). Platform tools use Mock; other configured tools execute normally. Optional `payload.mock`: `errors`/`results` keyed by platform tool name, `unsupported_apis` lists unavailable platform APIs. |
|
||||
| `list_pipelines` / `get_pipeline` / `create_pipeline` / `update_pipeline` / `delete_pipeline` | Manage pipelines |
|
||||
| `list_llm_models` / `get_llm_model` / `list_embedding_models` / `list_model_providers` | Inspect models & providers |
|
||||
| `list_knowledge_bases` / `get_knowledge_base` / `retrieve_knowledge_base` | RAG knowledge bases (incl. semantic search) |
|
||||
|
||||
@@ -27,7 +27,7 @@ const refRe = /(?:\]\(|`)(references\/[A-Za-z0-9_.\-/]+\.md)(?:\)|`)/g;
|
||||
|
||||
function validateStructuredItem(item: StructuredItem, requiredStrings: string[], requiredLists: string[]): string[] {
|
||||
const errors: string[] = [];
|
||||
const listKeys = item.path.includes("/cases/") && scalar(item.fields, "mode") === "probe"
|
||||
const listKeys = /[\\/]cases[\\/]/.test(item.path) && scalar(item.fields, "mode") === "probe"
|
||||
? requiredLists.filter((key) => key !== "env")
|
||||
: requiredLists;
|
||||
for (const key of requiredStrings) {
|
||||
|
||||
+2
-2
@@ -3,7 +3,7 @@ import { join } from "node:path";
|
||||
import type { ParsedYaml, Skill, StructuredItem, StructuredItemKind } from "./types.ts";
|
||||
import { fail } from "./cli.ts";
|
||||
|
||||
const frontmatterRe = /^---\n([\s\S]*?)\n---\n/;
|
||||
const frontmatterRe = /^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/;
|
||||
|
||||
export function statIsDirectory(path: string): boolean {
|
||||
try {
|
||||
@@ -107,7 +107,7 @@ export function globMarkdownRefs(skillPath: string): string[] {
|
||||
return readdirSync(refsDir)
|
||||
.filter((name) => name.endsWith(".md"))
|
||||
.sort()
|
||||
.map((name) => join("references", name));
|
||||
.map((name) => `references/${name}`);
|
||||
}
|
||||
|
||||
export function globYamlFiles(dir: string): string[] {
|
||||
|
||||
@@ -49,7 +49,16 @@ import {
|
||||
import { commandTroubleSearch } from "../src/commands/trouble.ts";
|
||||
import { commandValidate } from "../src/commands/validate.ts";
|
||||
import { commandIndex } from "../src/commands/skill.ts";
|
||||
import { loadEnv } from "../src/fs.ts";
|
||||
import { loadEnv, parseFrontmatter } from "../src/fs.ts";
|
||||
|
||||
test('frontmatter preserves metadata and body with LF and CRLF checkouts', () => {
|
||||
for (const newline of ['\n', '\r\n']) {
|
||||
const source = ['---', 'name: example', 'description: "Example skill"', '---', '# Body', ''].join(newline);
|
||||
const parsed = parseFrontmatter(source);
|
||||
assert.deepEqual(parsed.meta, { name: 'example', description: 'Example skill' });
|
||||
assert.equal(parsed.body, '# Body' + newline);
|
||||
}
|
||||
});
|
||||
import { repoRoot } from "../src/cli.ts";
|
||||
import {
|
||||
classifyDebugChatResult,
|
||||
@@ -2779,11 +2788,11 @@ test("test start creates a run handoff with a bounded report command", () => {
|
||||
assert.match(result.output, /bin\/lbs test plan pipeline-debug-chat/);
|
||||
assert.match(
|
||||
result.output,
|
||||
/bin\/lbs test run pipeline-debug-chat --run-id .+ --output reports\/evidence\/.+pipeline-debug-chat/,
|
||||
/bin\/lbs test run pipeline-debug-chat --run-id .+ --output reports[\\/]evidence[\\/].+pipeline-debug-chat/,
|
||||
);
|
||||
assert.match(
|
||||
result.output,
|
||||
/bin\/lbs test report pipeline-debug-chat --since ".+" --console-log reports\/evidence\/.+\/console\.log --evidence-dir reports\/evidence\/.+ --output reports\/.+pipeline-debug-chat\.md/,
|
||||
/bin\/lbs test report pipeline-debug-chat --since ".+" --console-log reports[\\/]evidence[\\/].+[\\/]console\.log --evidence-dir reports[\\/]evidence[\\/].+ --output reports[\\/].+pipeline-debug-chat\.md/,
|
||||
);
|
||||
assert.match(result.output, /Streaming completed/);
|
||||
});
|
||||
@@ -2891,15 +2900,15 @@ test("test run dry-run exposes case automation script and evidence paths", () =>
|
||||
assert.match(result.output, /scripts\/e2e\/pipeline-debug-chat\.mjs/);
|
||||
assert.match(
|
||||
result.output,
|
||||
/console_log: reports\/evidence\/run-123\/console\.log/,
|
||||
/console_log: reports[\\/]evidence[\\/]run-123[\\/]console\.log/,
|
||||
);
|
||||
assert.match(
|
||||
result.output,
|
||||
/automation_result_json: reports\/evidence\/run-123\/automation-result\.json/,
|
||||
/automation_result_json: reports[\\/]evidence[\\/]run-123[\\/]automation-result\.json/,
|
||||
);
|
||||
assert.match(
|
||||
result.output,
|
||||
/result_json: reports\/evidence\/run-123\/result\.json/,
|
||||
/result_json: reports[\\/]evidence[\\/]run-123[\\/]result\.json/,
|
||||
);
|
||||
assert.match(result.output, /LANGBOT_PIPELINE_URL/);
|
||||
});
|
||||
@@ -3260,7 +3269,7 @@ test("test run setup automation isolates evidence and reloads env", () => {
|
||||
assert.equal(plan.setup_automation.length, 1);
|
||||
assert.match(
|
||||
plan.setup_automation[0].evidence_dir,
|
||||
/setup\/01-write-setup-env$/,
|
||||
/setup[\\/]01-write-setup-env$/,
|
||||
);
|
||||
assert.match(
|
||||
plan.setup_automation[0].command,
|
||||
|
||||
Reference in New Issue
Block a user