mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-08-17 16:00:59 +00:00
feat: MCP server + in-repo skills (agent-friendly platform) (#2269)
* feat(api): support global API key from config.yaml (api.global_api_key) Accept a config-defined global API key anywhere a web-UI key is accepted (X-API-Key / Bearer), with no login session and no DB record. Useful for automated deployments and AI agents (HTTP API + MCP). Defaults to empty (disabled); does not require the lbk_ prefix. - templates/config.yaml: add api.global_api_key with security notes - service/apikey.py: verify_api_key checks global key first (constant-time) - docs/API_KEY_AUTH.md: document the global key + security guidance - tests: cover global-key match, prefix-free, fallback-to-db, disabled * feat(mcp): expose LangBot management as an MCP server at /mcp Add an MCP (Model Context Protocol) server so external AI agents can manage a LangBot instance. Reuses the same API-key auth as the HTTP API (including the config.yaml global API key). - pkg/api/mcp/server.py: FastMCP server wrapping the service layer; 21 curated tools across system/bots/pipelines/models/knowledge/mcp-servers/skills - pkg/api/mcp/mount.py: ASGI dispatcher fronting Quart; authenticates /mcp requests with an API key, runs the streamable-HTTP session manager lifespan - controller/main.py: serve the wrapped ASGI app via hypercorn (was run_task) - web: new 'MCP' tab in the API integration dialog showing endpoint, auth, and client config; i18n for 8 locales - tests/manual/mcp_smoke.py: e2e check (401 unauth, list tools, call tools) Tool surface is intentionally curated (not all ~25 route groups) to keep the agent surface small, safe, and maintainable. Extend deliberately. * feat(skills): add in-repo skills/ as the single source of truth Migrate the agent skills + QA/e2e test harness from the (now archived) langbot-app/langbot-skills repo into LangBot/skills/, and add four new skills. Migrated: - langbot-plugin-dev, langbot-testing (e2e), langbot-env-setup, langbot-skills-maintenance, langbot-eba-adapter-dev - the bin/lbs CLI (src/, test/, scripts/, schemas/, qa-agent-docs/) New: - langbot-dev core backend + web development - langbot-deploy Docker/K8s deployment + config.yaml + global API key - langbot-mcp-ops operating the LangBot MCP server (/mcp) - langbot-space-ops operating the Space marketplace MCP server - src/cli.ts repoRoot(): recognize the skills assets root (skills.index.json + bin/lbs) so the CLI works when nested inside the LangBot repo - README.md: unified skill catalog; skills.index.json regenerated Parity with source verified: bin/lbs validate + node test suite match the source repo (only the uncommitted .lbpkg build-artifact fixture differs). * docs(agents): document agent-facing surfaces + API/MCP/skills sync rule * docs(readme): add 'Built for AI Agents' section across all locales Highlight MCP server, in-repo skills (single source of truth), AGENTS.md sync rule, and llms.txt. Cross-link LangBot Space MCP marketplace. * style(mcp): fix ruff format + prettier lint in MCP server and API panel * style(web): prettier format MCP i18n locale entries * docs(skills): note MCP instance control in dev/testing skills All development-guidance skills now point to the LangBot instance MCP server (/mcp) and the Space marketplace MCP server, reusing API keys.
This commit is contained in:
Junyan Chin
@@ -0,0 +1,120 @@
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { cwd, exit } from "node:process";
|
||||
import { existsSync } from "node:fs";
|
||||
import type { CommandContext } from "./types.ts";
|
||||
|
||||
export function usage(): never {
|
||||
console.log(`Usage:
|
||||
bin/lbs [--root <path>] list
|
||||
bin/lbs [--root <path>] validate
|
||||
bin/lbs [--root <path>] index [--check]
|
||||
bin/lbs [--root <path>] new-skill <name> [--description <text>]
|
||||
bin/lbs [--root <path>] new-ref <skill> <name>
|
||||
|
||||
bin/lbs [--root <path>] env show [--json]
|
||||
bin/lbs [--root <path>] env doctor
|
||||
|
||||
bin/lbs [--root <path>] fixture list [skill] [--json]
|
||||
bin/lbs [--root <path>] fixture check [skill] [--json]
|
||||
|
||||
bin/lbs [--root <path>] log scan [--json] [--output <path>] [--backend-log <path>] [--frontend-log <path>] [--console-log <path>] [--case <case-id>] [--success-pattern <text>] [--failure-pattern <text>] [--expected-failure <text>] [--since <datetime>] [--until <datetime>] [--tail-lines <n>] [--no-auto-log] [--strict]
|
||||
bin/lbs [--root <path>] log watch [--json] [--backend-log <path>] [--case <case-id>] [--success-pattern <text>] [--failure-pattern <text>] [--expected-failure <text>] [--interval-ms <n>] [--duration-ms <n>] [--from-start] [--strict]
|
||||
bin/lbs [--root <path>] log guard start [--run-id <id>] [--output-dir <dir>] [--backend-log <path>] [--case <case-id>] [--json]
|
||||
bin/lbs [--root <path>] log guard stop --run-id <id> [--output-dir <dir>] [--session <path>] [--output <path>] [--case <case-id>] [--backend-log <path>] [--since <datetime>] [--until <datetime>] [--json] [--no-strict]
|
||||
|
||||
bin/lbs [--root <path>] case new <id> --title <text> [--skill langbot-testing] [--mode agent-browser|probe] [--area <area>] [--type smoke]
|
||||
bin/lbs [--root <path>] case list [skill] [--json] [--type <type>] [--area <area>] [--tag <tag>] [--priority p0|p1|p2] [--risk low|medium|high] [--automation] [--ci] [--ready] [--machine-ready]
|
||||
bin/lbs [--root <path>] case show [skill] <id>
|
||||
|
||||
bin/lbs [--root <path>] suite new <id> --title <text> [--skill langbot-testing] [--description <text>] [--type smoke] [--priority p2]
|
||||
bin/lbs [--root <path>] suite list [skill] [--json] [--type <type>] [--priority p0|p1|p2]
|
||||
bin/lbs [--root <path>] suite show [skill] <id>
|
||||
bin/lbs [--root <path>] suite plan [skill] <id> [--json]
|
||||
bin/lbs [--root <path>] suite start [skill] <id> [--run-id <id>] [--evidence-dir <dir>] [--output <path>] [--json]
|
||||
bin/lbs [--root <path>] suite run [skill] <id> [--run-id <id>] [--evidence-dir <dir>] [--output <path>] [--headed] [--dry-run] [--include-manual-check] [--include-not-ready] [--json]
|
||||
bin/lbs [--root <path>] suite report [skill] <id> [--run-id <id>] [--evidence-dir <dir>] [--output <path>] [--json]
|
||||
|
||||
bin/lbs [--root <path>] test plan [skill] <case-id> [--json]
|
||||
bin/lbs [--root <path>] test recommend [--file <path>] [--json]
|
||||
bin/lbs [--root <path>] test start [skill] <case-id> [--output <path>] [--json]
|
||||
bin/lbs [--root <path>] test run [skill] <case-id> [--output <dir>] [--run-id <id>] [--headed] [--dry-run] [--json]
|
||||
bin/lbs [--root <path>] test report [skill] <case-id> [--output <path>] [--json] [--backend-log <path>] [--frontend-log <path>] [--console-log <path>] [--evidence-dir <dir>] [--since <datetime>] [--until <datetime>] [--tail-lines <n>] [--no-auto-log]
|
||||
bin/lbs [--root <path>] test result [skill] <case-id> --result <pass|fail|blocked|env_issue|flaky> --reason <text> --evidence-dir <dir> [--evidence ui,console,backend_log] [--started-at <datetime>] [--finished-at <datetime>] [--run-id <id>] [--url <url>] [--browser-path <text>] [--report <path>] [--notes <text>] [--json]
|
||||
|
||||
bin/lbs [--root <path>] trouble list [skill]
|
||||
bin/lbs [--root <path>] trouble show [skill] <id>
|
||||
bin/lbs [--root <path>] trouble search <query>
|
||||
bin/lbs [--root <path>] trouble add <skill> --title <text> --symptom <text> --cause <text> --fix <text> [--id <id>] [--verify <text>]
|
||||
`);
|
||||
exit(2);
|
||||
}
|
||||
|
||||
export function fail(message: string): never {
|
||||
console.error(`ERROR: ${message}`);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
export function repoRoot(start: string): string {
|
||||
let current = resolve(start);
|
||||
while (true) {
|
||||
// The skills assets root is identified by skills.index.json (present at the
|
||||
// root of this assets tree). Check it first so that when the tree lives
|
||||
// inside a larger repo (e.g. LangBot/skills/), we stop at the assets root
|
||||
// and not at the outer repo's .git/README.md.
|
||||
if (existsSync(`${current}/skills.index.json`) && existsSync(`${current}/bin/lbs`)) {
|
||||
return current;
|
||||
}
|
||||
if (existsSync(`${current}/.git`) && existsSync(`${current}/README.md`)) {
|
||||
return current;
|
||||
}
|
||||
const parent = dirname(current);
|
||||
if (parent === current) return resolve(start);
|
||||
current = parent;
|
||||
}
|
||||
}
|
||||
|
||||
export function parseGlobalArgs(rawArgs: string[]): CommandContext {
|
||||
let root = repoRoot(cwd());
|
||||
const args = [...rawArgs];
|
||||
|
||||
for (let i = 0; i < args.length; ) {
|
||||
if (args[i] === "--root") {
|
||||
const value = args[i + 1];
|
||||
if (!value) fail("--root requires a path");
|
||||
root = resolve(value);
|
||||
args.splice(i, 2);
|
||||
continue;
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
|
||||
return { root, args };
|
||||
}
|
||||
|
||||
export function parseOptions(args: string[]): { positional: string[]; options: Record<string, string | boolean> } {
|
||||
const positional: string[] = [];
|
||||
const options: Record<string, string | boolean> = {};
|
||||
|
||||
for (let i = 0; i < args.length; i += 1) {
|
||||
const arg = args[i];
|
||||
if (arg.startsWith("--")) {
|
||||
const key = arg.slice(2);
|
||||
const value = args[i + 1];
|
||||
if (!value || value.startsWith("--")) {
|
||||
options[key] = true;
|
||||
} else {
|
||||
options[key] = value;
|
||||
i += 1;
|
||||
}
|
||||
} else {
|
||||
positional.push(arg);
|
||||
}
|
||||
}
|
||||
|
||||
return { positional, options };
|
||||
}
|
||||
|
||||
export function optionString(options: Record<string, string | boolean>, key: string): string | undefined {
|
||||
const value = options[key];
|
||||
return typeof value === "string" ? value : undefined;
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
import { existsSync, mkdirSync, writeFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import type { CommandContext } from "../types.ts";
|
||||
import { parseOptions, optionString, usage, fail } from "../cli.ts";
|
||||
import { caseModeValues } from "../constants.ts";
|
||||
import { boolValue, findStructuredItem, getSkill, listValue, loadStructuredItems, scalar, yamlList, yamlQuote } from "../fs.ts";
|
||||
import { caseAutomationReadiness, caseEnvReadiness, caseFixtureReadiness, caseManualReadiness, runtimeEnv } from "../readiness.ts";
|
||||
import { setupAutomationEntries } from "../setup-automation.ts";
|
||||
|
||||
function casePath(root: string, skillName: string, id: string): string {
|
||||
const skill = getSkill(root, skillName);
|
||||
const dir = join(skill.path, "cases");
|
||||
mkdirSync(dir, { recursive: true });
|
||||
return join(dir, `${id}.yaml`);
|
||||
}
|
||||
|
||||
export function commandCaseNew(ctx: CommandContext): number {
|
||||
const { positional, options } = parseOptions(ctx.args.slice(2));
|
||||
const id = positional[0];
|
||||
const title = optionString(options, "title");
|
||||
if (!id || !title) usage();
|
||||
|
||||
const skill = optionString(options, "skill") ?? "langbot-testing";
|
||||
const path = casePath(ctx.root, skill, id);
|
||||
if (existsSync(path)) fail(`case already exists: ${path}`);
|
||||
|
||||
const area = optionString(options, "area") ?? "general";
|
||||
const type = optionString(options, "type") ?? "smoke";
|
||||
const mode = optionString(options, "mode") ?? "agent-browser";
|
||||
if (!caseModeValues.includes(mode)) fail(`--mode must be one of ${caseModeValues.join(", ")}`);
|
||||
const isProbe = mode === "probe";
|
||||
|
||||
const text =
|
||||
`id: ${id}\n` +
|
||||
`title: ${yamlQuote(title)}\n` +
|
||||
`mode: ${mode}\n` +
|
||||
`area: ${area}\n` +
|
||||
`type: ${type}\n` +
|
||||
"priority: p2\n" +
|
||||
"risk: medium\n" +
|
||||
"ci_eligible: false\n" +
|
||||
"tags:\n" +
|
||||
yamlList([type]) +
|
||||
"\nskills:\n" +
|
||||
yamlList(["langbot-env-setup", skill]) +
|
||||
"\nenv:\n" +
|
||||
yamlList(isProbe ? [] : ["LANGBOT_FRONTEND_URL", "LANGBOT_BACKEND_URL"]) +
|
||||
"\nsteps:\n" +
|
||||
yamlList([isProbe ? "Describe the probe command, script, or diagnostic to run." : "Describe the user-visible action to perform."]) +
|
||||
"\nchecks:\n" +
|
||||
yamlList(isProbe
|
||||
? [
|
||||
"Probe: Describe the expected success signal.",
|
||||
"Evidence: Required logs, API diagnostics, or filesystem artifacts are written.",
|
||||
]
|
||||
: [
|
||||
"UI: Describe the user-visible success signal.",
|
||||
"Console: No unexpected frontend errors.",
|
||||
"Logs: Relevant backend processing completed when applicable.",
|
||||
]) +
|
||||
"\nevidence_required:\n" +
|
||||
yamlList(isProbe ? ["api_diagnostic"] : ["ui", "console"]) +
|
||||
"\ndiagnostics:\n" +
|
||||
yamlList([isProbe
|
||||
? "Use logs, API, or filesystem diagnostics to explain probe failures."
|
||||
: "Use API/curl/logs only to distinguish frontend failure from backend/runtime failure."]) +
|
||||
"\ntroubleshooting:\n" +
|
||||
yamlList([]) +
|
||||
"\n";
|
||||
|
||||
writeFileSync(path, text, "utf8");
|
||||
console.log(path);
|
||||
return 0;
|
||||
}
|
||||
|
||||
function caseRow(item: ReturnType<typeof loadStructuredItems>[number], root: string): Record<string, unknown> {
|
||||
const automation = scalar(item.fields, "automation");
|
||||
const env = runtimeEnv(root);
|
||||
const id = scalar(item.fields, "id");
|
||||
const row = {
|
||||
skill: item.skill,
|
||||
id,
|
||||
title: scalar(item.fields, "title"),
|
||||
mode: scalar(item.fields, "mode"),
|
||||
area: scalar(item.fields, "area"),
|
||||
type: scalar(item.fields, "type"),
|
||||
priority: scalar(item.fields, "priority"),
|
||||
risk: scalar(item.fields, "risk"),
|
||||
ci_eligible: boolValue(item.fields, "ci_eligible") ?? false,
|
||||
tags: listValue(item.fields, "tags"),
|
||||
env: listValue(item.fields, "env"),
|
||||
env_any: listValue(item.fields, "env_any"),
|
||||
preconditions: listValue(item.fields, "preconditions"),
|
||||
setup: listValue(item.fields, "setup"),
|
||||
setup_automation: setupAutomationEntries(item),
|
||||
setup_provides_env: listValue(item.fields, "setup_provides_env"),
|
||||
cleanup: listValue(item.fields, "cleanup"),
|
||||
evidence_required: listValue(item.fields, "evidence_required"),
|
||||
automation,
|
||||
automation_exists: automation ? existsSync(resolve(root, automation)) : false,
|
||||
env_readiness: caseEnvReadiness(item, env),
|
||||
automation_readiness: caseAutomationReadiness(item, env),
|
||||
fixture_readiness: caseFixtureReadiness(root, id),
|
||||
manual_readiness: caseManualReadiness(item),
|
||||
};
|
||||
return {
|
||||
...row,
|
||||
readiness: readinessLabel(row),
|
||||
};
|
||||
}
|
||||
|
||||
function hasTag(row: Record<string, unknown>, tag: string): boolean {
|
||||
const tags = row.tags;
|
||||
return Array.isArray(tags) && tags.includes(tag);
|
||||
}
|
||||
|
||||
function hasMissingReadiness(row: Record<string, unknown>): boolean {
|
||||
for (const key of ["env_readiness", "automation_readiness", "fixture_readiness"]) {
|
||||
const value = row[key] as Record<string, unknown> | undefined;
|
||||
if (value?.status === "missing") return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function hasManualCheck(row: Record<string, unknown>): boolean {
|
||||
const manual = row.manual_readiness as Record<string, unknown> | undefined;
|
||||
return manual?.status === "manual_check";
|
||||
}
|
||||
|
||||
function readinessLabel(row: Record<string, unknown>): string {
|
||||
if (hasMissingReadiness(row)) return "not-ready";
|
||||
return hasManualCheck(row) ? "manual-check" : "ready";
|
||||
}
|
||||
|
||||
export function commandCaseList(ctx: CommandContext): number {
|
||||
const { positional, options } = parseOptions(ctx.args.slice(2));
|
||||
const skill = positional[0];
|
||||
const rows = loadStructuredItems(ctx.root, "cases", skill)
|
||||
.map((item) => caseRow(item, ctx.root))
|
||||
.filter((row) => !optionString(options, "type") || row.type === optionString(options, "type"))
|
||||
.filter((row) => !optionString(options, "area") || row.area === optionString(options, "area"))
|
||||
.filter((row) => !optionString(options, "priority") || row.priority === optionString(options, "priority"))
|
||||
.filter((row) => !optionString(options, "risk") || row.risk === optionString(options, "risk"))
|
||||
.filter((row) => !optionString(options, "tag") || hasTag(row, optionString(options, "tag") ?? ""))
|
||||
.filter((row) => options.automation !== true || Boolean(row.automation))
|
||||
.filter((row) => options.ci !== true || row.ci_eligible === true)
|
||||
.filter((row) => options["machine-ready"] !== true || !hasMissingReadiness(row))
|
||||
.filter((row) => options.ready !== true || (!hasMissingReadiness(row) && !hasManualCheck(row)));
|
||||
|
||||
if (options.json === true) {
|
||||
console.log(JSON.stringify(rows, null, 2));
|
||||
return 0;
|
||||
}
|
||||
|
||||
for (const row of rows) {
|
||||
console.log([
|
||||
row.skill,
|
||||
row.id,
|
||||
row.type,
|
||||
row.area,
|
||||
row.priority,
|
||||
row.risk,
|
||||
row.ci_eligible ? "ci" : "manual",
|
||||
row.automation ? "automated" : "manual-path",
|
||||
row.readiness,
|
||||
row.title,
|
||||
].join("\t"));
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
export function commandCaseShow(ctx: CommandContext): number {
|
||||
const positional = ctx.args.slice(2);
|
||||
if (positional.length < 1 || positional.length > 2) usage();
|
||||
const item = positional.length === 1
|
||||
? findStructuredItem(ctx.root, "cases", positional[0])
|
||||
: findStructuredItem(ctx.root, "cases", positional[0], positional[1]);
|
||||
console.log(item.raw.trimEnd());
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { Socket } from "node:net";
|
||||
import type { CommandContext } from "../types.ts";
|
||||
import { parseOptions } from "../cli.ts";
|
||||
import { loadEnv } from "../fs.ts";
|
||||
import { requiredEnvKeys } from "../constants.ts";
|
||||
import { redactEnvValue } from "../readiness.ts";
|
||||
|
||||
export function commandEnvShow(ctx: CommandContext): number {
|
||||
const { options } = parseOptions(ctx.args.slice(2));
|
||||
const env = loadEnv(ctx.root);
|
||||
const outputEnv = Object.fromEntries(
|
||||
Object.entries(env).map(([key, value]) => [key, redactEnvValue(key, value)]),
|
||||
);
|
||||
if (options.json === true) {
|
||||
console.log(JSON.stringify(outputEnv, null, 2));
|
||||
return 0;
|
||||
}
|
||||
for (const key of Object.keys(outputEnv).sort()) {
|
||||
console.log(`${key}=${outputEnv[key]}`);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
async function checkUrl(label: string, url: string): Promise<{ ok: boolean; message: string }> {
|
||||
if (!url) return { ok: false, message: `${label}: missing` };
|
||||
const displayUrl = redactEnvValue(label, url);
|
||||
try {
|
||||
const response = await fetch(url, { method: "HEAD", signal: AbortSignal.timeout(2500) });
|
||||
return { ok: response.ok || response.status < 500, message: `${label}: ${displayUrl} -> HTTP ${response.status}` };
|
||||
} catch (error) {
|
||||
return { ok: false, message: `${label}: ${displayUrl} -> ${String(error).replace(/\s+/g, " ")}` };
|
||||
}
|
||||
}
|
||||
|
||||
function endpoint(url: string): { host: string; port: number } | null {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null;
|
||||
const port = parsed.port ? Number(parsed.port) : parsed.protocol === "https:" ? 443 : 80;
|
||||
return { host: parsed.hostname, port };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function checkTcpListener(url: string): Promise<{ ok: boolean; message: string } | null> {
|
||||
const target = endpoint(url);
|
||||
if (!target) return null;
|
||||
|
||||
return await new Promise((resolve) => {
|
||||
const socket = new Socket();
|
||||
let settled = false;
|
||||
const finish = (ok: boolean, detail: string) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
socket.destroy();
|
||||
resolve({
|
||||
ok,
|
||||
message: `${target.host}:${target.port} ${detail}`,
|
||||
});
|
||||
};
|
||||
|
||||
socket.setTimeout(1500);
|
||||
socket.once("connect", () => finish(true, "is listening"));
|
||||
socket.once("timeout", () => finish(false, "did not accept TCP connection before timeout"));
|
||||
socket.once("error", (error) => finish(false, `is not listening (${error.message})`));
|
||||
socket.connect(target.port, target.host);
|
||||
});
|
||||
}
|
||||
|
||||
function startupHint(label: string, env: Record<string, string>): string | null {
|
||||
if (label === "LANGBOT_BACKEND_URL" && env.LANGBOT_REPO) {
|
||||
return `start backend: cd ${env.LANGBOT_REPO} && uv run main.py`;
|
||||
}
|
||||
if (label === "LANGBOT_FRONTEND_URL" && env.LANGBOT_WEB_REPO) {
|
||||
return `start frontend: cd ${env.LANGBOT_WEB_REPO} && pnpm dev`;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function compareProxyPair(env: Record<string, string>, upper: string, lower: string): string | null {
|
||||
const upperValue = process.env[upper] ?? env[upper] ?? "";
|
||||
const lowerValue = process.env[lower] ?? env[lower] ?? "";
|
||||
if (upperValue && lowerValue && upperValue !== lowerValue) {
|
||||
return `${upper}/${lower}: mismatch (${redactEnvValue(upper, upperValue)} vs ${redactEnvValue(lower, lowerValue)})`;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function commandEnvDoctor(ctx: CommandContext): Promise<number> {
|
||||
const env = loadEnv(ctx.root);
|
||||
const failures: string[] = [];
|
||||
const warnings: string[] = [];
|
||||
|
||||
for (const key of requiredEnvKeys) {
|
||||
if (!env[key]) failures.push(`missing ${key}`);
|
||||
}
|
||||
|
||||
for (const [label, path] of [
|
||||
["LANGBOT_REPO", env.LANGBOT_REPO],
|
||||
["LANGBOT_WEB_REPO", env.LANGBOT_WEB_REPO],
|
||||
["LANGBOT_CHROMIUM_EXECUTABLE", env.LANGBOT_CHROMIUM_EXECUTABLE],
|
||||
]) {
|
||||
if (!path || !existsSync(path)) failures.push(`${label}: path does not exist (${path || "missing"})`);
|
||||
}
|
||||
|
||||
if (env.LANGBOT_BROWSER_PROFILE && !existsSync(env.LANGBOT_BROWSER_PROFILE)) {
|
||||
warnings.push(`LANGBOT_BROWSER_PROFILE: path does not exist yet (${env.LANGBOT_BROWSER_PROFILE})`);
|
||||
}
|
||||
|
||||
for (const mismatch of [
|
||||
compareProxyPair(env, "HTTP_PROXY", "http_proxy"),
|
||||
compareProxyPair(env, "HTTPS_PROXY", "https_proxy"),
|
||||
compareProxyPair(env, "ALL_PROXY", "all_proxy"),
|
||||
compareProxyPair(env, "NO_PROXY", "no_proxy"),
|
||||
]) {
|
||||
if (mismatch) failures.push(mismatch);
|
||||
}
|
||||
|
||||
for (const [label, result] of await Promise.all([
|
||||
checkUrl("LANGBOT_BACKEND_URL", env.LANGBOT_BACKEND_URL).then((result) => ["LANGBOT_BACKEND_URL", result] as const),
|
||||
checkUrl("LANGBOT_FRONTEND_URL", env.LANGBOT_FRONTEND_URL).then((result) => ["LANGBOT_FRONTEND_URL", result] as const),
|
||||
])) {
|
||||
if (result.ok) console.log(`OK: ${result.message}`);
|
||||
else {
|
||||
failures.push(result.message);
|
||||
const tcp = await checkTcpListener(env[label]);
|
||||
if (tcp && !tcp.ok) failures.push(`${label}: no HTTP service reachable because ${tcp.message}`);
|
||||
const hint = startupHint(label, env);
|
||||
if (hint) warnings.push(`${label}: ${hint}`);
|
||||
}
|
||||
}
|
||||
|
||||
for (const warning of warnings) console.log(`WARN: ${warning}`);
|
||||
for (const failure of failures) console.log(`FAIL: ${failure}`);
|
||||
if (failures.length > 0) return 1;
|
||||
console.log("OK: environment looks usable");
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
import type { CommandContext } from "../types.ts";
|
||||
import { parseOptions } from "../cli.ts";
|
||||
import { loadFixtureItems } from "../fixtures.ts";
|
||||
import { dirname, join } from "node:path";
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
|
||||
function fixtureRows(root: string, skill: string | undefined): ReturnType<typeof loadFixtureItems> {
|
||||
return loadFixtureItems(root, skill);
|
||||
}
|
||||
|
||||
function qaAgentRunnerSourceFindings(item: ReturnType<typeof loadFixtureItems>["items"][number]) {
|
||||
if (!item.checks.includes("qa_agent_runner_source") || !item.exists) return [];
|
||||
const root = dirname(item.absolute_path);
|
||||
const required = [
|
||||
"main.py",
|
||||
"components/agent_runner/default.yaml",
|
||||
"components/agent_runner/default.py",
|
||||
"assets/icon.svg",
|
||||
];
|
||||
const missing = required
|
||||
.filter((path) => !existsSync(join(root, path)))
|
||||
.map((path) => ({
|
||||
severity: "fail",
|
||||
kind: "fixture_check_missing_file",
|
||||
id: item.id,
|
||||
path: `${item.path.replace(/\/[^/]+$/, "")}/${path}`,
|
||||
}));
|
||||
if (missing.length > 0) return missing;
|
||||
|
||||
const manifest = readFileSync(item.absolute_path, "utf8");
|
||||
const runnerYaml = readFileSync(join(root, "components/agent_runner/default.yaml"), "utf8");
|
||||
const runnerPy = readFileSync(join(root, "components/agent_runner/default.py"), "utf8");
|
||||
const requiredText = [
|
||||
[manifest, "AgentRunner", "manifest.yaml"],
|
||||
[manifest, "QAAgentRunnerPlugin", "manifest.yaml"],
|
||||
[runnerYaml, "kind: AgentRunner", "components/agent_runner/default.yaml"],
|
||||
[runnerYaml, "DefaultAgentRunner", "components/agent_runner/default.yaml"],
|
||||
[runnerPy, "QA_AGENT_RUNNER_OK", "components/agent_runner/default.py"],
|
||||
[runnerPy, "QA_AGENT_RUNNER_CONTROLLED_FAILURE", "components/agent_runner/default.py"],
|
||||
];
|
||||
return requiredText
|
||||
.filter(([text, needle]) => !text.includes(needle))
|
||||
.map(([, needle, relativePath]) => ({
|
||||
severity: "fail",
|
||||
kind: "fixture_check_missing_text",
|
||||
id: item.id,
|
||||
path: `${item.path.replace(/\/[^/]+$/, "")}/${relativePath}`,
|
||||
detail: `missing ${needle}`,
|
||||
}));
|
||||
}
|
||||
|
||||
function zipPackageFindings(item: ReturnType<typeof loadFixtureItems>["items"][number]) {
|
||||
if (!item.checks.includes("zip_package") || !item.exists) return [];
|
||||
const header = readFileSync(item.absolute_path).subarray(0, 4).toString("binary");
|
||||
if (header === "PK\u0003\u0004" || header === "PK\u0005\u0006") return [];
|
||||
return [{
|
||||
severity: "fail",
|
||||
kind: "fixture_check_invalid_zip",
|
||||
id: item.id,
|
||||
path: item.path,
|
||||
}];
|
||||
}
|
||||
|
||||
export function commandFixtureList(ctx: CommandContext): number {
|
||||
const { positional, options } = parseOptions(ctx.args.slice(2));
|
||||
const skill = positional[0];
|
||||
const result = fixtureRows(ctx.root, skill);
|
||||
|
||||
if (options.json === true) {
|
||||
console.log(JSON.stringify(result.items, null, 2));
|
||||
return result.errors.length > 0 ? 1 : 0;
|
||||
}
|
||||
|
||||
for (const item of result.items) {
|
||||
console.log([
|
||||
item.skill,
|
||||
item.id,
|
||||
item.kind,
|
||||
item.exists ? "present" : "missing",
|
||||
item.path,
|
||||
item.title,
|
||||
].join("\t"));
|
||||
}
|
||||
for (const error of result.errors) console.error(`ERROR: ${error}`);
|
||||
return result.errors.length > 0 ? 1 : 0;
|
||||
}
|
||||
|
||||
export function commandFixtureCheck(ctx: CommandContext): number {
|
||||
const { positional, options } = parseOptions(ctx.args.slice(2));
|
||||
const skill = positional[0];
|
||||
const result = fixtureRows(ctx.root, skill);
|
||||
const findings = [
|
||||
...result.errors.map((error) => ({ severity: "fail", kind: "invalid_manifest", detail: error })),
|
||||
...result.items
|
||||
.filter((item) => !item.exists)
|
||||
.map((item) => ({
|
||||
severity: "fail",
|
||||
kind: "missing_fixture",
|
||||
id: item.id,
|
||||
path: item.path,
|
||||
absolute_path: item.absolute_path,
|
||||
})),
|
||||
...result.items.flatMap(qaAgentRunnerSourceFindings),
|
||||
...result.items.flatMap(zipPackageFindings),
|
||||
];
|
||||
const report = {
|
||||
status: findings.some((finding) => finding.severity === "fail") ? "fail" : "pass",
|
||||
fixture_count: result.items.length,
|
||||
findings,
|
||||
fixtures: result.items,
|
||||
};
|
||||
|
||||
if (options.json === true) {
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
} else {
|
||||
console.log(`# Fixture Check`);
|
||||
console.log("");
|
||||
console.log(`status: ${report.status}`);
|
||||
console.log(`fixture_count: ${report.fixture_count}`);
|
||||
console.log("");
|
||||
console.log("## Fixtures");
|
||||
for (const item of result.items) {
|
||||
console.log(`- ${item.id}: ${item.exists ? "present" : "missing"} (${item.path})`);
|
||||
}
|
||||
console.log("");
|
||||
console.log("## Findings");
|
||||
if (findings.length === 0) console.log("- None.");
|
||||
else for (const finding of findings) console.log(`- [${finding.severity}] ${finding.kind}: ${"detail" in finding ? finding.detail : finding.id}`);
|
||||
}
|
||||
|
||||
return report.status === "pass" ? 0 : 1;
|
||||
}
|
||||
@@ -0,0 +1,427 @@
|
||||
import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import type { CommandContext } from "../types.ts";
|
||||
import { optionString, parseOptions, usage } from "../cli.ts";
|
||||
import { findStructuredItem, loadEnv } from "../fs.ts";
|
||||
import {
|
||||
latestLangBotLogPath,
|
||||
logPatternContextFromStructuredItem,
|
||||
renderLogFinding,
|
||||
renderLogSuccessSignal,
|
||||
scanLogSources,
|
||||
scanLogText,
|
||||
strictLogGuardExitCode,
|
||||
type LogFinding,
|
||||
type LogGuardPatternContext,
|
||||
type LogGuardResult,
|
||||
type LogSuccessSignal,
|
||||
} from "../log-guard.ts";
|
||||
|
||||
type LogGuardSession = {
|
||||
source: "log-guard-session";
|
||||
run_id: string;
|
||||
started_at: string;
|
||||
started_at_local: string;
|
||||
backend_log: string;
|
||||
case_id: string;
|
||||
case_skill: string;
|
||||
};
|
||||
|
||||
type WatchSummary = {
|
||||
mode: "watch";
|
||||
status: string;
|
||||
path: string;
|
||||
started_at_local: string;
|
||||
finished_at_local: string;
|
||||
bytes_read: number;
|
||||
findings: LogFinding[];
|
||||
success_signals: LogSuccessSignal[];
|
||||
};
|
||||
|
||||
function pad2(value: number): string {
|
||||
return String(value).padStart(2, "0");
|
||||
}
|
||||
|
||||
function pad3(value: number): string {
|
||||
return String(value).padStart(3, "0");
|
||||
}
|
||||
|
||||
function localIsoWithOffset(date: Date): string {
|
||||
const offsetMinutes = -date.getTimezoneOffset();
|
||||
const sign = offsetMinutes >= 0 ? "+" : "-";
|
||||
const absoluteOffset = Math.abs(offsetMinutes);
|
||||
return [
|
||||
`${date.getFullYear()}-${pad2(date.getMonth() + 1)}-${pad2(date.getDate())}`,
|
||||
`T${pad2(date.getHours())}:${pad2(date.getMinutes())}:${pad2(date.getSeconds())}.${pad3(date.getMilliseconds())}`,
|
||||
`${sign}${pad2(Math.floor(absoluteOffset / 60))}:${pad2(absoluteOffset % 60)}`,
|
||||
].join("");
|
||||
}
|
||||
|
||||
function timestampSlug(localIso: string): string {
|
||||
return localIso
|
||||
.replace(/T/, "-")
|
||||
.replace(/[.:+]/g, "-")
|
||||
.replace(/[^A-Za-z0-9_-]+/g, "-")
|
||||
.replace(/-+/g, "-")
|
||||
.replace(/^-|-$/g, "");
|
||||
}
|
||||
|
||||
function writeOrPrint(content: string, output: string | undefined): void {
|
||||
if (!output) {
|
||||
console.log(content.trimEnd());
|
||||
return;
|
||||
}
|
||||
const path = resolve(output);
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
writeFileSync(path, content, "utf8");
|
||||
console.log(path);
|
||||
}
|
||||
|
||||
function positiveIntegerOption(options: Record<string, string | boolean>, key: string, fallback: number): number {
|
||||
const raw = optionString(options, key);
|
||||
if (!raw) return fallback;
|
||||
const parsed = Number.parseInt(raw, 10);
|
||||
if (!/^\d+$/.test(raw) || parsed <= 0) return fallback;
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function splitPatternList(value: string | undefined): string[] {
|
||||
if (!value) return [];
|
||||
return value
|
||||
.split(/\s*\|\s*|\s*,\s*/)
|
||||
.map((item) => item.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function patternContextFromOptions(root: string, options: Record<string, string | boolean>): LogGuardPatternContext {
|
||||
const caseId = optionString(options, "case");
|
||||
const base = caseId ? logPatternContextFromStructuredItem(findStructuredItem(root, "cases", caseId)) : {};
|
||||
return {
|
||||
successPatterns: [
|
||||
...(base.successPatterns ?? []),
|
||||
...splitPatternList(optionString(options, "success-pattern")),
|
||||
],
|
||||
failurePatterns: [
|
||||
...(base.failurePatterns ?? []),
|
||||
...splitPatternList(optionString(options, "failure-pattern")),
|
||||
],
|
||||
expectedFailures: [
|
||||
...(base.expectedFailures ?? []),
|
||||
...splitPatternList(optionString(options, "expected-failure")),
|
||||
],
|
||||
relatedTroubleshootingIds: base.relatedTroubleshootingIds ?? [],
|
||||
};
|
||||
}
|
||||
|
||||
function latestOrExplicitBackendLog(root: string, options: Record<string, string | boolean>): string {
|
||||
const explicit = optionString(options, "backend-log");
|
||||
if (explicit) return resolve(explicit);
|
||||
const auto = latestLangBotLogPath(loadEnv(root));
|
||||
return auto ? resolve(auto) : "";
|
||||
}
|
||||
|
||||
function renderSources(result: LogGuardResult): string[] {
|
||||
const lines: string[] = [];
|
||||
if (result.sources.length === 0) {
|
||||
lines.push("- sources: no log files provided; use --backend-log or configure LANGBOT_REPO.");
|
||||
return lines;
|
||||
}
|
||||
lines.push("- sources:");
|
||||
for (const source of result.sources) {
|
||||
const origin = source.auto_detected ? ", auto" : "";
|
||||
const total = source.total_line_count === undefined ? "" : `/${source.total_line_count}`;
|
||||
const range = source.start_line === undefined || source.end_line === undefined
|
||||
? ""
|
||||
: `, lines ${source.start_line}-${source.end_line}`;
|
||||
const timestamped = source.timestamped_line_count === undefined ? "" : `, ${source.timestamped_line_count} timestamped`;
|
||||
lines.push(` - ${source.source}: ${source.path} (${source.status}${origin}, ${source.line_count}${total} lines${range}${timestamped})`);
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
function renderLogGuardMarkdown(title: string, result: LogGuardResult, extra: string[] = []): string {
|
||||
const lines: string[] = [];
|
||||
lines.push(`# ${title}`);
|
||||
lines.push("");
|
||||
lines.push(`Generated: ${new Date().toISOString()}`);
|
||||
lines.push(`Status: ${result.status}`);
|
||||
lines.push(`Scan mode: ${result.scan.mode}`);
|
||||
if (result.scan.since) lines.push(`Since: ${result.scan.since}`);
|
||||
if (result.scan.until) lines.push(`Until: ${result.scan.until}`);
|
||||
if (result.scan.tail_lines !== undefined) lines.push(`Tail lines: ${result.scan.tail_lines}`);
|
||||
if (extra.length > 0) {
|
||||
lines.push("");
|
||||
lines.push("## Context");
|
||||
for (const item of extra) lines.push(`- ${item}`);
|
||||
}
|
||||
lines.push("");
|
||||
lines.push("## Sources");
|
||||
lines.push(...renderSources(result));
|
||||
if (result.scan.warnings.length > 0) {
|
||||
lines.push("");
|
||||
lines.push("## Scan Warnings");
|
||||
for (const warning of result.scan.warnings) lines.push(`- ${warning}`);
|
||||
}
|
||||
lines.push("");
|
||||
lines.push("## Findings");
|
||||
if (result.findings.length === 0) lines.push("- None.");
|
||||
else for (const finding of result.findings) lines.push(renderLogFinding(finding));
|
||||
lines.push("");
|
||||
lines.push("## Success Signals");
|
||||
if (result.success_signals.length === 0) lines.push("- None.");
|
||||
else for (const signal of result.success_signals) lines.push(renderLogSuccessSignal(signal));
|
||||
lines.push("");
|
||||
return `${lines.join("\n").trimEnd()}\n`;
|
||||
}
|
||||
|
||||
function statusFromEvents(findings: LogFinding[], successSignals: LogSuccessSignal[]): string {
|
||||
if (findings.some((finding) => finding.severity === "fail" || finding.severity === "missing_input")) return "fail";
|
||||
if (findings.some((finding) => finding.severity === "matched_troubleshooting" && finding.related_to_case !== false)) return "fail";
|
||||
if (findings.some((finding) => finding.severity === "env_issue")) return "env_issue";
|
||||
if (findings.some((finding) => finding.severity === "warning")) return "warning";
|
||||
if (successSignals.length > 0) return "pass";
|
||||
return "no_activity";
|
||||
}
|
||||
|
||||
function strictSummaryExitCode(status: string): number {
|
||||
return status === "fail" || status === "env_issue" ? 1 : 0;
|
||||
}
|
||||
|
||||
function sessionDir(options: Record<string, string | boolean>): string {
|
||||
return optionString(options, "output-dir") ?? join("reports", "log-guards");
|
||||
}
|
||||
|
||||
function sessionPath(options: Record<string, string | boolean>, runId: string): string {
|
||||
return join(sessionDir(options), `${runId}.json`);
|
||||
}
|
||||
|
||||
function readSession(options: Record<string, string | boolean>): LogGuardSession | undefined {
|
||||
const runId = optionString(options, "run-id");
|
||||
const explicitSession = optionString(options, "session");
|
||||
const path = explicitSession ? resolve(explicitSession) : runId ? resolve(sessionPath(options, runId)) : "";
|
||||
if (!path || !existsSync(path)) return undefined;
|
||||
return JSON.parse(readFileSync(path, "utf8")) as LogGuardSession;
|
||||
}
|
||||
|
||||
export function commandLogScan(ctx: CommandContext): number {
|
||||
const { positional, options } = parseOptions(ctx.args.slice(2));
|
||||
if (positional.length > 0) usage();
|
||||
|
||||
const result = scanLogSources(ctx.root, options, patternContextFromOptions(ctx.root, options));
|
||||
const output = optionString(options, "output");
|
||||
const content = options.json === true
|
||||
? `${JSON.stringify(result, null, 2)}\n`
|
||||
: renderLogGuardMarkdown("Log Guard Scan", result, [
|
||||
optionString(options, "case") ? `case: ${optionString(options, "case")}` : "case: none",
|
||||
options.strict === true ? "strict: yes" : "strict: no",
|
||||
]);
|
||||
writeOrPrint(content, output);
|
||||
return options.strict === true ? strictLogGuardExitCode(result) : 0;
|
||||
}
|
||||
|
||||
export async function commandLogWatch(ctx: CommandContext): Promise<number> {
|
||||
const { positional, options } = parseOptions(ctx.args.slice(2));
|
||||
if (positional.length > 0) usage();
|
||||
|
||||
const path = latestOrExplicitBackendLog(ctx.root, options);
|
||||
if (!path) {
|
||||
console.error("ERROR: no backend log found; pass --backend-log or configure LANGBOT_REPO.");
|
||||
return 1;
|
||||
}
|
||||
if (!existsSync(path)) {
|
||||
console.error(`ERROR: backend log does not exist: ${path}`);
|
||||
return 1;
|
||||
}
|
||||
|
||||
const context = patternContextFromOptions(ctx.root, options);
|
||||
const intervalMs = positiveIntegerOption(options, "interval-ms", 1000);
|
||||
const durationMs = optionString(options, "duration-ms")
|
||||
? positiveIntegerOption(options, "duration-ms", 0)
|
||||
: 0;
|
||||
const startedAtLocal = localIsoWithOffset(new Date());
|
||||
const findings: LogFinding[] = [];
|
||||
const successSignals: LogSuccessSignal[] = [];
|
||||
let bytesRead = 0;
|
||||
let offset = options["from-start"] === true ? 0 : statSync(path).size;
|
||||
let baseLineNumber = options["from-start"] === true
|
||||
? 0
|
||||
: readFileSync(path).subarray(0, offset).toString("utf8").split(/\r?\n/).length - 1;
|
||||
let carry = "";
|
||||
|
||||
if (options.json !== true) {
|
||||
console.log(`# Log Guard Watch`);
|
||||
console.log(`Path: ${path}`);
|
||||
console.log(`Started: ${startedAtLocal}`);
|
||||
console.log(`Mode: ${options["from-start"] === true ? "from-start" : "new-lines"}`);
|
||||
}
|
||||
|
||||
const startedMs = Date.now();
|
||||
let stopRequested = false;
|
||||
const stop = (): void => {
|
||||
stopRequested = true;
|
||||
};
|
||||
process.once("SIGINT", stop);
|
||||
process.once("SIGTERM", stop);
|
||||
|
||||
const poll = (): void => {
|
||||
const buffer = readFileSync(path);
|
||||
if (buffer.length < offset) {
|
||||
offset = 0;
|
||||
baseLineNumber = 0;
|
||||
carry = "";
|
||||
}
|
||||
if (buffer.length === offset) return;
|
||||
|
||||
const chunk = buffer.subarray(offset).toString("utf8");
|
||||
offset = buffer.length;
|
||||
bytesRead += Buffer.byteLength(chunk);
|
||||
const text = `${carry}${chunk}`;
|
||||
const hasCompleteLine = /\r?\n$/.test(text);
|
||||
const lastNewline = Math.max(text.lastIndexOf("\n"), text.lastIndexOf("\r"));
|
||||
if (!hasCompleteLine && lastNewline === -1) {
|
||||
carry = text;
|
||||
return;
|
||||
}
|
||||
|
||||
const complete = hasCompleteLine ? text : text.slice(0, lastNewline + 1);
|
||||
carry = hasCompleteLine ? "" : text.slice(lastNewline + 1);
|
||||
if (!complete) return;
|
||||
|
||||
const result = scanLogText(ctx.root, "backend", path, complete, {}, context, baseLineNumber, false);
|
||||
baseLineNumber += complete.split(/\r?\n/).length - 1;
|
||||
findings.push(...result.findings);
|
||||
successSignals.push(...result.success_signals);
|
||||
|
||||
if (options.json !== true) {
|
||||
for (const finding of result.findings) console.log(renderLogFinding(finding));
|
||||
for (const signal of result.success_signals) console.log(renderLogSuccessSignal(signal));
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
do {
|
||||
poll();
|
||||
if (stopRequested) break;
|
||||
if (durationMs > 0 && Date.now() - startedMs >= durationMs) break;
|
||||
await delay(Math.min(intervalMs, durationMs > 0 ? Math.max(1, durationMs - (Date.now() - startedMs)) : intervalMs));
|
||||
} while (!stopRequested);
|
||||
|
||||
if (carry) {
|
||||
const result = scanLogText(ctx.root, "backend", path, carry, {}, context, baseLineNumber, false);
|
||||
findings.push(...result.findings);
|
||||
successSignals.push(...result.success_signals);
|
||||
if (options.json !== true) {
|
||||
for (const finding of result.findings) console.log(renderLogFinding(finding));
|
||||
for (const signal of result.success_signals) console.log(renderLogSuccessSignal(signal));
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
process.off("SIGINT", stop);
|
||||
process.off("SIGTERM", stop);
|
||||
}
|
||||
|
||||
const summary: WatchSummary = {
|
||||
mode: "watch",
|
||||
status: statusFromEvents(findings, successSignals),
|
||||
path,
|
||||
started_at_local: startedAtLocal,
|
||||
finished_at_local: localIsoWithOffset(new Date()),
|
||||
bytes_read: bytesRead,
|
||||
findings,
|
||||
success_signals: successSignals,
|
||||
};
|
||||
|
||||
if (options.json === true) {
|
||||
console.log(JSON.stringify(summary, null, 2));
|
||||
} else {
|
||||
console.log(`Status: ${summary.status}`);
|
||||
console.log(`Bytes read: ${summary.bytes_read}`);
|
||||
}
|
||||
return options.strict === true ? strictSummaryExitCode(summary.status) : 0;
|
||||
}
|
||||
|
||||
export function commandLogGuard(ctx: CommandContext): number {
|
||||
const sub = ctx.args[2];
|
||||
if (sub === "start") return commandLogGuardStart(ctx);
|
||||
if (sub === "stop") return commandLogGuardStop(ctx);
|
||||
usage();
|
||||
}
|
||||
|
||||
function commandLogGuardStart(ctx: CommandContext): number {
|
||||
const { positional, options } = parseOptions(ctx.args.slice(3));
|
||||
if (positional.length > 0) usage();
|
||||
|
||||
const now = new Date();
|
||||
const startedAtLocal = localIsoWithOffset(now);
|
||||
const runId = optionString(options, "run-id") ?? `log-guard-${timestampSlug(startedAtLocal)}`;
|
||||
const caseId = optionString(options, "case") ?? "";
|
||||
const caseItem = caseId ? findStructuredItem(ctx.root, "cases", caseId) : undefined;
|
||||
const session: LogGuardSession = {
|
||||
source: "log-guard-session",
|
||||
run_id: runId,
|
||||
started_at: now.toISOString(),
|
||||
started_at_local: startedAtLocal,
|
||||
backend_log: latestOrExplicitBackendLog(ctx.root, options),
|
||||
case_id: caseId,
|
||||
case_skill: caseItem?.skill ?? "",
|
||||
};
|
||||
const path = resolve(sessionPath(options, runId));
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
writeFileSync(path, `${JSON.stringify(session, null, 2)}\n`, "utf8");
|
||||
|
||||
const result = {
|
||||
...session,
|
||||
path,
|
||||
stop_command: `bin/lbs log guard stop --run-id ${runId} --output-dir ${sessionDir(options)}`,
|
||||
};
|
||||
if (options.json === true) console.log(JSON.stringify(result, null, 2));
|
||||
else {
|
||||
console.log(`# Log Guard Session`);
|
||||
console.log(`Run: ${runId}`);
|
||||
console.log(`Started: ${startedAtLocal}`);
|
||||
console.log(`Session: ${path}`);
|
||||
if (session.backend_log) console.log(`Backend log: ${session.backend_log}`);
|
||||
if (session.case_id) console.log(`Case: ${session.case_id}`);
|
||||
console.log(`Stop: ${result.stop_command}`);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
function commandLogGuardStop(ctx: CommandContext): number {
|
||||
const { positional, options } = parseOptions(ctx.args.slice(3));
|
||||
if (positional.length > 0) usage();
|
||||
|
||||
const session = readSession(options);
|
||||
if (!session) {
|
||||
console.error("ERROR: log guard session not found; pass --run-id with --output-dir or --session.");
|
||||
return 1;
|
||||
}
|
||||
|
||||
const now = new Date();
|
||||
const scanOptions: Record<string, string | boolean> = {
|
||||
...options,
|
||||
since: optionString(options, "since") ?? session.started_at_local,
|
||||
until: optionString(options, "until") ?? localIsoWithOffset(now),
|
||||
};
|
||||
if (session.backend_log && typeof scanOptions["backend-log"] !== "string") {
|
||||
scanOptions["backend-log"] = session.backend_log;
|
||||
}
|
||||
|
||||
const caseId = optionString(options, "case") ?? session.case_id;
|
||||
const context = caseId
|
||||
? logPatternContextFromStructuredItem(findStructuredItem(ctx.root, "cases", caseId))
|
||||
: patternContextFromOptions(ctx.root, options);
|
||||
const result = scanLogSources(ctx.root, scanOptions, context);
|
||||
const output = optionString(options, "output") ?? join(sessionDir(options), `${session.run_id}.md`);
|
||||
const content = options.json === true
|
||||
? `${JSON.stringify({ session, result }, null, 2)}\n`
|
||||
: renderLogGuardMarkdown("Log Guard Report", result, [
|
||||
`run_id: ${session.run_id}`,
|
||||
`started: ${session.started_at_local}`,
|
||||
`finished: ${scanOptions.until}`,
|
||||
caseId ? `case: ${caseId}` : "case: none",
|
||||
]);
|
||||
writeOrPrint(content, options.json === true ? optionString(options, "output") : output);
|
||||
return options["no-strict"] === true ? 0 : strictLogGuardExitCode(result);
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import type { CommandContext } from "../types.ts";
|
||||
import { fail, optionString, parseOptions, usage } from "../cli.ts";
|
||||
import { loadFixtureItems } from "../fixtures.ts";
|
||||
import { boolValue, getSkill, globMarkdownRefs, listValue, loadSkills, loadStructuredItems, scalar, skillsRoot } from "../fs.ts";
|
||||
|
||||
export function commandList(ctx: CommandContext): number {
|
||||
for (const skill of loadSkills(ctx.root)) {
|
||||
console.log(`${skill.directory}\t${skill.name}\t${skill.description}`);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
function buildIndexData(root: string): Record<string, unknown> {
|
||||
const caseSummary = (item: ReturnType<typeof loadStructuredItems>[number]) => ({
|
||||
id: scalar(item.fields, "id"),
|
||||
title: scalar(item.fields, "title"),
|
||||
mode: scalar(item.fields, "mode"),
|
||||
area: scalar(item.fields, "area"),
|
||||
type: scalar(item.fields, "type"),
|
||||
priority: scalar(item.fields, "priority"),
|
||||
risk: scalar(item.fields, "risk"),
|
||||
ci_eligible: boolValue(item.fields, "ci_eligible") ?? false,
|
||||
tags: listValue(item.fields, "tags"),
|
||||
automation: scalar(item.fields, "automation"),
|
||||
setup_automation: listValue(item.fields, "setup_automation"),
|
||||
setup_provides_env: listValue(item.fields, "setup_provides_env"),
|
||||
evidence_required: listValue(item.fields, "evidence_required"),
|
||||
});
|
||||
const troubleshootingSummary = (item: ReturnType<typeof loadStructuredItems>[number]) => ({
|
||||
id: scalar(item.fields, "id"),
|
||||
title: scalar(item.fields, "title"),
|
||||
category: scalar(item.fields, "category") || "product",
|
||||
related_cases: listValue(item.fields, "related_cases"),
|
||||
});
|
||||
const suiteSummary = (item: ReturnType<typeof loadStructuredItems>[number]) => ({
|
||||
id: scalar(item.fields, "id"),
|
||||
title: scalar(item.fields, "title"),
|
||||
description: scalar(item.fields, "description"),
|
||||
type: scalar(item.fields, "type"),
|
||||
priority: scalar(item.fields, "priority"),
|
||||
tags: listValue(item.fields, "tags"),
|
||||
cases: listValue(item.fields, "cases"),
|
||||
});
|
||||
return {
|
||||
generated_by: "lbs",
|
||||
skills: loadSkills(root).map((skill) => ({
|
||||
directory: skill.directory,
|
||||
name: skill.name,
|
||||
description: skill.description,
|
||||
references: globMarkdownRefs(skill.path),
|
||||
cases: loadStructuredItems(root, "cases", skill.directory).map((item) => scalar(item.fields, "id")),
|
||||
case_summaries: loadStructuredItems(root, "cases", skill.directory).map(caseSummary),
|
||||
suites: loadStructuredItems(root, "suites", skill.directory).map((item) => scalar(item.fields, "id")),
|
||||
suite_summaries: loadStructuredItems(root, "suites", skill.directory).map(suiteSummary),
|
||||
fixtures: loadFixtureItems(root, skill.directory).items.map((item) => ({
|
||||
id: item.id,
|
||||
title: item.title,
|
||||
kind: item.kind,
|
||||
path: item.path,
|
||||
related_cases: item.related_cases,
|
||||
})),
|
||||
troubleshooting: loadStructuredItems(root, "troubleshooting", skill.directory).map((item) => scalar(item.fields, "id")),
|
||||
troubleshooting_summaries: loadStructuredItems(root, "troubleshooting", skill.directory).map(troubleshootingSummary),
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
export function commandIndex(ctx: CommandContext): number {
|
||||
const { options } = parseOptions(ctx.args.slice(1));
|
||||
const data = buildIndexData(ctx.root);
|
||||
const out = join(ctx.root, "skills.index.json");
|
||||
const content = `${JSON.stringify(data, null, 2)}\n`;
|
||||
if (options.check === true) {
|
||||
if (!existsSync(out)) {
|
||||
console.error(`ERROR: missing index: ${out}`);
|
||||
return 1;
|
||||
}
|
||||
if (readFileSync(out, "utf8") !== content) {
|
||||
console.error(`ERROR: index is stale: ${out}`);
|
||||
return 1;
|
||||
}
|
||||
console.log(`OK ${out}`);
|
||||
return 0;
|
||||
}
|
||||
writeFileSync(out, content, "utf8");
|
||||
console.log(out);
|
||||
return 0;
|
||||
}
|
||||
|
||||
export function commandNewSkill(ctx: CommandContext): number {
|
||||
const { positional, options } = parseOptions(ctx.args.slice(1));
|
||||
const name = positional[0];
|
||||
if (!name) usage();
|
||||
|
||||
const skillDir = join(skillsRoot(ctx.root), name);
|
||||
const skillMd = join(skillDir, "SKILL.md");
|
||||
if (existsSync(skillMd)) fail(`skill already exists: ${skillDir}`);
|
||||
|
||||
mkdirSync(skillDir, { recursive: true });
|
||||
const description = optionString(options, "description") ?? `Use when working with ${name}.`;
|
||||
const text =
|
||||
`---\nname: ${name}\ndescription: ${description}\n---\n\n` +
|
||||
`# ${name}\n\n` +
|
||||
"Add concise routing and workflow instructions here.\n";
|
||||
writeFileSync(skillMd, text, "utf8");
|
||||
console.log(skillMd);
|
||||
return 0;
|
||||
}
|
||||
|
||||
export function commandNewRef(ctx: CommandContext): number {
|
||||
const skill = ctx.args[1];
|
||||
const rawName = ctx.args[2];
|
||||
if (!skill || !rawName) usage();
|
||||
|
||||
const skillDir = getSkill(ctx.root, skill).path;
|
||||
const refsDir = join(skillDir, "references");
|
||||
mkdirSync(refsDir, { recursive: true });
|
||||
const name = rawName.endsWith(".md") ? rawName : `${rawName}.md`;
|
||||
const refPath = join(refsDir, name);
|
||||
if (existsSync(refPath)) fail(`reference already exists: ${refPath}`);
|
||||
|
||||
const title = name.replace(/\.md$/, "").replace(/-/g, " ").replace(/\b\w/g, (char) => char.toUpperCase());
|
||||
writeFileSync(refPath, `# ${title}\n\nAdd concise reusable instructions here.\n`, "utf8");
|
||||
console.log(refPath);
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,704 @@
|
||||
import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { execPath } from "node:process";
|
||||
import type { CommandContext, StructuredItem } from "../types.ts";
|
||||
import { fail, optionString, parseOptions, usage } from "../cli.ts";
|
||||
import { findStructuredItem, getSkill, listValue, loadStructuredItems, scalar, yamlList, yamlQuote } from "../fs.ts";
|
||||
import { caseAutomationReadiness, caseEnvReadiness, caseFixtureReadiness, caseManualReadiness, runtimeEnv } from "../readiness.ts";
|
||||
import { lbsScriptPath, setupAutomationEntries } from "../setup-automation.ts";
|
||||
|
||||
function suitePath(root: string, skillName: string, id: string): string {
|
||||
const skill = getSkill(root, skillName);
|
||||
const dir = join(skill.path, "suites");
|
||||
mkdirSync(dir, { recursive: true });
|
||||
return join(dir, `${id}.yaml`);
|
||||
}
|
||||
|
||||
function caseItemById(root: string, id: string): StructuredItem {
|
||||
return findStructuredItem(root, "cases", id);
|
||||
}
|
||||
|
||||
function suiteCaseSummary(root: string, id: string): Record<string, unknown> {
|
||||
const item = caseItemById(root, id);
|
||||
const env = runtimeEnv(root);
|
||||
const caseId = scalar(item.fields, "id");
|
||||
return {
|
||||
skill: item.skill,
|
||||
id: caseId,
|
||||
title: scalar(item.fields, "title"),
|
||||
mode: scalar(item.fields, "mode"),
|
||||
area: scalar(item.fields, "area"),
|
||||
type: scalar(item.fields, "type"),
|
||||
priority: scalar(item.fields, "priority"),
|
||||
risk: scalar(item.fields, "risk"),
|
||||
tags: listValue(item.fields, "tags"),
|
||||
preconditions: listValue(item.fields, "preconditions"),
|
||||
setup: listValue(item.fields, "setup"),
|
||||
setup_automation: setupAutomationEntries(item),
|
||||
setup_provides_env: listValue(item.fields, "setup_provides_env"),
|
||||
automation: scalar(item.fields, "automation"),
|
||||
evidence_required: listValue(item.fields, "evidence_required"),
|
||||
env_readiness: caseEnvReadiness(item, env),
|
||||
automation_readiness: caseAutomationReadiness(item, env),
|
||||
fixture_readiness: caseFixtureReadiness(root, caseId),
|
||||
manual_readiness: caseManualReadiness(item),
|
||||
};
|
||||
}
|
||||
|
||||
function suiteSummary(item: StructuredItem): Record<string, string | string[]> {
|
||||
return {
|
||||
skill: item.skill,
|
||||
id: scalar(item.fields, "id"),
|
||||
title: scalar(item.fields, "title"),
|
||||
description: scalar(item.fields, "description"),
|
||||
type: scalar(item.fields, "type"),
|
||||
priority: scalar(item.fields, "priority"),
|
||||
tags: listValue(item.fields, "tags"),
|
||||
cases: listValue(item.fields, "cases"),
|
||||
};
|
||||
}
|
||||
|
||||
function findSuite(root: string, args: string[]): StructuredItem {
|
||||
if (args.length < 1 || args.length > 2) usage();
|
||||
return args.length === 1
|
||||
? findStructuredItem(root, "suites", args[0])
|
||||
: findStructuredItem(root, "suites", args[0], args[1]);
|
||||
}
|
||||
|
||||
function pad2(value: number): string {
|
||||
return String(value).padStart(2, "0");
|
||||
}
|
||||
|
||||
function pad3(value: number): string {
|
||||
return String(value).padStart(3, "0");
|
||||
}
|
||||
|
||||
function localIsoWithOffset(date: Date): string {
|
||||
const offsetMinutes = -date.getTimezoneOffset();
|
||||
const sign = offsetMinutes >= 0 ? "+" : "-";
|
||||
const absoluteOffset = Math.abs(offsetMinutes);
|
||||
return [
|
||||
`${date.getFullYear()}-${pad2(date.getMonth() + 1)}-${pad2(date.getDate())}`,
|
||||
`T${pad2(date.getHours())}:${pad2(date.getMinutes())}:${pad2(date.getSeconds())}.${pad3(date.getMilliseconds())}`,
|
||||
`${sign}${pad2(Math.floor(absoluteOffset / 60))}:${pad2(absoluteOffset % 60)}`,
|
||||
].join("");
|
||||
}
|
||||
|
||||
function timestampSlug(localIso: string): string {
|
||||
return localIso
|
||||
.replace(/T/, "-")
|
||||
.replace(/[.:+]/g, "-")
|
||||
.replace(/[^A-Za-z0-9_-]+/g, "-")
|
||||
.replace(/-+/g, "-")
|
||||
.replace(/^-|-$/g, "");
|
||||
}
|
||||
|
||||
function writeOrPrint(content: string, output: string | undefined): void {
|
||||
if (!output) {
|
||||
console.log(content.trimEnd());
|
||||
return;
|
||||
}
|
||||
const path = resolve(output);
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
writeFileSync(path, content, "utf8");
|
||||
console.log(path);
|
||||
}
|
||||
|
||||
function suiteCases(root: string, item: StructuredItem): Record<string, unknown>[] {
|
||||
return listValue(item.fields, "cases").map((id) => suiteCaseSummary(root, id));
|
||||
}
|
||||
|
||||
function statusOf(caseItem: Record<string, unknown>, key: string): string {
|
||||
const value = caseItem[key] as Record<string, unknown> | undefined;
|
||||
return typeof value?.status === "string" ? value.status : "not_required";
|
||||
}
|
||||
|
||||
function readinessSummary(cases: Array<Record<string, unknown>>): Record<string, unknown> {
|
||||
const missingEnv = cases.filter((item) => statusOf(item, "env_readiness") === "missing").map((item) => item.id);
|
||||
const missingAutomation = cases.filter((item) => statusOf(item, "automation_readiness") === "missing").map((item) => item.id);
|
||||
const missingFixture = cases.filter((item) => statusOf(item, "fixture_readiness") === "missing").map((item) => item.id);
|
||||
const manualCheck = cases.filter((item) => statusOf(item, "manual_readiness") === "manual_check").map((item) => item.id);
|
||||
const missingCount = missingEnv.length + missingAutomation.length + missingFixture.length;
|
||||
return {
|
||||
status: missingCount > 0 ? "missing" : manualCheck.length > 0 ? "manual_check" : "ready",
|
||||
missing_env_cases: missingEnv,
|
||||
missing_automation_env_cases: missingAutomation,
|
||||
missing_fixture_cases: missingFixture,
|
||||
manual_check_cases: manualCheck,
|
||||
};
|
||||
}
|
||||
|
||||
function hasProbeCases(cases: Array<Record<string, unknown>>): boolean {
|
||||
return cases.some((caseItem) => caseItem.mode === "probe");
|
||||
}
|
||||
|
||||
function suiteReportGuidance(cases: Array<Record<string, unknown>>): string {
|
||||
return hasProbeCases(cases)
|
||||
? "Run each case according to its mode; probe cases may collect non-UI evidence, while agent-browser cases still require browser/UI execution."
|
||||
: "Run each case through browser/UI first; use test report with the evidence directory and backend log window after execution.";
|
||||
}
|
||||
|
||||
function suiteResultPolicy(cases: Array<Record<string, unknown>>): string[] {
|
||||
if (hasProbeCases(cases)) {
|
||||
return [
|
||||
"A suite is not pass unless every case has a result and required evidence for the same run window.",
|
||||
"agent-browser cases require UI/browser results; probe cases are judged by their declared checks and required evidence.",
|
||||
"blocked and env_issue are not product pass; report them separately.",
|
||||
];
|
||||
}
|
||||
|
||||
return [
|
||||
"A suite is not pass unless every case has a UI/browser result and required evidence for the same run window.",
|
||||
"blocked and env_issue are not product pass; report them separately.",
|
||||
];
|
||||
}
|
||||
|
||||
function suiteEvidencePolicy(cases: Array<Record<string, unknown>>): string[] {
|
||||
if (hasProbeCases(cases)) {
|
||||
return [
|
||||
"Run each case according to its mode. Agent-browser cases use browser/UI; probe cases use their declared probe steps or automation.",
|
||||
"Use each case evidence_dir for screenshots, console.log, network.log, automation-result.json, result.json, and any probe artifacts.",
|
||||
"After case execution and report review, run each result_command_template with the final status and collected evidence.",
|
||||
"After per-case result.json files exist, run the suite report command to aggregate them.",
|
||||
"blocked and env_issue are not product pass; they must be reported separately from pass.",
|
||||
];
|
||||
}
|
||||
|
||||
return [
|
||||
"Run each case through browser/UI. API/curl/log diagnostics cannot make a UI case pass by themselves.",
|
||||
"Use each case evidence_dir for screenshots, console.log, network.log, automation-result.json, and final result.json.",
|
||||
"After case execution and report review, run each result_command_template with the final status and collected evidence.",
|
||||
"After per-case result.json files exist, run the suite report command to aggregate them.",
|
||||
"blocked and env_issue are not product pass; they must be reported separately from pass.",
|
||||
];
|
||||
}
|
||||
|
||||
function buildSuitePlan(root: string, item: StructuredItem): Record<string, unknown> {
|
||||
const suite = suiteSummary(item);
|
||||
const cases = suiteCases(root, item);
|
||||
return {
|
||||
...suite,
|
||||
cases,
|
||||
readiness: readinessSummary(cases),
|
||||
commands: cases.map((caseItem) => ({
|
||||
id: caseItem.id,
|
||||
plan: `bin/lbs test plan ${caseItem.id}`,
|
||||
start: `bin/lbs test start ${caseItem.id}`,
|
||||
automation: caseItem.automation ? `bin/lbs test run ${caseItem.id} --dry-run` : "",
|
||||
})),
|
||||
report_guidance: suiteReportGuidance(cases),
|
||||
};
|
||||
}
|
||||
|
||||
export function commandSuiteNew(ctx: CommandContext): number {
|
||||
const { positional, options } = parseOptions(ctx.args.slice(2));
|
||||
const id = positional[0];
|
||||
const title = optionString(options, "title");
|
||||
if (!id || !title) usage();
|
||||
|
||||
const skill = optionString(options, "skill") ?? "langbot-testing";
|
||||
const path = suitePath(ctx.root, skill, id);
|
||||
if (existsSync(path)) fail(`suite already exists: ${path}`);
|
||||
|
||||
const text =
|
||||
`id: ${id}\n` +
|
||||
`title: ${yamlQuote(title)}\n` +
|
||||
`description: ${yamlQuote(optionString(options, "description") ?? "Describe when to run this suite.")}\n` +
|
||||
`type: ${optionString(options, "type") ?? "smoke"}\n` +
|
||||
`priority: ${optionString(options, "priority") ?? "p2"}\n` +
|
||||
"tags:\n" +
|
||||
yamlList([optionString(options, "type") ?? "smoke"]) +
|
||||
"\ncases:\n" +
|
||||
yamlList(["webui-login-state"]) +
|
||||
"\n";
|
||||
|
||||
writeFileSync(path, text, "utf8");
|
||||
console.log(path);
|
||||
return 0;
|
||||
}
|
||||
|
||||
export function commandSuiteList(ctx: CommandContext): number {
|
||||
const { positional, options } = parseOptions(ctx.args.slice(2));
|
||||
const skill = positional[0];
|
||||
const rows = loadStructuredItems(ctx.root, "suites", skill)
|
||||
.map(suiteSummary)
|
||||
.filter((row) => !optionString(options, "type") || row.type === optionString(options, "type"))
|
||||
.filter((row) => !optionString(options, "priority") || row.priority === optionString(options, "priority"));
|
||||
|
||||
if (options.json === true) {
|
||||
console.log(JSON.stringify(rows, null, 2));
|
||||
return 0;
|
||||
}
|
||||
|
||||
for (const row of rows) {
|
||||
console.log([
|
||||
row.skill,
|
||||
row.id,
|
||||
row.type,
|
||||
row.priority,
|
||||
Array.isArray(row.cases) ? row.cases.length : 0,
|
||||
row.title,
|
||||
].join("\t"));
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
export function commandSuiteShow(ctx: CommandContext): number {
|
||||
const item = findSuite(ctx.root, ctx.args.slice(2));
|
||||
console.log(item.raw.trimEnd());
|
||||
return 0;
|
||||
}
|
||||
|
||||
export function commandSuitePlan(ctx: CommandContext): number {
|
||||
const { positional: args, options } = parseOptions(ctx.args.slice(2));
|
||||
const item = findSuite(ctx.root, args);
|
||||
const plan = buildSuitePlan(ctx.root, item);
|
||||
const suite = suiteSummary(item);
|
||||
const cases = suiteCases(ctx.root, item);
|
||||
|
||||
if (options.json === true) {
|
||||
console.log(JSON.stringify(plan, null, 2));
|
||||
return 0;
|
||||
}
|
||||
|
||||
console.log(`# Suite Plan: ${suite.id}`);
|
||||
console.log("");
|
||||
console.log(`Title: ${suite.title}`);
|
||||
console.log(`Type: ${suite.type}`);
|
||||
console.log(`Priority: ${suite.priority}`);
|
||||
console.log(`Description: ${suite.description}`);
|
||||
console.log("");
|
||||
const readiness = readinessSummary(cases);
|
||||
console.log("## Readiness");
|
||||
console.log(`Status: ${readiness.status}`);
|
||||
for (const [key, value] of Object.entries(readiness)) {
|
||||
if (key === "status" || !Array.isArray(value) || value.length === 0) continue;
|
||||
console.log(`- ${key}: ${value.join(", ")}`);
|
||||
}
|
||||
console.log("");
|
||||
console.log("## Cases");
|
||||
for (const [index, caseItem] of cases.entries()) {
|
||||
console.log(`${index + 1}. ${caseItem.id} [${caseItem.priority}/${caseItem.risk}] ${caseItem.title}`);
|
||||
console.log(` - plan: bin/lbs test plan ${caseItem.id}`);
|
||||
console.log(` - start: bin/lbs test start ${caseItem.id}`);
|
||||
if (caseItem.automation) console.log(` - automation dry-run: bin/lbs test run ${caseItem.id} --dry-run`);
|
||||
console.log(` - evidence: ${Array.isArray(caseItem.evidence_required) ? caseItem.evidence_required.join(", ") : ""}`);
|
||||
const envReadiness = caseItem.env_readiness as Record<string, unknown>;
|
||||
const automationReadiness = caseItem.automation_readiness as Record<string, unknown>;
|
||||
const fixtureReadiness = caseItem.fixture_readiness as Record<string, unknown>;
|
||||
const manualReadiness = caseItem.manual_readiness as Record<string, unknown>;
|
||||
const missing: string[] = [];
|
||||
if (Array.isArray(envReadiness.missing) && envReadiness.missing.length > 0) missing.push(`env=${envReadiness.missing.join(",")}`);
|
||||
if (Array.isArray(automationReadiness.missing) && automationReadiness.missing.length > 0) missing.push(`automation_env=${automationReadiness.missing.join(",")}`);
|
||||
if (Array.isArray(fixtureReadiness.missing) && fixtureReadiness.missing.length > 0) missing.push(`fixture=${fixtureReadiness.missing.join(",")}`);
|
||||
const manualLabel = manualReadiness.status === "manual_check" ? " manual_check" : "";
|
||||
console.log(` - readiness: ${missing.length === 0 ? `ready${manualLabel}` : `missing ${missing.join(" ")}`}`);
|
||||
const preconditions = caseItem.preconditions;
|
||||
if (Array.isArray(preconditions) && preconditions.length > 0) console.log(` - preconditions: ${preconditions.length}`);
|
||||
const setupAutomation = caseItem.setup_automation;
|
||||
if (Array.isArray(setupAutomation) && setupAutomation.length > 0) console.log(` - setup automation: ${setupAutomation.length}`);
|
||||
}
|
||||
console.log("");
|
||||
console.log("## Result Policy");
|
||||
for (const policy of suiteResultPolicy(cases)) console.log(`- ${policy}`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
function suiteStartPath(root: string, path: string): string {
|
||||
return resolve(root, path);
|
||||
}
|
||||
|
||||
function ensureDirectory(root: string, path: string, label: string): void {
|
||||
const resolvedPath = suiteStartPath(root, path);
|
||||
if (existsSync(resolvedPath) && !statSync(resolvedPath).isDirectory()) {
|
||||
fail(`${label} exists and is not a directory: ${resolvedPath}`);
|
||||
}
|
||||
mkdirSync(resolvedPath, { recursive: true });
|
||||
}
|
||||
|
||||
function buildSuiteStart(
|
||||
root: string,
|
||||
item: StructuredItem,
|
||||
args: string[],
|
||||
options: Record<string, string | boolean>,
|
||||
): Record<string, unknown> {
|
||||
const now = new Date();
|
||||
const startedAtLocal = localIsoWithOffset(now);
|
||||
const suite = suiteSummary(item);
|
||||
const suiteId = String(suite.id);
|
||||
const runId = optionString(options, "run-id") ?? `${timestampSlug(startedAtLocal)}-${suiteId}`;
|
||||
const evidenceRoot = optionString(options, "evidence-dir") ?? join("reports", "evidence", runId);
|
||||
const reportPath = join("reports", `${runId}.md`);
|
||||
const manifestPath = join(evidenceRoot, "suite-start.json");
|
||||
const handoffPath = join(evidenceRoot, "suite-start.md");
|
||||
const cases = suiteCases(root, item).map((caseItem) => {
|
||||
const caseId = String(caseItem.id);
|
||||
const caseRunId = `${runId}-${caseId}`;
|
||||
const evidenceDir = join(evidenceRoot, caseId);
|
||||
const consoleLog = join(evidenceDir, "console.log");
|
||||
const caseReportPath = join("reports", `${caseRunId}.md`);
|
||||
return {
|
||||
...caseItem,
|
||||
run_id: caseRunId,
|
||||
evidence_dir: evidenceDir,
|
||||
plan_command: `bin/lbs test plan ${caseId}`,
|
||||
start_command: `bin/lbs test start ${caseId}`,
|
||||
automation_command: caseItem.automation
|
||||
? `bin/lbs test run ${caseId} --run-id ${caseRunId} --output ${evidenceDir}`
|
||||
: "",
|
||||
report_command: caseItem.automation
|
||||
? `bin/lbs test report ${caseId} --since "${startedAtLocal}" --console-log ${consoleLog} --evidence-dir ${evidenceDir} --output ${caseReportPath}`
|
||||
: `bin/lbs test report ${caseId} --since "${startedAtLocal}" --evidence-dir ${evidenceDir} --output ${caseReportPath}`,
|
||||
result_command_template: `bin/lbs test result ${caseId} --result <status> --reason "<short reason>" --evidence-dir ${evidenceDir} --run-id ${caseRunId} --started-at "${startedAtLocal}" --evidence ${Array.isArray(caseItem.evidence_required) ? caseItem.evidence_required.join(",") : ""}`,
|
||||
};
|
||||
});
|
||||
|
||||
const locator = args.join(" ");
|
||||
return {
|
||||
run_id: runId,
|
||||
started_at: now.toISOString(),
|
||||
started_at_local: startedAtLocal,
|
||||
suite,
|
||||
evidence_root: evidenceRoot,
|
||||
manifest_path: manifestPath,
|
||||
handoff_path: handoffPath,
|
||||
cases,
|
||||
suite_report_path: reportPath,
|
||||
plan_command: `bin/lbs suite plan ${locator}`,
|
||||
report_command: `bin/lbs suite report ${locator} --run-id ${runId} --evidence-dir ${evidenceRoot} --output ${reportPath}`,
|
||||
evidence_policy: suiteEvidencePolicy(cases),
|
||||
};
|
||||
}
|
||||
|
||||
function writeSuiteStartArtifacts(root: string, start: Record<string, unknown>, rendered: string): void {
|
||||
const evidenceRoot = String(start.evidence_root || "");
|
||||
if (!evidenceRoot) return;
|
||||
|
||||
ensureDirectory(root, evidenceRoot, "suite evidence directory");
|
||||
for (const caseItem of start.cases as Array<Record<string, unknown>>) {
|
||||
const evidenceDir = String(caseItem.evidence_dir || "");
|
||||
if (evidenceDir) ensureDirectory(root, evidenceDir, "case evidence directory");
|
||||
}
|
||||
|
||||
const manifestPath = String(start.manifest_path || "");
|
||||
if (manifestPath) {
|
||||
const path = suiteStartPath(root, manifestPath);
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
writeFileSync(path, `${JSON.stringify(start, null, 2)}\n`, "utf8");
|
||||
}
|
||||
|
||||
const handoffPath = String(start.handoff_path || "");
|
||||
if (handoffPath) {
|
||||
const path = suiteStartPath(root, handoffPath);
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
writeFileSync(path, rendered, "utf8");
|
||||
}
|
||||
}
|
||||
|
||||
function renderSuiteStart(start: Record<string, unknown>): string {
|
||||
const suite = start.suite as Record<string, unknown>;
|
||||
const cases = start.cases as Array<Record<string, unknown>>;
|
||||
const lines: string[] = [];
|
||||
lines.push(`# Suite Start: ${suite.id}`);
|
||||
lines.push("");
|
||||
lines.push(`Run: ${start.run_id}`);
|
||||
lines.push(`Started: ${start.started_at_local}`);
|
||||
lines.push(`Title: ${suite.title}`);
|
||||
lines.push(`Evidence root: ${start.evidence_root}`);
|
||||
lines.push("");
|
||||
lines.push("## Commands");
|
||||
lines.push(`- plan: ${start.plan_command}`);
|
||||
lines.push(`- report: ${start.report_command}`);
|
||||
lines.push("");
|
||||
lines.push("## Cases");
|
||||
for (const [index, caseItem] of cases.entries()) {
|
||||
lines.push(`${index + 1}. ${caseItem.id} [${caseItem.priority}/${caseItem.risk}] ${caseItem.title}`);
|
||||
lines.push(` - evidence_dir: ${caseItem.evidence_dir}`);
|
||||
lines.push(` - plan: ${caseItem.plan_command}`);
|
||||
if (caseItem.automation_command) lines.push(` - automation: ${caseItem.automation_command}`);
|
||||
else lines.push(` - manual start: ${caseItem.start_command}`);
|
||||
lines.push(` - report: ${caseItem.report_command}`);
|
||||
lines.push(` - result template: ${caseItem.result_command_template}`);
|
||||
}
|
||||
lines.push("");
|
||||
lines.push("## Evidence Policy");
|
||||
for (const item of start.evidence_policy as string[]) lines.push(`- ${item}`);
|
||||
return `${lines.join("\n").trimEnd()}\n`;
|
||||
}
|
||||
|
||||
export function commandSuiteStart(ctx: CommandContext): number {
|
||||
const { positional: args, options } = parseOptions(ctx.args.slice(2));
|
||||
const item = findSuite(ctx.root, args);
|
||||
const start = buildSuiteStart(ctx.root, item, args, options);
|
||||
const rendered = renderSuiteStart(start);
|
||||
writeSuiteStartArtifacts(ctx.root, start, rendered);
|
||||
const content = options.json === true ? `${JSON.stringify(start, null, 2)}\n` : rendered;
|
||||
writeOrPrint(content, optionString(options, "output"));
|
||||
return 0;
|
||||
}
|
||||
|
||||
function suiteRunCaseArgs(root: string, caseItem: Record<string, unknown>, headed: boolean): string[] {
|
||||
const args = [
|
||||
lbsScriptPath(),
|
||||
"--root",
|
||||
root,
|
||||
"test",
|
||||
"run",
|
||||
String(caseItem.id),
|
||||
"--run-id",
|
||||
String(caseItem.run_id),
|
||||
"--output",
|
||||
String(caseItem.evidence_dir),
|
||||
];
|
||||
if (headed) args.push("--headed");
|
||||
return args;
|
||||
}
|
||||
|
||||
function suiteReportExitCode(status: string): number {
|
||||
if (status === "pass") return 0;
|
||||
if (status === "blocked" || status === "env_issue" || status === "flaky") return 2;
|
||||
return 1;
|
||||
}
|
||||
|
||||
function outputTail(value: string | Buffer | null | undefined): string {
|
||||
return String(value ?? "").trim().slice(-4000);
|
||||
}
|
||||
|
||||
function executionProblemStatus(executions: Array<Record<string, unknown>>): string {
|
||||
const statuses = executions.map((item) => String(item.status));
|
||||
if (statuses.includes("nonzero")) return "fail";
|
||||
if (statuses.includes("skipped")) return "incomplete";
|
||||
return "";
|
||||
}
|
||||
|
||||
function missingReadinessReason(caseItem: Record<string, unknown>): string {
|
||||
const labels: Array<[string, string]> = [
|
||||
["env", "env_readiness"],
|
||||
["automation_env", "automation_readiness"],
|
||||
["fixture", "fixture_readiness"],
|
||||
];
|
||||
const missing = labels.flatMap(([label, key]) => {
|
||||
const value = caseItem[key] as Record<string, unknown> | undefined;
|
||||
if (value?.status !== "missing") return [];
|
||||
const names = Array.isArray(value.missing) ? value.missing.filter((item): item is string => typeof item === "string") : [];
|
||||
return [`${label}=${names.length > 0 ? names.join(",") : "missing"}`];
|
||||
});
|
||||
return missing.length > 0
|
||||
? `case readiness missing (${missing.join(" ")}); rerun with --include-not-ready after fixing or intentionally accepting readiness gaps`
|
||||
: "";
|
||||
}
|
||||
|
||||
export function commandSuiteRun(ctx: CommandContext): number {
|
||||
const { positional: args, options } = parseOptions(ctx.args.slice(2));
|
||||
const item = findSuite(ctx.root, args);
|
||||
const start = buildSuiteStart(ctx.root, item, args, options);
|
||||
const renderedStart = renderSuiteStart(start);
|
||||
const dryRun = options["dry-run"] === true;
|
||||
if (!dryRun) writeSuiteStartArtifacts(ctx.root, start, renderedStart);
|
||||
|
||||
const executions = [];
|
||||
for (const caseItem of start.cases as Array<Record<string, unknown>>) {
|
||||
if (statusOf(caseItem, "manual_readiness") === "manual_check" && options["include-manual-check"] !== true) {
|
||||
executions.push({ id: caseItem.id, status: "skipped", reason: "case requires manual_check; rerun with --include-manual-check after confirming preconditions" });
|
||||
continue;
|
||||
}
|
||||
const missingReadiness = missingReadinessReason(caseItem);
|
||||
if (missingReadiness && options["include-not-ready"] !== true) {
|
||||
executions.push({ id: caseItem.id, status: "skipped", reason: missingReadiness });
|
||||
continue;
|
||||
}
|
||||
if (!caseItem.automation) {
|
||||
executions.push({ id: caseItem.id, status: "skipped", reason: "case has no automation" });
|
||||
continue;
|
||||
}
|
||||
const runArgs = suiteRunCaseArgs(ctx.root, caseItem, options.headed === true);
|
||||
if (dryRun) {
|
||||
executions.push({ id: caseItem.id, status: "planned", reason: "dry-run; case automation not executed", command: [execPath, ...runArgs].join(" ") });
|
||||
continue;
|
||||
}
|
||||
if (options.json !== true) console.log(`Suite case: ${caseItem.id}`);
|
||||
const result = spawnSync(execPath, runArgs, {
|
||||
cwd: ctx.root,
|
||||
encoding: "utf8",
|
||||
stdio: options.json === true ? "pipe" : "inherit",
|
||||
});
|
||||
const status = result.error ? 1 : result.status ?? 1;
|
||||
executions.push({
|
||||
id: caseItem.id,
|
||||
status: status === 0 ? "ok" : "nonzero",
|
||||
exit_status: status,
|
||||
reason: result.error?.message || "",
|
||||
stdout: outputTail(result.stdout),
|
||||
stderr: outputTail(result.stderr),
|
||||
});
|
||||
}
|
||||
|
||||
const report = buildSuiteReport(ctx.root, item, {
|
||||
...options,
|
||||
"run-id": String(start.run_id),
|
||||
"evidence-dir": String(start.evidence_root),
|
||||
}, executions);
|
||||
const payload = {
|
||||
run_id: start.run_id,
|
||||
evidence_root: start.evidence_root,
|
||||
executions,
|
||||
report,
|
||||
};
|
||||
const content = options.json === true
|
||||
? `${JSON.stringify(payload, null, 2)}\n`
|
||||
: renderSuiteReport(report);
|
||||
writeOrPrint(content, optionString(options, "output") ?? (options.json === true || dryRun ? undefined : String(start.suite_report_path || "")));
|
||||
return dryRun ? 0 : suiteReportExitCode(String(report.status));
|
||||
}
|
||||
|
||||
function arrayField(data: Record<string, unknown>, key: string): string[] {
|
||||
const value = data[key];
|
||||
return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") : [];
|
||||
}
|
||||
|
||||
function readCaseResult(evidenceDir: string, caseId: string, expectedRunId: string, requiredEvidence: string[]): Record<string, unknown> {
|
||||
const resultPath = join(evidenceDir, "result.json");
|
||||
if (!existsSync(resultPath)) {
|
||||
return { status: "missing", path: resultPath, reason: "result.json not found" };
|
||||
}
|
||||
try {
|
||||
const parsed = JSON.parse(readFileSync(resultPath, "utf8")) as Record<string, unknown>;
|
||||
if (parsed.case_id !== caseId) {
|
||||
return {
|
||||
status: "invalid",
|
||||
path: resultPath,
|
||||
reason: `result.json case_id mismatch: expected ${caseId}, got ${String(parsed.case_id ?? "missing")}`,
|
||||
};
|
||||
}
|
||||
if (expectedRunId && parsed.run_id !== expectedRunId) {
|
||||
return {
|
||||
status: "invalid",
|
||||
path: resultPath,
|
||||
reason: `result.json run_id mismatch: expected ${expectedRunId}, got ${String(parsed.run_id ?? "missing")}`,
|
||||
};
|
||||
}
|
||||
const collected = arrayField(parsed, "evidence_collected");
|
||||
const missing = requiredEvidence.filter((item) => !collected.includes(item));
|
||||
return {
|
||||
status: typeof parsed.status === "string" ? parsed.status : "invalid",
|
||||
path: resultPath,
|
||||
reason: typeof parsed.reason === "string" ? parsed.reason : "",
|
||||
started_at_local: typeof parsed.started_at_local === "string" ? parsed.started_at_local : "",
|
||||
finished_at_local: typeof parsed.finished_at_local === "string" ? parsed.finished_at_local : "",
|
||||
url: typeof parsed.url === "string" ? parsed.url : "",
|
||||
evidence_collected: collected,
|
||||
evidence_required: requiredEvidence,
|
||||
evidence_missing: missing,
|
||||
evidence_status: missing.length === 0 ? "complete" : "incomplete",
|
||||
};
|
||||
} catch (error) {
|
||||
return { status: "invalid", path: resultPath, reason: String(error) };
|
||||
}
|
||||
}
|
||||
|
||||
function suiteStatus(caseResults: Array<Record<string, unknown>>): string {
|
||||
const statuses = caseResults.map((item) => String(item.status));
|
||||
if (statuses.length === 0) return "not_run";
|
||||
if (statuses.includes("fail") || statuses.includes("invalid")) return "fail";
|
||||
if (statuses.includes("missing")) return "incomplete";
|
||||
if (caseResults.some((item) => item.status === "pass" && item.evidence_status !== "complete")) return "incomplete";
|
||||
if (statuses.every((status) => status === "pass")) return "pass";
|
||||
if (statuses.includes("blocked")) return "blocked";
|
||||
if (statuses.includes("env_issue")) return "env_issue";
|
||||
if (statuses.includes("flaky")) return "flaky";
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
function buildSuiteReport(
|
||||
root: string,
|
||||
item: StructuredItem,
|
||||
options: Record<string, string | boolean>,
|
||||
executions: Array<Record<string, unknown>> = [],
|
||||
): Record<string, unknown> {
|
||||
const suite = suiteSummary(item);
|
||||
const runId = optionString(options, "run-id") ?? "";
|
||||
const evidenceRoot = optionString(options, "evidence-dir") ?? (runId ? join("reports", "evidence", runId) : "");
|
||||
const cases = suiteCases(root, item).map((caseItem) => {
|
||||
const caseId = String(caseItem.id);
|
||||
const expectedCaseRunId = runId ? `${runId}-${caseId}` : "";
|
||||
const evidenceDir = evidenceRoot ? join(evidenceRoot, caseId) : "";
|
||||
const requiredEvidence = Array.isArray(caseItem.evidence_required) ? caseItem.evidence_required : [];
|
||||
const result = evidenceDir
|
||||
? readCaseResult(evidenceDir, caseId, expectedCaseRunId, requiredEvidence)
|
||||
: { status: "missing", path: "", reason: "Set --evidence-dir or --run-id to locate case result.json files" };
|
||||
return {
|
||||
...caseItem,
|
||||
evidence_dir: evidenceDir,
|
||||
result,
|
||||
};
|
||||
});
|
||||
const counts: Record<string, number> = {};
|
||||
for (const item of cases) {
|
||||
const status = String((item.result as Record<string, unknown>).status);
|
||||
counts[status] = (counts[status] ?? 0) + 1;
|
||||
}
|
||||
|
||||
const resultStatus = suiteStatus(cases.map((item) => item.result as Record<string, unknown>));
|
||||
const executionStatus = executionProblemStatus(executions);
|
||||
return {
|
||||
generated_at: new Date().toISOString(),
|
||||
run_id: runId,
|
||||
suite,
|
||||
evidence_root: evidenceRoot,
|
||||
status: executionStatus || resultStatus,
|
||||
counts,
|
||||
cases,
|
||||
execution_status: executionStatus || "ok",
|
||||
decision_policy: [
|
||||
"pass requires every case result to be pass.",
|
||||
"suite run pass also requires every attempted execution to finish ok.",
|
||||
"blocked and env_issue are not product pass.",
|
||||
"pass results missing required evidence keep the suite incomplete.",
|
||||
"result.json must match the expected case_id and suite case run_id.",
|
||||
"missing or invalid result.json means the suite is incomplete or failed to collect evidence.",
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function renderSuiteReport(report: Record<string, unknown>): string {
|
||||
const suite = report.suite as Record<string, unknown>;
|
||||
const cases = report.cases as Array<Record<string, unknown>>;
|
||||
const counts = report.counts as Record<string, number>;
|
||||
const lines: string[] = [];
|
||||
lines.push(`# Suite Report: ${suite.id}`);
|
||||
lines.push("");
|
||||
lines.push(`Generated: ${report.generated_at}`);
|
||||
if (report.run_id) lines.push(`Run: ${report.run_id}`);
|
||||
lines.push(`Title: ${suite.title}`);
|
||||
lines.push(`Status: ${report.status}`);
|
||||
lines.push(`Evidence root: ${report.evidence_root || "not provided"}`);
|
||||
lines.push("");
|
||||
lines.push("## Counts");
|
||||
for (const key of Object.keys(counts).sort()) lines.push(`- ${key}: ${counts[key]}`);
|
||||
if (Object.keys(counts).length === 0) lines.push("- None.");
|
||||
lines.push("");
|
||||
lines.push("## Cases");
|
||||
for (const caseItem of cases) {
|
||||
const result = caseItem.result as Record<string, unknown>;
|
||||
lines.push(`- ${caseItem.id}: ${result.status} - ${result.reason || "no reason"}`);
|
||||
if (Array.isArray(result.evidence_missing) && result.evidence_missing.length > 0) {
|
||||
lines.push(` evidence_missing: ${result.evidence_missing.join(", ")}`);
|
||||
}
|
||||
if (caseItem.evidence_dir) lines.push(` evidence_dir: ${caseItem.evidence_dir}`);
|
||||
if (result.path) lines.push(` result_json: ${result.path}`);
|
||||
}
|
||||
lines.push("");
|
||||
lines.push("## Decision Policy");
|
||||
for (const item of report.decision_policy as string[]) lines.push(`- ${item}`);
|
||||
return `${lines.join("\n").trimEnd()}\n`;
|
||||
}
|
||||
|
||||
export function commandSuiteReport(ctx: CommandContext): number {
|
||||
const { positional: args, options } = parseOptions(ctx.args.slice(2));
|
||||
const item = findSuite(ctx.root, args);
|
||||
const report = buildSuiteReport(ctx.root, item, options);
|
||||
const content = options.json === true ? `${JSON.stringify(report, null, 2)}\n` : renderSuiteReport(report);
|
||||
writeOrPrint(content, optionString(options, "output"));
|
||||
return 0;
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,95 @@
|
||||
import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import type { CommandContext } from "../types.ts";
|
||||
import { fail, optionString, parseOptions, usage } from "../cli.ts";
|
||||
import { findStructuredItem, getSkill, loadStructuredItems, scalar, slugify, todayIso, yamlList, yamlQuote } from "../fs.ts";
|
||||
|
||||
function troubleshootingYamlPath(root: string, skillName: string, id: string): string {
|
||||
const skill = getSkill(root, skillName);
|
||||
const dir = join(skill.path, "troubleshooting");
|
||||
mkdirSync(dir, { recursive: true });
|
||||
return join(dir, `${id}.yaml`);
|
||||
}
|
||||
|
||||
function legacyTroubleshootingPath(root: string, skillName: string): string {
|
||||
const skill = getSkill(root, skillName);
|
||||
const refsDir = join(skill.path, "references");
|
||||
mkdirSync(refsDir, { recursive: true });
|
||||
const path = join(refsDir, "troubleshooting.md");
|
||||
if (!existsSync(path)) writeFileSync(path, "# Troubleshooting\n\n", "utf8");
|
||||
return path;
|
||||
}
|
||||
|
||||
export function commandTroubleList(ctx: CommandContext): number {
|
||||
const skill = ctx.args[2];
|
||||
const yamlItems = loadStructuredItems(ctx.root, "troubleshooting", skill);
|
||||
for (const item of yamlItems) {
|
||||
console.log(`${item.skill}\t${scalar(item.fields, "id")}\t${scalar(item.fields, "title")}`);
|
||||
}
|
||||
|
||||
if (skill && yamlItems.length === 0) {
|
||||
const legacyPath = legacyTroubleshootingPath(ctx.root, skill);
|
||||
const text = readFileSync(legacyPath, "utf8");
|
||||
const headings = Array.from(text.matchAll(/^##\s+(.+)$/gm)).map((match) => match[1]);
|
||||
for (const heading of headings) console.log(`${skill}\tlegacy\t${heading}`);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
export function commandTroubleShow(ctx: CommandContext): number {
|
||||
const positional = ctx.args.slice(2);
|
||||
if (positional.length < 1 || positional.length > 2) usage();
|
||||
const item = positional.length === 1
|
||||
? findStructuredItem(ctx.root, "troubleshooting", positional[0])
|
||||
: findStructuredItem(ctx.root, "troubleshooting", positional[0], positional[1]);
|
||||
console.log(item.raw.trimEnd());
|
||||
return 0;
|
||||
}
|
||||
|
||||
export function commandTroubleSearch(ctx: CommandContext): number {
|
||||
const query = ctx.args[2]?.toLowerCase();
|
||||
if (!query) usage();
|
||||
const items = loadStructuredItems(ctx.root, "troubleshooting").filter((item) => item.raw.toLowerCase().includes(query));
|
||||
for (const item of items) {
|
||||
console.log(`${item.skill}\t${scalar(item.fields, "id")}\t${scalar(item.fields, "title")}`);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
export function commandTroubleAdd(ctx: CommandContext): number {
|
||||
const skill = ctx.args[2];
|
||||
if (!skill) usage();
|
||||
const { options } = parseOptions(ctx.args.slice(3));
|
||||
for (const key of ["title", "symptom", "cause", "fix"]) {
|
||||
if (!optionString(options, key)) fail(`--${key} is required`);
|
||||
}
|
||||
|
||||
const title = optionString(options, "title") ?? "";
|
||||
const symptom = optionString(options, "symptom") ?? "";
|
||||
const id = optionString(options, "id") ?? slugify(title);
|
||||
const path = troubleshootingYamlPath(ctx.root, skill, id);
|
||||
if (existsSync(path)) fail(`troubleshooting entry already exists: ${path}`);
|
||||
|
||||
const text =
|
||||
`id: ${id}\n` +
|
||||
`title: ${yamlQuote(title)}\n` +
|
||||
`date: ${todayIso()}\n` +
|
||||
"symptoms:\n" +
|
||||
yamlList([symptom]) +
|
||||
"\npatterns:\n" +
|
||||
yamlList([symptom]) +
|
||||
"\nlikely_causes:\n" +
|
||||
yamlList([optionString(options, "cause") ?? ""]) +
|
||||
"\nfix_steps:\n" +
|
||||
yamlList([optionString(options, "fix") ?? ""]) +
|
||||
"\nverification: " +
|
||||
yamlQuote(optionString(options, "verify") ?? "Add the command, UI signal, or log line that proves the fix worked.") +
|
||||
"\nrelated_cases:\n" +
|
||||
yamlList([]) +
|
||||
"\n";
|
||||
|
||||
writeFileSync(path, text, "utf8");
|
||||
appendFileSync(legacyTroubleshootingPath(ctx.root, skill), `\n## ${id}: ${title}\n\nSee \`../troubleshooting/${id}.yaml\`.\n`, "utf8");
|
||||
console.log(path);
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,394 @@
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { stderr } from "node:process";
|
||||
import type { Skill, StructuredItem } from "../types.ts";
|
||||
import { loadFixtureItems } from "../fixtures.ts";
|
||||
import {
|
||||
caseEvidenceValues,
|
||||
caseModeValues,
|
||||
casePriorityValues,
|
||||
caseRequiredLists,
|
||||
caseRequiredStrings,
|
||||
caseRiskValues,
|
||||
caseTypeValues,
|
||||
requiredEnvKeys,
|
||||
suiteRequiredLists,
|
||||
suiteRequiredStrings,
|
||||
suiteTypeValues,
|
||||
troubleRequiredLists,
|
||||
troubleRequiredStrings,
|
||||
troubleshootingCategoryValues,
|
||||
} from "../constants.ts";
|
||||
import { boolValue, envExamplePath, envPath, listValue, loadSkills, loadStructuredItems, parseEnvFile, scalar } from "../fs.ts";
|
||||
import { envKeyPattern, isEnvAnyGroup, splitEnvAnyGroup } from "../env-groups.ts";
|
||||
import { parseSetupAutomationEntry, validateSetupAutomationEntry } from "../setup-automation.ts";
|
||||
|
||||
const refRe = /(?:\]\(|`)(references\/[A-Za-z0-9_.\-/]+\.md)(?:\)|`)/g;
|
||||
|
||||
function validateStructuredItem(item: StructuredItem, requiredStrings: string[], requiredLists: string[]): string[] {
|
||||
const errors: string[] = [];
|
||||
const listKeys = item.path.includes("/cases/") && scalar(item.fields, "mode") === "probe"
|
||||
? requiredLists.filter((key) => key !== "env")
|
||||
: requiredLists;
|
||||
for (const key of requiredStrings) {
|
||||
if (!scalar(item.fields, key)) errors.push(`${item.path}: missing '${key}'`);
|
||||
}
|
||||
for (const key of listKeys) {
|
||||
if (listValue(item.fields, key).length === 0) errors.push(`${item.path}: missing '${key}' entries`);
|
||||
}
|
||||
const id = scalar(item.fields, "id");
|
||||
if (id && !/^[a-z0-9][a-z0-9_-]*$/.test(id)) {
|
||||
errors.push(`${item.path}: id must use lowercase letters, digits, dashes, or underscores`);
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
function validateEnum(item: StructuredItem, key: string, values: string[]): string[] {
|
||||
const value = scalar(item.fields, key);
|
||||
if (!value) return [];
|
||||
return values.includes(value) ? [] : [`${item.path}: '${key}' must be one of ${values.join(", ")}`];
|
||||
}
|
||||
|
||||
function validateListEnum(item: StructuredItem, key: string, values: string[]): string[] {
|
||||
const allowed = new Set(values);
|
||||
return listValue(item.fields, key)
|
||||
.filter((value) => !allowed.has(value))
|
||||
.map((value) => `${item.path}: '${key}' contains unsupported value '${value}'`);
|
||||
}
|
||||
|
||||
function validateDuplicateListValues(item: StructuredItem, keys: string[]): string[] {
|
||||
const errors: string[] = [];
|
||||
for (const key of keys) {
|
||||
const seen = new Set<string>();
|
||||
for (const value of listValue(item.fields, key)) {
|
||||
if (seen.has(value)) errors.push(`${item.path}: '${key}' contains duplicate value '${value}'`);
|
||||
seen.add(value);
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
function validateEnvKeyList(item: StructuredItem, key: string): string[] {
|
||||
return listValue(item.fields, key)
|
||||
.filter((value) => !envKeyPattern.test(value))
|
||||
.map((value) => `${item.path}: '${key}' contains invalid env key '${value}'`);
|
||||
}
|
||||
|
||||
function validateEnvKeyScalar(item: StructuredItem, key: string): string[] {
|
||||
const value = scalar(item.fields, key);
|
||||
if (!value) return [];
|
||||
return envKeyPattern.test(value)
|
||||
? []
|
||||
: [`${item.path}: '${key}' contains invalid env key '${value}'`];
|
||||
}
|
||||
|
||||
function validateJsonScalar(item: StructuredItem, key: string): string[] {
|
||||
const value = scalar(item.fields, key);
|
||||
if (!value) return [];
|
||||
try {
|
||||
JSON.parse(value);
|
||||
return [];
|
||||
} catch (error) {
|
||||
return [`${item.path}: '${key}' must be valid JSON: ${(error as Error).message}`];
|
||||
}
|
||||
}
|
||||
|
||||
function validateEnvAnyList(item: StructuredItem, key: string): string[] {
|
||||
return listValue(item.fields, key)
|
||||
.filter((value) => !isEnvAnyGroup(value))
|
||||
.map((value) => `${item.path}: '${key}' contains invalid env any-group '${value}'`);
|
||||
}
|
||||
|
||||
function validateCaseItem(root: string, item: StructuredItem, skillNames: Set<string>, troubleIds: Set<string>, caseIds: Set<string>): string[] {
|
||||
const errors = [
|
||||
...validateEnum(item, "mode", caseModeValues),
|
||||
...validateEnum(item, "type", caseTypeValues),
|
||||
...validateEnum(item, "priority", casePriorityValues),
|
||||
...validateEnum(item, "risk", caseRiskValues),
|
||||
...validateListEnum(item, "evidence_required", caseEvidenceValues),
|
||||
...validateDuplicateListValues(item, [
|
||||
"tags",
|
||||
"skills",
|
||||
"env",
|
||||
"env_any",
|
||||
"automation_env",
|
||||
"automation_env_any",
|
||||
"setup_automation",
|
||||
"setup_provides_env",
|
||||
"evidence_required",
|
||||
"troubleshooting",
|
||||
]),
|
||||
...validateEnvKeyList(item, "env"),
|
||||
...validateEnvAnyList(item, "env_any"),
|
||||
...validateEnvKeyList(item, "automation_env"),
|
||||
...validateEnvAnyList(item, "automation_env_any"),
|
||||
...validateEnvKeyList(item, "setup_provides_env"),
|
||||
...validateEnvKeyScalar(item, "automation_pipeline_url_env"),
|
||||
...validateEnvKeyScalar(item, "automation_pipeline_name_env"),
|
||||
...validateJsonScalar(item, "automation_filesystem_checks_json"),
|
||||
...listValue(item.fields, "setup_automation").flatMap((entry) => (
|
||||
validateSetupAutomationEntry(root, entry, caseIds).map((error) => `${item.path}: ${error}`)
|
||||
)),
|
||||
];
|
||||
|
||||
if (boolValue(item.fields, "ci_eligible") === undefined) {
|
||||
errors.push(`${item.path}: missing or invalid boolean 'ci_eligible'`);
|
||||
}
|
||||
|
||||
for (const skill of listValue(item.fields, "skills")) {
|
||||
if (!skillNames.has(skill)) errors.push(`${item.path}: references unknown skill '${skill}'`);
|
||||
}
|
||||
|
||||
for (const id of listValue(item.fields, "troubleshooting")) {
|
||||
if (!troubleIds.has(id)) errors.push(`${item.path}: references unknown troubleshooting '${id}'`);
|
||||
}
|
||||
|
||||
const automation = scalar(item.fields, "automation");
|
||||
if (!automation && listValue(item.fields, "automation_env").length > 0) {
|
||||
errors.push(`${item.path}: 'automation_env' requires 'automation'`);
|
||||
}
|
||||
if (!automation && listValue(item.fields, "automation_env_any").length > 0) {
|
||||
errors.push(`${item.path}: 'automation_env_any' requires 'automation'`);
|
||||
}
|
||||
if (!automation && (scalar(item.fields, "automation_pipeline_url_env") || scalar(item.fields, "automation_pipeline_name_env"))) {
|
||||
errors.push(`${item.path}: automation pipeline env aliases require 'automation'`);
|
||||
}
|
||||
if (listValue(item.fields, "setup_provides_env").length > 0 && listValue(item.fields, "setup_automation").length === 0) {
|
||||
errors.push(`${item.path}: 'setup_provides_env' requires 'setup_automation'`);
|
||||
}
|
||||
for (const key of ["automation_pipeline_url_env", "automation_pipeline_name_env"]) {
|
||||
const value = scalar(item.fields, key);
|
||||
if (!value) continue;
|
||||
const declared = new Set([
|
||||
...listValue(item.fields, "env"),
|
||||
...listValue(item.fields, "env_any").flatMap(splitEnvAnyGroup),
|
||||
...listValue(item.fields, "automation_env"),
|
||||
...listValue(item.fields, "automation_env_any").flatMap(splitEnvAnyGroup),
|
||||
]);
|
||||
if (!declared.has(value)) {
|
||||
errors.push(`${item.path}: '${key}' value '${value}' must be listed in env, env_any, automation_env, or automation_env_any`);
|
||||
}
|
||||
}
|
||||
if (automation && !existsSync(join(root, automation))) {
|
||||
errors.push(`${item.path}: automation script does not exist: ${automation}`);
|
||||
}
|
||||
for (const entry of listValue(item.fields, "setup_automation")) {
|
||||
const spec = parseSetupAutomationEntry(entry);
|
||||
if (spec.kind === "case" && spec.target === scalar(item.fields, "id")) {
|
||||
errors.push(`${item.path}: setup_automation cannot reference the same case '${spec.target}'`);
|
||||
}
|
||||
}
|
||||
|
||||
const timeout = scalar(item.fields, "automation_response_timeout_ms");
|
||||
if (timeout && (!/^\d+$/.test(timeout) || Number.parseInt(timeout, 10) <= 0)) {
|
||||
errors.push(`${item.path}: 'automation_response_timeout_ms' must be a positive integer string`);
|
||||
}
|
||||
const streamOutput = scalar(item.fields, "automation_stream_output");
|
||||
if (streamOutput && !["0", "1", "false", "true"].includes(streamOutput)) {
|
||||
errors.push(`${item.path}: 'automation_stream_output' must be one of 0, 1, false, or true`);
|
||||
}
|
||||
const imageBase64Fixture = scalar(item.fields, "automation_image_base64_fixture");
|
||||
if (imageBase64Fixture && !existsSync(join(root, imageBase64Fixture))) {
|
||||
errors.push(`${item.path}: automation image fixture does not exist: ${imageBase64Fixture}`);
|
||||
}
|
||||
|
||||
return errors;
|
||||
}
|
||||
|
||||
function validateSetupAutomationCycles(caseItems: StructuredItem[]): string[] {
|
||||
const byId = new Map(caseItems.map((item) => [scalar(item.fields, "id"), item]));
|
||||
const visiting = new Set<string>();
|
||||
const visited = new Set<string>();
|
||||
const errors: string[] = [];
|
||||
|
||||
function visit(id: string, path: string[]): void {
|
||||
if (visited.has(id)) return;
|
||||
if (visiting.has(id)) {
|
||||
const cycle = [...path.slice(path.indexOf(id)), id].join(" -> ");
|
||||
const item = byId.get(id);
|
||||
errors.push(`${item?.path ?? id}: setup_automation case cycle detected: ${cycle}`);
|
||||
return;
|
||||
}
|
||||
const item = byId.get(id);
|
||||
if (!item) return;
|
||||
visiting.add(id);
|
||||
for (const entry of listValue(item.fields, "setup_automation")) {
|
||||
const spec = parseSetupAutomationEntry(entry);
|
||||
if (spec.kind === "case") visit(spec.target, [...path, spec.target]);
|
||||
}
|
||||
visiting.delete(id);
|
||||
visited.add(id);
|
||||
}
|
||||
|
||||
for (const id of byId.keys()) visit(id, [id]);
|
||||
return errors;
|
||||
}
|
||||
|
||||
function validateTroubleshootingItem(item: StructuredItem, caseIds: Set<string>): string[] {
|
||||
const errors = [
|
||||
...validateEnum(item, "category", troubleshootingCategoryValues),
|
||||
...validateDuplicateListValues(item, ["symptoms", "patterns", "likely_causes", "fix_steps", "related_cases"]),
|
||||
];
|
||||
for (const id of listValue(item.fields, "related_cases")) {
|
||||
if (!caseIds.has(id)) errors.push(`${item.path}: references unknown case '${id}'`);
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
function validateFixtures(root: string, caseIds: Set<string>): string[] {
|
||||
const { items, errors } = loadFixtureItems(root);
|
||||
const result = [...errors];
|
||||
const seen = new Map<string, string>();
|
||||
for (const item of items) {
|
||||
if (!/^[a-z0-9][a-z0-9_-]*$/.test(item.id)) {
|
||||
result.push(`${item.manifest_path}: fixture id '${item.id}' must use lowercase letters, digits, dashes, or underscores`);
|
||||
}
|
||||
if (seen.has(item.id)) {
|
||||
result.push(`${item.manifest_path}: duplicate fixture id '${item.id}' also used by ${seen.get(item.id)}`);
|
||||
} else {
|
||||
seen.set(item.id, item.manifest_path);
|
||||
}
|
||||
if (!item.exists) result.push(`${item.manifest_path}: fixture path does not exist: ${item.path}`);
|
||||
for (const caseId of item.related_cases) {
|
||||
if (!caseIds.has(caseId)) result.push(`${item.manifest_path}: fixture '${item.id}' references unknown case '${caseId}'`);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function validateSuiteItem(item: StructuredItem, caseIds: Set<string>): string[] {
|
||||
const errors = [
|
||||
...validateEnum(item, "type", suiteTypeValues),
|
||||
...validateEnum(item, "priority", casePriorityValues),
|
||||
...validateDuplicateListValues(item, ["tags", "cases"]),
|
||||
];
|
||||
for (const id of listValue(item.fields, "cases")) {
|
||||
if (!caseIds.has(id)) errors.push(`${item.path}: references unknown case '${id}'`);
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
function validateDuplicateIds(items: StructuredItem[], label: string): string[] {
|
||||
const errors: string[] = [];
|
||||
const seen = new Map<string, string>();
|
||||
for (const item of items) {
|
||||
const id = scalar(item.fields, "id");
|
||||
if (!id) continue;
|
||||
const key = `${item.skill}:${id}`;
|
||||
if (seen.has(key)) errors.push(`${item.path}: duplicate ${label} id '${id}' also used by ${seen.get(key)}`);
|
||||
else seen.set(key, item.path);
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
function validateGlobalDuplicateIds(items: StructuredItem[], label: string): string[] {
|
||||
const errors: string[] = [];
|
||||
const seen = new Map<string, string>();
|
||||
for (const item of items) {
|
||||
const id = scalar(item.fields, "id");
|
||||
if (!id) continue;
|
||||
if (seen.has(id)) errors.push(`${item.path}: duplicate global ${label} id '${id}' also used by ${seen.get(id)}`);
|
||||
else seen.set(id, item.path);
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
function validateEnv(root: string): string[] {
|
||||
const path = envPath(root);
|
||||
const examplePath = envExamplePath(root);
|
||||
const errors: string[] = [];
|
||||
if (!existsSync(path)) return [`${path}: missing shared env file`];
|
||||
const env = parseEnvFile(path);
|
||||
for (const key of requiredEnvKeys) {
|
||||
if (!(key in env)) errors.push(`${path}: missing ${key}`);
|
||||
}
|
||||
if (!existsSync(examplePath)) {
|
||||
errors.push(`${examplePath}: missing env template`);
|
||||
} else {
|
||||
const example = parseEnvFile(examplePath);
|
||||
for (const key of requiredEnvKeys) {
|
||||
if (!(key in example)) errors.push(`${examplePath}: missing template key ${key}`);
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
function validateSchemas(root: string): string[] {
|
||||
const errors: string[] = [];
|
||||
for (const name of ["case.schema.json", "suite.schema.json", "troubleshooting.schema.json", "skill-index.schema.json"]) {
|
||||
const path = join(root, "schemas", name);
|
||||
if (!existsSync(path)) {
|
||||
errors.push(`${path}: missing schema`);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
JSON.parse(readFileSync(path, "utf8"));
|
||||
} catch (error) {
|
||||
errors.push(`${path}: invalid JSON schema (${String(error)})`);
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
function validateSkill(skill: Skill): string[] {
|
||||
const errors: string[] = [];
|
||||
if (!skill.name) errors.push(`${skill.path}: missing frontmatter name`);
|
||||
if (!skill.description) errors.push(`${skill.path}: missing frontmatter description`);
|
||||
if (skill.name && skill.name !== skill.directory) {
|
||||
errors.push(`${skill.path}: name '${skill.name}' does not match directory '${skill.directory}'`);
|
||||
}
|
||||
|
||||
const refs = new Set<string>();
|
||||
for (const match of skill.body.matchAll(refRe)) refs.add(match[1]);
|
||||
for (const ref of Array.from(refs).sort()) {
|
||||
if (!existsSync(join(skill.path, ref))) {
|
||||
errors.push(`${skill.path}: referenced file does not exist: ${ref}`);
|
||||
}
|
||||
}
|
||||
|
||||
const legacyTroubleshooting = join(skill.path, "references", "troubleshooting.md");
|
||||
if (existsSync(legacyTroubleshooting)) {
|
||||
const text = readFileSync(legacyTroubleshooting, "utf8");
|
||||
if (text.includes("\n## ") && !text.includes("### Symptom")) {
|
||||
errors.push(`${legacyTroubleshooting}: troubleshooting entries should include '### Symptom'`);
|
||||
}
|
||||
}
|
||||
|
||||
return errors;
|
||||
}
|
||||
|
||||
export function commandValidate(root: string): number {
|
||||
const skills = loadSkills(root);
|
||||
const caseItems = loadStructuredItems(root, "cases");
|
||||
const suiteItems = loadStructuredItems(root, "suites");
|
||||
const troubleItems = loadStructuredItems(root, "troubleshooting");
|
||||
const skillNames = new Set(skills.map((skill) => skill.name));
|
||||
const caseIds = new Set(caseItems.map((item) => scalar(item.fields, "id")).filter(Boolean));
|
||||
const troubleIds = new Set(troubleItems.map((item) => scalar(item.fields, "id")).filter(Boolean));
|
||||
const errors = [
|
||||
...validateEnv(root),
|
||||
...validateSchemas(root),
|
||||
...skills.flatMap(validateSkill),
|
||||
...caseItems.flatMap((item) => validateStructuredItem(item, caseRequiredStrings, caseRequiredLists)),
|
||||
...caseItems.flatMap((item) => validateCaseItem(root, item, skillNames, troubleIds, caseIds)),
|
||||
...validateSetupAutomationCycles(caseItems),
|
||||
...suiteItems.flatMap((item) => validateStructuredItem(item, suiteRequiredStrings, suiteRequiredLists)),
|
||||
...suiteItems.flatMap((item) => validateSuiteItem(item, caseIds)),
|
||||
...troubleItems.flatMap((item) => validateStructuredItem(item, troubleRequiredStrings, troubleRequiredLists)),
|
||||
...troubleItems.flatMap((item) => validateTroubleshootingItem(item, caseIds)),
|
||||
...validateFixtures(root, caseIds),
|
||||
...validateDuplicateIds(caseItems, "case"),
|
||||
...validateDuplicateIds(suiteItems, "suite"),
|
||||
...validateDuplicateIds(troubleItems, "troubleshooting"),
|
||||
...validateGlobalDuplicateIds(caseItems, "case"),
|
||||
...validateGlobalDuplicateIds(suiteItems, "suite"),
|
||||
...validateGlobalDuplicateIds(troubleItems, "troubleshooting"),
|
||||
];
|
||||
|
||||
if (errors.length > 0) {
|
||||
for (const error of errors) stderr.write(`ERROR: ${error}\n`);
|
||||
return 1;
|
||||
}
|
||||
console.log("OK");
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
export const requiredEnvKeys = [
|
||||
"LANGBOT_FRONTEND_URL",
|
||||
"LANGBOT_BACKEND_URL",
|
||||
"LANGBOT_DEV_FRONTEND_URL",
|
||||
"LANGBOT_REPO",
|
||||
"LANGBOT_WEB_REPO",
|
||||
"LANGBOT_BROWSER_PROFILE",
|
||||
"LANGBOT_CHROMIUM_EXECUTABLE",
|
||||
];
|
||||
|
||||
export const caseModeValues = ["agent-browser", "probe"];
|
||||
export const caseTypeValues = ["smoke", "regression", "feature", "provider", "exploratory"];
|
||||
export const casePriorityValues = ["p0", "p1", "p2"];
|
||||
export const caseRiskValues = ["low", "medium", "high"];
|
||||
export const caseEvidenceValues = [
|
||||
"ui",
|
||||
"screenshot",
|
||||
"console",
|
||||
"network",
|
||||
"backend_log",
|
||||
"frontend_log",
|
||||
"api_diagnostic",
|
||||
"filesystem",
|
||||
];
|
||||
export const testResultStatusValues = ["pass", "fail", "blocked", "env_issue", "flaky"];
|
||||
export const troubleshootingCategoryValues = ["product", "env_issue", "external_dependency", "blocked", "flaky"];
|
||||
export const suiteTypeValues = ["smoke", "regression", "release_gate", "exploratory"];
|
||||
export const suiteRequiredStrings = ["id", "title", "description", "type", "priority"];
|
||||
export const suiteRequiredLists = ["tags", "cases"];
|
||||
|
||||
export const caseRequiredStrings = ["id", "title", "mode", "area", "type", "priority", "risk"];
|
||||
export const caseRequiredLists = ["tags", "skills", "env", "steps", "checks", "evidence_required"];
|
||||
export const troubleRequiredStrings = ["id", "title", "verification"];
|
||||
export const troubleRequiredLists = ["symptoms", "patterns", "likely_causes", "fix_steps"];
|
||||
@@ -0,0 +1,10 @@
|
||||
export const envKeyPattern = /^[A-Z][A-Z0-9_]*$/;
|
||||
|
||||
export function splitEnvAnyGroup(value: string): string[] {
|
||||
return value.split("|").map((item) => item.trim()).filter(Boolean);
|
||||
}
|
||||
|
||||
export function isEnvAnyGroup(value: string): boolean {
|
||||
const keys = splitEnvAnyGroup(value);
|
||||
return keys.length >= 2 && new Set(keys).size === keys.length && keys.every((key) => envKeyPattern.test(key));
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { loadSkills } from "./fs.ts";
|
||||
|
||||
export type FixtureItem = {
|
||||
skill: string;
|
||||
manifest_path: string;
|
||||
id: string;
|
||||
title: string;
|
||||
path: string;
|
||||
kind: string;
|
||||
related_cases: string[];
|
||||
checks: string[];
|
||||
absolute_path: string;
|
||||
exists: boolean;
|
||||
};
|
||||
|
||||
export type FixtureLoadResult = {
|
||||
items: FixtureItem[];
|
||||
errors: string[];
|
||||
};
|
||||
|
||||
function stringField(data: Record<string, unknown>, key: string): string {
|
||||
const value = data[key];
|
||||
return typeof value === "string" ? value : "";
|
||||
}
|
||||
|
||||
function stringList(data: Record<string, unknown>, key: string): string[] {
|
||||
const value = data[key];
|
||||
return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") : [];
|
||||
}
|
||||
|
||||
export function loadFixtureItems(root: string, skillFilter?: string): FixtureLoadResult {
|
||||
const items: FixtureItem[] = [];
|
||||
const errors: string[] = [];
|
||||
const skills = loadSkills(root).filter((skill) => !skillFilter || skill.directory === skillFilter || skill.name === skillFilter);
|
||||
|
||||
for (const skill of skills) {
|
||||
const manifestPath = join(skill.path, "fixtures", "fixtures.json");
|
||||
if (!existsSync(manifestPath)) continue;
|
||||
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(readFileSync(manifestPath, "utf8"));
|
||||
} catch (error) {
|
||||
errors.push(`${manifestPath}: invalid fixture manifest JSON (${String(error)})`);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!Array.isArray(parsed)) {
|
||||
errors.push(`${manifestPath}: fixture manifest must be a JSON array`);
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const [index, entry] of parsed.entries()) {
|
||||
if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
|
||||
errors.push(`${manifestPath}: fixture entry ${index} must be an object`);
|
||||
continue;
|
||||
}
|
||||
const data = entry as Record<string, unknown>;
|
||||
const id = stringField(data, "id");
|
||||
const title = stringField(data, "title");
|
||||
const path = stringField(data, "path");
|
||||
const kind = stringField(data, "kind") || "file";
|
||||
if (!id || !title || !path) {
|
||||
errors.push(`${manifestPath}: fixture entry ${index} must include id, title, and path`);
|
||||
continue;
|
||||
}
|
||||
const absolutePath = join(skill.path, path);
|
||||
items.push({
|
||||
skill: skill.directory,
|
||||
manifest_path: manifestPath,
|
||||
id,
|
||||
title,
|
||||
path,
|
||||
kind,
|
||||
related_cases: stringList(data, "related_cases"),
|
||||
checks: stringList(data, "checks"),
|
||||
absolute_path: absolutePath,
|
||||
exists: existsSync(absolutePath),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return { items, errors };
|
||||
}
|
||||
@@ -0,0 +1,226 @@
|
||||
import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import type { ParsedYaml, Skill, StructuredItem, StructuredItemKind } from "./types.ts";
|
||||
import { fail } from "./cli.ts";
|
||||
|
||||
const frontmatterRe = /^---\n([\s\S]*?)\n---\n/;
|
||||
|
||||
export function statIsDirectory(path: string): boolean {
|
||||
try {
|
||||
return statSync(path).isDirectory();
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function skillsRoot(root: string): string {
|
||||
const nested = join(root, "skills");
|
||||
return existsSync(nested) && statIsDirectory(nested) ? nested : root;
|
||||
}
|
||||
|
||||
export function envPath(root: string): string {
|
||||
return join(skillsRoot(root), ".env");
|
||||
}
|
||||
|
||||
export function envLocalPath(root: string): string {
|
||||
return join(skillsRoot(root), ".env.local");
|
||||
}
|
||||
|
||||
export function envExamplePath(root: string): string {
|
||||
return join(skillsRoot(root), ".env.example");
|
||||
}
|
||||
|
||||
export function loadEnv(root: string): Record<string, string> {
|
||||
return {
|
||||
...parseEnvFile(envPath(root)),
|
||||
...parseEnvFile(envLocalPath(root)),
|
||||
};
|
||||
}
|
||||
|
||||
export function listDirectories(root: string): string[] {
|
||||
return readdirSync(root)
|
||||
.filter((name) => !name.startsWith("."))
|
||||
.filter((name) => statIsDirectory(join(root, name)))
|
||||
.sort();
|
||||
}
|
||||
|
||||
export function parseFrontmatter(text: string): { meta: Record<string, string>; body: string } {
|
||||
const match = text.match(frontmatterRe);
|
||||
if (!match) return { meta: {}, body: text };
|
||||
|
||||
const meta: Record<string, string> = {};
|
||||
for (const line of match[1].split("\n")) {
|
||||
const sep = line.indexOf(":");
|
||||
if (sep === -1) continue;
|
||||
const key = line.slice(0, sep).trim();
|
||||
const value = line.slice(sep + 1).trim().replace(/^["']|["']$/g, "");
|
||||
meta[key] = value;
|
||||
}
|
||||
|
||||
return { meta, body: text.slice(match[0].length) };
|
||||
}
|
||||
|
||||
export function loadSkills(root: string): Skill[] {
|
||||
const skills: Skill[] = [];
|
||||
const base = skillsRoot(root);
|
||||
for (const directory of listDirectories(base)) {
|
||||
const skillPath = join(base, directory);
|
||||
const skillMd = join(skillPath, "SKILL.md");
|
||||
if (!existsSync(skillMd)) continue;
|
||||
const text = readFileSync(skillMd, "utf8");
|
||||
const { meta, body } = parseFrontmatter(text);
|
||||
skills.push({
|
||||
path: skillPath,
|
||||
directory,
|
||||
name: meta.name ?? "",
|
||||
description: meta.description ?? "",
|
||||
body,
|
||||
});
|
||||
}
|
||||
return skills;
|
||||
}
|
||||
|
||||
export function getSkill(root: string, skillName: string): Skill {
|
||||
const skill = loadSkills(root).find((item) => item.directory === skillName || item.name === skillName);
|
||||
if (!skill) fail(`unknown skill: ${skillName}`);
|
||||
return skill;
|
||||
}
|
||||
|
||||
export function parseEnvFile(path: string): Record<string, string> {
|
||||
if (!existsSync(path)) return {};
|
||||
const env: Record<string, string> = {};
|
||||
for (const rawLine of readFileSync(path, "utf8").split(/\r?\n/)) {
|
||||
const line = rawLine.trim();
|
||||
if (!line || line.startsWith("#")) continue;
|
||||
const sep = line.indexOf("=");
|
||||
if (sep === -1) continue;
|
||||
const key = line.slice(0, sep).trim();
|
||||
const value = line.slice(sep + 1).trim().replace(/^["']|["']$/g, "");
|
||||
env[key] = value;
|
||||
}
|
||||
return env;
|
||||
}
|
||||
|
||||
export function globMarkdownRefs(skillPath: string): string[] {
|
||||
const refsDir = join(skillPath, "references");
|
||||
if (!existsSync(refsDir)) return [];
|
||||
return readdirSync(refsDir)
|
||||
.filter((name) => name.endsWith(".md"))
|
||||
.sort()
|
||||
.map((name) => join("references", name));
|
||||
}
|
||||
|
||||
export function globYamlFiles(dir: string): string[] {
|
||||
if (!existsSync(dir)) return [];
|
||||
return readdirSync(dir)
|
||||
.filter((name) => name.endsWith(".yaml") || name.endsWith(".yml"))
|
||||
.sort()
|
||||
.map((name) => join(dir, name));
|
||||
}
|
||||
|
||||
function unquote(value: string): string {
|
||||
return value.trim().replace(/^["']|["']$/g, "");
|
||||
}
|
||||
|
||||
function parseScalarValue(value: string): string | boolean {
|
||||
const trimmed = value.trim();
|
||||
if (/^["'].*["']$/.test(trimmed)) return unquote(trimmed);
|
||||
if (trimmed === "true") return true;
|
||||
if (trimmed === "false") return false;
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
export function parseYamlLite(text: string): ParsedYaml {
|
||||
const fields: ParsedYaml = {};
|
||||
let currentList: string | null = null;
|
||||
|
||||
for (const rawLine of text.split(/\r?\n/)) {
|
||||
const line = rawLine.replace(/\s+$/, "");
|
||||
if (!line.trim() || line.trim().startsWith("#")) continue;
|
||||
|
||||
const pair = line.match(/^([A-Za-z0-9_]+):\s*(.*)$/);
|
||||
if (pair) {
|
||||
const key = pair[1];
|
||||
const value = pair[2];
|
||||
if (value === "") {
|
||||
fields[key] = [];
|
||||
currentList = key;
|
||||
} else {
|
||||
fields[key] = parseScalarValue(value);
|
||||
currentList = null;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const item = line.match(/^\s*-\s*(.*)$/);
|
||||
if (item && currentList) {
|
||||
const existing = fields[currentList];
|
||||
if (Array.isArray(existing)) existing.push(unquote(item[1]));
|
||||
}
|
||||
}
|
||||
|
||||
return fields;
|
||||
}
|
||||
|
||||
export function scalar(fields: ParsedYaml, key: string): string {
|
||||
const value = fields[key];
|
||||
return typeof value === "string" ? value : "";
|
||||
}
|
||||
|
||||
export function boolValue(fields: ParsedYaml, key: string): boolean | undefined {
|
||||
const value = fields[key];
|
||||
return typeof value === "boolean" ? value : undefined;
|
||||
}
|
||||
|
||||
export function listValue(fields: ParsedYaml, key: string): string[] {
|
||||
const value = fields[key];
|
||||
return Array.isArray(value) ? value : [];
|
||||
}
|
||||
|
||||
export function yamlQuote(value: string): string {
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
|
||||
export function yamlList(values: string[]): string {
|
||||
return values.map((value) => ` - ${yamlQuote(value)}`).join("\n");
|
||||
}
|
||||
|
||||
export function loadStructuredItems(root: string, kind: StructuredItemKind, skillFilter?: string): StructuredItem[] {
|
||||
const skills = skillFilter ? [getSkill(root, skillFilter)] : loadSkills(root);
|
||||
const items: StructuredItem[] = [];
|
||||
for (const skill of skills) {
|
||||
for (const path of globYamlFiles(join(skill.path, kind))) {
|
||||
const raw = readFileSync(path, "utf8");
|
||||
items.push({ path, skill: skill.directory, fields: parseYamlLite(raw), raw });
|
||||
}
|
||||
}
|
||||
return items.sort((a, b) => `${a.skill}:${scalar(a.fields, "id")}`.localeCompare(`${b.skill}:${scalar(b.fields, "id")}`));
|
||||
}
|
||||
|
||||
export function findStructuredItem(
|
||||
root: string,
|
||||
kind: StructuredItemKind,
|
||||
skillOrId: string,
|
||||
maybeId?: string,
|
||||
): StructuredItem {
|
||||
const skillFilter = maybeId ? skillOrId : undefined;
|
||||
const id = maybeId ?? skillOrId;
|
||||
const matches = loadStructuredItems(root, kind, skillFilter).filter((item) => scalar(item.fields, "id") === id);
|
||||
if (matches.length === 0) fail(`unknown ${kind.slice(0, -1)}: ${id}`);
|
||||
if (matches.length > 1) {
|
||||
fail(`ambiguous ${kind.slice(0, -1)} '${id}', specify skill: ${matches.map((item) => item.skill).join(", ")}`);
|
||||
}
|
||||
return matches[0];
|
||||
}
|
||||
|
||||
export function slugify(input: string): string {
|
||||
return input
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9\u4e00-\u9fa5]+/g, "-")
|
||||
.replace(/^-+|-+$/g, "");
|
||||
}
|
||||
|
||||
export function todayIso(): string {
|
||||
return new Date().toISOString().slice(0, 10);
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { argv, exit } from "node:process";
|
||||
import { parseGlobalArgs, usage } from "./cli.ts";
|
||||
import { commandCaseList, commandCaseNew, commandCaseShow } from "./commands/case.ts";
|
||||
import { commandEnvDoctor, commandEnvShow } from "./commands/env.ts";
|
||||
import { commandFixtureCheck, commandFixtureList } from "./commands/fixture.ts";
|
||||
import { commandIndex, commandList, commandNewRef, commandNewSkill } from "./commands/skill.ts";
|
||||
import { commandLogGuard, commandLogScan, commandLogWatch } from "./commands/log.ts";
|
||||
import { commandSuiteList, commandSuiteNew, commandSuitePlan, commandSuiteReport, commandSuiteRun, commandSuiteShow, commandSuiteStart } from "./commands/suite.ts";
|
||||
import { commandTestPlan, commandTestRecommend, commandTestReport, commandTestResult, commandTestRun, commandTestStart } from "./commands/test.ts";
|
||||
import { commandTroubleAdd, commandTroubleList, commandTroubleSearch, commandTroubleShow } from "./commands/trouble.ts";
|
||||
import { commandValidate } from "./commands/validate.ts";
|
||||
|
||||
async function main(): Promise<number> {
|
||||
const ctx = parseGlobalArgs(argv.slice(2));
|
||||
const command = ctx.args[0];
|
||||
if (!command) usage();
|
||||
|
||||
if (command === "list") return commandList(ctx);
|
||||
if (command === "validate") return commandValidate(ctx.root);
|
||||
if (command === "index") return commandIndex(ctx);
|
||||
if (command === "new-skill") return commandNewSkill(ctx);
|
||||
if (command === "new-ref") return commandNewRef(ctx);
|
||||
|
||||
if (command === "env") {
|
||||
const sub = ctx.args[1];
|
||||
if (sub === "show") return commandEnvShow(ctx);
|
||||
if (sub === "doctor") return await commandEnvDoctor(ctx);
|
||||
}
|
||||
|
||||
if (command === "fixture") {
|
||||
const sub = ctx.args[1];
|
||||
if (sub === "list") return commandFixtureList(ctx);
|
||||
if (sub === "check") return commandFixtureCheck(ctx);
|
||||
}
|
||||
|
||||
if (command === "log") {
|
||||
const sub = ctx.args[1];
|
||||
if (sub === "scan") return commandLogScan(ctx);
|
||||
if (sub === "watch") return await commandLogWatch(ctx);
|
||||
if (sub === "guard") return commandLogGuard(ctx);
|
||||
}
|
||||
|
||||
if (command === "case") {
|
||||
const sub = ctx.args[1];
|
||||
if (sub === "new") return commandCaseNew(ctx);
|
||||
if (sub === "list") return commandCaseList(ctx);
|
||||
if (sub === "show") return commandCaseShow(ctx);
|
||||
}
|
||||
|
||||
if (command === "suite") {
|
||||
const sub = ctx.args[1];
|
||||
if (sub === "new") return commandSuiteNew(ctx);
|
||||
if (sub === "list") return commandSuiteList(ctx);
|
||||
if (sub === "show") return commandSuiteShow(ctx);
|
||||
if (sub === "plan") return commandSuitePlan(ctx);
|
||||
if (sub === "start") return commandSuiteStart(ctx);
|
||||
if (sub === "run") return commandSuiteRun(ctx);
|
||||
if (sub === "report") return commandSuiteReport(ctx);
|
||||
}
|
||||
|
||||
if (command === "test") {
|
||||
const sub = ctx.args[1];
|
||||
if (sub === "plan") return commandTestPlan(ctx);
|
||||
if (sub === "recommend") return commandTestRecommend(ctx);
|
||||
if (sub === "start") return commandTestStart(ctx);
|
||||
if (sub === "run") return commandTestRun(ctx);
|
||||
if (sub === "report") return commandTestReport(ctx);
|
||||
if (sub === "result") return commandTestResult(ctx);
|
||||
}
|
||||
|
||||
if (command === "trouble") {
|
||||
const sub = ctx.args[1];
|
||||
if (sub === "list") return commandTroubleList(ctx);
|
||||
if (sub === "show") return commandTroubleShow(ctx);
|
||||
if (sub === "search") return commandTroubleSearch(ctx);
|
||||
if (sub === "add") return commandTroubleAdd(ctx);
|
||||
}
|
||||
|
||||
usage();
|
||||
}
|
||||
|
||||
exit(await main());
|
||||
@@ -0,0 +1,805 @@
|
||||
import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import type { StructuredItem } from "./types.ts";
|
||||
import { listValue, loadEnv, loadStructuredItems, scalar } from "./fs.ts";
|
||||
|
||||
export type LogSourceName = "backend" | "frontend" | "console";
|
||||
export type FindingSeverity =
|
||||
| "fail"
|
||||
| "warning"
|
||||
| "matched_troubleshooting"
|
||||
| "env_issue"
|
||||
| "ignored_expected_issue"
|
||||
| "missing_input";
|
||||
|
||||
export type LogFinding = {
|
||||
source: LogSourceName;
|
||||
path: string;
|
||||
severity: FindingSeverity;
|
||||
kind: string;
|
||||
pattern: string;
|
||||
line?: number;
|
||||
excerpt?: string;
|
||||
troubleshooting_id?: string;
|
||||
troubleshooting_title?: string;
|
||||
related_to_case?: boolean;
|
||||
};
|
||||
|
||||
export type LogLine = {
|
||||
number: number;
|
||||
text: string;
|
||||
};
|
||||
|
||||
export type LogSuccessSignal = {
|
||||
source: LogSourceName;
|
||||
path: string;
|
||||
kind: "case_success_pattern";
|
||||
pattern: string;
|
||||
line?: number;
|
||||
excerpt?: string;
|
||||
};
|
||||
|
||||
export type LogScanMode =
|
||||
| "whole-file"
|
||||
| "since"
|
||||
| "until"
|
||||
| "since+until"
|
||||
| "tail-lines"
|
||||
| "since+tail-lines"
|
||||
| "until+tail-lines"
|
||||
| "since+until+tail-lines";
|
||||
|
||||
export type LogScanConfig = {
|
||||
mode: LogScanMode;
|
||||
since?: string;
|
||||
since_epoch_ms?: number;
|
||||
until?: string;
|
||||
until_epoch_ms?: number;
|
||||
tail_lines?: number;
|
||||
warnings: string[];
|
||||
};
|
||||
|
||||
export type LogSourceSummary = {
|
||||
source: LogSourceName;
|
||||
path: string;
|
||||
status: "scanned" | "missing" | "auto_not_found";
|
||||
line_count: number;
|
||||
total_line_count?: number;
|
||||
start_line?: number;
|
||||
end_line?: number;
|
||||
timestamped_line_count?: number;
|
||||
auto_detected?: boolean;
|
||||
};
|
||||
|
||||
export type LogGuardPatternContext = {
|
||||
successPatterns?: string[];
|
||||
failurePatterns?: string[];
|
||||
expectedFailures?: string[];
|
||||
relatedTroubleshootingIds?: string[];
|
||||
};
|
||||
|
||||
export type LogGuardResult = {
|
||||
status: string;
|
||||
scan: LogScanConfig;
|
||||
sources: LogSourceSummary[];
|
||||
success_signals: LogSuccessSignal[];
|
||||
findings: LogFinding[];
|
||||
};
|
||||
|
||||
export type AutomationResultEvidence = {
|
||||
status: "not_provided" | "missing" | "invalid" | "loaded";
|
||||
path?: string;
|
||||
result?: string;
|
||||
reason?: string;
|
||||
started_at?: string;
|
||||
started_at_local?: string;
|
||||
finished_at?: string;
|
||||
finished_at_local?: string;
|
||||
url?: string;
|
||||
prompt?: string;
|
||||
expected_text?: string;
|
||||
};
|
||||
|
||||
type MutableScanState = {
|
||||
findings: LogFinding[];
|
||||
successSignals: LogSuccessSignal[];
|
||||
seenFindings: Set<string>;
|
||||
seenSuccessSignals: Set<string>;
|
||||
};
|
||||
|
||||
const secretAssignmentRe = /\b(api[_-]?key|authorization|credential|jwt|oauth|password|secret|token)\s*[:=]\s*["']?([^"',\s]+)/gi;
|
||||
const bearerSecretRe = /\bbearer\s+[A-Za-z0-9._~+/=-]{8,}/i;
|
||||
const openAiStyleSecretRe = /\bsk-[A-Za-z0-9_-]{6,}\b/i;
|
||||
|
||||
const unexpectedPatterns: Array<{
|
||||
kind: string;
|
||||
pattern: string;
|
||||
regex: RegExp;
|
||||
severity: FindingSeverity;
|
||||
sources?: LogSourceName[];
|
||||
}> = [
|
||||
{ kind: "python_traceback", pattern: "Traceback", regex: /\bTraceback(?: \(most recent call last\))?/i, severity: "fail" },
|
||||
{
|
||||
kind: "unretrieved_task_exception",
|
||||
pattern: "Task exception was never retrieved",
|
||||
regex: /Task exception was never retrieved/i,
|
||||
severity: "fail",
|
||||
},
|
||||
{
|
||||
kind: "unawaited_coroutine",
|
||||
pattern: "RuntimeWarning: coroutine .* was never awaited",
|
||||
regex: /RuntimeWarning:\s+coroutine .* was never awaited/i,
|
||||
severity: "fail",
|
||||
},
|
||||
{
|
||||
kind: "unclosed_client_session",
|
||||
pattern: "Unclosed client session",
|
||||
regex: /Unclosed client session/i,
|
||||
severity: "fail",
|
||||
},
|
||||
{ kind: "unclosed_connector", pattern: "Unclosed connector", regex: /Unclosed connector/i, severity: "fail" },
|
||||
{ kind: "key_error", pattern: "KeyError", regex: /(^|[^A-Za-z])KeyError(?:\b|:)/, severity: "fail" },
|
||||
{ kind: "type_error", pattern: "TypeError", regex: /(^|[^A-Za-z])TypeError(?:\b|:)/, severity: "fail" },
|
||||
{
|
||||
kind: "attribute_error",
|
||||
pattern: "AttributeError",
|
||||
regex: /(^|[^A-Za-z])AttributeError(?:\b|:)/,
|
||||
severity: "fail",
|
||||
},
|
||||
{
|
||||
kind: "frontend_uncaught_error",
|
||||
pattern: "Uncaught frontend error",
|
||||
regex: /\bUncaught (?:[A-Za-z]*Error|Exception)|Unhandled(?: promise rejection|Rejection)/i,
|
||||
severity: "fail",
|
||||
sources: ["console", "frontend"],
|
||||
},
|
||||
{
|
||||
kind: "http_5xx",
|
||||
pattern: "HTTP 5xx resource failure",
|
||||
regex: /Failed to load resource: the server responded with a status of 5\d\d|HTTP\/\d(?:\.\d)?\s+5\d\d/i,
|
||||
severity: "fail",
|
||||
},
|
||||
{ kind: "error_log", pattern: "ERROR or CRITICAL log line", regex: /\b(?:ERROR|CRITICAL)\b/, severity: "warning" },
|
||||
];
|
||||
|
||||
export function logPatternContextFromStructuredItem(item: StructuredItem): LogGuardPatternContext {
|
||||
return {
|
||||
successPatterns: listValue(item.fields, "success_patterns"),
|
||||
failurePatterns: listValue(item.fields, "failure_patterns"),
|
||||
expectedFailures: listValue(item.fields, "expected_failures"),
|
||||
relatedTroubleshootingIds: listValue(item.fields, "troubleshooting"),
|
||||
};
|
||||
}
|
||||
|
||||
export function scanStructuredLogSources(
|
||||
root: string,
|
||||
item: StructuredItem,
|
||||
options: Record<string, string | boolean>,
|
||||
): LogGuardResult {
|
||||
return scanLogSources(root, options, logPatternContextFromStructuredItem(item));
|
||||
}
|
||||
|
||||
export function scanLogSources(
|
||||
root: string,
|
||||
options: Record<string, string | boolean>,
|
||||
context: LogGuardPatternContext = {},
|
||||
): LogGuardResult {
|
||||
const env = loadEnv(root);
|
||||
const scan = parseScanConfig(optionsWithEvidenceWindow(options));
|
||||
const configuredSources: Array<{ source: LogSourceName; option: string }> = [
|
||||
{ source: "backend", option: "backend-log" },
|
||||
{ source: "frontend", option: "frontend-log" },
|
||||
{ source: "console", option: "console-log" },
|
||||
];
|
||||
const sources: LogSourceSummary[] = [];
|
||||
const state: MutableScanState = {
|
||||
findings: [],
|
||||
successSignals: [],
|
||||
seenFindings: new Set(),
|
||||
seenSuccessSignals: new Set(),
|
||||
};
|
||||
|
||||
for (const warning of scan.warnings) {
|
||||
addFinding(state, {
|
||||
source: "backend",
|
||||
path: "log-scan-options",
|
||||
severity: "missing_input",
|
||||
kind: "invalid_log_scan_option",
|
||||
pattern: warning,
|
||||
});
|
||||
}
|
||||
|
||||
for (const configured of configuredSources) {
|
||||
const explicitPath = options[configured.option];
|
||||
const autoPath = configured.source === "backend" && options["no-auto-log"] !== true
|
||||
? latestLangBotLogPath(env)
|
||||
: null;
|
||||
const rawPath = typeof explicitPath === "string" ? explicitPath : autoPath;
|
||||
const autoDetected = typeof explicitPath !== "string" && rawPath === autoPath;
|
||||
if (!rawPath) {
|
||||
if (configured.source === "backend" && options["no-auto-log"] !== true) {
|
||||
const logsDir = env.LANGBOT_REPO ? join(env.LANGBOT_REPO, "data", "logs") : "LANGBOT_REPO/data/logs";
|
||||
sources.push({ source: "backend", path: join(logsDir, "langbot-*.log"), status: "auto_not_found", line_count: 0, auto_detected: true });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const path = resolve(rawPath);
|
||||
if (!existsSync(path)) {
|
||||
sources.push({ source: configured.source, path, status: "missing", line_count: 0 });
|
||||
if (!autoDetected) {
|
||||
addFinding(state, {
|
||||
source: configured.source,
|
||||
path,
|
||||
severity: "missing_input",
|
||||
kind: "missing_log_file",
|
||||
pattern: `${configured.option} path does not exist`,
|
||||
});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const text = readFileSync(path, "utf8");
|
||||
scanLogTextIntoState(root, configured.source, path, text, scan, context, sources, state);
|
||||
}
|
||||
|
||||
finalizeMissingSuccessSignal(context, sources, state);
|
||||
return buildLogGuardResult(scan, sources, state);
|
||||
}
|
||||
|
||||
export function scanLogText(
|
||||
root: string,
|
||||
source: LogSourceName,
|
||||
path: string,
|
||||
text: string,
|
||||
options: Record<string, string | boolean> = {},
|
||||
context: LogGuardPatternContext = {},
|
||||
baseLineNumber = 0,
|
||||
includeMissingSuccessSignal = true,
|
||||
): LogGuardResult {
|
||||
const scan = parseScanConfig(options);
|
||||
const sources: LogSourceSummary[] = [];
|
||||
const state: MutableScanState = {
|
||||
findings: [],
|
||||
successSignals: [],
|
||||
seenFindings: new Set(),
|
||||
seenSuccessSignals: new Set(),
|
||||
};
|
||||
|
||||
scanLogTextIntoState(root, source, resolve(path), text, scan, context, sources, state, baseLineNumber);
|
||||
if (includeMissingSuccessSignal) finalizeMissingSuccessSignal(context, sources, state);
|
||||
return buildLogGuardResult(scan, sources, state);
|
||||
}
|
||||
|
||||
function scanLogTextIntoState(
|
||||
root: string,
|
||||
source: LogSourceName,
|
||||
path: string,
|
||||
text: string,
|
||||
scan: LogScanConfig,
|
||||
context: LogGuardPatternContext,
|
||||
sources: LogSourceSummary[],
|
||||
state: MutableScanState,
|
||||
baseLineNumber = 0,
|
||||
): void {
|
||||
const allLines = text.split(/\r?\n/).map((line, index) => ({ number: baseLineNumber + index + 1, text: line }));
|
||||
const selected = selectLinesForScan(allLines, scan);
|
||||
sources.push({
|
||||
source,
|
||||
path,
|
||||
status: "scanned",
|
||||
line_count: selected.lines.length,
|
||||
total_line_count: allLines.length,
|
||||
start_line: selected.lines[0]?.number,
|
||||
end_line: selected.lines[selected.lines.length - 1]?.number,
|
||||
timestamped_line_count: selected.timestampedLineCount,
|
||||
});
|
||||
|
||||
scanUnexpectedPatterns(state, source, path, selected.lines, context.expectedFailures ?? []);
|
||||
scanCaseDeclaredPatterns(
|
||||
state,
|
||||
source,
|
||||
path,
|
||||
selected.lines,
|
||||
context.successPatterns ?? [],
|
||||
context.failurePatterns ?? [],
|
||||
context.expectedFailures ?? [],
|
||||
);
|
||||
scanTroubleshootingPatterns(
|
||||
state,
|
||||
source,
|
||||
path,
|
||||
selected.lines,
|
||||
loadStructuredItems(root, "troubleshooting"),
|
||||
new Set(context.relatedTroubleshootingIds ?? []),
|
||||
context.expectedFailures ?? [],
|
||||
);
|
||||
}
|
||||
|
||||
function buildLogGuardResult(scan: LogScanConfig, sources: LogSourceSummary[], state: MutableScanState): LogGuardResult {
|
||||
const scannedCount = sources.filter((source) => source.status === "scanned").length;
|
||||
const status = scannedCount === 0 && state.findings.length === 0
|
||||
? "not_run"
|
||||
: state.findings.some((finding) => finding.severity === "fail" || finding.severity === "missing_input")
|
||||
? "fail"
|
||||
: state.findings.some((finding) => finding.severity === "matched_troubleshooting" && finding.related_to_case !== false)
|
||||
? "fail"
|
||||
: state.findings.some((finding) => finding.severity === "env_issue")
|
||||
? "env_issue"
|
||||
: state.findings.some((finding) => finding.severity === "warning")
|
||||
? "warning"
|
||||
: "pass";
|
||||
|
||||
return { status, scan, sources, success_signals: state.successSignals, findings: state.findings };
|
||||
}
|
||||
|
||||
function finalizeMissingSuccessSignal(
|
||||
context: LogGuardPatternContext,
|
||||
sources: LogSourceSummary[],
|
||||
state: MutableScanState,
|
||||
): void {
|
||||
const scannedCount = sources.filter((source) => source.status === "scanned").length;
|
||||
const successPatterns = context.successPatterns ?? [];
|
||||
if (scannedCount > 0 && successPatterns.length > 0 && state.successSignals.length === 0) {
|
||||
addFinding(state, {
|
||||
source: "backend",
|
||||
path: "case-success-patterns",
|
||||
severity: "warning",
|
||||
kind: "missing_success_signal",
|
||||
pattern: successPatterns.join(" | "),
|
||||
excerpt: "No declared success_patterns matched the scanned log window.",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function shouldTreatAssignmentValueAsSecret(value: string): boolean {
|
||||
const normalized = value.trim().replace(/^["']|["']$/g, "");
|
||||
const lower = normalized.toLowerCase();
|
||||
if (!normalized) return false;
|
||||
if (["error", "invalid", "missing", "none", "null", "undefined", "redacted", "[redacted]"].includes(lower)) {
|
||||
return false;
|
||||
}
|
||||
if (/^(error|invalid|missing|none|null|undefined)\b/i.test(normalized)) return false;
|
||||
if (/^(your-|<|\$\{|example-|placeholder)/i.test(normalized)) return false;
|
||||
if (openAiStyleSecretRe.test(normalized)) return true;
|
||||
return normalized.length >= 8 && /[A-Za-z0-9]/.test(normalized);
|
||||
}
|
||||
|
||||
function redactSecretAssignments(text: string): string {
|
||||
return text.replace(secretAssignmentRe, (match, key: string, value: string) => {
|
||||
if (!shouldTreatAssignmentValueAsSecret(value)) return match;
|
||||
return match.replace(value, "[redacted]");
|
||||
});
|
||||
}
|
||||
|
||||
export function redactSecrets(text: string): string {
|
||||
return redactSecretAssignments(text
|
||||
.replace(/(\bauthorization\s*[:=]\s*bearer\s+)[A-Za-z0-9._~+/=-]+/gi, "$1[redacted]")
|
||||
.replace(/\bbearer\s+[A-Za-z0-9._~+/=-]{8,}/gi, "Bearer [redacted]")
|
||||
.replace(/\bsk-[A-Za-z0-9_-]{6,}\b/g, "[redacted]"));
|
||||
}
|
||||
|
||||
function hasSecretLeak(line: string): boolean {
|
||||
secretAssignmentRe.lastIndex = 0;
|
||||
const hasSecretAssignment = Array.from(line.matchAll(secretAssignmentRe))
|
||||
.some((match) => shouldTreatAssignmentValueAsSecret(match[2] ?? ""));
|
||||
return hasSecretAssignment || bearerSecretRe.test(line) || openAiStyleSecretRe.test(line);
|
||||
}
|
||||
|
||||
function findingKey(finding: LogFinding): string {
|
||||
return [
|
||||
finding.source,
|
||||
finding.path,
|
||||
finding.kind,
|
||||
finding.pattern,
|
||||
finding.line ?? "",
|
||||
finding.troubleshooting_id ?? "",
|
||||
].join("\0");
|
||||
}
|
||||
|
||||
function addFinding(state: MutableScanState, finding: LogFinding): void {
|
||||
const key = findingKey(finding);
|
||||
if (state.seenFindings.has(key)) return;
|
||||
state.seenFindings.add(key);
|
||||
state.findings.push(finding);
|
||||
}
|
||||
|
||||
function successSignalKey(signal: LogSuccessSignal): string {
|
||||
return [signal.source, signal.path, signal.pattern, signal.line ?? ""].join("\0");
|
||||
}
|
||||
|
||||
function addSuccessSignal(state: MutableScanState, signal: LogSuccessSignal): void {
|
||||
const key = successSignalKey(signal);
|
||||
if (state.seenSuccessSignals.has(key)) return;
|
||||
state.seenSuccessSignals.add(key);
|
||||
state.successSignals.push(signal);
|
||||
}
|
||||
|
||||
function isExpectedFinding(finding: LogFinding, expectedFailures: string[]): boolean {
|
||||
if (finding.kind === "secret_leak" || finding.severity === "missing_input") return false;
|
||||
const haystack = [
|
||||
finding.kind,
|
||||
finding.pattern,
|
||||
finding.troubleshooting_id ?? "",
|
||||
finding.troubleshooting_title ?? "",
|
||||
finding.excerpt ?? "",
|
||||
].join("\n").toLowerCase();
|
||||
return expectedFailures.some((item) => item && haystack.includes(item.toLowerCase()));
|
||||
}
|
||||
|
||||
function withExpectedSeverity(finding: LogFinding, expectedFailures: string[]): LogFinding {
|
||||
if (!isExpectedFinding(finding, expectedFailures)) return finding;
|
||||
return { ...finding, severity: "ignored_expected_issue" };
|
||||
}
|
||||
|
||||
function scanUnexpectedPatterns(
|
||||
state: MutableScanState,
|
||||
source: LogSourceName,
|
||||
path: string,
|
||||
lines: LogLine[],
|
||||
expectedFailures: string[],
|
||||
): void {
|
||||
for (const line of lines) {
|
||||
for (const pattern of unexpectedPatterns) {
|
||||
if (pattern.sources && !pattern.sources.includes(source)) continue;
|
||||
if (!pattern.regex.test(line.text)) continue;
|
||||
addFinding(state, withExpectedSeverity({
|
||||
source,
|
||||
path,
|
||||
severity: pattern.severity,
|
||||
kind: pattern.kind,
|
||||
pattern: pattern.pattern,
|
||||
line: line.number,
|
||||
excerpt: redactSecrets(line.text.trim()),
|
||||
}, expectedFailures));
|
||||
}
|
||||
|
||||
if (hasSecretLeak(line.text)) {
|
||||
addFinding(state, {
|
||||
source,
|
||||
path,
|
||||
severity: "fail",
|
||||
kind: "secret_leak",
|
||||
pattern: "secret-like value in logs",
|
||||
line: line.number,
|
||||
excerpt: redactSecrets(line.text.trim()),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function scanTroubleshootingPatterns(
|
||||
state: MutableScanState,
|
||||
source: LogSourceName,
|
||||
path: string,
|
||||
lines: LogLine[],
|
||||
troubles: StructuredItem[],
|
||||
relatedIds: Set<string>,
|
||||
expectedFailures: string[],
|
||||
): void {
|
||||
for (const entry of troubles) {
|
||||
const id = scalar(entry.fields, "id");
|
||||
const title = scalar(entry.fields, "title");
|
||||
const category = scalar(entry.fields, "category");
|
||||
for (const pattern of listValue(entry.fields, "patterns")) {
|
||||
const needle = pattern.toLowerCase();
|
||||
if (!needle) continue;
|
||||
let matchesForPattern = 0;
|
||||
for (const line of lines) {
|
||||
if (!line.text.toLowerCase().includes(needle)) continue;
|
||||
if (id === "plugin-runtime-timeout" && isModelRouteUnavailableText(line.text)) continue;
|
||||
addFinding(state, withExpectedSeverity({
|
||||
source,
|
||||
path,
|
||||
severity: category === "env_issue" ? "env_issue" : "matched_troubleshooting",
|
||||
kind: "troubleshooting_pattern",
|
||||
pattern,
|
||||
line: line.number,
|
||||
excerpt: redactSecrets(line.text.trim()),
|
||||
troubleshooting_id: id,
|
||||
troubleshooting_title: title,
|
||||
related_to_case: relatedIds.has(id),
|
||||
}, expectedFailures));
|
||||
matchesForPattern += 1;
|
||||
if (matchesForPattern >= 3) break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function isModelRouteUnavailableText(text: string): boolean {
|
||||
return /model_not_found|no available channel for model|invalid api key|当前分组上游负载已饱和/i.test(text);
|
||||
}
|
||||
|
||||
function scanCaseDeclaredPatterns(
|
||||
state: MutableScanState,
|
||||
source: LogSourceName,
|
||||
path: string,
|
||||
lines: LogLine[],
|
||||
successPatterns: string[],
|
||||
failurePatterns: string[],
|
||||
expectedFailures: string[],
|
||||
): void {
|
||||
for (const pattern of successPatterns) {
|
||||
const needle = pattern.toLowerCase();
|
||||
if (!needle) continue;
|
||||
let matchesForPattern = 0;
|
||||
for (const line of lines) {
|
||||
if (!line.text.toLowerCase().includes(needle)) continue;
|
||||
addSuccessSignal(state, {
|
||||
source,
|
||||
path,
|
||||
kind: "case_success_pattern",
|
||||
pattern,
|
||||
line: line.number,
|
||||
excerpt: redactSecrets(line.text.trim()),
|
||||
});
|
||||
matchesForPattern += 1;
|
||||
if (matchesForPattern >= 3) break;
|
||||
}
|
||||
}
|
||||
|
||||
for (const pattern of failurePatterns) {
|
||||
const needle = pattern.toLowerCase();
|
||||
if (!needle) continue;
|
||||
let matchesForPattern = 0;
|
||||
for (const line of lines) {
|
||||
if (!line.text.toLowerCase().includes(needle)) continue;
|
||||
addFinding(state, withExpectedSeverity({
|
||||
source,
|
||||
path,
|
||||
severity: "fail",
|
||||
kind: "case_failure_pattern",
|
||||
pattern,
|
||||
line: line.number,
|
||||
excerpt: redactSecrets(line.text.trim()),
|
||||
}, expectedFailures));
|
||||
matchesForPattern += 1;
|
||||
if (matchesForPattern >= 3) break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function optionsWithEvidenceWindow(options: Record<string, string | boolean>): Record<string, string | boolean> {
|
||||
if (typeof options.since === "string" && typeof options.until === "string") {
|
||||
return options;
|
||||
}
|
||||
|
||||
const evidenceDir = evidenceDirFromOptions(options);
|
||||
if (!evidenceDir) return options;
|
||||
|
||||
const resultPath = automationResultPath(evidenceDir);
|
||||
if (!existsSync(resultPath)) return options;
|
||||
|
||||
try {
|
||||
const result = JSON.parse(readFileSync(resultPath, "utf8")) as Record<string, unknown>;
|
||||
const enriched = { ...options };
|
||||
const startedAt = stringField(result, "started_at_local") ?? stringField(result, "started_at");
|
||||
const finishedAt = stringField(result, "finished_at_local") ?? stringField(result, "finished_at");
|
||||
|
||||
if (typeof enriched.since !== "string" && startedAt) {
|
||||
enriched.since = startedAt;
|
||||
}
|
||||
if (typeof enriched.until !== "string" && finishedAt) {
|
||||
enriched.until = finishedAt;
|
||||
}
|
||||
return enriched;
|
||||
} catch {
|
||||
return options;
|
||||
}
|
||||
}
|
||||
|
||||
function stringField(data: Record<string, unknown>, key: string): string | undefined {
|
||||
const value = data[key];
|
||||
return typeof value === "string" && value.trim() ? value : undefined;
|
||||
}
|
||||
|
||||
function evidenceDirFromOptions(options: Record<string, string | boolean>): string | undefined {
|
||||
const explicit = typeof options["evidence-dir"] === "string" ? options["evidence-dir"] : undefined;
|
||||
if (explicit) return resolve(explicit);
|
||||
const consoleLog = typeof options["console-log"] === "string" ? options["console-log"] : undefined;
|
||||
return consoleLog ? dirname(resolve(consoleLog)) : undefined;
|
||||
}
|
||||
|
||||
function automationResultPath(evidenceDir: string): string {
|
||||
const primary = join(evidenceDir, "automation-result.json");
|
||||
if (existsSync(primary)) return primary;
|
||||
return join(evidenceDir, "result.json");
|
||||
}
|
||||
|
||||
export function readAutomationResultEvidence(options: Record<string, string | boolean>): AutomationResultEvidence {
|
||||
const evidenceDir = evidenceDirFromOptions(options);
|
||||
if (!evidenceDir) return { status: "not_provided" };
|
||||
|
||||
const resultPath = automationResultPath(evidenceDir);
|
||||
if (!existsSync(resultPath)) return { status: "missing", path: resultPath };
|
||||
|
||||
try {
|
||||
const result = JSON.parse(readFileSync(resultPath, "utf8")) as Record<string, unknown>;
|
||||
if (result.source === "final") {
|
||||
return {
|
||||
status: "not_provided",
|
||||
path: resultPath,
|
||||
reason: "only final result.json is present; automation-result.json was not found",
|
||||
};
|
||||
}
|
||||
return {
|
||||
status: "loaded",
|
||||
path: resultPath,
|
||||
result: stringField(result, "status"),
|
||||
reason: stringField(result, "reason"),
|
||||
started_at: stringField(result, "started_at"),
|
||||
started_at_local: stringField(result, "started_at_local"),
|
||||
finished_at: stringField(result, "finished_at"),
|
||||
finished_at_local: stringField(result, "finished_at_local"),
|
||||
url: stringField(result, "url"),
|
||||
prompt: redactSecrets(stringField(result, "prompt") ?? ""),
|
||||
expected_text: stringField(result, "expected_text"),
|
||||
};
|
||||
} catch (error) {
|
||||
return { status: "invalid", path: resultPath, reason: String(error) };
|
||||
}
|
||||
}
|
||||
|
||||
export function latestLangBotLogPath(env: Record<string, string>): string | null {
|
||||
const repo = env.LANGBOT_REPO;
|
||||
if (!repo) return null;
|
||||
const logsDir = join(repo, "data", "logs");
|
||||
if (!existsSync(logsDir)) return null;
|
||||
|
||||
const candidates = readdirSync(logsDir)
|
||||
.filter((name) => /^langbot-.*\.log$/.test(name))
|
||||
.map((name) => join(logsDir, name))
|
||||
.filter((path) => {
|
||||
try {
|
||||
return statSync(path).isFile();
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
})
|
||||
.sort((a, b) => statSync(b).mtimeMs - statSync(a).mtimeMs);
|
||||
|
||||
return candidates[0] ?? null;
|
||||
}
|
||||
|
||||
export function parseScanConfig(options: Record<string, string | boolean>): LogScanConfig {
|
||||
const warnings: string[] = [];
|
||||
const sinceInput = typeof options.since === "string" ? options.since : undefined;
|
||||
const sinceMs = sinceInput ? Date.parse(sinceInput) : undefined;
|
||||
const untilInput = typeof options.until === "string" ? options.until : undefined;
|
||||
const untilMs = untilInput ? Date.parse(untilInput) : undefined;
|
||||
const tailInput = typeof options["tail-lines"] === "string" ? options["tail-lines"] : undefined;
|
||||
let tailLines: number | undefined;
|
||||
|
||||
if (sinceInput && Number.isNaN(sinceMs)) {
|
||||
warnings.push(`--since is not a valid date/time: ${sinceInput}`);
|
||||
}
|
||||
if (untilInput && Number.isNaN(untilMs)) {
|
||||
warnings.push(`--until is not a valid date/time: ${untilInput}`);
|
||||
}
|
||||
|
||||
if (tailInput) {
|
||||
const parsed = Number.parseInt(tailInput, 10);
|
||||
if (!/^\d+$/.test(tailInput) || parsed <= 0) {
|
||||
warnings.push(`--tail-lines must be a positive integer: ${tailInput}`);
|
||||
} else {
|
||||
tailLines = parsed;
|
||||
}
|
||||
}
|
||||
|
||||
const hasSince = sinceInput !== undefined && sinceMs !== undefined && !Number.isNaN(sinceMs);
|
||||
const hasUntil = untilInput !== undefined && untilMs !== undefined && !Number.isNaN(untilMs);
|
||||
const hasTail = tailLines !== undefined;
|
||||
let mode: LogScanMode = "whole-file";
|
||||
if (hasSince && hasUntil && hasTail) {
|
||||
mode = "since+until+tail-lines";
|
||||
} else if (hasSince && hasUntil) {
|
||||
mode = "since+until";
|
||||
} else if (hasSince && hasTail) {
|
||||
mode = "since+tail-lines";
|
||||
} else if (hasUntil && hasTail) {
|
||||
mode = "until+tail-lines";
|
||||
} else if (hasSince) {
|
||||
mode = "since";
|
||||
} else if (hasUntil) {
|
||||
mode = "until";
|
||||
} else if (hasTail) {
|
||||
mode = "tail-lines";
|
||||
}
|
||||
|
||||
return {
|
||||
mode,
|
||||
since: hasSince ? sinceInput : undefined,
|
||||
since_epoch_ms: hasSince ? sinceMs : undefined,
|
||||
until: hasUntil ? untilInput : undefined,
|
||||
until_epoch_ms: hasUntil ? untilMs : undefined,
|
||||
tail_lines: tailLines,
|
||||
warnings,
|
||||
};
|
||||
}
|
||||
|
||||
function selectLinesForScan(lines: LogLine[], scan: LogScanConfig): { lines: LogLine[]; timestampedLineCount: number } {
|
||||
let selected = lines;
|
||||
let timestampedLineCount = 0;
|
||||
|
||||
if (scan.since_epoch_ms !== undefined || scan.until_epoch_ms !== undefined) {
|
||||
const offsetMinutes =
|
||||
timezoneOffsetMinutes(scan.since)
|
||||
?? timezoneOffsetMinutes(scan.until)
|
||||
?? -new Date(scan.since_epoch_ms ?? scan.until_epoch_ms ?? Date.now()).getTimezoneOffset();
|
||||
const yearHint = new Date(scan.since_epoch_ms ?? scan.until_epoch_ms ?? Date.now()).getUTCFullYear();
|
||||
let includeCurrentBlock = false;
|
||||
const filtered = lines.filter((line) => {
|
||||
const timestamp = parseLogLineTimestampMs(line.text, yearHint, offsetMinutes);
|
||||
if (timestamp !== null) {
|
||||
timestampedLineCount += 1;
|
||||
includeCurrentBlock =
|
||||
(scan.since_epoch_ms === undefined || timestamp >= scan.since_epoch_ms)
|
||||
&& (scan.until_epoch_ms === undefined || timestamp <= scan.until_epoch_ms);
|
||||
return includeCurrentBlock;
|
||||
}
|
||||
return includeCurrentBlock;
|
||||
});
|
||||
selected = timestampedLineCount === 0 ? lines : filtered;
|
||||
} else {
|
||||
timestampedLineCount = lines.reduce((count, line) => (
|
||||
parseLogLineTimestampMs(line.text, new Date().getFullYear(), -new Date().getTimezoneOffset()) === null
|
||||
? count
|
||||
: count + 1
|
||||
), 0);
|
||||
}
|
||||
|
||||
if (scan.tail_lines !== undefined && selected.length > scan.tail_lines) {
|
||||
selected = selected.slice(-scan.tail_lines);
|
||||
}
|
||||
|
||||
return { lines: selected, timestampedLineCount };
|
||||
}
|
||||
|
||||
function timezoneOffsetMinutes(input: string | undefined): number | null {
|
||||
if (!input) return null;
|
||||
if (/[zZ]$/.test(input)) return 0;
|
||||
const match = input.match(/([+-])(\d{2}):?(\d{2})$/);
|
||||
if (!match) return null;
|
||||
const sign = match[1] === "-" ? -1 : 1;
|
||||
return sign * (Number.parseInt(match[2], 10) * 60 + Number.parseInt(match[3], 10));
|
||||
}
|
||||
|
||||
function parseLogLineTimestampMs(line: string, yearHint: number, offsetMinutes: number): number | null {
|
||||
const fullIso = line.match(/^\[?(\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(?:\.\d{1,3})?(?:Z|[+-]\d{2}:?\d{2})?)\]?/);
|
||||
if (fullIso) {
|
||||
const timestamp = Date.parse(fullIso[1].replace(" ", "T"));
|
||||
return Number.isNaN(timestamp) ? null : timestamp;
|
||||
}
|
||||
|
||||
const langBot = line.match(/^\[(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2})\.(\d{3})\]/);
|
||||
if (!langBot) return null;
|
||||
const [, month, day, hour, minute, second, millisecond] = langBot;
|
||||
return Date.UTC(
|
||||
yearHint,
|
||||
Number.parseInt(month, 10) - 1,
|
||||
Number.parseInt(day, 10),
|
||||
Number.parseInt(hour, 10),
|
||||
Number.parseInt(minute, 10),
|
||||
Number.parseInt(second, 10),
|
||||
Number.parseInt(millisecond, 10),
|
||||
) - offsetMinutes * 60 * 1000;
|
||||
}
|
||||
|
||||
export function renderLogFinding(finding: LogFinding): string {
|
||||
const location = finding.line ? `${finding.source}:${finding.line}` : finding.source;
|
||||
const trouble = finding.troubleshooting_id ? ` (${finding.troubleshooting_id})` : "";
|
||||
const related = finding.related_to_case === true ? ", related" : "";
|
||||
const excerpt = finding.excerpt ? ` - ${finding.excerpt}` : "";
|
||||
return `- [${finding.severity}] ${location}: ${finding.kind}${trouble}${related}; pattern: ${finding.pattern}${excerpt}`;
|
||||
}
|
||||
|
||||
export function renderLogSuccessSignal(signal: LogSuccessSignal): string {
|
||||
const location = signal.line ? `${signal.source}:${signal.line}` : signal.source;
|
||||
const excerpt = signal.excerpt ? ` - ${signal.excerpt}` : "";
|
||||
return `- ${location}: ${signal.pattern}${excerpt}`;
|
||||
}
|
||||
|
||||
export function strictLogGuardExitCode(result: LogGuardResult): number {
|
||||
return result.status === "fail" || result.status === "env_issue" ? 1 : 0;
|
||||
}
|
||||
@@ -0,0 +1,251 @@
|
||||
import { env as processEnv } from "node:process";
|
||||
import type { StructuredItem } from "./types.ts";
|
||||
import { loadFixtureItems } from "./fixtures.ts";
|
||||
import { listValue, loadEnv, scalar } from "./fs.ts";
|
||||
import { splitEnvAnyGroup } from "./env-groups.ts";
|
||||
|
||||
type EnvSource = Record<string, string | undefined>;
|
||||
|
||||
export type EnvReadiness = {
|
||||
status: "ready" | "missing" | "not_required";
|
||||
required: string[];
|
||||
configured: string[];
|
||||
missing: string[];
|
||||
values: Record<string, string>;
|
||||
};
|
||||
|
||||
export type AutomationReadiness = EnvReadiness & {
|
||||
script: string;
|
||||
defaulted: string[];
|
||||
pipeline_env_required: boolean;
|
||||
env_aliases: Array<{
|
||||
target: string;
|
||||
source: string;
|
||||
configured: boolean;
|
||||
}>;
|
||||
};
|
||||
|
||||
export type ManualReadiness = {
|
||||
status: "manual_check" | "not_required";
|
||||
preconditions: string[];
|
||||
setup: string[];
|
||||
cleanup: string[];
|
||||
};
|
||||
|
||||
export type FixtureReadiness = {
|
||||
status: "ready" | "missing" | "not_required";
|
||||
required: Array<{
|
||||
id: string;
|
||||
kind: string;
|
||||
path: string;
|
||||
exists: boolean;
|
||||
}>;
|
||||
missing: string[];
|
||||
};
|
||||
|
||||
const secretKeyRe = /(?:api[_-]?key|authorization|bearer|credential|jwt|oauth|password|secret|token)/i;
|
||||
|
||||
export function redactEnvValue(key: string, value: string): string {
|
||||
if (!value) return "";
|
||||
if (secretKeyRe.test(key)) return "[redacted]";
|
||||
return value.replace(/(https?:\/\/)([^:@/\s]+):([^@/\s]+)@/i, "$1[redacted]@");
|
||||
}
|
||||
|
||||
export function runtimeEnv(root: string): Record<string, string> {
|
||||
const result: Record<string, string> = { ...loadEnv(root) };
|
||||
for (const [key, value] of Object.entries(processEnv)) {
|
||||
if (typeof value === "string") result[key] = value;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function envReadiness(
|
||||
keys: string[],
|
||||
env: EnvSource,
|
||||
defaults: Record<string, string> = {},
|
||||
anyGroups: string[] = [],
|
||||
providedBySetup: Set<string> = new Set(),
|
||||
): EnvReadiness {
|
||||
const required = [...keys];
|
||||
const configured = required.filter((key) => Boolean(env[key]) || Boolean(defaults[key]) || providedBySetup.has(key));
|
||||
const missing = required.filter((key) => !env[key] && !defaults[key] && !providedBySetup.has(key));
|
||||
const values: Record<string, string> = Object.fromEntries(
|
||||
required.map((key) => [key, redactEnvValue(key, env[key] ?? defaults[key] ?? setupProvidedValue(key, providedBySetup))]),
|
||||
);
|
||||
|
||||
for (const group of anyGroups) {
|
||||
const keysInGroup = splitEnvAnyGroup(group);
|
||||
required.push(group);
|
||||
const configuredKeys = keysInGroup.filter((key) => Boolean(env[key]) || Boolean(defaults[key]) || providedBySetup.has(key));
|
||||
if (configuredKeys.length === 0) missing.push(group);
|
||||
else configured.push(...configuredKeys);
|
||||
for (const key of keysInGroup) {
|
||||
values[key] = redactEnvValue(key, env[key] ?? defaults[key] ?? setupProvidedValue(key, providedBySetup));
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
status: required.length === 0 ? "not_required" : missing.length === 0 ? "ready" : "missing",
|
||||
required,
|
||||
configured: Array.from(new Set(configured)),
|
||||
missing,
|
||||
values,
|
||||
};
|
||||
}
|
||||
|
||||
function setupProvidedValue(key: string, providedBySetup: Set<string>): string {
|
||||
return providedBySetup.has(key) ? "[provided by setup_automation]" : "";
|
||||
}
|
||||
|
||||
export function setupProvidedEnv(item: StructuredItem): Set<string> {
|
||||
return new Set(listValue(item.fields, "setup_provides_env"));
|
||||
}
|
||||
|
||||
export function automationEnvDefaults(item: StructuredItem, env: EnvSource = processEnv): Record<string, string> {
|
||||
const mapping: Array<[string, string]> = [
|
||||
["automation_prompt", "LANGBOT_E2E_PROMPT"],
|
||||
["automation_prompts_json", "LANGBOT_E2E_PROMPTS_JSON"],
|
||||
["automation_expected_text", "LANGBOT_E2E_EXPECTED_TEXT"],
|
||||
["automation_response_timeout_ms", "LANGBOT_E2E_RESPONSE_TIMEOUT_MS"],
|
||||
["automation_stream_output", "LANGBOT_E2E_STREAM_OUTPUT"],
|
||||
["automation_image_base64_fixture", "LANGBOT_E2E_IMAGE_BASE64_PATH"],
|
||||
["automation_runner_config_patch_json", "LANGBOT_E2E_RUNNER_CONFIG_PATCH_JSON"],
|
||||
["automation_restore_runner_config", "LANGBOT_E2E_RESTORE_RUNNER_CONFIG"],
|
||||
["automation_expected_runner_id", "LANGBOT_E2E_EXPECTED_RUNNER_ID"],
|
||||
["automation_reset_debug_chat", "LANGBOT_E2E_RESET_DEBUG_CHAT"],
|
||||
["automation_debug_chat_session_type", "LANGBOT_E2E_DEBUG_CHAT_SESSION_TYPE"],
|
||||
["automation_filesystem_checks_json", "LANGBOT_E2E_FILESYSTEM_CHECKS_JSON"],
|
||||
["automation_plugin_package", "LANGBOT_E2E_PLUGIN_PACKAGE"],
|
||||
["automation_expected_plugin_id", "LANGBOT_E2E_EXPECTED_PLUGIN_ID"],
|
||||
["automation_expected_tool", "LANGBOT_E2E_EXPECTED_TOOL"],
|
||||
];
|
||||
const defaults: Record<string, string> = {};
|
||||
for (const [field, envKey] of mapping) {
|
||||
const value = scalar(item.fields, field);
|
||||
if (value) defaults[envKey] = expandEnvRefs(value, env);
|
||||
}
|
||||
const failurePatterns = listValue(item.fields, "failure_patterns");
|
||||
if (failurePatterns.length > 0) defaults.LANGBOT_E2E_FAILURE_SIGNALS = failurePatterns.join("\n");
|
||||
return defaults;
|
||||
}
|
||||
|
||||
function expandEnvRefs(value: string, env: EnvSource): string {
|
||||
return value.replace(/\$\{([A-Z][A-Z0-9_]*)\}|\$([A-Z][A-Z0-9_]*)/g, (_match, braced, bare) => {
|
||||
return env[braced || bare] || "";
|
||||
});
|
||||
}
|
||||
|
||||
export function caseEnvReadiness(item: StructuredItem, env: EnvSource): EnvReadiness {
|
||||
const aliasSources = new Set(automationEnvAliases(item, env).map((alias) => alias.source));
|
||||
const provided = setupProvidedEnv(item);
|
||||
return envReadiness(
|
||||
listValue(item.fields, "env").filter((key) => !aliasSources.has(key)),
|
||||
env,
|
||||
{},
|
||||
listValue(item.fields, "env_any"),
|
||||
provided,
|
||||
);
|
||||
}
|
||||
|
||||
function automationEnvAliases(item: StructuredItem, env: EnvSource): Array<{
|
||||
target: string;
|
||||
source: string;
|
||||
configured: boolean;
|
||||
}> {
|
||||
const provided = setupProvidedEnv(item);
|
||||
const mapping: Array<[string, string]> = [
|
||||
["automation_pipeline_url_env", "LANGBOT_E2E_PIPELINE_URL"],
|
||||
["automation_pipeline_name_env", "LANGBOT_E2E_PIPELINE_NAME"],
|
||||
];
|
||||
return mapping
|
||||
.map(([field, target]) => {
|
||||
const source = scalar(item.fields, field);
|
||||
return source ? { target, source, configured: Boolean(env[source]) || provided.has(source) } : null;
|
||||
})
|
||||
.filter((item): item is { target: string; source: string; configured: boolean } => item !== null);
|
||||
}
|
||||
|
||||
export function automationPipelineEnvRequired(item: StructuredItem): boolean {
|
||||
return Boolean(scalar(item.fields, "automation_pipeline_url_env") || scalar(item.fields, "automation_pipeline_name_env"));
|
||||
}
|
||||
|
||||
export function caseAutomationReadiness(item: StructuredItem, env: EnvSource): AutomationReadiness {
|
||||
const script = scalar(item.fields, "automation");
|
||||
const aliases = automationEnvAliases(item, env);
|
||||
const aliasSources = new Set(aliases.map((alias) => alias.source));
|
||||
const defaults = automationEnvDefaults(item, env);
|
||||
const provided = setupProvidedEnv(item);
|
||||
const requiredKeys = listValue(item.fields, "automation_env").filter((key) => !aliasSources.has(key));
|
||||
const readiness = envReadiness(requiredKeys, env, defaults, listValue(item.fields, "automation_env_any"), provided);
|
||||
const defaulted = requiredKeys.filter((key) => !env[key] && Boolean(defaults[key]));
|
||||
const aliasConfigured = aliases.some((alias) => alias.configured);
|
||||
const aliasMissing = automationPipelineEnvRequired(item) && !aliasConfigured
|
||||
? [aliases.map((alias) => alias.source).join("|")]
|
||||
: [];
|
||||
const missing = [...readiness.missing, ...aliasMissing].filter(Boolean);
|
||||
const configured = [
|
||||
...readiness.configured,
|
||||
...aliases.filter((alias) => alias.configured).map((alias) => alias.source),
|
||||
];
|
||||
const values = {
|
||||
...readiness.values,
|
||||
...Object.fromEntries(aliases.map((alias) => [
|
||||
alias.source,
|
||||
redactEnvValue(alias.source, env[alias.source] ?? setupProvidedValue(alias.source, provided)),
|
||||
])),
|
||||
};
|
||||
return {
|
||||
...readiness,
|
||||
status: script ? missing.length === 0 ? "ready" : "missing" : "not_required",
|
||||
script,
|
||||
defaulted,
|
||||
required: [...readiness.required, ...aliases.map((alias) => alias.source)],
|
||||
configured,
|
||||
missing,
|
||||
values,
|
||||
pipeline_env_required: automationPipelineEnvRequired(item),
|
||||
env_aliases: aliases,
|
||||
};
|
||||
}
|
||||
|
||||
export function resolvedAutomationEnvOverrides(item: StructuredItem, env: EnvSource): Record<string, string> {
|
||||
const overrides: Record<string, string> = {};
|
||||
for (const alias of automationEnvAliases(item, env)) {
|
||||
const value = env[alias.source];
|
||||
if (value) overrides[alias.target] = value;
|
||||
}
|
||||
for (const [key, value] of Object.entries(automationEnvDefaults(item, env))) {
|
||||
overrides[key] = expandEnvRefs(value, env);
|
||||
}
|
||||
if (automationPipelineEnvRequired(item)) overrides.LANGBOT_E2E_PIPELINE_REQUIRED = "1";
|
||||
return overrides;
|
||||
}
|
||||
|
||||
export function caseManualReadiness(item: StructuredItem): ManualReadiness {
|
||||
const preconditions = listValue(item.fields, "preconditions");
|
||||
const setup = listValue(item.fields, "setup");
|
||||
const cleanup = listValue(item.fields, "cleanup");
|
||||
return {
|
||||
status: preconditions.length > 0 || setup.length > 0 ? "manual_check" : "not_required",
|
||||
preconditions,
|
||||
setup,
|
||||
cleanup,
|
||||
};
|
||||
}
|
||||
|
||||
export function caseFixtureReadiness(root: string, caseId: string): FixtureReadiness {
|
||||
const fixtures = loadFixtureItems(root).items
|
||||
.filter((item) => item.related_cases.includes(caseId))
|
||||
.map((item) => ({
|
||||
id: item.id,
|
||||
kind: item.kind,
|
||||
path: item.path,
|
||||
exists: item.exists,
|
||||
}));
|
||||
const missing = fixtures.filter((item) => !item.exists).map((item) => item.id);
|
||||
return {
|
||||
status: fixtures.length === 0 ? "not_required" : missing.length === 0 ? "ready" : "missing",
|
||||
required: fixtures,
|
||||
missing,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { basename, dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { listValue } from "./fs.ts";
|
||||
import type { StructuredItem } from "./types.ts";
|
||||
|
||||
export type SetupAutomationSpec = {
|
||||
entry: string;
|
||||
kind: "case" | "node";
|
||||
target: string;
|
||||
args: string[];
|
||||
};
|
||||
|
||||
export function setupAutomationEntries(item: StructuredItem): string[] {
|
||||
return listValue(item.fields, "setup_automation");
|
||||
}
|
||||
|
||||
export function parseSetupAutomationEntry(entry: string): SetupAutomationSpec {
|
||||
const trimmed = entry.trim();
|
||||
if (trimmed.startsWith("case:")) {
|
||||
return {
|
||||
entry,
|
||||
kind: "case",
|
||||
target: trimmed.slice("case:".length).trim(),
|
||||
args: [],
|
||||
};
|
||||
}
|
||||
if (trimmed.startsWith("node:")) {
|
||||
const words = trimmed.slice("node:".length).trim().split(/\s+/).filter(Boolean);
|
||||
return {
|
||||
entry,
|
||||
kind: "node",
|
||||
target: words[0] ?? "",
|
||||
args: words.slice(1),
|
||||
};
|
||||
}
|
||||
return {
|
||||
entry,
|
||||
kind: "case",
|
||||
target: "",
|
||||
args: [],
|
||||
};
|
||||
}
|
||||
|
||||
export function validateSetupAutomationEntry(root: string, entry: string, caseIds: Set<string>): string[] {
|
||||
const spec = parseSetupAutomationEntry(entry);
|
||||
const errors: string[] = [];
|
||||
if (!entry.startsWith("case:") && !entry.startsWith("node:")) {
|
||||
return [`setup_automation entry must start with 'case:' or 'node:': ${entry}`];
|
||||
}
|
||||
if (!spec.target) errors.push(`setup_automation entry is missing a target: ${entry}`);
|
||||
if (spec.kind === "case") {
|
||||
if (spec.args.length > 0) errors.push(`setup_automation case entries cannot include args: ${entry}`);
|
||||
if (spec.target && !/^[a-z0-9][a-z0-9_-]*$/.test(spec.target)) {
|
||||
errors.push(`setup_automation case target must be a case id: ${entry}`);
|
||||
} else if (spec.target && !caseIds.has(spec.target)) {
|
||||
errors.push(`setup_automation references unknown case '${spec.target}'`);
|
||||
}
|
||||
}
|
||||
if (spec.kind === "node") {
|
||||
if (spec.target.startsWith("/") || spec.target.includes("..") || !spec.target.startsWith("scripts/")) {
|
||||
errors.push(`setup_automation node target must be a repository scripts/ path: ${entry}`);
|
||||
}
|
||||
if (spec.target && !/\.(mjs|js|ts)$/.test(spec.target)) {
|
||||
errors.push(`setup_automation node target must be a Node script: ${entry}`);
|
||||
}
|
||||
if (spec.target && !existsSync(join(root, spec.target))) {
|
||||
errors.push(`setup_automation node script does not exist: ${spec.target}`);
|
||||
}
|
||||
for (const arg of spec.args) {
|
||||
if (!/^--[A-Za-z0-9][A-Za-z0-9_-]*(?:=[A-Za-z0-9_./:@-]+)?$/.test(arg)) {
|
||||
errors.push(`setup_automation node arg must be a simple --flag or --key=value: ${entry}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
export function setupAutomationEvidenceName(index: number, spec: SetupAutomationSpec): string {
|
||||
const target = spec.kind === "case" ? spec.target : basename(spec.target).replace(/\.[^.]+$/, "");
|
||||
return `${String(index + 1).padStart(2, "0")}-${target.replace(/[^A-Za-z0-9_-]+/g, "-")}`;
|
||||
}
|
||||
|
||||
export function setupAutomationScriptPath(root: string, spec: SetupAutomationSpec): string {
|
||||
return spec.kind === "node" && spec.target ? resolve(root, spec.target) : "";
|
||||
}
|
||||
|
||||
export function lbsScriptPath(): string {
|
||||
return resolve(dirname(fileURLToPath(import.meta.url)), "lbs.ts");
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
export type Skill = {
|
||||
path: string;
|
||||
directory: string;
|
||||
name: string;
|
||||
description: string;
|
||||
body: string;
|
||||
};
|
||||
|
||||
export type CommandContext = {
|
||||
root: string;
|
||||
args: string[];
|
||||
};
|
||||
|
||||
export type ParsedYamlValue = string | boolean | string[];
|
||||
|
||||
export type ParsedYaml = Record<string, ParsedYamlValue>;
|
||||
|
||||
export type StructuredItem = {
|
||||
path: string;
|
||||
skill: string;
|
||||
fields: ParsedYaml;
|
||||
raw: string;
|
||||
};
|
||||
|
||||
export type StructuredItemKind = "cases" | "suites" | "troubleshooting";
|
||||
Reference in New Issue
Block a user