mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-09-01 07:07:14 +00:00
feat: MCP server + in-repo skills (agent-friendly platform) (#2269)
* feat(api): support global API key from config.yaml (api.global_api_key) Accept a config-defined global API key anywhere a web-UI key is accepted (X-API-Key / Bearer), with no login session and no DB record. Useful for automated deployments and AI agents (HTTP API + MCP). Defaults to empty (disabled); does not require the lbk_ prefix. - templates/config.yaml: add api.global_api_key with security notes - service/apikey.py: verify_api_key checks global key first (constant-time) - docs/API_KEY_AUTH.md: document the global key + security guidance - tests: cover global-key match, prefix-free, fallback-to-db, disabled * feat(mcp): expose LangBot management as an MCP server at /mcp Add an MCP (Model Context Protocol) server so external AI agents can manage a LangBot instance. Reuses the same API-key auth as the HTTP API (including the config.yaml global API key). - pkg/api/mcp/server.py: FastMCP server wrapping the service layer; 21 curated tools across system/bots/pipelines/models/knowledge/mcp-servers/skills - pkg/api/mcp/mount.py: ASGI dispatcher fronting Quart; authenticates /mcp requests with an API key, runs the streamable-HTTP session manager lifespan - controller/main.py: serve the wrapped ASGI app via hypercorn (was run_task) - web: new 'MCP' tab in the API integration dialog showing endpoint, auth, and client config; i18n for 8 locales - tests/manual/mcp_smoke.py: e2e check (401 unauth, list tools, call tools) Tool surface is intentionally curated (not all ~25 route groups) to keep the agent surface small, safe, and maintainable. Extend deliberately. * feat(skills): add in-repo skills/ as the single source of truth Migrate the agent skills + QA/e2e test harness from the (now archived) langbot-app/langbot-skills repo into LangBot/skills/, and add four new skills. Migrated: - langbot-plugin-dev, langbot-testing (e2e), langbot-env-setup, langbot-skills-maintenance, langbot-eba-adapter-dev - the bin/lbs CLI (src/, test/, scripts/, schemas/, qa-agent-docs/) New: - langbot-dev core backend + web development - langbot-deploy Docker/K8s deployment + config.yaml + global API key - langbot-mcp-ops operating the LangBot MCP server (/mcp) - langbot-space-ops operating the Space marketplace MCP server - src/cli.ts repoRoot(): recognize the skills assets root (skills.index.json + bin/lbs) so the CLI works when nested inside the LangBot repo - README.md: unified skill catalog; skills.index.json regenerated Parity with source verified: bin/lbs validate + node test suite match the source repo (only the uncommitted .lbpkg build-artifact fixture differs). * docs(agents): document agent-facing surfaces + API/MCP/skills sync rule * docs(readme): add 'Built for AI Agents' section across all locales Highlight MCP server, in-repo skills (single source of truth), AGENTS.md sync rule, and llms.txt. Cross-link LangBot Space MCP marketplace. * style(mcp): fix ruff format + prettier lint in MCP server and API panel * style(web): prettier format MCP i18n locale entries * docs(skills): note MCP instance control in dev/testing skills All development-guidance skills now point to the LangBot instance MCP server (/mcp) and the Space marketplace MCP server, reusing API keys.
This commit is contained in:
Junyan Chin
@@ -0,0 +1,180 @@
|
||||
import { existsSync, mkdirSync, writeFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import type { CommandContext } from "../types.ts";
|
||||
import { parseOptions, optionString, usage, fail } from "../cli.ts";
|
||||
import { caseModeValues } from "../constants.ts";
|
||||
import { boolValue, findStructuredItem, getSkill, listValue, loadStructuredItems, scalar, yamlList, yamlQuote } from "../fs.ts";
|
||||
import { caseAutomationReadiness, caseEnvReadiness, caseFixtureReadiness, caseManualReadiness, runtimeEnv } from "../readiness.ts";
|
||||
import { setupAutomationEntries } from "../setup-automation.ts";
|
||||
|
||||
function casePath(root: string, skillName: string, id: string): string {
|
||||
const skill = getSkill(root, skillName);
|
||||
const dir = join(skill.path, "cases");
|
||||
mkdirSync(dir, { recursive: true });
|
||||
return join(dir, `${id}.yaml`);
|
||||
}
|
||||
|
||||
export function commandCaseNew(ctx: CommandContext): number {
|
||||
const { positional, options } = parseOptions(ctx.args.slice(2));
|
||||
const id = positional[0];
|
||||
const title = optionString(options, "title");
|
||||
if (!id || !title) usage();
|
||||
|
||||
const skill = optionString(options, "skill") ?? "langbot-testing";
|
||||
const path = casePath(ctx.root, skill, id);
|
||||
if (existsSync(path)) fail(`case already exists: ${path}`);
|
||||
|
||||
const area = optionString(options, "area") ?? "general";
|
||||
const type = optionString(options, "type") ?? "smoke";
|
||||
const mode = optionString(options, "mode") ?? "agent-browser";
|
||||
if (!caseModeValues.includes(mode)) fail(`--mode must be one of ${caseModeValues.join(", ")}`);
|
||||
const isProbe = mode === "probe";
|
||||
|
||||
const text =
|
||||
`id: ${id}\n` +
|
||||
`title: ${yamlQuote(title)}\n` +
|
||||
`mode: ${mode}\n` +
|
||||
`area: ${area}\n` +
|
||||
`type: ${type}\n` +
|
||||
"priority: p2\n" +
|
||||
"risk: medium\n" +
|
||||
"ci_eligible: false\n" +
|
||||
"tags:\n" +
|
||||
yamlList([type]) +
|
||||
"\nskills:\n" +
|
||||
yamlList(["langbot-env-setup", skill]) +
|
||||
"\nenv:\n" +
|
||||
yamlList(isProbe ? [] : ["LANGBOT_FRONTEND_URL", "LANGBOT_BACKEND_URL"]) +
|
||||
"\nsteps:\n" +
|
||||
yamlList([isProbe ? "Describe the probe command, script, or diagnostic to run." : "Describe the user-visible action to perform."]) +
|
||||
"\nchecks:\n" +
|
||||
yamlList(isProbe
|
||||
? [
|
||||
"Probe: Describe the expected success signal.",
|
||||
"Evidence: Required logs, API diagnostics, or filesystem artifacts are written.",
|
||||
]
|
||||
: [
|
||||
"UI: Describe the user-visible success signal.",
|
||||
"Console: No unexpected frontend errors.",
|
||||
"Logs: Relevant backend processing completed when applicable.",
|
||||
]) +
|
||||
"\nevidence_required:\n" +
|
||||
yamlList(isProbe ? ["api_diagnostic"] : ["ui", "console"]) +
|
||||
"\ndiagnostics:\n" +
|
||||
yamlList([isProbe
|
||||
? "Use logs, API, or filesystem diagnostics to explain probe failures."
|
||||
: "Use API/curl/logs only to distinguish frontend failure from backend/runtime failure."]) +
|
||||
"\ntroubleshooting:\n" +
|
||||
yamlList([]) +
|
||||
"\n";
|
||||
|
||||
writeFileSync(path, text, "utf8");
|
||||
console.log(path);
|
||||
return 0;
|
||||
}
|
||||
|
||||
function caseRow(item: ReturnType<typeof loadStructuredItems>[number], root: string): Record<string, unknown> {
|
||||
const automation = scalar(item.fields, "automation");
|
||||
const env = runtimeEnv(root);
|
||||
const id = scalar(item.fields, "id");
|
||||
const row = {
|
||||
skill: item.skill,
|
||||
id,
|
||||
title: scalar(item.fields, "title"),
|
||||
mode: scalar(item.fields, "mode"),
|
||||
area: scalar(item.fields, "area"),
|
||||
type: scalar(item.fields, "type"),
|
||||
priority: scalar(item.fields, "priority"),
|
||||
risk: scalar(item.fields, "risk"),
|
||||
ci_eligible: boolValue(item.fields, "ci_eligible") ?? false,
|
||||
tags: listValue(item.fields, "tags"),
|
||||
env: listValue(item.fields, "env"),
|
||||
env_any: listValue(item.fields, "env_any"),
|
||||
preconditions: listValue(item.fields, "preconditions"),
|
||||
setup: listValue(item.fields, "setup"),
|
||||
setup_automation: setupAutomationEntries(item),
|
||||
setup_provides_env: listValue(item.fields, "setup_provides_env"),
|
||||
cleanup: listValue(item.fields, "cleanup"),
|
||||
evidence_required: listValue(item.fields, "evidence_required"),
|
||||
automation,
|
||||
automation_exists: automation ? existsSync(resolve(root, automation)) : false,
|
||||
env_readiness: caseEnvReadiness(item, env),
|
||||
automation_readiness: caseAutomationReadiness(item, env),
|
||||
fixture_readiness: caseFixtureReadiness(root, id),
|
||||
manual_readiness: caseManualReadiness(item),
|
||||
};
|
||||
return {
|
||||
...row,
|
||||
readiness: readinessLabel(row),
|
||||
};
|
||||
}
|
||||
|
||||
function hasTag(row: Record<string, unknown>, tag: string): boolean {
|
||||
const tags = row.tags;
|
||||
return Array.isArray(tags) && tags.includes(tag);
|
||||
}
|
||||
|
||||
function hasMissingReadiness(row: Record<string, unknown>): boolean {
|
||||
for (const key of ["env_readiness", "automation_readiness", "fixture_readiness"]) {
|
||||
const value = row[key] as Record<string, unknown> | undefined;
|
||||
if (value?.status === "missing") return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function hasManualCheck(row: Record<string, unknown>): boolean {
|
||||
const manual = row.manual_readiness as Record<string, unknown> | undefined;
|
||||
return manual?.status === "manual_check";
|
||||
}
|
||||
|
||||
function readinessLabel(row: Record<string, unknown>): string {
|
||||
if (hasMissingReadiness(row)) return "not-ready";
|
||||
return hasManualCheck(row) ? "manual-check" : "ready";
|
||||
}
|
||||
|
||||
export function commandCaseList(ctx: CommandContext): number {
|
||||
const { positional, options } = parseOptions(ctx.args.slice(2));
|
||||
const skill = positional[0];
|
||||
const rows = loadStructuredItems(ctx.root, "cases", skill)
|
||||
.map((item) => caseRow(item, ctx.root))
|
||||
.filter((row) => !optionString(options, "type") || row.type === optionString(options, "type"))
|
||||
.filter((row) => !optionString(options, "area") || row.area === optionString(options, "area"))
|
||||
.filter((row) => !optionString(options, "priority") || row.priority === optionString(options, "priority"))
|
||||
.filter((row) => !optionString(options, "risk") || row.risk === optionString(options, "risk"))
|
||||
.filter((row) => !optionString(options, "tag") || hasTag(row, optionString(options, "tag") ?? ""))
|
||||
.filter((row) => options.automation !== true || Boolean(row.automation))
|
||||
.filter((row) => options.ci !== true || row.ci_eligible === true)
|
||||
.filter((row) => options["machine-ready"] !== true || !hasMissingReadiness(row))
|
||||
.filter((row) => options.ready !== true || (!hasMissingReadiness(row) && !hasManualCheck(row)));
|
||||
|
||||
if (options.json === true) {
|
||||
console.log(JSON.stringify(rows, null, 2));
|
||||
return 0;
|
||||
}
|
||||
|
||||
for (const row of rows) {
|
||||
console.log([
|
||||
row.skill,
|
||||
row.id,
|
||||
row.type,
|
||||
row.area,
|
||||
row.priority,
|
||||
row.risk,
|
||||
row.ci_eligible ? "ci" : "manual",
|
||||
row.automation ? "automated" : "manual-path",
|
||||
row.readiness,
|
||||
row.title,
|
||||
].join("\t"));
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
export function commandCaseShow(ctx: CommandContext): number {
|
||||
const positional = ctx.args.slice(2);
|
||||
if (positional.length < 1 || positional.length > 2) usage();
|
||||
const item = positional.length === 1
|
||||
? findStructuredItem(ctx.root, "cases", positional[0])
|
||||
: findStructuredItem(ctx.root, "cases", positional[0], positional[1]);
|
||||
console.log(item.raw.trimEnd());
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { Socket } from "node:net";
|
||||
import type { CommandContext } from "../types.ts";
|
||||
import { parseOptions } from "../cli.ts";
|
||||
import { loadEnv } from "../fs.ts";
|
||||
import { requiredEnvKeys } from "../constants.ts";
|
||||
import { redactEnvValue } from "../readiness.ts";
|
||||
|
||||
export function commandEnvShow(ctx: CommandContext): number {
|
||||
const { options } = parseOptions(ctx.args.slice(2));
|
||||
const env = loadEnv(ctx.root);
|
||||
const outputEnv = Object.fromEntries(
|
||||
Object.entries(env).map(([key, value]) => [key, redactEnvValue(key, value)]),
|
||||
);
|
||||
if (options.json === true) {
|
||||
console.log(JSON.stringify(outputEnv, null, 2));
|
||||
return 0;
|
||||
}
|
||||
for (const key of Object.keys(outputEnv).sort()) {
|
||||
console.log(`${key}=${outputEnv[key]}`);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
async function checkUrl(label: string, url: string): Promise<{ ok: boolean; message: string }> {
|
||||
if (!url) return { ok: false, message: `${label}: missing` };
|
||||
const displayUrl = redactEnvValue(label, url);
|
||||
try {
|
||||
const response = await fetch(url, { method: "HEAD", signal: AbortSignal.timeout(2500) });
|
||||
return { ok: response.ok || response.status < 500, message: `${label}: ${displayUrl} -> HTTP ${response.status}` };
|
||||
} catch (error) {
|
||||
return { ok: false, message: `${label}: ${displayUrl} -> ${String(error).replace(/\s+/g, " ")}` };
|
||||
}
|
||||
}
|
||||
|
||||
function endpoint(url: string): { host: string; port: number } | null {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null;
|
||||
const port = parsed.port ? Number(parsed.port) : parsed.protocol === "https:" ? 443 : 80;
|
||||
return { host: parsed.hostname, port };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function checkTcpListener(url: string): Promise<{ ok: boolean; message: string } | null> {
|
||||
const target = endpoint(url);
|
||||
if (!target) return null;
|
||||
|
||||
return await new Promise((resolve) => {
|
||||
const socket = new Socket();
|
||||
let settled = false;
|
||||
const finish = (ok: boolean, detail: string) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
socket.destroy();
|
||||
resolve({
|
||||
ok,
|
||||
message: `${target.host}:${target.port} ${detail}`,
|
||||
});
|
||||
};
|
||||
|
||||
socket.setTimeout(1500);
|
||||
socket.once("connect", () => finish(true, "is listening"));
|
||||
socket.once("timeout", () => finish(false, "did not accept TCP connection before timeout"));
|
||||
socket.once("error", (error) => finish(false, `is not listening (${error.message})`));
|
||||
socket.connect(target.port, target.host);
|
||||
});
|
||||
}
|
||||
|
||||
function startupHint(label: string, env: Record<string, string>): string | null {
|
||||
if (label === "LANGBOT_BACKEND_URL" && env.LANGBOT_REPO) {
|
||||
return `start backend: cd ${env.LANGBOT_REPO} && uv run main.py`;
|
||||
}
|
||||
if (label === "LANGBOT_FRONTEND_URL" && env.LANGBOT_WEB_REPO) {
|
||||
return `start frontend: cd ${env.LANGBOT_WEB_REPO} && pnpm dev`;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function compareProxyPair(env: Record<string, string>, upper: string, lower: string): string | null {
|
||||
const upperValue = process.env[upper] ?? env[upper] ?? "";
|
||||
const lowerValue = process.env[lower] ?? env[lower] ?? "";
|
||||
if (upperValue && lowerValue && upperValue !== lowerValue) {
|
||||
return `${upper}/${lower}: mismatch (${redactEnvValue(upper, upperValue)} vs ${redactEnvValue(lower, lowerValue)})`;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function commandEnvDoctor(ctx: CommandContext): Promise<number> {
|
||||
const env = loadEnv(ctx.root);
|
||||
const failures: string[] = [];
|
||||
const warnings: string[] = [];
|
||||
|
||||
for (const key of requiredEnvKeys) {
|
||||
if (!env[key]) failures.push(`missing ${key}`);
|
||||
}
|
||||
|
||||
for (const [label, path] of [
|
||||
["LANGBOT_REPO", env.LANGBOT_REPO],
|
||||
["LANGBOT_WEB_REPO", env.LANGBOT_WEB_REPO],
|
||||
["LANGBOT_CHROMIUM_EXECUTABLE", env.LANGBOT_CHROMIUM_EXECUTABLE],
|
||||
]) {
|
||||
if (!path || !existsSync(path)) failures.push(`${label}: path does not exist (${path || "missing"})`);
|
||||
}
|
||||
|
||||
if (env.LANGBOT_BROWSER_PROFILE && !existsSync(env.LANGBOT_BROWSER_PROFILE)) {
|
||||
warnings.push(`LANGBOT_BROWSER_PROFILE: path does not exist yet (${env.LANGBOT_BROWSER_PROFILE})`);
|
||||
}
|
||||
|
||||
for (const mismatch of [
|
||||
compareProxyPair(env, "HTTP_PROXY", "http_proxy"),
|
||||
compareProxyPair(env, "HTTPS_PROXY", "https_proxy"),
|
||||
compareProxyPair(env, "ALL_PROXY", "all_proxy"),
|
||||
compareProxyPair(env, "NO_PROXY", "no_proxy"),
|
||||
]) {
|
||||
if (mismatch) failures.push(mismatch);
|
||||
}
|
||||
|
||||
for (const [label, result] of await Promise.all([
|
||||
checkUrl("LANGBOT_BACKEND_URL", env.LANGBOT_BACKEND_URL).then((result) => ["LANGBOT_BACKEND_URL", result] as const),
|
||||
checkUrl("LANGBOT_FRONTEND_URL", env.LANGBOT_FRONTEND_URL).then((result) => ["LANGBOT_FRONTEND_URL", result] as const),
|
||||
])) {
|
||||
if (result.ok) console.log(`OK: ${result.message}`);
|
||||
else {
|
||||
failures.push(result.message);
|
||||
const tcp = await checkTcpListener(env[label]);
|
||||
if (tcp && !tcp.ok) failures.push(`${label}: no HTTP service reachable because ${tcp.message}`);
|
||||
const hint = startupHint(label, env);
|
||||
if (hint) warnings.push(`${label}: ${hint}`);
|
||||
}
|
||||
}
|
||||
|
||||
for (const warning of warnings) console.log(`WARN: ${warning}`);
|
||||
for (const failure of failures) console.log(`FAIL: ${failure}`);
|
||||
if (failures.length > 0) return 1;
|
||||
console.log("OK: environment looks usable");
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
import type { CommandContext } from "../types.ts";
|
||||
import { parseOptions } from "../cli.ts";
|
||||
import { loadFixtureItems } from "../fixtures.ts";
|
||||
import { dirname, join } from "node:path";
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
|
||||
function fixtureRows(root: string, skill: string | undefined): ReturnType<typeof loadFixtureItems> {
|
||||
return loadFixtureItems(root, skill);
|
||||
}
|
||||
|
||||
function qaAgentRunnerSourceFindings(item: ReturnType<typeof loadFixtureItems>["items"][number]) {
|
||||
if (!item.checks.includes("qa_agent_runner_source") || !item.exists) return [];
|
||||
const root = dirname(item.absolute_path);
|
||||
const required = [
|
||||
"main.py",
|
||||
"components/agent_runner/default.yaml",
|
||||
"components/agent_runner/default.py",
|
||||
"assets/icon.svg",
|
||||
];
|
||||
const missing = required
|
||||
.filter((path) => !existsSync(join(root, path)))
|
||||
.map((path) => ({
|
||||
severity: "fail",
|
||||
kind: "fixture_check_missing_file",
|
||||
id: item.id,
|
||||
path: `${item.path.replace(/\/[^/]+$/, "")}/${path}`,
|
||||
}));
|
||||
if (missing.length > 0) return missing;
|
||||
|
||||
const manifest = readFileSync(item.absolute_path, "utf8");
|
||||
const runnerYaml = readFileSync(join(root, "components/agent_runner/default.yaml"), "utf8");
|
||||
const runnerPy = readFileSync(join(root, "components/agent_runner/default.py"), "utf8");
|
||||
const requiredText = [
|
||||
[manifest, "AgentRunner", "manifest.yaml"],
|
||||
[manifest, "QAAgentRunnerPlugin", "manifest.yaml"],
|
||||
[runnerYaml, "kind: AgentRunner", "components/agent_runner/default.yaml"],
|
||||
[runnerYaml, "DefaultAgentRunner", "components/agent_runner/default.yaml"],
|
||||
[runnerPy, "QA_AGENT_RUNNER_OK", "components/agent_runner/default.py"],
|
||||
[runnerPy, "QA_AGENT_RUNNER_CONTROLLED_FAILURE", "components/agent_runner/default.py"],
|
||||
];
|
||||
return requiredText
|
||||
.filter(([text, needle]) => !text.includes(needle))
|
||||
.map(([, needle, relativePath]) => ({
|
||||
severity: "fail",
|
||||
kind: "fixture_check_missing_text",
|
||||
id: item.id,
|
||||
path: `${item.path.replace(/\/[^/]+$/, "")}/${relativePath}`,
|
||||
detail: `missing ${needle}`,
|
||||
}));
|
||||
}
|
||||
|
||||
function zipPackageFindings(item: ReturnType<typeof loadFixtureItems>["items"][number]) {
|
||||
if (!item.checks.includes("zip_package") || !item.exists) return [];
|
||||
const header = readFileSync(item.absolute_path).subarray(0, 4).toString("binary");
|
||||
if (header === "PK\u0003\u0004" || header === "PK\u0005\u0006") return [];
|
||||
return [{
|
||||
severity: "fail",
|
||||
kind: "fixture_check_invalid_zip",
|
||||
id: item.id,
|
||||
path: item.path,
|
||||
}];
|
||||
}
|
||||
|
||||
export function commandFixtureList(ctx: CommandContext): number {
|
||||
const { positional, options } = parseOptions(ctx.args.slice(2));
|
||||
const skill = positional[0];
|
||||
const result = fixtureRows(ctx.root, skill);
|
||||
|
||||
if (options.json === true) {
|
||||
console.log(JSON.stringify(result.items, null, 2));
|
||||
return result.errors.length > 0 ? 1 : 0;
|
||||
}
|
||||
|
||||
for (const item of result.items) {
|
||||
console.log([
|
||||
item.skill,
|
||||
item.id,
|
||||
item.kind,
|
||||
item.exists ? "present" : "missing",
|
||||
item.path,
|
||||
item.title,
|
||||
].join("\t"));
|
||||
}
|
||||
for (const error of result.errors) console.error(`ERROR: ${error}`);
|
||||
return result.errors.length > 0 ? 1 : 0;
|
||||
}
|
||||
|
||||
export function commandFixtureCheck(ctx: CommandContext): number {
|
||||
const { positional, options } = parseOptions(ctx.args.slice(2));
|
||||
const skill = positional[0];
|
||||
const result = fixtureRows(ctx.root, skill);
|
||||
const findings = [
|
||||
...result.errors.map((error) => ({ severity: "fail", kind: "invalid_manifest", detail: error })),
|
||||
...result.items
|
||||
.filter((item) => !item.exists)
|
||||
.map((item) => ({
|
||||
severity: "fail",
|
||||
kind: "missing_fixture",
|
||||
id: item.id,
|
||||
path: item.path,
|
||||
absolute_path: item.absolute_path,
|
||||
})),
|
||||
...result.items.flatMap(qaAgentRunnerSourceFindings),
|
||||
...result.items.flatMap(zipPackageFindings),
|
||||
];
|
||||
const report = {
|
||||
status: findings.some((finding) => finding.severity === "fail") ? "fail" : "pass",
|
||||
fixture_count: result.items.length,
|
||||
findings,
|
||||
fixtures: result.items,
|
||||
};
|
||||
|
||||
if (options.json === true) {
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
} else {
|
||||
console.log(`# Fixture Check`);
|
||||
console.log("");
|
||||
console.log(`status: ${report.status}`);
|
||||
console.log(`fixture_count: ${report.fixture_count}`);
|
||||
console.log("");
|
||||
console.log("## Fixtures");
|
||||
for (const item of result.items) {
|
||||
console.log(`- ${item.id}: ${item.exists ? "present" : "missing"} (${item.path})`);
|
||||
}
|
||||
console.log("");
|
||||
console.log("## Findings");
|
||||
if (findings.length === 0) console.log("- None.");
|
||||
else for (const finding of findings) console.log(`- [${finding.severity}] ${finding.kind}: ${"detail" in finding ? finding.detail : finding.id}`);
|
||||
}
|
||||
|
||||
return report.status === "pass" ? 0 : 1;
|
||||
}
|
||||
@@ -0,0 +1,427 @@
|
||||
import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import type { CommandContext } from "../types.ts";
|
||||
import { optionString, parseOptions, usage } from "../cli.ts";
|
||||
import { findStructuredItem, loadEnv } from "../fs.ts";
|
||||
import {
|
||||
latestLangBotLogPath,
|
||||
logPatternContextFromStructuredItem,
|
||||
renderLogFinding,
|
||||
renderLogSuccessSignal,
|
||||
scanLogSources,
|
||||
scanLogText,
|
||||
strictLogGuardExitCode,
|
||||
type LogFinding,
|
||||
type LogGuardPatternContext,
|
||||
type LogGuardResult,
|
||||
type LogSuccessSignal,
|
||||
} from "../log-guard.ts";
|
||||
|
||||
type LogGuardSession = {
|
||||
source: "log-guard-session";
|
||||
run_id: string;
|
||||
started_at: string;
|
||||
started_at_local: string;
|
||||
backend_log: string;
|
||||
case_id: string;
|
||||
case_skill: string;
|
||||
};
|
||||
|
||||
type WatchSummary = {
|
||||
mode: "watch";
|
||||
status: string;
|
||||
path: string;
|
||||
started_at_local: string;
|
||||
finished_at_local: string;
|
||||
bytes_read: number;
|
||||
findings: LogFinding[];
|
||||
success_signals: LogSuccessSignal[];
|
||||
};
|
||||
|
||||
function pad2(value: number): string {
|
||||
return String(value).padStart(2, "0");
|
||||
}
|
||||
|
||||
function pad3(value: number): string {
|
||||
return String(value).padStart(3, "0");
|
||||
}
|
||||
|
||||
function localIsoWithOffset(date: Date): string {
|
||||
const offsetMinutes = -date.getTimezoneOffset();
|
||||
const sign = offsetMinutes >= 0 ? "+" : "-";
|
||||
const absoluteOffset = Math.abs(offsetMinutes);
|
||||
return [
|
||||
`${date.getFullYear()}-${pad2(date.getMonth() + 1)}-${pad2(date.getDate())}`,
|
||||
`T${pad2(date.getHours())}:${pad2(date.getMinutes())}:${pad2(date.getSeconds())}.${pad3(date.getMilliseconds())}`,
|
||||
`${sign}${pad2(Math.floor(absoluteOffset / 60))}:${pad2(absoluteOffset % 60)}`,
|
||||
].join("");
|
||||
}
|
||||
|
||||
function timestampSlug(localIso: string): string {
|
||||
return localIso
|
||||
.replace(/T/, "-")
|
||||
.replace(/[.:+]/g, "-")
|
||||
.replace(/[^A-Za-z0-9_-]+/g, "-")
|
||||
.replace(/-+/g, "-")
|
||||
.replace(/^-|-$/g, "");
|
||||
}
|
||||
|
||||
function writeOrPrint(content: string, output: string | undefined): void {
|
||||
if (!output) {
|
||||
console.log(content.trimEnd());
|
||||
return;
|
||||
}
|
||||
const path = resolve(output);
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
writeFileSync(path, content, "utf8");
|
||||
console.log(path);
|
||||
}
|
||||
|
||||
function positiveIntegerOption(options: Record<string, string | boolean>, key: string, fallback: number): number {
|
||||
const raw = optionString(options, key);
|
||||
if (!raw) return fallback;
|
||||
const parsed = Number.parseInt(raw, 10);
|
||||
if (!/^\d+$/.test(raw) || parsed <= 0) return fallback;
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function splitPatternList(value: string | undefined): string[] {
|
||||
if (!value) return [];
|
||||
return value
|
||||
.split(/\s*\|\s*|\s*,\s*/)
|
||||
.map((item) => item.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function patternContextFromOptions(root: string, options: Record<string, string | boolean>): LogGuardPatternContext {
|
||||
const caseId = optionString(options, "case");
|
||||
const base = caseId ? logPatternContextFromStructuredItem(findStructuredItem(root, "cases", caseId)) : {};
|
||||
return {
|
||||
successPatterns: [
|
||||
...(base.successPatterns ?? []),
|
||||
...splitPatternList(optionString(options, "success-pattern")),
|
||||
],
|
||||
failurePatterns: [
|
||||
...(base.failurePatterns ?? []),
|
||||
...splitPatternList(optionString(options, "failure-pattern")),
|
||||
],
|
||||
expectedFailures: [
|
||||
...(base.expectedFailures ?? []),
|
||||
...splitPatternList(optionString(options, "expected-failure")),
|
||||
],
|
||||
relatedTroubleshootingIds: base.relatedTroubleshootingIds ?? [],
|
||||
};
|
||||
}
|
||||
|
||||
function latestOrExplicitBackendLog(root: string, options: Record<string, string | boolean>): string {
|
||||
const explicit = optionString(options, "backend-log");
|
||||
if (explicit) return resolve(explicit);
|
||||
const auto = latestLangBotLogPath(loadEnv(root));
|
||||
return auto ? resolve(auto) : "";
|
||||
}
|
||||
|
||||
function renderSources(result: LogGuardResult): string[] {
|
||||
const lines: string[] = [];
|
||||
if (result.sources.length === 0) {
|
||||
lines.push("- sources: no log files provided; use --backend-log or configure LANGBOT_REPO.");
|
||||
return lines;
|
||||
}
|
||||
lines.push("- sources:");
|
||||
for (const source of result.sources) {
|
||||
const origin = source.auto_detected ? ", auto" : "";
|
||||
const total = source.total_line_count === undefined ? "" : `/${source.total_line_count}`;
|
||||
const range = source.start_line === undefined || source.end_line === undefined
|
||||
? ""
|
||||
: `, lines ${source.start_line}-${source.end_line}`;
|
||||
const timestamped = source.timestamped_line_count === undefined ? "" : `, ${source.timestamped_line_count} timestamped`;
|
||||
lines.push(` - ${source.source}: ${source.path} (${source.status}${origin}, ${source.line_count}${total} lines${range}${timestamped})`);
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
function renderLogGuardMarkdown(title: string, result: LogGuardResult, extra: string[] = []): string {
|
||||
const lines: string[] = [];
|
||||
lines.push(`# ${title}`);
|
||||
lines.push("");
|
||||
lines.push(`Generated: ${new Date().toISOString()}`);
|
||||
lines.push(`Status: ${result.status}`);
|
||||
lines.push(`Scan mode: ${result.scan.mode}`);
|
||||
if (result.scan.since) lines.push(`Since: ${result.scan.since}`);
|
||||
if (result.scan.until) lines.push(`Until: ${result.scan.until}`);
|
||||
if (result.scan.tail_lines !== undefined) lines.push(`Tail lines: ${result.scan.tail_lines}`);
|
||||
if (extra.length > 0) {
|
||||
lines.push("");
|
||||
lines.push("## Context");
|
||||
for (const item of extra) lines.push(`- ${item}`);
|
||||
}
|
||||
lines.push("");
|
||||
lines.push("## Sources");
|
||||
lines.push(...renderSources(result));
|
||||
if (result.scan.warnings.length > 0) {
|
||||
lines.push("");
|
||||
lines.push("## Scan Warnings");
|
||||
for (const warning of result.scan.warnings) lines.push(`- ${warning}`);
|
||||
}
|
||||
lines.push("");
|
||||
lines.push("## Findings");
|
||||
if (result.findings.length === 0) lines.push("- None.");
|
||||
else for (const finding of result.findings) lines.push(renderLogFinding(finding));
|
||||
lines.push("");
|
||||
lines.push("## Success Signals");
|
||||
if (result.success_signals.length === 0) lines.push("- None.");
|
||||
else for (const signal of result.success_signals) lines.push(renderLogSuccessSignal(signal));
|
||||
lines.push("");
|
||||
return `${lines.join("\n").trimEnd()}\n`;
|
||||
}
|
||||
|
||||
function statusFromEvents(findings: LogFinding[], successSignals: LogSuccessSignal[]): string {
|
||||
if (findings.some((finding) => finding.severity === "fail" || finding.severity === "missing_input")) return "fail";
|
||||
if (findings.some((finding) => finding.severity === "matched_troubleshooting" && finding.related_to_case !== false)) return "fail";
|
||||
if (findings.some((finding) => finding.severity === "env_issue")) return "env_issue";
|
||||
if (findings.some((finding) => finding.severity === "warning")) return "warning";
|
||||
if (successSignals.length > 0) return "pass";
|
||||
return "no_activity";
|
||||
}
|
||||
|
||||
function strictSummaryExitCode(status: string): number {
|
||||
return status === "fail" || status === "env_issue" ? 1 : 0;
|
||||
}
|
||||
|
||||
function sessionDir(options: Record<string, string | boolean>): string {
|
||||
return optionString(options, "output-dir") ?? join("reports", "log-guards");
|
||||
}
|
||||
|
||||
function sessionPath(options: Record<string, string | boolean>, runId: string): string {
|
||||
return join(sessionDir(options), `${runId}.json`);
|
||||
}
|
||||
|
||||
function readSession(options: Record<string, string | boolean>): LogGuardSession | undefined {
|
||||
const runId = optionString(options, "run-id");
|
||||
const explicitSession = optionString(options, "session");
|
||||
const path = explicitSession ? resolve(explicitSession) : runId ? resolve(sessionPath(options, runId)) : "";
|
||||
if (!path || !existsSync(path)) return undefined;
|
||||
return JSON.parse(readFileSync(path, "utf8")) as LogGuardSession;
|
||||
}
|
||||
|
||||
export function commandLogScan(ctx: CommandContext): number {
|
||||
const { positional, options } = parseOptions(ctx.args.slice(2));
|
||||
if (positional.length > 0) usage();
|
||||
|
||||
const result = scanLogSources(ctx.root, options, patternContextFromOptions(ctx.root, options));
|
||||
const output = optionString(options, "output");
|
||||
const content = options.json === true
|
||||
? `${JSON.stringify(result, null, 2)}\n`
|
||||
: renderLogGuardMarkdown("Log Guard Scan", result, [
|
||||
optionString(options, "case") ? `case: ${optionString(options, "case")}` : "case: none",
|
||||
options.strict === true ? "strict: yes" : "strict: no",
|
||||
]);
|
||||
writeOrPrint(content, output);
|
||||
return options.strict === true ? strictLogGuardExitCode(result) : 0;
|
||||
}
|
||||
|
||||
export async function commandLogWatch(ctx: CommandContext): Promise<number> {
|
||||
const { positional, options } = parseOptions(ctx.args.slice(2));
|
||||
if (positional.length > 0) usage();
|
||||
|
||||
const path = latestOrExplicitBackendLog(ctx.root, options);
|
||||
if (!path) {
|
||||
console.error("ERROR: no backend log found; pass --backend-log or configure LANGBOT_REPO.");
|
||||
return 1;
|
||||
}
|
||||
if (!existsSync(path)) {
|
||||
console.error(`ERROR: backend log does not exist: ${path}`);
|
||||
return 1;
|
||||
}
|
||||
|
||||
const context = patternContextFromOptions(ctx.root, options);
|
||||
const intervalMs = positiveIntegerOption(options, "interval-ms", 1000);
|
||||
const durationMs = optionString(options, "duration-ms")
|
||||
? positiveIntegerOption(options, "duration-ms", 0)
|
||||
: 0;
|
||||
const startedAtLocal = localIsoWithOffset(new Date());
|
||||
const findings: LogFinding[] = [];
|
||||
const successSignals: LogSuccessSignal[] = [];
|
||||
let bytesRead = 0;
|
||||
let offset = options["from-start"] === true ? 0 : statSync(path).size;
|
||||
let baseLineNumber = options["from-start"] === true
|
||||
? 0
|
||||
: readFileSync(path).subarray(0, offset).toString("utf8").split(/\r?\n/).length - 1;
|
||||
let carry = "";
|
||||
|
||||
if (options.json !== true) {
|
||||
console.log(`# Log Guard Watch`);
|
||||
console.log(`Path: ${path}`);
|
||||
console.log(`Started: ${startedAtLocal}`);
|
||||
console.log(`Mode: ${options["from-start"] === true ? "from-start" : "new-lines"}`);
|
||||
}
|
||||
|
||||
const startedMs = Date.now();
|
||||
let stopRequested = false;
|
||||
const stop = (): void => {
|
||||
stopRequested = true;
|
||||
};
|
||||
process.once("SIGINT", stop);
|
||||
process.once("SIGTERM", stop);
|
||||
|
||||
const poll = (): void => {
|
||||
const buffer = readFileSync(path);
|
||||
if (buffer.length < offset) {
|
||||
offset = 0;
|
||||
baseLineNumber = 0;
|
||||
carry = "";
|
||||
}
|
||||
if (buffer.length === offset) return;
|
||||
|
||||
const chunk = buffer.subarray(offset).toString("utf8");
|
||||
offset = buffer.length;
|
||||
bytesRead += Buffer.byteLength(chunk);
|
||||
const text = `${carry}${chunk}`;
|
||||
const hasCompleteLine = /\r?\n$/.test(text);
|
||||
const lastNewline = Math.max(text.lastIndexOf("\n"), text.lastIndexOf("\r"));
|
||||
if (!hasCompleteLine && lastNewline === -1) {
|
||||
carry = text;
|
||||
return;
|
||||
}
|
||||
|
||||
const complete = hasCompleteLine ? text : text.slice(0, lastNewline + 1);
|
||||
carry = hasCompleteLine ? "" : text.slice(lastNewline + 1);
|
||||
if (!complete) return;
|
||||
|
||||
const result = scanLogText(ctx.root, "backend", path, complete, {}, context, baseLineNumber, false);
|
||||
baseLineNumber += complete.split(/\r?\n/).length - 1;
|
||||
findings.push(...result.findings);
|
||||
successSignals.push(...result.success_signals);
|
||||
|
||||
if (options.json !== true) {
|
||||
for (const finding of result.findings) console.log(renderLogFinding(finding));
|
||||
for (const signal of result.success_signals) console.log(renderLogSuccessSignal(signal));
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
do {
|
||||
poll();
|
||||
if (stopRequested) break;
|
||||
if (durationMs > 0 && Date.now() - startedMs >= durationMs) break;
|
||||
await delay(Math.min(intervalMs, durationMs > 0 ? Math.max(1, durationMs - (Date.now() - startedMs)) : intervalMs));
|
||||
} while (!stopRequested);
|
||||
|
||||
if (carry) {
|
||||
const result = scanLogText(ctx.root, "backend", path, carry, {}, context, baseLineNumber, false);
|
||||
findings.push(...result.findings);
|
||||
successSignals.push(...result.success_signals);
|
||||
if (options.json !== true) {
|
||||
for (const finding of result.findings) console.log(renderLogFinding(finding));
|
||||
for (const signal of result.success_signals) console.log(renderLogSuccessSignal(signal));
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
process.off("SIGINT", stop);
|
||||
process.off("SIGTERM", stop);
|
||||
}
|
||||
|
||||
const summary: WatchSummary = {
|
||||
mode: "watch",
|
||||
status: statusFromEvents(findings, successSignals),
|
||||
path,
|
||||
started_at_local: startedAtLocal,
|
||||
finished_at_local: localIsoWithOffset(new Date()),
|
||||
bytes_read: bytesRead,
|
||||
findings,
|
||||
success_signals: successSignals,
|
||||
};
|
||||
|
||||
if (options.json === true) {
|
||||
console.log(JSON.stringify(summary, null, 2));
|
||||
} else {
|
||||
console.log(`Status: ${summary.status}`);
|
||||
console.log(`Bytes read: ${summary.bytes_read}`);
|
||||
}
|
||||
return options.strict === true ? strictSummaryExitCode(summary.status) : 0;
|
||||
}
|
||||
|
||||
export function commandLogGuard(ctx: CommandContext): number {
|
||||
const sub = ctx.args[2];
|
||||
if (sub === "start") return commandLogGuardStart(ctx);
|
||||
if (sub === "stop") return commandLogGuardStop(ctx);
|
||||
usage();
|
||||
}
|
||||
|
||||
function commandLogGuardStart(ctx: CommandContext): number {
|
||||
const { positional, options } = parseOptions(ctx.args.slice(3));
|
||||
if (positional.length > 0) usage();
|
||||
|
||||
const now = new Date();
|
||||
const startedAtLocal = localIsoWithOffset(now);
|
||||
const runId = optionString(options, "run-id") ?? `log-guard-${timestampSlug(startedAtLocal)}`;
|
||||
const caseId = optionString(options, "case") ?? "";
|
||||
const caseItem = caseId ? findStructuredItem(ctx.root, "cases", caseId) : undefined;
|
||||
const session: LogGuardSession = {
|
||||
source: "log-guard-session",
|
||||
run_id: runId,
|
||||
started_at: now.toISOString(),
|
||||
started_at_local: startedAtLocal,
|
||||
backend_log: latestOrExplicitBackendLog(ctx.root, options),
|
||||
case_id: caseId,
|
||||
case_skill: caseItem?.skill ?? "",
|
||||
};
|
||||
const path = resolve(sessionPath(options, runId));
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
writeFileSync(path, `${JSON.stringify(session, null, 2)}\n`, "utf8");
|
||||
|
||||
const result = {
|
||||
...session,
|
||||
path,
|
||||
stop_command: `bin/lbs log guard stop --run-id ${runId} --output-dir ${sessionDir(options)}`,
|
||||
};
|
||||
if (options.json === true) console.log(JSON.stringify(result, null, 2));
|
||||
else {
|
||||
console.log(`# Log Guard Session`);
|
||||
console.log(`Run: ${runId}`);
|
||||
console.log(`Started: ${startedAtLocal}`);
|
||||
console.log(`Session: ${path}`);
|
||||
if (session.backend_log) console.log(`Backend log: ${session.backend_log}`);
|
||||
if (session.case_id) console.log(`Case: ${session.case_id}`);
|
||||
console.log(`Stop: ${result.stop_command}`);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
function commandLogGuardStop(ctx: CommandContext): number {
|
||||
const { positional, options } = parseOptions(ctx.args.slice(3));
|
||||
if (positional.length > 0) usage();
|
||||
|
||||
const session = readSession(options);
|
||||
if (!session) {
|
||||
console.error("ERROR: log guard session not found; pass --run-id with --output-dir or --session.");
|
||||
return 1;
|
||||
}
|
||||
|
||||
const now = new Date();
|
||||
const scanOptions: Record<string, string | boolean> = {
|
||||
...options,
|
||||
since: optionString(options, "since") ?? session.started_at_local,
|
||||
until: optionString(options, "until") ?? localIsoWithOffset(now),
|
||||
};
|
||||
if (session.backend_log && typeof scanOptions["backend-log"] !== "string") {
|
||||
scanOptions["backend-log"] = session.backend_log;
|
||||
}
|
||||
|
||||
const caseId = optionString(options, "case") ?? session.case_id;
|
||||
const context = caseId
|
||||
? logPatternContextFromStructuredItem(findStructuredItem(ctx.root, "cases", caseId))
|
||||
: patternContextFromOptions(ctx.root, options);
|
||||
const result = scanLogSources(ctx.root, scanOptions, context);
|
||||
const output = optionString(options, "output") ?? join(sessionDir(options), `${session.run_id}.md`);
|
||||
const content = options.json === true
|
||||
? `${JSON.stringify({ session, result }, null, 2)}\n`
|
||||
: renderLogGuardMarkdown("Log Guard Report", result, [
|
||||
`run_id: ${session.run_id}`,
|
||||
`started: ${session.started_at_local}`,
|
||||
`finished: ${scanOptions.until}`,
|
||||
caseId ? `case: ${caseId}` : "case: none",
|
||||
]);
|
||||
writeOrPrint(content, options.json === true ? optionString(options, "output") : output);
|
||||
return options["no-strict"] === true ? 0 : strictLogGuardExitCode(result);
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import type { CommandContext } from "../types.ts";
|
||||
import { fail, optionString, parseOptions, usage } from "../cli.ts";
|
||||
import { loadFixtureItems } from "../fixtures.ts";
|
||||
import { boolValue, getSkill, globMarkdownRefs, listValue, loadSkills, loadStructuredItems, scalar, skillsRoot } from "../fs.ts";
|
||||
|
||||
export function commandList(ctx: CommandContext): number {
|
||||
for (const skill of loadSkills(ctx.root)) {
|
||||
console.log(`${skill.directory}\t${skill.name}\t${skill.description}`);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
function buildIndexData(root: string): Record<string, unknown> {
|
||||
const caseSummary = (item: ReturnType<typeof loadStructuredItems>[number]) => ({
|
||||
id: scalar(item.fields, "id"),
|
||||
title: scalar(item.fields, "title"),
|
||||
mode: scalar(item.fields, "mode"),
|
||||
area: scalar(item.fields, "area"),
|
||||
type: scalar(item.fields, "type"),
|
||||
priority: scalar(item.fields, "priority"),
|
||||
risk: scalar(item.fields, "risk"),
|
||||
ci_eligible: boolValue(item.fields, "ci_eligible") ?? false,
|
||||
tags: listValue(item.fields, "tags"),
|
||||
automation: scalar(item.fields, "automation"),
|
||||
setup_automation: listValue(item.fields, "setup_automation"),
|
||||
setup_provides_env: listValue(item.fields, "setup_provides_env"),
|
||||
evidence_required: listValue(item.fields, "evidence_required"),
|
||||
});
|
||||
const troubleshootingSummary = (item: ReturnType<typeof loadStructuredItems>[number]) => ({
|
||||
id: scalar(item.fields, "id"),
|
||||
title: scalar(item.fields, "title"),
|
||||
category: scalar(item.fields, "category") || "product",
|
||||
related_cases: listValue(item.fields, "related_cases"),
|
||||
});
|
||||
const suiteSummary = (item: ReturnType<typeof loadStructuredItems>[number]) => ({
|
||||
id: scalar(item.fields, "id"),
|
||||
title: scalar(item.fields, "title"),
|
||||
description: scalar(item.fields, "description"),
|
||||
type: scalar(item.fields, "type"),
|
||||
priority: scalar(item.fields, "priority"),
|
||||
tags: listValue(item.fields, "tags"),
|
||||
cases: listValue(item.fields, "cases"),
|
||||
});
|
||||
return {
|
||||
generated_by: "lbs",
|
||||
skills: loadSkills(root).map((skill) => ({
|
||||
directory: skill.directory,
|
||||
name: skill.name,
|
||||
description: skill.description,
|
||||
references: globMarkdownRefs(skill.path),
|
||||
cases: loadStructuredItems(root, "cases", skill.directory).map((item) => scalar(item.fields, "id")),
|
||||
case_summaries: loadStructuredItems(root, "cases", skill.directory).map(caseSummary),
|
||||
suites: loadStructuredItems(root, "suites", skill.directory).map((item) => scalar(item.fields, "id")),
|
||||
suite_summaries: loadStructuredItems(root, "suites", skill.directory).map(suiteSummary),
|
||||
fixtures: loadFixtureItems(root, skill.directory).items.map((item) => ({
|
||||
id: item.id,
|
||||
title: item.title,
|
||||
kind: item.kind,
|
||||
path: item.path,
|
||||
related_cases: item.related_cases,
|
||||
})),
|
||||
troubleshooting: loadStructuredItems(root, "troubleshooting", skill.directory).map((item) => scalar(item.fields, "id")),
|
||||
troubleshooting_summaries: loadStructuredItems(root, "troubleshooting", skill.directory).map(troubleshootingSummary),
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
export function commandIndex(ctx: CommandContext): number {
|
||||
const { options } = parseOptions(ctx.args.slice(1));
|
||||
const data = buildIndexData(ctx.root);
|
||||
const out = join(ctx.root, "skills.index.json");
|
||||
const content = `${JSON.stringify(data, null, 2)}\n`;
|
||||
if (options.check === true) {
|
||||
if (!existsSync(out)) {
|
||||
console.error(`ERROR: missing index: ${out}`);
|
||||
return 1;
|
||||
}
|
||||
if (readFileSync(out, "utf8") !== content) {
|
||||
console.error(`ERROR: index is stale: ${out}`);
|
||||
return 1;
|
||||
}
|
||||
console.log(`OK ${out}`);
|
||||
return 0;
|
||||
}
|
||||
writeFileSync(out, content, "utf8");
|
||||
console.log(out);
|
||||
return 0;
|
||||
}
|
||||
|
||||
export function commandNewSkill(ctx: CommandContext): number {
|
||||
const { positional, options } = parseOptions(ctx.args.slice(1));
|
||||
const name = positional[0];
|
||||
if (!name) usage();
|
||||
|
||||
const skillDir = join(skillsRoot(ctx.root), name);
|
||||
const skillMd = join(skillDir, "SKILL.md");
|
||||
if (existsSync(skillMd)) fail(`skill already exists: ${skillDir}`);
|
||||
|
||||
mkdirSync(skillDir, { recursive: true });
|
||||
const description = optionString(options, "description") ?? `Use when working with ${name}.`;
|
||||
const text =
|
||||
`---\nname: ${name}\ndescription: ${description}\n---\n\n` +
|
||||
`# ${name}\n\n` +
|
||||
"Add concise routing and workflow instructions here.\n";
|
||||
writeFileSync(skillMd, text, "utf8");
|
||||
console.log(skillMd);
|
||||
return 0;
|
||||
}
|
||||
|
||||
export function commandNewRef(ctx: CommandContext): number {
|
||||
const skill = ctx.args[1];
|
||||
const rawName = ctx.args[2];
|
||||
if (!skill || !rawName) usage();
|
||||
|
||||
const skillDir = getSkill(ctx.root, skill).path;
|
||||
const refsDir = join(skillDir, "references");
|
||||
mkdirSync(refsDir, { recursive: true });
|
||||
const name = rawName.endsWith(".md") ? rawName : `${rawName}.md`;
|
||||
const refPath = join(refsDir, name);
|
||||
if (existsSync(refPath)) fail(`reference already exists: ${refPath}`);
|
||||
|
||||
const title = name.replace(/\.md$/, "").replace(/-/g, " ").replace(/\b\w/g, (char) => char.toUpperCase());
|
||||
writeFileSync(refPath, `# ${title}\n\nAdd concise reusable instructions here.\n`, "utf8");
|
||||
console.log(refPath);
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,704 @@
|
||||
import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { execPath } from "node:process";
|
||||
import type { CommandContext, StructuredItem } from "../types.ts";
|
||||
import { fail, optionString, parseOptions, usage } from "../cli.ts";
|
||||
import { findStructuredItem, getSkill, listValue, loadStructuredItems, scalar, yamlList, yamlQuote } from "../fs.ts";
|
||||
import { caseAutomationReadiness, caseEnvReadiness, caseFixtureReadiness, caseManualReadiness, runtimeEnv } from "../readiness.ts";
|
||||
import { lbsScriptPath, setupAutomationEntries } from "../setup-automation.ts";
|
||||
|
||||
function suitePath(root: string, skillName: string, id: string): string {
|
||||
const skill = getSkill(root, skillName);
|
||||
const dir = join(skill.path, "suites");
|
||||
mkdirSync(dir, { recursive: true });
|
||||
return join(dir, `${id}.yaml`);
|
||||
}
|
||||
|
||||
function caseItemById(root: string, id: string): StructuredItem {
|
||||
return findStructuredItem(root, "cases", id);
|
||||
}
|
||||
|
||||
function suiteCaseSummary(root: string, id: string): Record<string, unknown> {
|
||||
const item = caseItemById(root, id);
|
||||
const env = runtimeEnv(root);
|
||||
const caseId = scalar(item.fields, "id");
|
||||
return {
|
||||
skill: item.skill,
|
||||
id: caseId,
|
||||
title: scalar(item.fields, "title"),
|
||||
mode: scalar(item.fields, "mode"),
|
||||
area: scalar(item.fields, "area"),
|
||||
type: scalar(item.fields, "type"),
|
||||
priority: scalar(item.fields, "priority"),
|
||||
risk: scalar(item.fields, "risk"),
|
||||
tags: listValue(item.fields, "tags"),
|
||||
preconditions: listValue(item.fields, "preconditions"),
|
||||
setup: listValue(item.fields, "setup"),
|
||||
setup_automation: setupAutomationEntries(item),
|
||||
setup_provides_env: listValue(item.fields, "setup_provides_env"),
|
||||
automation: scalar(item.fields, "automation"),
|
||||
evidence_required: listValue(item.fields, "evidence_required"),
|
||||
env_readiness: caseEnvReadiness(item, env),
|
||||
automation_readiness: caseAutomationReadiness(item, env),
|
||||
fixture_readiness: caseFixtureReadiness(root, caseId),
|
||||
manual_readiness: caseManualReadiness(item),
|
||||
};
|
||||
}
|
||||
|
||||
function suiteSummary(item: StructuredItem): Record<string, string | string[]> {
|
||||
return {
|
||||
skill: item.skill,
|
||||
id: scalar(item.fields, "id"),
|
||||
title: scalar(item.fields, "title"),
|
||||
description: scalar(item.fields, "description"),
|
||||
type: scalar(item.fields, "type"),
|
||||
priority: scalar(item.fields, "priority"),
|
||||
tags: listValue(item.fields, "tags"),
|
||||
cases: listValue(item.fields, "cases"),
|
||||
};
|
||||
}
|
||||
|
||||
function findSuite(root: string, args: string[]): StructuredItem {
|
||||
if (args.length < 1 || args.length > 2) usage();
|
||||
return args.length === 1
|
||||
? findStructuredItem(root, "suites", args[0])
|
||||
: findStructuredItem(root, "suites", args[0], args[1]);
|
||||
}
|
||||
|
||||
function pad2(value: number): string {
|
||||
return String(value).padStart(2, "0");
|
||||
}
|
||||
|
||||
function pad3(value: number): string {
|
||||
return String(value).padStart(3, "0");
|
||||
}
|
||||
|
||||
function localIsoWithOffset(date: Date): string {
|
||||
const offsetMinutes = -date.getTimezoneOffset();
|
||||
const sign = offsetMinutes >= 0 ? "+" : "-";
|
||||
const absoluteOffset = Math.abs(offsetMinutes);
|
||||
return [
|
||||
`${date.getFullYear()}-${pad2(date.getMonth() + 1)}-${pad2(date.getDate())}`,
|
||||
`T${pad2(date.getHours())}:${pad2(date.getMinutes())}:${pad2(date.getSeconds())}.${pad3(date.getMilliseconds())}`,
|
||||
`${sign}${pad2(Math.floor(absoluteOffset / 60))}:${pad2(absoluteOffset % 60)}`,
|
||||
].join("");
|
||||
}
|
||||
|
||||
function timestampSlug(localIso: string): string {
|
||||
return localIso
|
||||
.replace(/T/, "-")
|
||||
.replace(/[.:+]/g, "-")
|
||||
.replace(/[^A-Za-z0-9_-]+/g, "-")
|
||||
.replace(/-+/g, "-")
|
||||
.replace(/^-|-$/g, "");
|
||||
}
|
||||
|
||||
function writeOrPrint(content: string, output: string | undefined): void {
|
||||
if (!output) {
|
||||
console.log(content.trimEnd());
|
||||
return;
|
||||
}
|
||||
const path = resolve(output);
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
writeFileSync(path, content, "utf8");
|
||||
console.log(path);
|
||||
}
|
||||
|
||||
function suiteCases(root: string, item: StructuredItem): Record<string, unknown>[] {
|
||||
return listValue(item.fields, "cases").map((id) => suiteCaseSummary(root, id));
|
||||
}
|
||||
|
||||
function statusOf(caseItem: Record<string, unknown>, key: string): string {
|
||||
const value = caseItem[key] as Record<string, unknown> | undefined;
|
||||
return typeof value?.status === "string" ? value.status : "not_required";
|
||||
}
|
||||
|
||||
function readinessSummary(cases: Array<Record<string, unknown>>): Record<string, unknown> {
|
||||
const missingEnv = cases.filter((item) => statusOf(item, "env_readiness") === "missing").map((item) => item.id);
|
||||
const missingAutomation = cases.filter((item) => statusOf(item, "automation_readiness") === "missing").map((item) => item.id);
|
||||
const missingFixture = cases.filter((item) => statusOf(item, "fixture_readiness") === "missing").map((item) => item.id);
|
||||
const manualCheck = cases.filter((item) => statusOf(item, "manual_readiness") === "manual_check").map((item) => item.id);
|
||||
const missingCount = missingEnv.length + missingAutomation.length + missingFixture.length;
|
||||
return {
|
||||
status: missingCount > 0 ? "missing" : manualCheck.length > 0 ? "manual_check" : "ready",
|
||||
missing_env_cases: missingEnv,
|
||||
missing_automation_env_cases: missingAutomation,
|
||||
missing_fixture_cases: missingFixture,
|
||||
manual_check_cases: manualCheck,
|
||||
};
|
||||
}
|
||||
|
||||
function hasProbeCases(cases: Array<Record<string, unknown>>): boolean {
|
||||
return cases.some((caseItem) => caseItem.mode === "probe");
|
||||
}
|
||||
|
||||
function suiteReportGuidance(cases: Array<Record<string, unknown>>): string {
|
||||
return hasProbeCases(cases)
|
||||
? "Run each case according to its mode; probe cases may collect non-UI evidence, while agent-browser cases still require browser/UI execution."
|
||||
: "Run each case through browser/UI first; use test report with the evidence directory and backend log window after execution.";
|
||||
}
|
||||
|
||||
function suiteResultPolicy(cases: Array<Record<string, unknown>>): string[] {
|
||||
if (hasProbeCases(cases)) {
|
||||
return [
|
||||
"A suite is not pass unless every case has a result and required evidence for the same run window.",
|
||||
"agent-browser cases require UI/browser results; probe cases are judged by their declared checks and required evidence.",
|
||||
"blocked and env_issue are not product pass; report them separately.",
|
||||
];
|
||||
}
|
||||
|
||||
return [
|
||||
"A suite is not pass unless every case has a UI/browser result and required evidence for the same run window.",
|
||||
"blocked and env_issue are not product pass; report them separately.",
|
||||
];
|
||||
}
|
||||
|
||||
function suiteEvidencePolicy(cases: Array<Record<string, unknown>>): string[] {
|
||||
if (hasProbeCases(cases)) {
|
||||
return [
|
||||
"Run each case according to its mode. Agent-browser cases use browser/UI; probe cases use their declared probe steps or automation.",
|
||||
"Use each case evidence_dir for screenshots, console.log, network.log, automation-result.json, result.json, and any probe artifacts.",
|
||||
"After case execution and report review, run each result_command_template with the final status and collected evidence.",
|
||||
"After per-case result.json files exist, run the suite report command to aggregate them.",
|
||||
"blocked and env_issue are not product pass; they must be reported separately from pass.",
|
||||
];
|
||||
}
|
||||
|
||||
return [
|
||||
"Run each case through browser/UI. API/curl/log diagnostics cannot make a UI case pass by themselves.",
|
||||
"Use each case evidence_dir for screenshots, console.log, network.log, automation-result.json, and final result.json.",
|
||||
"After case execution and report review, run each result_command_template with the final status and collected evidence.",
|
||||
"After per-case result.json files exist, run the suite report command to aggregate them.",
|
||||
"blocked and env_issue are not product pass; they must be reported separately from pass.",
|
||||
];
|
||||
}
|
||||
|
||||
function buildSuitePlan(root: string, item: StructuredItem): Record<string, unknown> {
|
||||
const suite = suiteSummary(item);
|
||||
const cases = suiteCases(root, item);
|
||||
return {
|
||||
...suite,
|
||||
cases,
|
||||
readiness: readinessSummary(cases),
|
||||
commands: cases.map((caseItem) => ({
|
||||
id: caseItem.id,
|
||||
plan: `bin/lbs test plan ${caseItem.id}`,
|
||||
start: `bin/lbs test start ${caseItem.id}`,
|
||||
automation: caseItem.automation ? `bin/lbs test run ${caseItem.id} --dry-run` : "",
|
||||
})),
|
||||
report_guidance: suiteReportGuidance(cases),
|
||||
};
|
||||
}
|
||||
|
||||
export function commandSuiteNew(ctx: CommandContext): number {
|
||||
const { positional, options } = parseOptions(ctx.args.slice(2));
|
||||
const id = positional[0];
|
||||
const title = optionString(options, "title");
|
||||
if (!id || !title) usage();
|
||||
|
||||
const skill = optionString(options, "skill") ?? "langbot-testing";
|
||||
const path = suitePath(ctx.root, skill, id);
|
||||
if (existsSync(path)) fail(`suite already exists: ${path}`);
|
||||
|
||||
const text =
|
||||
`id: ${id}\n` +
|
||||
`title: ${yamlQuote(title)}\n` +
|
||||
`description: ${yamlQuote(optionString(options, "description") ?? "Describe when to run this suite.")}\n` +
|
||||
`type: ${optionString(options, "type") ?? "smoke"}\n` +
|
||||
`priority: ${optionString(options, "priority") ?? "p2"}\n` +
|
||||
"tags:\n" +
|
||||
yamlList([optionString(options, "type") ?? "smoke"]) +
|
||||
"\ncases:\n" +
|
||||
yamlList(["webui-login-state"]) +
|
||||
"\n";
|
||||
|
||||
writeFileSync(path, text, "utf8");
|
||||
console.log(path);
|
||||
return 0;
|
||||
}
|
||||
|
||||
export function commandSuiteList(ctx: CommandContext): number {
|
||||
const { positional, options } = parseOptions(ctx.args.slice(2));
|
||||
const skill = positional[0];
|
||||
const rows = loadStructuredItems(ctx.root, "suites", skill)
|
||||
.map(suiteSummary)
|
||||
.filter((row) => !optionString(options, "type") || row.type === optionString(options, "type"))
|
||||
.filter((row) => !optionString(options, "priority") || row.priority === optionString(options, "priority"));
|
||||
|
||||
if (options.json === true) {
|
||||
console.log(JSON.stringify(rows, null, 2));
|
||||
return 0;
|
||||
}
|
||||
|
||||
for (const row of rows) {
|
||||
console.log([
|
||||
row.skill,
|
||||
row.id,
|
||||
row.type,
|
||||
row.priority,
|
||||
Array.isArray(row.cases) ? row.cases.length : 0,
|
||||
row.title,
|
||||
].join("\t"));
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
export function commandSuiteShow(ctx: CommandContext): number {
|
||||
const item = findSuite(ctx.root, ctx.args.slice(2));
|
||||
console.log(item.raw.trimEnd());
|
||||
return 0;
|
||||
}
|
||||
|
||||
export function commandSuitePlan(ctx: CommandContext): number {
|
||||
const { positional: args, options } = parseOptions(ctx.args.slice(2));
|
||||
const item = findSuite(ctx.root, args);
|
||||
const plan = buildSuitePlan(ctx.root, item);
|
||||
const suite = suiteSummary(item);
|
||||
const cases = suiteCases(ctx.root, item);
|
||||
|
||||
if (options.json === true) {
|
||||
console.log(JSON.stringify(plan, null, 2));
|
||||
return 0;
|
||||
}
|
||||
|
||||
console.log(`# Suite Plan: ${suite.id}`);
|
||||
console.log("");
|
||||
console.log(`Title: ${suite.title}`);
|
||||
console.log(`Type: ${suite.type}`);
|
||||
console.log(`Priority: ${suite.priority}`);
|
||||
console.log(`Description: ${suite.description}`);
|
||||
console.log("");
|
||||
const readiness = readinessSummary(cases);
|
||||
console.log("## Readiness");
|
||||
console.log(`Status: ${readiness.status}`);
|
||||
for (const [key, value] of Object.entries(readiness)) {
|
||||
if (key === "status" || !Array.isArray(value) || value.length === 0) continue;
|
||||
console.log(`- ${key}: ${value.join(", ")}`);
|
||||
}
|
||||
console.log("");
|
||||
console.log("## Cases");
|
||||
for (const [index, caseItem] of cases.entries()) {
|
||||
console.log(`${index + 1}. ${caseItem.id} [${caseItem.priority}/${caseItem.risk}] ${caseItem.title}`);
|
||||
console.log(` - plan: bin/lbs test plan ${caseItem.id}`);
|
||||
console.log(` - start: bin/lbs test start ${caseItem.id}`);
|
||||
if (caseItem.automation) console.log(` - automation dry-run: bin/lbs test run ${caseItem.id} --dry-run`);
|
||||
console.log(` - evidence: ${Array.isArray(caseItem.evidence_required) ? caseItem.evidence_required.join(", ") : ""}`);
|
||||
const envReadiness = caseItem.env_readiness as Record<string, unknown>;
|
||||
const automationReadiness = caseItem.automation_readiness as Record<string, unknown>;
|
||||
const fixtureReadiness = caseItem.fixture_readiness as Record<string, unknown>;
|
||||
const manualReadiness = caseItem.manual_readiness as Record<string, unknown>;
|
||||
const missing: string[] = [];
|
||||
if (Array.isArray(envReadiness.missing) && envReadiness.missing.length > 0) missing.push(`env=${envReadiness.missing.join(",")}`);
|
||||
if (Array.isArray(automationReadiness.missing) && automationReadiness.missing.length > 0) missing.push(`automation_env=${automationReadiness.missing.join(",")}`);
|
||||
if (Array.isArray(fixtureReadiness.missing) && fixtureReadiness.missing.length > 0) missing.push(`fixture=${fixtureReadiness.missing.join(",")}`);
|
||||
const manualLabel = manualReadiness.status === "manual_check" ? " manual_check" : "";
|
||||
console.log(` - readiness: ${missing.length === 0 ? `ready${manualLabel}` : `missing ${missing.join(" ")}`}`);
|
||||
const preconditions = caseItem.preconditions;
|
||||
if (Array.isArray(preconditions) && preconditions.length > 0) console.log(` - preconditions: ${preconditions.length}`);
|
||||
const setupAutomation = caseItem.setup_automation;
|
||||
if (Array.isArray(setupAutomation) && setupAutomation.length > 0) console.log(` - setup automation: ${setupAutomation.length}`);
|
||||
}
|
||||
console.log("");
|
||||
console.log("## Result Policy");
|
||||
for (const policy of suiteResultPolicy(cases)) console.log(`- ${policy}`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
function suiteStartPath(root: string, path: string): string {
|
||||
return resolve(root, path);
|
||||
}
|
||||
|
||||
function ensureDirectory(root: string, path: string, label: string): void {
|
||||
const resolvedPath = suiteStartPath(root, path);
|
||||
if (existsSync(resolvedPath) && !statSync(resolvedPath).isDirectory()) {
|
||||
fail(`${label} exists and is not a directory: ${resolvedPath}`);
|
||||
}
|
||||
mkdirSync(resolvedPath, { recursive: true });
|
||||
}
|
||||
|
||||
function buildSuiteStart(
|
||||
root: string,
|
||||
item: StructuredItem,
|
||||
args: string[],
|
||||
options: Record<string, string | boolean>,
|
||||
): Record<string, unknown> {
|
||||
const now = new Date();
|
||||
const startedAtLocal = localIsoWithOffset(now);
|
||||
const suite = suiteSummary(item);
|
||||
const suiteId = String(suite.id);
|
||||
const runId = optionString(options, "run-id") ?? `${timestampSlug(startedAtLocal)}-${suiteId}`;
|
||||
const evidenceRoot = optionString(options, "evidence-dir") ?? join("reports", "evidence", runId);
|
||||
const reportPath = join("reports", `${runId}.md`);
|
||||
const manifestPath = join(evidenceRoot, "suite-start.json");
|
||||
const handoffPath = join(evidenceRoot, "suite-start.md");
|
||||
const cases = suiteCases(root, item).map((caseItem) => {
|
||||
const caseId = String(caseItem.id);
|
||||
const caseRunId = `${runId}-${caseId}`;
|
||||
const evidenceDir = join(evidenceRoot, caseId);
|
||||
const consoleLog = join(evidenceDir, "console.log");
|
||||
const caseReportPath = join("reports", `${caseRunId}.md`);
|
||||
return {
|
||||
...caseItem,
|
||||
run_id: caseRunId,
|
||||
evidence_dir: evidenceDir,
|
||||
plan_command: `bin/lbs test plan ${caseId}`,
|
||||
start_command: `bin/lbs test start ${caseId}`,
|
||||
automation_command: caseItem.automation
|
||||
? `bin/lbs test run ${caseId} --run-id ${caseRunId} --output ${evidenceDir}`
|
||||
: "",
|
||||
report_command: caseItem.automation
|
||||
? `bin/lbs test report ${caseId} --since "${startedAtLocal}" --console-log ${consoleLog} --evidence-dir ${evidenceDir} --output ${caseReportPath}`
|
||||
: `bin/lbs test report ${caseId} --since "${startedAtLocal}" --evidence-dir ${evidenceDir} --output ${caseReportPath}`,
|
||||
result_command_template: `bin/lbs test result ${caseId} --result <status> --reason "<short reason>" --evidence-dir ${evidenceDir} --run-id ${caseRunId} --started-at "${startedAtLocal}" --evidence ${Array.isArray(caseItem.evidence_required) ? caseItem.evidence_required.join(",") : ""}`,
|
||||
};
|
||||
});
|
||||
|
||||
const locator = args.join(" ");
|
||||
return {
|
||||
run_id: runId,
|
||||
started_at: now.toISOString(),
|
||||
started_at_local: startedAtLocal,
|
||||
suite,
|
||||
evidence_root: evidenceRoot,
|
||||
manifest_path: manifestPath,
|
||||
handoff_path: handoffPath,
|
||||
cases,
|
||||
suite_report_path: reportPath,
|
||||
plan_command: `bin/lbs suite plan ${locator}`,
|
||||
report_command: `bin/lbs suite report ${locator} --run-id ${runId} --evidence-dir ${evidenceRoot} --output ${reportPath}`,
|
||||
evidence_policy: suiteEvidencePolicy(cases),
|
||||
};
|
||||
}
|
||||
|
||||
function writeSuiteStartArtifacts(root: string, start: Record<string, unknown>, rendered: string): void {
|
||||
const evidenceRoot = String(start.evidence_root || "");
|
||||
if (!evidenceRoot) return;
|
||||
|
||||
ensureDirectory(root, evidenceRoot, "suite evidence directory");
|
||||
for (const caseItem of start.cases as Array<Record<string, unknown>>) {
|
||||
const evidenceDir = String(caseItem.evidence_dir || "");
|
||||
if (evidenceDir) ensureDirectory(root, evidenceDir, "case evidence directory");
|
||||
}
|
||||
|
||||
const manifestPath = String(start.manifest_path || "");
|
||||
if (manifestPath) {
|
||||
const path = suiteStartPath(root, manifestPath);
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
writeFileSync(path, `${JSON.stringify(start, null, 2)}\n`, "utf8");
|
||||
}
|
||||
|
||||
const handoffPath = String(start.handoff_path || "");
|
||||
if (handoffPath) {
|
||||
const path = suiteStartPath(root, handoffPath);
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
writeFileSync(path, rendered, "utf8");
|
||||
}
|
||||
}
|
||||
|
||||
function renderSuiteStart(start: Record<string, unknown>): string {
|
||||
const suite = start.suite as Record<string, unknown>;
|
||||
const cases = start.cases as Array<Record<string, unknown>>;
|
||||
const lines: string[] = [];
|
||||
lines.push(`# Suite Start: ${suite.id}`);
|
||||
lines.push("");
|
||||
lines.push(`Run: ${start.run_id}`);
|
||||
lines.push(`Started: ${start.started_at_local}`);
|
||||
lines.push(`Title: ${suite.title}`);
|
||||
lines.push(`Evidence root: ${start.evidence_root}`);
|
||||
lines.push("");
|
||||
lines.push("## Commands");
|
||||
lines.push(`- plan: ${start.plan_command}`);
|
||||
lines.push(`- report: ${start.report_command}`);
|
||||
lines.push("");
|
||||
lines.push("## Cases");
|
||||
for (const [index, caseItem] of cases.entries()) {
|
||||
lines.push(`${index + 1}. ${caseItem.id} [${caseItem.priority}/${caseItem.risk}] ${caseItem.title}`);
|
||||
lines.push(` - evidence_dir: ${caseItem.evidence_dir}`);
|
||||
lines.push(` - plan: ${caseItem.plan_command}`);
|
||||
if (caseItem.automation_command) lines.push(` - automation: ${caseItem.automation_command}`);
|
||||
else lines.push(` - manual start: ${caseItem.start_command}`);
|
||||
lines.push(` - report: ${caseItem.report_command}`);
|
||||
lines.push(` - result template: ${caseItem.result_command_template}`);
|
||||
}
|
||||
lines.push("");
|
||||
lines.push("## Evidence Policy");
|
||||
for (const item of start.evidence_policy as string[]) lines.push(`- ${item}`);
|
||||
return `${lines.join("\n").trimEnd()}\n`;
|
||||
}
|
||||
|
||||
export function commandSuiteStart(ctx: CommandContext): number {
|
||||
const { positional: args, options } = parseOptions(ctx.args.slice(2));
|
||||
const item = findSuite(ctx.root, args);
|
||||
const start = buildSuiteStart(ctx.root, item, args, options);
|
||||
const rendered = renderSuiteStart(start);
|
||||
writeSuiteStartArtifacts(ctx.root, start, rendered);
|
||||
const content = options.json === true ? `${JSON.stringify(start, null, 2)}\n` : rendered;
|
||||
writeOrPrint(content, optionString(options, "output"));
|
||||
return 0;
|
||||
}
|
||||
|
||||
function suiteRunCaseArgs(root: string, caseItem: Record<string, unknown>, headed: boolean): string[] {
|
||||
const args = [
|
||||
lbsScriptPath(),
|
||||
"--root",
|
||||
root,
|
||||
"test",
|
||||
"run",
|
||||
String(caseItem.id),
|
||||
"--run-id",
|
||||
String(caseItem.run_id),
|
||||
"--output",
|
||||
String(caseItem.evidence_dir),
|
||||
];
|
||||
if (headed) args.push("--headed");
|
||||
return args;
|
||||
}
|
||||
|
||||
function suiteReportExitCode(status: string): number {
|
||||
if (status === "pass") return 0;
|
||||
if (status === "blocked" || status === "env_issue" || status === "flaky") return 2;
|
||||
return 1;
|
||||
}
|
||||
|
||||
function outputTail(value: string | Buffer | null | undefined): string {
|
||||
return String(value ?? "").trim().slice(-4000);
|
||||
}
|
||||
|
||||
function executionProblemStatus(executions: Array<Record<string, unknown>>): string {
|
||||
const statuses = executions.map((item) => String(item.status));
|
||||
if (statuses.includes("nonzero")) return "fail";
|
||||
if (statuses.includes("skipped")) return "incomplete";
|
||||
return "";
|
||||
}
|
||||
|
||||
function missingReadinessReason(caseItem: Record<string, unknown>): string {
|
||||
const labels: Array<[string, string]> = [
|
||||
["env", "env_readiness"],
|
||||
["automation_env", "automation_readiness"],
|
||||
["fixture", "fixture_readiness"],
|
||||
];
|
||||
const missing = labels.flatMap(([label, key]) => {
|
||||
const value = caseItem[key] as Record<string, unknown> | undefined;
|
||||
if (value?.status !== "missing") return [];
|
||||
const names = Array.isArray(value.missing) ? value.missing.filter((item): item is string => typeof item === "string") : [];
|
||||
return [`${label}=${names.length > 0 ? names.join(",") : "missing"}`];
|
||||
});
|
||||
return missing.length > 0
|
||||
? `case readiness missing (${missing.join(" ")}); rerun with --include-not-ready after fixing or intentionally accepting readiness gaps`
|
||||
: "";
|
||||
}
|
||||
|
||||
export function commandSuiteRun(ctx: CommandContext): number {
|
||||
const { positional: args, options } = parseOptions(ctx.args.slice(2));
|
||||
const item = findSuite(ctx.root, args);
|
||||
const start = buildSuiteStart(ctx.root, item, args, options);
|
||||
const renderedStart = renderSuiteStart(start);
|
||||
const dryRun = options["dry-run"] === true;
|
||||
if (!dryRun) writeSuiteStartArtifacts(ctx.root, start, renderedStart);
|
||||
|
||||
const executions = [];
|
||||
for (const caseItem of start.cases as Array<Record<string, unknown>>) {
|
||||
if (statusOf(caseItem, "manual_readiness") === "manual_check" && options["include-manual-check"] !== true) {
|
||||
executions.push({ id: caseItem.id, status: "skipped", reason: "case requires manual_check; rerun with --include-manual-check after confirming preconditions" });
|
||||
continue;
|
||||
}
|
||||
const missingReadiness = missingReadinessReason(caseItem);
|
||||
if (missingReadiness && options["include-not-ready"] !== true) {
|
||||
executions.push({ id: caseItem.id, status: "skipped", reason: missingReadiness });
|
||||
continue;
|
||||
}
|
||||
if (!caseItem.automation) {
|
||||
executions.push({ id: caseItem.id, status: "skipped", reason: "case has no automation" });
|
||||
continue;
|
||||
}
|
||||
const runArgs = suiteRunCaseArgs(ctx.root, caseItem, options.headed === true);
|
||||
if (dryRun) {
|
||||
executions.push({ id: caseItem.id, status: "planned", reason: "dry-run; case automation not executed", command: [execPath, ...runArgs].join(" ") });
|
||||
continue;
|
||||
}
|
||||
if (options.json !== true) console.log(`Suite case: ${caseItem.id}`);
|
||||
const result = spawnSync(execPath, runArgs, {
|
||||
cwd: ctx.root,
|
||||
encoding: "utf8",
|
||||
stdio: options.json === true ? "pipe" : "inherit",
|
||||
});
|
||||
const status = result.error ? 1 : result.status ?? 1;
|
||||
executions.push({
|
||||
id: caseItem.id,
|
||||
status: status === 0 ? "ok" : "nonzero",
|
||||
exit_status: status,
|
||||
reason: result.error?.message || "",
|
||||
stdout: outputTail(result.stdout),
|
||||
stderr: outputTail(result.stderr),
|
||||
});
|
||||
}
|
||||
|
||||
const report = buildSuiteReport(ctx.root, item, {
|
||||
...options,
|
||||
"run-id": String(start.run_id),
|
||||
"evidence-dir": String(start.evidence_root),
|
||||
}, executions);
|
||||
const payload = {
|
||||
run_id: start.run_id,
|
||||
evidence_root: start.evidence_root,
|
||||
executions,
|
||||
report,
|
||||
};
|
||||
const content = options.json === true
|
||||
? `${JSON.stringify(payload, null, 2)}\n`
|
||||
: renderSuiteReport(report);
|
||||
writeOrPrint(content, optionString(options, "output") ?? (options.json === true || dryRun ? undefined : String(start.suite_report_path || "")));
|
||||
return dryRun ? 0 : suiteReportExitCode(String(report.status));
|
||||
}
|
||||
|
||||
function arrayField(data: Record<string, unknown>, key: string): string[] {
|
||||
const value = data[key];
|
||||
return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") : [];
|
||||
}
|
||||
|
||||
function readCaseResult(evidenceDir: string, caseId: string, expectedRunId: string, requiredEvidence: string[]): Record<string, unknown> {
|
||||
const resultPath = join(evidenceDir, "result.json");
|
||||
if (!existsSync(resultPath)) {
|
||||
return { status: "missing", path: resultPath, reason: "result.json not found" };
|
||||
}
|
||||
try {
|
||||
const parsed = JSON.parse(readFileSync(resultPath, "utf8")) as Record<string, unknown>;
|
||||
if (parsed.case_id !== caseId) {
|
||||
return {
|
||||
status: "invalid",
|
||||
path: resultPath,
|
||||
reason: `result.json case_id mismatch: expected ${caseId}, got ${String(parsed.case_id ?? "missing")}`,
|
||||
};
|
||||
}
|
||||
if (expectedRunId && parsed.run_id !== expectedRunId) {
|
||||
return {
|
||||
status: "invalid",
|
||||
path: resultPath,
|
||||
reason: `result.json run_id mismatch: expected ${expectedRunId}, got ${String(parsed.run_id ?? "missing")}`,
|
||||
};
|
||||
}
|
||||
const collected = arrayField(parsed, "evidence_collected");
|
||||
const missing = requiredEvidence.filter((item) => !collected.includes(item));
|
||||
return {
|
||||
status: typeof parsed.status === "string" ? parsed.status : "invalid",
|
||||
path: resultPath,
|
||||
reason: typeof parsed.reason === "string" ? parsed.reason : "",
|
||||
started_at_local: typeof parsed.started_at_local === "string" ? parsed.started_at_local : "",
|
||||
finished_at_local: typeof parsed.finished_at_local === "string" ? parsed.finished_at_local : "",
|
||||
url: typeof parsed.url === "string" ? parsed.url : "",
|
||||
evidence_collected: collected,
|
||||
evidence_required: requiredEvidence,
|
||||
evidence_missing: missing,
|
||||
evidence_status: missing.length === 0 ? "complete" : "incomplete",
|
||||
};
|
||||
} catch (error) {
|
||||
return { status: "invalid", path: resultPath, reason: String(error) };
|
||||
}
|
||||
}
|
||||
|
||||
function suiteStatus(caseResults: Array<Record<string, unknown>>): string {
|
||||
const statuses = caseResults.map((item) => String(item.status));
|
||||
if (statuses.length === 0) return "not_run";
|
||||
if (statuses.includes("fail") || statuses.includes("invalid")) return "fail";
|
||||
if (statuses.includes("missing")) return "incomplete";
|
||||
if (caseResults.some((item) => item.status === "pass" && item.evidence_status !== "complete")) return "incomplete";
|
||||
if (statuses.every((status) => status === "pass")) return "pass";
|
||||
if (statuses.includes("blocked")) return "blocked";
|
||||
if (statuses.includes("env_issue")) return "env_issue";
|
||||
if (statuses.includes("flaky")) return "flaky";
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
function buildSuiteReport(
|
||||
root: string,
|
||||
item: StructuredItem,
|
||||
options: Record<string, string | boolean>,
|
||||
executions: Array<Record<string, unknown>> = [],
|
||||
): Record<string, unknown> {
|
||||
const suite = suiteSummary(item);
|
||||
const runId = optionString(options, "run-id") ?? "";
|
||||
const evidenceRoot = optionString(options, "evidence-dir") ?? (runId ? join("reports", "evidence", runId) : "");
|
||||
const cases = suiteCases(root, item).map((caseItem) => {
|
||||
const caseId = String(caseItem.id);
|
||||
const expectedCaseRunId = runId ? `${runId}-${caseId}` : "";
|
||||
const evidenceDir = evidenceRoot ? join(evidenceRoot, caseId) : "";
|
||||
const requiredEvidence = Array.isArray(caseItem.evidence_required) ? caseItem.evidence_required : [];
|
||||
const result = evidenceDir
|
||||
? readCaseResult(evidenceDir, caseId, expectedCaseRunId, requiredEvidence)
|
||||
: { status: "missing", path: "", reason: "Set --evidence-dir or --run-id to locate case result.json files" };
|
||||
return {
|
||||
...caseItem,
|
||||
evidence_dir: evidenceDir,
|
||||
result,
|
||||
};
|
||||
});
|
||||
const counts: Record<string, number> = {};
|
||||
for (const item of cases) {
|
||||
const status = String((item.result as Record<string, unknown>).status);
|
||||
counts[status] = (counts[status] ?? 0) + 1;
|
||||
}
|
||||
|
||||
const resultStatus = suiteStatus(cases.map((item) => item.result as Record<string, unknown>));
|
||||
const executionStatus = executionProblemStatus(executions);
|
||||
return {
|
||||
generated_at: new Date().toISOString(),
|
||||
run_id: runId,
|
||||
suite,
|
||||
evidence_root: evidenceRoot,
|
||||
status: executionStatus || resultStatus,
|
||||
counts,
|
||||
cases,
|
||||
execution_status: executionStatus || "ok",
|
||||
decision_policy: [
|
||||
"pass requires every case result to be pass.",
|
||||
"suite run pass also requires every attempted execution to finish ok.",
|
||||
"blocked and env_issue are not product pass.",
|
||||
"pass results missing required evidence keep the suite incomplete.",
|
||||
"result.json must match the expected case_id and suite case run_id.",
|
||||
"missing or invalid result.json means the suite is incomplete or failed to collect evidence.",
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function renderSuiteReport(report: Record<string, unknown>): string {
|
||||
const suite = report.suite as Record<string, unknown>;
|
||||
const cases = report.cases as Array<Record<string, unknown>>;
|
||||
const counts = report.counts as Record<string, number>;
|
||||
const lines: string[] = [];
|
||||
lines.push(`# Suite Report: ${suite.id}`);
|
||||
lines.push("");
|
||||
lines.push(`Generated: ${report.generated_at}`);
|
||||
if (report.run_id) lines.push(`Run: ${report.run_id}`);
|
||||
lines.push(`Title: ${suite.title}`);
|
||||
lines.push(`Status: ${report.status}`);
|
||||
lines.push(`Evidence root: ${report.evidence_root || "not provided"}`);
|
||||
lines.push("");
|
||||
lines.push("## Counts");
|
||||
for (const key of Object.keys(counts).sort()) lines.push(`- ${key}: ${counts[key]}`);
|
||||
if (Object.keys(counts).length === 0) lines.push("- None.");
|
||||
lines.push("");
|
||||
lines.push("## Cases");
|
||||
for (const caseItem of cases) {
|
||||
const result = caseItem.result as Record<string, unknown>;
|
||||
lines.push(`- ${caseItem.id}: ${result.status} - ${result.reason || "no reason"}`);
|
||||
if (Array.isArray(result.evidence_missing) && result.evidence_missing.length > 0) {
|
||||
lines.push(` evidence_missing: ${result.evidence_missing.join(", ")}`);
|
||||
}
|
||||
if (caseItem.evidence_dir) lines.push(` evidence_dir: ${caseItem.evidence_dir}`);
|
||||
if (result.path) lines.push(` result_json: ${result.path}`);
|
||||
}
|
||||
lines.push("");
|
||||
lines.push("## Decision Policy");
|
||||
for (const item of report.decision_policy as string[]) lines.push(`- ${item}`);
|
||||
return `${lines.join("\n").trimEnd()}\n`;
|
||||
}
|
||||
|
||||
export function commandSuiteReport(ctx: CommandContext): number {
|
||||
const { positional: args, options } = parseOptions(ctx.args.slice(2));
|
||||
const item = findSuite(ctx.root, args);
|
||||
const report = buildSuiteReport(ctx.root, item, options);
|
||||
const content = options.json === true ? `${JSON.stringify(report, null, 2)}\n` : renderSuiteReport(report);
|
||||
writeOrPrint(content, optionString(options, "output"));
|
||||
return 0;
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,95 @@
|
||||
import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import type { CommandContext } from "../types.ts";
|
||||
import { fail, optionString, parseOptions, usage } from "../cli.ts";
|
||||
import { findStructuredItem, getSkill, loadStructuredItems, scalar, slugify, todayIso, yamlList, yamlQuote } from "../fs.ts";
|
||||
|
||||
function troubleshootingYamlPath(root: string, skillName: string, id: string): string {
|
||||
const skill = getSkill(root, skillName);
|
||||
const dir = join(skill.path, "troubleshooting");
|
||||
mkdirSync(dir, { recursive: true });
|
||||
return join(dir, `${id}.yaml`);
|
||||
}
|
||||
|
||||
function legacyTroubleshootingPath(root: string, skillName: string): string {
|
||||
const skill = getSkill(root, skillName);
|
||||
const refsDir = join(skill.path, "references");
|
||||
mkdirSync(refsDir, { recursive: true });
|
||||
const path = join(refsDir, "troubleshooting.md");
|
||||
if (!existsSync(path)) writeFileSync(path, "# Troubleshooting\n\n", "utf8");
|
||||
return path;
|
||||
}
|
||||
|
||||
export function commandTroubleList(ctx: CommandContext): number {
|
||||
const skill = ctx.args[2];
|
||||
const yamlItems = loadStructuredItems(ctx.root, "troubleshooting", skill);
|
||||
for (const item of yamlItems) {
|
||||
console.log(`${item.skill}\t${scalar(item.fields, "id")}\t${scalar(item.fields, "title")}`);
|
||||
}
|
||||
|
||||
if (skill && yamlItems.length === 0) {
|
||||
const legacyPath = legacyTroubleshootingPath(ctx.root, skill);
|
||||
const text = readFileSync(legacyPath, "utf8");
|
||||
const headings = Array.from(text.matchAll(/^##\s+(.+)$/gm)).map((match) => match[1]);
|
||||
for (const heading of headings) console.log(`${skill}\tlegacy\t${heading}`);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
export function commandTroubleShow(ctx: CommandContext): number {
|
||||
const positional = ctx.args.slice(2);
|
||||
if (positional.length < 1 || positional.length > 2) usage();
|
||||
const item = positional.length === 1
|
||||
? findStructuredItem(ctx.root, "troubleshooting", positional[0])
|
||||
: findStructuredItem(ctx.root, "troubleshooting", positional[0], positional[1]);
|
||||
console.log(item.raw.trimEnd());
|
||||
return 0;
|
||||
}
|
||||
|
||||
export function commandTroubleSearch(ctx: CommandContext): number {
|
||||
const query = ctx.args[2]?.toLowerCase();
|
||||
if (!query) usage();
|
||||
const items = loadStructuredItems(ctx.root, "troubleshooting").filter((item) => item.raw.toLowerCase().includes(query));
|
||||
for (const item of items) {
|
||||
console.log(`${item.skill}\t${scalar(item.fields, "id")}\t${scalar(item.fields, "title")}`);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
export function commandTroubleAdd(ctx: CommandContext): number {
|
||||
const skill = ctx.args[2];
|
||||
if (!skill) usage();
|
||||
const { options } = parseOptions(ctx.args.slice(3));
|
||||
for (const key of ["title", "symptom", "cause", "fix"]) {
|
||||
if (!optionString(options, key)) fail(`--${key} is required`);
|
||||
}
|
||||
|
||||
const title = optionString(options, "title") ?? "";
|
||||
const symptom = optionString(options, "symptom") ?? "";
|
||||
const id = optionString(options, "id") ?? slugify(title);
|
||||
const path = troubleshootingYamlPath(ctx.root, skill, id);
|
||||
if (existsSync(path)) fail(`troubleshooting entry already exists: ${path}`);
|
||||
|
||||
const text =
|
||||
`id: ${id}\n` +
|
||||
`title: ${yamlQuote(title)}\n` +
|
||||
`date: ${todayIso()}\n` +
|
||||
"symptoms:\n" +
|
||||
yamlList([symptom]) +
|
||||
"\npatterns:\n" +
|
||||
yamlList([symptom]) +
|
||||
"\nlikely_causes:\n" +
|
||||
yamlList([optionString(options, "cause") ?? ""]) +
|
||||
"\nfix_steps:\n" +
|
||||
yamlList([optionString(options, "fix") ?? ""]) +
|
||||
"\nverification: " +
|
||||
yamlQuote(optionString(options, "verify") ?? "Add the command, UI signal, or log line that proves the fix worked.") +
|
||||
"\nrelated_cases:\n" +
|
||||
yamlList([]) +
|
||||
"\n";
|
||||
|
||||
writeFileSync(path, text, "utf8");
|
||||
appendFileSync(legacyTroubleshootingPath(ctx.root, skill), `\n## ${id}: ${title}\n\nSee \`../troubleshooting/${id}.yaml\`.\n`, "utf8");
|
||||
console.log(path);
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,394 @@
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { stderr } from "node:process";
|
||||
import type { Skill, StructuredItem } from "../types.ts";
|
||||
import { loadFixtureItems } from "../fixtures.ts";
|
||||
import {
|
||||
caseEvidenceValues,
|
||||
caseModeValues,
|
||||
casePriorityValues,
|
||||
caseRequiredLists,
|
||||
caseRequiredStrings,
|
||||
caseRiskValues,
|
||||
caseTypeValues,
|
||||
requiredEnvKeys,
|
||||
suiteRequiredLists,
|
||||
suiteRequiredStrings,
|
||||
suiteTypeValues,
|
||||
troubleRequiredLists,
|
||||
troubleRequiredStrings,
|
||||
troubleshootingCategoryValues,
|
||||
} from "../constants.ts";
|
||||
import { boolValue, envExamplePath, envPath, listValue, loadSkills, loadStructuredItems, parseEnvFile, scalar } from "../fs.ts";
|
||||
import { envKeyPattern, isEnvAnyGroup, splitEnvAnyGroup } from "../env-groups.ts";
|
||||
import { parseSetupAutomationEntry, validateSetupAutomationEntry } from "../setup-automation.ts";
|
||||
|
||||
const refRe = /(?:\]\(|`)(references\/[A-Za-z0-9_.\-/]+\.md)(?:\)|`)/g;
|
||||
|
||||
function validateStructuredItem(item: StructuredItem, requiredStrings: string[], requiredLists: string[]): string[] {
|
||||
const errors: string[] = [];
|
||||
const listKeys = item.path.includes("/cases/") && scalar(item.fields, "mode") === "probe"
|
||||
? requiredLists.filter((key) => key !== "env")
|
||||
: requiredLists;
|
||||
for (const key of requiredStrings) {
|
||||
if (!scalar(item.fields, key)) errors.push(`${item.path}: missing '${key}'`);
|
||||
}
|
||||
for (const key of listKeys) {
|
||||
if (listValue(item.fields, key).length === 0) errors.push(`${item.path}: missing '${key}' entries`);
|
||||
}
|
||||
const id = scalar(item.fields, "id");
|
||||
if (id && !/^[a-z0-9][a-z0-9_-]*$/.test(id)) {
|
||||
errors.push(`${item.path}: id must use lowercase letters, digits, dashes, or underscores`);
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
function validateEnum(item: StructuredItem, key: string, values: string[]): string[] {
|
||||
const value = scalar(item.fields, key);
|
||||
if (!value) return [];
|
||||
return values.includes(value) ? [] : [`${item.path}: '${key}' must be one of ${values.join(", ")}`];
|
||||
}
|
||||
|
||||
function validateListEnum(item: StructuredItem, key: string, values: string[]): string[] {
|
||||
const allowed = new Set(values);
|
||||
return listValue(item.fields, key)
|
||||
.filter((value) => !allowed.has(value))
|
||||
.map((value) => `${item.path}: '${key}' contains unsupported value '${value}'`);
|
||||
}
|
||||
|
||||
function validateDuplicateListValues(item: StructuredItem, keys: string[]): string[] {
|
||||
const errors: string[] = [];
|
||||
for (const key of keys) {
|
||||
const seen = new Set<string>();
|
||||
for (const value of listValue(item.fields, key)) {
|
||||
if (seen.has(value)) errors.push(`${item.path}: '${key}' contains duplicate value '${value}'`);
|
||||
seen.add(value);
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
function validateEnvKeyList(item: StructuredItem, key: string): string[] {
|
||||
return listValue(item.fields, key)
|
||||
.filter((value) => !envKeyPattern.test(value))
|
||||
.map((value) => `${item.path}: '${key}' contains invalid env key '${value}'`);
|
||||
}
|
||||
|
||||
function validateEnvKeyScalar(item: StructuredItem, key: string): string[] {
|
||||
const value = scalar(item.fields, key);
|
||||
if (!value) return [];
|
||||
return envKeyPattern.test(value)
|
||||
? []
|
||||
: [`${item.path}: '${key}' contains invalid env key '${value}'`];
|
||||
}
|
||||
|
||||
function validateJsonScalar(item: StructuredItem, key: string): string[] {
|
||||
const value = scalar(item.fields, key);
|
||||
if (!value) return [];
|
||||
try {
|
||||
JSON.parse(value);
|
||||
return [];
|
||||
} catch (error) {
|
||||
return [`${item.path}: '${key}' must be valid JSON: ${(error as Error).message}`];
|
||||
}
|
||||
}
|
||||
|
||||
function validateEnvAnyList(item: StructuredItem, key: string): string[] {
|
||||
return listValue(item.fields, key)
|
||||
.filter((value) => !isEnvAnyGroup(value))
|
||||
.map((value) => `${item.path}: '${key}' contains invalid env any-group '${value}'`);
|
||||
}
|
||||
|
||||
function validateCaseItem(root: string, item: StructuredItem, skillNames: Set<string>, troubleIds: Set<string>, caseIds: Set<string>): string[] {
|
||||
const errors = [
|
||||
...validateEnum(item, "mode", caseModeValues),
|
||||
...validateEnum(item, "type", caseTypeValues),
|
||||
...validateEnum(item, "priority", casePriorityValues),
|
||||
...validateEnum(item, "risk", caseRiskValues),
|
||||
...validateListEnum(item, "evidence_required", caseEvidenceValues),
|
||||
...validateDuplicateListValues(item, [
|
||||
"tags",
|
||||
"skills",
|
||||
"env",
|
||||
"env_any",
|
||||
"automation_env",
|
||||
"automation_env_any",
|
||||
"setup_automation",
|
||||
"setup_provides_env",
|
||||
"evidence_required",
|
||||
"troubleshooting",
|
||||
]),
|
||||
...validateEnvKeyList(item, "env"),
|
||||
...validateEnvAnyList(item, "env_any"),
|
||||
...validateEnvKeyList(item, "automation_env"),
|
||||
...validateEnvAnyList(item, "automation_env_any"),
|
||||
...validateEnvKeyList(item, "setup_provides_env"),
|
||||
...validateEnvKeyScalar(item, "automation_pipeline_url_env"),
|
||||
...validateEnvKeyScalar(item, "automation_pipeline_name_env"),
|
||||
...validateJsonScalar(item, "automation_filesystem_checks_json"),
|
||||
...listValue(item.fields, "setup_automation").flatMap((entry) => (
|
||||
validateSetupAutomationEntry(root, entry, caseIds).map((error) => `${item.path}: ${error}`)
|
||||
)),
|
||||
];
|
||||
|
||||
if (boolValue(item.fields, "ci_eligible") === undefined) {
|
||||
errors.push(`${item.path}: missing or invalid boolean 'ci_eligible'`);
|
||||
}
|
||||
|
||||
for (const skill of listValue(item.fields, "skills")) {
|
||||
if (!skillNames.has(skill)) errors.push(`${item.path}: references unknown skill '${skill}'`);
|
||||
}
|
||||
|
||||
for (const id of listValue(item.fields, "troubleshooting")) {
|
||||
if (!troubleIds.has(id)) errors.push(`${item.path}: references unknown troubleshooting '${id}'`);
|
||||
}
|
||||
|
||||
const automation = scalar(item.fields, "automation");
|
||||
if (!automation && listValue(item.fields, "automation_env").length > 0) {
|
||||
errors.push(`${item.path}: 'automation_env' requires 'automation'`);
|
||||
}
|
||||
if (!automation && listValue(item.fields, "automation_env_any").length > 0) {
|
||||
errors.push(`${item.path}: 'automation_env_any' requires 'automation'`);
|
||||
}
|
||||
if (!automation && (scalar(item.fields, "automation_pipeline_url_env") || scalar(item.fields, "automation_pipeline_name_env"))) {
|
||||
errors.push(`${item.path}: automation pipeline env aliases require 'automation'`);
|
||||
}
|
||||
if (listValue(item.fields, "setup_provides_env").length > 0 && listValue(item.fields, "setup_automation").length === 0) {
|
||||
errors.push(`${item.path}: 'setup_provides_env' requires 'setup_automation'`);
|
||||
}
|
||||
for (const key of ["automation_pipeline_url_env", "automation_pipeline_name_env"]) {
|
||||
const value = scalar(item.fields, key);
|
||||
if (!value) continue;
|
||||
const declared = new Set([
|
||||
...listValue(item.fields, "env"),
|
||||
...listValue(item.fields, "env_any").flatMap(splitEnvAnyGroup),
|
||||
...listValue(item.fields, "automation_env"),
|
||||
...listValue(item.fields, "automation_env_any").flatMap(splitEnvAnyGroup),
|
||||
]);
|
||||
if (!declared.has(value)) {
|
||||
errors.push(`${item.path}: '${key}' value '${value}' must be listed in env, env_any, automation_env, or automation_env_any`);
|
||||
}
|
||||
}
|
||||
if (automation && !existsSync(join(root, automation))) {
|
||||
errors.push(`${item.path}: automation script does not exist: ${automation}`);
|
||||
}
|
||||
for (const entry of listValue(item.fields, "setup_automation")) {
|
||||
const spec = parseSetupAutomationEntry(entry);
|
||||
if (spec.kind === "case" && spec.target === scalar(item.fields, "id")) {
|
||||
errors.push(`${item.path}: setup_automation cannot reference the same case '${spec.target}'`);
|
||||
}
|
||||
}
|
||||
|
||||
const timeout = scalar(item.fields, "automation_response_timeout_ms");
|
||||
if (timeout && (!/^\d+$/.test(timeout) || Number.parseInt(timeout, 10) <= 0)) {
|
||||
errors.push(`${item.path}: 'automation_response_timeout_ms' must be a positive integer string`);
|
||||
}
|
||||
const streamOutput = scalar(item.fields, "automation_stream_output");
|
||||
if (streamOutput && !["0", "1", "false", "true"].includes(streamOutput)) {
|
||||
errors.push(`${item.path}: 'automation_stream_output' must be one of 0, 1, false, or true`);
|
||||
}
|
||||
const imageBase64Fixture = scalar(item.fields, "automation_image_base64_fixture");
|
||||
if (imageBase64Fixture && !existsSync(join(root, imageBase64Fixture))) {
|
||||
errors.push(`${item.path}: automation image fixture does not exist: ${imageBase64Fixture}`);
|
||||
}
|
||||
|
||||
return errors;
|
||||
}
|
||||
|
||||
function validateSetupAutomationCycles(caseItems: StructuredItem[]): string[] {
|
||||
const byId = new Map(caseItems.map((item) => [scalar(item.fields, "id"), item]));
|
||||
const visiting = new Set<string>();
|
||||
const visited = new Set<string>();
|
||||
const errors: string[] = [];
|
||||
|
||||
function visit(id: string, path: string[]): void {
|
||||
if (visited.has(id)) return;
|
||||
if (visiting.has(id)) {
|
||||
const cycle = [...path.slice(path.indexOf(id)), id].join(" -> ");
|
||||
const item = byId.get(id);
|
||||
errors.push(`${item?.path ?? id}: setup_automation case cycle detected: ${cycle}`);
|
||||
return;
|
||||
}
|
||||
const item = byId.get(id);
|
||||
if (!item) return;
|
||||
visiting.add(id);
|
||||
for (const entry of listValue(item.fields, "setup_automation")) {
|
||||
const spec = parseSetupAutomationEntry(entry);
|
||||
if (spec.kind === "case") visit(spec.target, [...path, spec.target]);
|
||||
}
|
||||
visiting.delete(id);
|
||||
visited.add(id);
|
||||
}
|
||||
|
||||
for (const id of byId.keys()) visit(id, [id]);
|
||||
return errors;
|
||||
}
|
||||
|
||||
function validateTroubleshootingItem(item: StructuredItem, caseIds: Set<string>): string[] {
|
||||
const errors = [
|
||||
...validateEnum(item, "category", troubleshootingCategoryValues),
|
||||
...validateDuplicateListValues(item, ["symptoms", "patterns", "likely_causes", "fix_steps", "related_cases"]),
|
||||
];
|
||||
for (const id of listValue(item.fields, "related_cases")) {
|
||||
if (!caseIds.has(id)) errors.push(`${item.path}: references unknown case '${id}'`);
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
function validateFixtures(root: string, caseIds: Set<string>): string[] {
|
||||
const { items, errors } = loadFixtureItems(root);
|
||||
const result = [...errors];
|
||||
const seen = new Map<string, string>();
|
||||
for (const item of items) {
|
||||
if (!/^[a-z0-9][a-z0-9_-]*$/.test(item.id)) {
|
||||
result.push(`${item.manifest_path}: fixture id '${item.id}' must use lowercase letters, digits, dashes, or underscores`);
|
||||
}
|
||||
if (seen.has(item.id)) {
|
||||
result.push(`${item.manifest_path}: duplicate fixture id '${item.id}' also used by ${seen.get(item.id)}`);
|
||||
} else {
|
||||
seen.set(item.id, item.manifest_path);
|
||||
}
|
||||
if (!item.exists) result.push(`${item.manifest_path}: fixture path does not exist: ${item.path}`);
|
||||
for (const caseId of item.related_cases) {
|
||||
if (!caseIds.has(caseId)) result.push(`${item.manifest_path}: fixture '${item.id}' references unknown case '${caseId}'`);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function validateSuiteItem(item: StructuredItem, caseIds: Set<string>): string[] {
|
||||
const errors = [
|
||||
...validateEnum(item, "type", suiteTypeValues),
|
||||
...validateEnum(item, "priority", casePriorityValues),
|
||||
...validateDuplicateListValues(item, ["tags", "cases"]),
|
||||
];
|
||||
for (const id of listValue(item.fields, "cases")) {
|
||||
if (!caseIds.has(id)) errors.push(`${item.path}: references unknown case '${id}'`);
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
function validateDuplicateIds(items: StructuredItem[], label: string): string[] {
|
||||
const errors: string[] = [];
|
||||
const seen = new Map<string, string>();
|
||||
for (const item of items) {
|
||||
const id = scalar(item.fields, "id");
|
||||
if (!id) continue;
|
||||
const key = `${item.skill}:${id}`;
|
||||
if (seen.has(key)) errors.push(`${item.path}: duplicate ${label} id '${id}' also used by ${seen.get(key)}`);
|
||||
else seen.set(key, item.path);
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
function validateGlobalDuplicateIds(items: StructuredItem[], label: string): string[] {
|
||||
const errors: string[] = [];
|
||||
const seen = new Map<string, string>();
|
||||
for (const item of items) {
|
||||
const id = scalar(item.fields, "id");
|
||||
if (!id) continue;
|
||||
if (seen.has(id)) errors.push(`${item.path}: duplicate global ${label} id '${id}' also used by ${seen.get(id)}`);
|
||||
else seen.set(id, item.path);
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
function validateEnv(root: string): string[] {
|
||||
const path = envPath(root);
|
||||
const examplePath = envExamplePath(root);
|
||||
const errors: string[] = [];
|
||||
if (!existsSync(path)) return [`${path}: missing shared env file`];
|
||||
const env = parseEnvFile(path);
|
||||
for (const key of requiredEnvKeys) {
|
||||
if (!(key in env)) errors.push(`${path}: missing ${key}`);
|
||||
}
|
||||
if (!existsSync(examplePath)) {
|
||||
errors.push(`${examplePath}: missing env template`);
|
||||
} else {
|
||||
const example = parseEnvFile(examplePath);
|
||||
for (const key of requiredEnvKeys) {
|
||||
if (!(key in example)) errors.push(`${examplePath}: missing template key ${key}`);
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
function validateSchemas(root: string): string[] {
|
||||
const errors: string[] = [];
|
||||
for (const name of ["case.schema.json", "suite.schema.json", "troubleshooting.schema.json", "skill-index.schema.json"]) {
|
||||
const path = join(root, "schemas", name);
|
||||
if (!existsSync(path)) {
|
||||
errors.push(`${path}: missing schema`);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
JSON.parse(readFileSync(path, "utf8"));
|
||||
} catch (error) {
|
||||
errors.push(`${path}: invalid JSON schema (${String(error)})`);
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
function validateSkill(skill: Skill): string[] {
|
||||
const errors: string[] = [];
|
||||
if (!skill.name) errors.push(`${skill.path}: missing frontmatter name`);
|
||||
if (!skill.description) errors.push(`${skill.path}: missing frontmatter description`);
|
||||
if (skill.name && skill.name !== skill.directory) {
|
||||
errors.push(`${skill.path}: name '${skill.name}' does not match directory '${skill.directory}'`);
|
||||
}
|
||||
|
||||
const refs = new Set<string>();
|
||||
for (const match of skill.body.matchAll(refRe)) refs.add(match[1]);
|
||||
for (const ref of Array.from(refs).sort()) {
|
||||
if (!existsSync(join(skill.path, ref))) {
|
||||
errors.push(`${skill.path}: referenced file does not exist: ${ref}`);
|
||||
}
|
||||
}
|
||||
|
||||
const legacyTroubleshooting = join(skill.path, "references", "troubleshooting.md");
|
||||
if (existsSync(legacyTroubleshooting)) {
|
||||
const text = readFileSync(legacyTroubleshooting, "utf8");
|
||||
if (text.includes("\n## ") && !text.includes("### Symptom")) {
|
||||
errors.push(`${legacyTroubleshooting}: troubleshooting entries should include '### Symptom'`);
|
||||
}
|
||||
}
|
||||
|
||||
return errors;
|
||||
}
|
||||
|
||||
export function commandValidate(root: string): number {
|
||||
const skills = loadSkills(root);
|
||||
const caseItems = loadStructuredItems(root, "cases");
|
||||
const suiteItems = loadStructuredItems(root, "suites");
|
||||
const troubleItems = loadStructuredItems(root, "troubleshooting");
|
||||
const skillNames = new Set(skills.map((skill) => skill.name));
|
||||
const caseIds = new Set(caseItems.map((item) => scalar(item.fields, "id")).filter(Boolean));
|
||||
const troubleIds = new Set(troubleItems.map((item) => scalar(item.fields, "id")).filter(Boolean));
|
||||
const errors = [
|
||||
...validateEnv(root),
|
||||
...validateSchemas(root),
|
||||
...skills.flatMap(validateSkill),
|
||||
...caseItems.flatMap((item) => validateStructuredItem(item, caseRequiredStrings, caseRequiredLists)),
|
||||
...caseItems.flatMap((item) => validateCaseItem(root, item, skillNames, troubleIds, caseIds)),
|
||||
...validateSetupAutomationCycles(caseItems),
|
||||
...suiteItems.flatMap((item) => validateStructuredItem(item, suiteRequiredStrings, suiteRequiredLists)),
|
||||
...suiteItems.flatMap((item) => validateSuiteItem(item, caseIds)),
|
||||
...troubleItems.flatMap((item) => validateStructuredItem(item, troubleRequiredStrings, troubleRequiredLists)),
|
||||
...troubleItems.flatMap((item) => validateTroubleshootingItem(item, caseIds)),
|
||||
...validateFixtures(root, caseIds),
|
||||
...validateDuplicateIds(caseItems, "case"),
|
||||
...validateDuplicateIds(suiteItems, "suite"),
|
||||
...validateDuplicateIds(troubleItems, "troubleshooting"),
|
||||
...validateGlobalDuplicateIds(caseItems, "case"),
|
||||
...validateGlobalDuplicateIds(suiteItems, "suite"),
|
||||
...validateGlobalDuplicateIds(troubleItems, "troubleshooting"),
|
||||
];
|
||||
|
||||
if (errors.length > 0) {
|
||||
for (const error of errors) stderr.write(`ERROR: ${error}\n`);
|
||||
return 1;
|
||||
}
|
||||
console.log("OK");
|
||||
return 0;
|
||||
}
|
||||
Reference in New Issue
Block a user