mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-10-28 19:35:27 +08:00
fix(docs): generate complete agent-readable Markdown exports (#121)
Co-authored-by: dadachann <[email protected]>
This commit is contained in:
Hyu
@@ -32,6 +32,24 @@ npm run dev
|
||||
|
||||
文档构建为 `dist/public` 下的静态站点，并托管在 Cloudflare Pages。部署前会运行类型检查、文档契约测试及完整静态构建。
|
||||
|
||||
### Agent 可读导出
|
||||
|
||||
`npm run build` 会在 `/docs` 基路径和站点地图处理完成后生成 Agent 文本。
|
||||
每个已发布页面都有 `.md` 副本，HTML 使用 `rel="alternate" type="text/markdown"`
|
||||
声明该地址，不改变可视界面。
|
||||
|
||||
- `/docs/llms.txt`、`/docs/llms-full.txt`：全部语言的索引与完整正文。
|
||||
- `/docs/{en,zh,ja}/llms.txt`、`llms-full.txt`：对应语言的索引与正文。
|
||||
- 各目录的 `llm.txt` 是 `llms.txt` 的兼容别名。
|
||||
- `/docs/openapi/service-api-{en,zh,ja}.json`：完整 API 规范。
|
||||
|
||||
生成器直接解析源 MDX 的 Markdown AST，保留提示标题和图片地址；每个 API
|
||||
操作包含请求、响应、鉴权及递归引用的 schema。遇到未支持的语义 JSX 或可执行
|
||||
表达式会中止构建，避免静默丢失正文。请修改 MDX/OpenAPI 源文件，不要编辑
|
||||
`dist/public` 中的生成结果。`npm test` 检查转换逻辑，构建后的
|
||||
`npm run test:static` 检查站点地图覆盖、HTML 发现链接、内部链接与 API 契约。
|
||||
已有完成基路径处理的构建可使用 `npm run generate:agent-docs` 单独更新文本。
|
||||
|
||||
### 一些规范化标准
|
||||
|
||||
- 文件夹和文件的命名：**一律使用全小写，单词直接`-`隔开，如**`plugin-intro.mdx`
|
||||
|
||||
@@ -32,6 +32,27 @@ Place images in the `images` directory, then reference them using the absolute p
|
||||
|
||||
The documentation is built as a static site in `dist/public` and hosted on Cloudflare Pages. Type checking, documentation contract tests, and a full static build run before deployment.
|
||||
|
||||
### Agent-readable exports
|
||||
|
||||
`npm run build` generates agent text after finalizing the `/docs` base path and
|
||||
sitemaps. Every published page has a canonical `.md` companion, advertised by an
|
||||
HTML `rel="alternate" type="text/markdown"` link without changing the visible UI.
|
||||
|
||||
- `/docs/llms.txt` and `/docs/llms-full.txt`: all-language index and complete text.
|
||||
- `/docs/{en,zh,ja}/llms.txt` and `llms-full.txt`: language-specific exports.
|
||||
- `llm.txt` is an identical compatibility alias for each `llms.txt` index.
|
||||
- `/docs/openapi/service-api-{en,zh,ja}.json`: complete API specifications.
|
||||
|
||||
The generator reads canonical MDX with a Markdown AST (not compiled image
|
||||
variables), preserves callout titles and literal image URLs, and exports every
|
||||
API operation with its request/response contract, security and transitive schema
|
||||
references. Unknown semantic JSX or executable expressions fail the build rather
|
||||
than silently losing content. Generated files stay in `dist/public`; edit source
|
||||
MDX/OpenAPI, not generated Markdown. `npm test` covers conversion, and
|
||||
`npm run test:static` verifies sitemap coverage, discovery, links and API contracts
|
||||
after a build. Run `npm run generate:agent-docs` to regenerate text in an existing
|
||||
finalized build.
|
||||
|
||||
### Some Standardization Guidelines
|
||||
|
||||
- Folder and file naming: **Use all lowercase, separate words with `-`, such as** `plugin-intro.mdx`
|
||||
|
||||
Generated
+7
-1
@@ -26,9 +26,15 @@
|
||||
"@types/node": "^24.10.0",
|
||||
"@types/react": "^19.2.2",
|
||||
"@types/react-dom": "^19.2.2",
|
||||
"remark-gfm": "^4.0.1",
|
||||
"remark-mdx": "^3.1.1",
|
||||
"remark-parse": "^11.0.0",
|
||||
"remark-stringify": "^11.0.0",
|
||||
"tailwindcss": "^4.3.3",
|
||||
"typescript": "^5.9.3",
|
||||
"vite": "^8.2.2"
|
||||
"unified": "^11.0.5",
|
||||
"vite": "^8.2.2",
|
||||
"yaml": "^2.9.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=24"
|
||||
|
||||
+11
-4
@@ -11,14 +11,15 @@
|
||||
"prepare:fumapress": "node scripts/prepare-fumapress.mjs",
|
||||
"build": "fumapress build",
|
||||
"prebuild": "npm run prepare:fumapress",
|
||||
"postbuild": "npm run finalize:base-path && npm run generate:sitemap-alternates && npm run normalize:static-paths && npm run test:static",
|
||||
"postbuild": "npm run finalize:base-path && npm run generate:sitemap-alternates && npm run normalize:static-paths && npm run generate:agent-docs && npm run test:static",
|
||||
"generate:agent-docs": "node scripts/generate-agent-docs.mjs",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"normalize:static-paths": "node scripts/normalize-static-paths.mjs",
|
||||
"test:static": "node --test tests/fumapress-static-build.test.mjs",
|
||||
"test:static": "node --test tests/fumapress-static-build.test.mjs tests/agent-static-build.test.mjs",
|
||||
"generate:sitemap-alternates": "node scripts/generate-alternate-sitemap.mjs",
|
||||
"sync:articles": "node scripts/sync-blog-articles.mjs",
|
||||
"check:articles": "node scripts/sync-blog-articles.mjs --check",
|
||||
"test": "node --test tests/blog-articles.test.mjs tests/fumapress-migration.test.mjs tests/seo.test.mjs && python3 -m unittest discover -s tests -p 'test_*.py'",
|
||||
"test": "node --test tests/agent-docs.test.mjs tests/blog-articles.test.mjs tests/fumapress-migration.test.mjs tests/seo.test.mjs && python3 -m unittest discover -s tests -p 'test_*.py'",
|
||||
"finalize:base-path": "node scripts/finalize-base-path.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
@@ -42,8 +43,14 @@
|
||||
"@types/node": "^24.10.0",
|
||||
"@types/react": "^19.2.2",
|
||||
"@types/react-dom": "^19.2.2",
|
||||
"remark-gfm": "^4.0.1",
|
||||
"remark-mdx": "^3.1.1",
|
||||
"remark-parse": "^11.0.0",
|
||||
"remark-stringify": "^11.0.0",
|
||||
"tailwindcss": "^4.3.3",
|
||||
"typescript": "^5.9.3",
|
||||
"vite": "^8.2.2"
|
||||
"unified": "^11.0.5",
|
||||
"vite": "^8.2.2",
|
||||
"yaml": "^2.9.0"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -379,6 +379,7 @@ export default defineConfig({
|
||||
const defaultLocale = alternates.includes("en") ? "en" : alternates[0];
|
||||
return <>
|
||||
<link rel="canonical" href={`${SITE_URL}${page.url}`} />
|
||||
<link rel="alternate" type="text/markdown" href={`${SITE_URL}${page.url}.md`} />
|
||||
{alternates.map((candidate) => (
|
||||
<link
|
||||
key={candidate}
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
#!/usr/bin/env node
|
||||
// Agent exports use canonical sources, not renderer-processed MDX: the latter
|
||||
// replaces images with JS variables and returns no text for virtual API pages.
|
||||
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { unified } from "unified";
|
||||
import remarkParse from "remark-parse";
|
||||
import remarkMdx from "remark-mdx";
|
||||
import remarkGfm from "remark-gfm";
|
||||
import remarkStringify from "remark-stringify";
|
||||
import { parse as parseYaml } from "yaml";
|
||||
import { collectMdxDocuments, normalizeMdxContent } from "./prepare-fumapress.mjs";
|
||||
|
||||
const ROOT = path.resolve(import.meta.dirname, "..");
|
||||
export const BASE = "https://langbot.app/docs";
|
||||
export const LOCALES = ["en", "zh", "ja"];
|
||||
const METHODS = /^(get|put|post|delete|patch|options|head|trace)$/;
|
||||
const encode = (value) => value.split("/").map(encodeURIComponent).join("/");
|
||||
const slug = (value) => value.replace(/\s+/g, "-").toLowerCase();
|
||||
const pageRoute = (file) => file.replace(/\.mdx$/, "").replace(/\/index$/, "");
|
||||
const text = (value) => ({ type: "text", value });
|
||||
const paragraph = (children) => ({ type: "paragraph", children });
|
||||
const parser = unified().use(remarkParse).use(remarkMdx).use(remarkGfm);
|
||||
const printer = unified().use(remarkStringify, { fences: true, bullet: "-" }).use(remarkGfm);
|
||||
|
||||
export function splitFrontmatter(source) {
|
||||
const match = source.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/);
|
||||
if (!match) throw new Error("Documentation source requires frontmatter");
|
||||
return { metadata: parseYaml(match[1]), body: source.slice(match[0].length) };
|
||||
}
|
||||
|
||||
export function canonicalLink(value, document, routes, redirects = [], seen = new Set()) {
|
||||
if (!value || value.startsWith("#")) return value;
|
||||
let local = value;
|
||||
const absolute = /^(?:[a-z][a-z\d+.-]*:|\/\/)/i.test(value);
|
||||
if (absolute) {
|
||||
// Example URLs can contain prose punctuation; only parse our docs hosts.
|
||||
if (!/^(?:https?:)?\/\/(?:docs\.langbot\.app|langbot\.app)(?:\/|$)/i.test(value)) return value;
|
||||
const url = new URL(value, BASE);
|
||||
if (url.hostname === "docs.langbot.app") local = `${url.pathname}${url.search}${url.hash}`;
|
||||
else if (url.origin === "https://langbot.app" && (url.pathname === "/docs" || url.pathname.startsWith("/docs/"))) {
|
||||
local = `${url.pathname.slice(5) || "/"}${url.search}${url.hash}`;
|
||||
} else return value; // Marketing, API hosts, mailto and other external links.
|
||||
}
|
||||
local = local.replace(/^\/docs(?=\/|$)/, "");
|
||||
const suffixAt = local.search(/[?#]/);
|
||||
const suffix = suffixAt < 0 ? "" : local.slice(suffixAt);
|
||||
let pathname = decodeURIComponent(suffixAt < 0 ? local : local.slice(0, suffixAt));
|
||||
pathname = pathname.startsWith("/") ? pathname.slice(1) : path.posix.join(path.posix.dirname(document), pathname);
|
||||
pathname = path.posix.normalize(pathname).replace(/^\.\//, "").replace(/\/$/, "");
|
||||
if (pathname === "." || pathname === "" || LOCALES.includes(pathname)) pathname = `${LOCALES.includes(pathname) ? pathname : document.split("/")[0]}/insight/guide`;
|
||||
const candidate = pathname.replace(/\.(?:html|mdx|md)$/, "").replace(/\/index$/, "");
|
||||
if (routes.has(candidate)) return `${BASE}/${encode(candidate)}.md${suffix}`;
|
||||
// Honor the same legacy aliases as the human site, without inventing
|
||||
// locale fallbacks for documents that have no translation.
|
||||
for (const redirect of redirects) {
|
||||
const wildcard = redirect.source.match(/^(.*\/):([A-Za-z][A-Za-z0-9_]*)\*$/);
|
||||
const matches = redirect.source === `/${candidate}` || (wildcard && `/${candidate}`.startsWith(wildcard[1]));
|
||||
if (matches && redirect.destination !== redirect.source) {
|
||||
if (seen.has(candidate)) throw new Error(`Redirect cycle: ${candidate}`);
|
||||
seen.add(candidate);
|
||||
const destination = wildcard
|
||||
? redirect.destination.replace(`:${wildcard[2]}*`, `/${candidate}`.slice(wildcard[1].length))
|
||||
: redirect.destination;
|
||||
return canonicalLink(`${destination}${suffix}`, document, routes, redirects, seen);
|
||||
}
|
||||
}
|
||||
return `${BASE}/${encode(pathname)}${suffix}`;
|
||||
}
|
||||
|
||||
export function renderMdx(source, document, routes, redirects = []) {
|
||||
const { metadata, body } = splitFrontmatter(source);
|
||||
const tree = parser.parse(normalizeMdxContent(body, document));
|
||||
const link = (value) => canonicalLink(value, document, routes, redirects);
|
||||
function transform(node) {
|
||||
if (["mdxjsEsm", "mdxFlowExpression", "mdxTextExpression"].includes(node.type)) {
|
||||
if (/^\s*\/\*[\s\S]*\*\/\s*$/.test(node.value)) return [];
|
||||
// Never silently drop an executable source value that might contain docs.
|
||||
throw new Error(`${document}: unsupported ${node.type}: ${node.value}`);
|
||||
}
|
||||
if (node.children) node.children = node.children.flatMap(transform);
|
||||
if (["link", "image", "definition"].includes(node.type)) node.url = link(node.url);
|
||||
if (!node.type.startsWith("mdxJsx")) return [node];
|
||||
const attributes = Object.fromEntries((node.attributes ?? [])
|
||||
.filter((attribute) => attribute.type === "mdxJsxAttribute")
|
||||
.map((attribute) => [attribute.name, attribute.value]));
|
||||
const children = node.children ?? [];
|
||||
if (node.name === "img") {
|
||||
if (typeof attributes.src !== "string") throw new Error(`${document}: image requires a literal src`);
|
||||
const image = { type: "image", url: link(attributes.src), alt: attributes.alt ?? "", title: attributes.title ?? null };
|
||||
return [node.type === "mdxJsxFlowElement" ? paragraph([image]) : image];
|
||||
}
|
||||
if (node.name === "a" && typeof attributes.href === "string") {
|
||||
const anchor = { type: "link", url: link(attributes.href), children };
|
||||
return [node.type === "mdxJsxFlowElement" ? paragraph([anchor]) : anchor];
|
||||
}
|
||||
if (node.name === "br") return [node.type === "mdxJsxFlowElement" ? paragraph([text("")]) : { type: "break" }];
|
||||
if (node.name === "summary") return [paragraph([{ type: "strong", children }])];
|
||||
if (node.name === "Callout") {
|
||||
return [{ type: "blockquote", children: [paragraph([text(`${attributes.title ?? attributes.type ?? "Note"}:`)]), ...children] }];
|
||||
}
|
||||
if (["Card", "Step", "Tab"].includes(node.name)) {
|
||||
if (typeof attributes.title !== "string") throw new Error(`${document}: ${node.name} requires a literal title`);
|
||||
const title = typeof attributes.href === "string"
|
||||
? { type: "link", url: link(attributes.href), children: [text(attributes.title)] }
|
||||
: { type: "strong", children: [text(attributes.title)] };
|
||||
return [paragraph([title]), ...children];
|
||||
}
|
||||
// Audited source wrappers; fail closed when new semantic components appear.
|
||||
if (["div", "details", "span", "Steps", "Tabs", "CardGroup"].includes(node.name)) return children;
|
||||
throw new Error(`${document}: unsupported JSX component ${node.name}`);
|
||||
}
|
||||
tree.children = tree.children.flatMap(transform);
|
||||
const title = metadata.title ?? pageRoute(document);
|
||||
const url = `${BASE}/${encode(pageRoute(document))}`;
|
||||
const description = metadata.description ? `${metadata.description}\n\n` : "";
|
||||
return { title, description: metadata.description ?? "", url, markdown: `# ${title}\n\nSource: ${url}\n\n${description}${printer.stringify(tree)}` };
|
||||
}
|
||||
|
||||
export function operationBundle(spec, apiPath, method) {
|
||||
const item = spec.paths[apiPath];
|
||||
const operation = item[method];
|
||||
const common = Object.fromEntries(Object.entries(item).filter(([key]) => !METHODS.test(key)));
|
||||
const bundle = {
|
||||
openapi: spec.openapi,
|
||||
info: spec.info,
|
||||
servers: operation.servers ?? item.servers ?? spec.servers ?? [],
|
||||
security: operation.security ?? spec.security ?? [],
|
||||
paths: { [apiPath]: { ...common, [method]: operation } },
|
||||
components: {},
|
||||
};
|
||||
const visited = new Set();
|
||||
function include(ref) {
|
||||
if (visited.has(ref)) return;
|
||||
visited.add(ref);
|
||||
if (!ref.startsWith("#/components/")) throw new Error(`Unsupported OpenAPI reference: ${ref}`);
|
||||
const keys = ref.slice(2).split("/").map((key) => key.replaceAll("~1", "/").replaceAll("~0", "~"));
|
||||
const value = keys.reduce((node, key) => node?.[key], spec);
|
||||
if (value === undefined) throw new Error(`Missing OpenAPI reference: ${ref}`);
|
||||
let parent = bundle;
|
||||
for (const key of keys.slice(0, -1)) parent = parent[key] ??= {};
|
||||
parent[keys.at(-1)] = value;
|
||||
visit(value);
|
||||
}
|
||||
function visit(value) {
|
||||
if (!value || typeof value !== "object") return;
|
||||
if (value.$ref) include(value.$ref);
|
||||
for (const child of Object.values(value)) visit(child);
|
||||
}
|
||||
visit(bundle);
|
||||
for (const alternative of bundle.security) for (const name of Object.keys(alternative)) {
|
||||
include(`#/components/securitySchemes/${name.replaceAll("~", "~0").replaceAll("/", "~1")}`);
|
||||
}
|
||||
return bundle;
|
||||
}
|
||||
|
||||
export function renderOperation(spec, apiPath, method, locale) {
|
||||
const operation = spec.paths[apiPath][method];
|
||||
if (!operation.summary) throw new Error(`Missing summary for ${method} ${apiPath}`);
|
||||
const route = `${locale}/api-reference/${slug(operation.tags?.[0] ?? "unknown")}/${slug(operation.summary)}`;
|
||||
const url = `${BASE}/${encode(route)}`;
|
||||
const bundle = operationBundle(spec, apiPath, method);
|
||||
const auth = bundle.security.length === 0 ? "No authentication required." :
|
||||
`Authentication (OpenAPI security alternatives; OR between entries, AND within each entry): ${JSON.stringify(bundle.security)}. See components.securitySchemes below for header names, schemes and scopes.`;
|
||||
const markdown = `# ${operation.summary}\n\nSource: ${url}\n\n## ${method.toUpperCase()} ${apiPath}\n\n${operation.description ?? ""}\n\n${auth}\n\nServers: ${bundle.servers.map((server) => server.url).join(", ") || "Relative to your LangBot instance"}. Use your own LangBot instance, not the documentation host.\n\n[Complete ${locale} OpenAPI specification](${BASE}/openapi/service-api-${locale}.json)\n\n## Operation contract\n\nThe OpenAPI subset below preserves path/operation parameters, request bodies, response codes, media types, examples and schemas. All referenced components are included transitively; resolve local JSON pointers against this document. The operation is preserved verbatim, including security overrides.\n\n\`\`\`json\n${JSON.stringify(bundle, null, 2)}\n\`\`\`\n`;
|
||||
return { route, title: operation.summary, description: `${method.toUpperCase()} ${apiPath}`, url, markdown };
|
||||
}
|
||||
|
||||
function indexText(pages, locale) {
|
||||
const languages = locale ? [locale] : LOCALES;
|
||||
const prefix = locale ? `${locale}/` : "";
|
||||
const lines = ["# LangBot Documentation", "", "> Canonical documentation for LangBot: deployment, messaging integrations, plugins, development and HTTP API reference.", "", "## Agent resources", "", `- [Complete documentation text](${BASE}/${prefix}llms-full.txt)`, `- [All-language index](${BASE}/llms.txt)`, ""];
|
||||
for (const language of languages) {
|
||||
lines.push(`## ${language}`, "", `- [${language} documentation index](${BASE}/${language}/llms.txt)`, `- [${language} complete documentation text](${BASE}/${language}/llms-full.txt)`, `- [${language} OpenAPI specification](${BASE}/openapi/service-api-${language}.json)`, "");
|
||||
for (const page of pages.filter((page) => page.route.startsWith(`${language}/`))) {
|
||||
const label = page.title.replace(/[\[\]\\]/g, "\\$&").replace(/\s+/g, " ");
|
||||
lines.push(`- [${label}](${page.url}.md)${page.description ? `: ${page.description.replace(/\s+/g, " ")}` : ""}`);
|
||||
}
|
||||
lines.push("");
|
||||
}
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
export async function generateAgentDocs({ root = ROOT, output = path.join(root, "dist/public") } = {}) {
|
||||
const documents = await collectMdxDocuments(root);
|
||||
const docs = JSON.parse(await readFile(path.join(root, "docs.json"), "utf8"));
|
||||
const pages = [];
|
||||
for (const locale of LOCALES) {
|
||||
const spec = JSON.parse(await readFile(path.join(root, `openapi/service-api-${locale}.json`), "utf8"));
|
||||
for (const [apiPath, item] of Object.entries(spec.paths)) for (const method of Object.keys(item).filter((key) => METHODS.test(key))) {
|
||||
pages.push(renderOperation(spec, apiPath, method, locale));
|
||||
}
|
||||
}
|
||||
const routes = new Set([...documents.map(pageRoute), ...pages.map((page) => page.route)]);
|
||||
for (const document of documents) {
|
||||
pages.push({ route: pageRoute(document), ...renderMdx(await readFile(path.join(root, document), "utf8"), document, routes, docs.redirects) });
|
||||
}
|
||||
pages.sort((a, b) => a.route.localeCompare(b.route, "en"));
|
||||
if (new Set(pages.map((page) => page.route)).size !== pages.length) throw new Error("Duplicate agent page routes");
|
||||
// Use the real static sitemap as the acceptance boundary, not a guessed count.
|
||||
const sitemap = await readFile(path.join(output, "sitemap.xml"), "utf8");
|
||||
const actual = [...sitemap.matchAll(/<loc>([^<]+)<\/loc>/g)].map((match) => decodeURIComponent(match[1].replace(/\/$/, ""))).sort();
|
||||
const expected = pages.map((page) => decodeURIComponent(page.url)).sort();
|
||||
if (JSON.stringify(actual) !== JSON.stringify(expected)) throw new Error("Agent sources do not match the exported sitemap");
|
||||
for (const page of pages) {
|
||||
const target = path.join(output, `${page.route}.md`);
|
||||
await mkdir(path.dirname(target), { recursive: true });
|
||||
await writeFile(target, page.markdown);
|
||||
}
|
||||
for (const locale of [null, ...LOCALES]) {
|
||||
const prefix = locale ? `${locale}/` : "";
|
||||
const selected = pages.filter((page) => !locale || page.route.startsWith(prefix));
|
||||
const index = indexText(selected, locale);
|
||||
await writeFile(path.join(output, `${prefix}llms.txt`), index);
|
||||
await writeFile(path.join(output, `${prefix}llm.txt`), index);
|
||||
await writeFile(path.join(output, `${prefix}llms-full.txt`), `${selected.map((page) => page.markdown.trim()).join("\n\n---\n\n")}\n`);
|
||||
}
|
||||
return { pages: pages.length, documents: documents.length, operations: pages.length - documents.length };
|
||||
}
|
||||
|
||||
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
console.log(await generateAgentDocs());
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { canonicalLink, renderMdx, operationBundle, BASE } from "../scripts/generate-agent-docs.mjs";
|
||||
import { unified } from "unified";
|
||||
import remarkParse from "remark-parse";
|
||||
|
||||
const routes = new Set(["zh/insight/guide", "zh/usage/models/readme"]);
|
||||
const document = "zh/insight/guide.mdx";
|
||||
test("canonical links preserve external/example URLs without trying to repair them", () => {
|
||||
for (const url of ["http://127.0.0.1:5300。", "mailto:[email protected]", "https://space.langbot.app/cloud", "https://langbot.app/zh/roadmap"]) {
|
||||
assert.equal(canonicalLink(url, document, routes), url);
|
||||
}
|
||||
});
|
||||
test("canonical links handle base paths, Unicode, legacy hosts, relative routes and images", () => {
|
||||
for (const url of ["/zh/usage/models/readme", "/docs/zh/usage/models/readme", "../usage/models/readme.html", "https://docs.langbot.app/zh/usage/models/readme.html", `${BASE}/zh/usage/models/readme.md`]) {
|
||||
assert.equal(canonicalLink(`${url}?q=1#model`, document, routes), `${BASE}/zh/usage/models/readme.md?q=1#model`);
|
||||
}
|
||||
assert.equal(canonicalLink("../../images/测试.png", document, routes), `${BASE}/images/%E6%B5%8B%E8%AF%95.png`);
|
||||
assert.equal(canonicalLink("#local", document, routes), "#local");
|
||||
});
|
||||
test("MDX exports preserve prose, wrapper content, literal images and code samples", () => {
|
||||
const source = '---\ntitle: "Test"\ndescription: "Description"\n---\n\n<Warning>\nDanger **now**.\n</Warning>\n\n<Accordion title="More">\nMore text.\n</Accordion>\n\n<img src="/images/example.png" alt="Example" />\n\n[Models](/zh/usage/models/readme.html)\n\n```python\nprint("/images/foo", "/zh/usage/models/readme.html", "<Info>")\n```\n';
|
||||
const result = renderMdx(source, document, routes).markdown;
|
||||
assert.match(result, /Danger \*\*now\*\*/);
|
||||
assert.match(result, /\*\*More\*\*/);
|
||||
assert.match(result, /More text/);
|
||||
assert.ok(result.includes(``));
|
||||
assert.ok(result.includes(`[Models](${BASE}/zh/usage/models/readme.md)`));
|
||||
const tree = unified().use(remarkParse).parse(result);
|
||||
assert.equal(tree.children.find((node) => node.type === "code").value, 'print("/images/foo", "/zh/usage/models/readme.html", "<Info>")');
|
||||
assert.doesNotMatch(result, /<\/?(?:Warning|Accordion)|__img/);
|
||||
});
|
||||
test("semantic JSX preserves titles and card links; unknown components fail closed", () => {
|
||||
const source = '---\ntitle: Test\n---\n<Callout title="Important" type="warning">\nRead this.\n</Callout>\n\n<Card title="Models" href="/zh/usage/models/readme">\nConfigure models.\n</Card>\n\n<Step title="Install">\nRun setup.\n</Step>\n\n<Tab title="Linux">\nUse shell.\n</Tab>\n';
|
||||
const result = renderMdx(source, document, routes).markdown;
|
||||
for (const title of ["Important", "Install", "Linux"]) assert.ok(result.includes(title));
|
||||
assert.ok(result.includes(`[Models](${BASE}/zh/usage/models/readme.md)`));
|
||||
assert.throws(() => renderMdx('---\ntitle: Test\n---\n<Unknown title="Lost" />', document, routes), /unsupported JSX/);
|
||||
});
|
||||
test("fixed redirect chains resolve with cycle detection", () => {
|
||||
const redirects = [{ source: "/old", destination: "/older" }, { source: "/older", destination: "/zh/usage/models/readme" }];
|
||||
assert.equal(canonicalLink("/old#section", document, routes, redirects), `${BASE}/zh/usage/models/readme.md#section`);
|
||||
assert.throws(() => canonicalLink("/old", document, routes, [...redirects.slice(0, 1), { source: "/older", destination: "/old" }]), /Redirect cycle/);
|
||||
});
|
||||
test("legacy wildcard redirects resolve through multiple rules", () => {
|
||||
const redirects = [{ source: "/deploy/:slug*", destination: "/zh/deploy/:slug*" }, { source: "/zh/deploy/:slug*", destination: "/zh/usage/:slug*" }];
|
||||
assert.equal(canonicalLink("/deploy/models/readme#model", document, routes, redirects), `${BASE}/zh/usage/models/readme.md#model`);
|
||||
});
|
||||
test("OpenAPI closure preserves recursive refs, path parameters and security alternatives", () => {
|
||||
const spec = { openapi: "3.0.3", info: { title: "Test", version: "1" }, security: [{ Key: [] }], components: { securitySchemes: { Key: { type: "apiKey", in: "header", name: "X-Key" } }, schemas: { Node: { type: "object", properties: { next: { $ref: "#/components/schemas/Node" } } } } }, paths: { "/items/{id}": { parameters: [{ name: "id", in: "path", required: true, schema: { type: "string" } }], get: { responses: { 200: { description: "OK", content: { "application/json": { schema: { $ref: "#/components/schemas/Node" } } } } } } } } };
|
||||
const bundle = operationBundle(spec, "/items/{id}", "get");
|
||||
assert.deepEqual(bundle.paths, spec.paths);
|
||||
assert.deepEqual(bundle.components, spec.components);
|
||||
assert.deepEqual(bundle.security, spec.security);
|
||||
const bad = structuredClone(spec);
|
||||
delete bad.components.schemas.Node;
|
||||
assert.throws(() => operationBundle(bad, "/items/{id}", "get"), /Missing OpenAPI reference/);
|
||||
});
|
||||
@@ -0,0 +1,107 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { readFile, stat } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import test from "node:test";
|
||||
import { unified } from "unified";
|
||||
import remarkParse from "remark-parse";
|
||||
|
||||
const root = path.resolve(import.meta.dirname, "..");
|
||||
const output = process.env.AGENT_TEST_OUTPUT ?? path.join(root, "dist/public");
|
||||
const base = "https://langbot.app/docs";
|
||||
const locales = ["en", "zh", "ja"];
|
||||
const read = (file) => readFile(path.join(output, file), "utf8");
|
||||
const slug = (value) => value.replace(/\s+/g, "-").toLowerCase();
|
||||
const encode = (value) => value.split("/").map(encodeURIComponent).join("/");
|
||||
const sitemap = await read("sitemap.xml");
|
||||
const urls = [...sitemap.matchAll(/<loc>([^<]+)<\/loc>/g)].map((m) => m[1].replace(/\/$/, ""));
|
||||
const relative = (url) => decodeURIComponent(url.slice(base.length + 1));
|
||||
function links(node, result = []) {
|
||||
if (["link", "image", "definition"].includes(node.type)) result.push(node.url);
|
||||
for (const child of node.children ?? []) links(child, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
test("every sitemap page has meaningful canonical Markdown and HTML discovery", async () => {
|
||||
assert.equal(urls.length, 479);
|
||||
for (const url of urls) {
|
||||
const route = relative(url);
|
||||
const text = await read(`${route}.md`);
|
||||
assert.ok(text.length > 100, `${route}: empty Markdown`);
|
||||
assert.ok(text.includes(url), `${route}: missing canonical URL`);
|
||||
assert.doesNotMatch(text, /__img\d+/, route);
|
||||
const html = await read(`${route}/index.html`);
|
||||
assert.ok(html.includes(`rel="alternate" type="text/markdown" href="${url}.md"`), `${route}: no Markdown discovery`);
|
||||
}
|
||||
});
|
||||
|
||||
test("all index and full-text exports cover exactly the published pages", async () => {
|
||||
for (const locale of [null, ...locales]) {
|
||||
const prefix = locale ? `${locale}/` : "";
|
||||
const index = await read(`${prefix}llms.txt`);
|
||||
const full = await read(`${prefix}llms-full.txt`);
|
||||
assert.equal(await read(`${prefix}llm.txt`), index);
|
||||
const expected = urls.filter((url) => !locale || url.startsWith(`${base}/${locale}/`));
|
||||
const actual = links(unified().use(remarkParse).parse(index)).filter((url) => url.endsWith(".md"));
|
||||
assert.deepEqual(actual.sort(), expected.map((url) => `${url}.md`).sort());
|
||||
for (const url of expected) assert.ok(full.includes((await read(`${relative(url)}.md`)).trim()), `${prefix}full export missing ${url}`);
|
||||
for (const lang of locale ? [locale] : locales) assert.ok(index.includes(`${base}/openapi/service-api-${lang}.json`));
|
||||
assert.doesNotMatch(index, /https:\/\/docs\.langbot\.app/);
|
||||
assert.ok((await stat(path.join(output, `${prefix}llms-full.txt`))).size < 25 * 1024 * 1024);
|
||||
}
|
||||
});
|
||||
|
||||
test("every API operation preserves the exact OpenAPI contract and auth override", async () => {
|
||||
let count = 0;
|
||||
for (const locale of locales) {
|
||||
const source = JSON.parse(await readFile(path.join(root, `openapi/service-api-${locale}.json`), "utf8"));
|
||||
assert.deepEqual(JSON.parse(await read(`openapi/service-api-${locale}.json`)), source);
|
||||
for (const [apiPath, pathItem] of Object.entries(source.paths)) {
|
||||
for (const [method, operation] of Object.entries(pathItem)) {
|
||||
if (!/^(get|put|post|delete|patch|options|head|trace)$/.test(method)) continue;
|
||||
count++;
|
||||
const route = `${locale}/api-reference/${slug(operation.tags?.[0] ?? "unknown")}/${slug(operation.summary)}`;
|
||||
const text = await read(`${route}.md`);
|
||||
assert.ok(text.includes(`${method.toUpperCase()} ${apiPath}`), route);
|
||||
assert.ok(text.includes(`${base}/${encode(route)}`), route);
|
||||
assert.ok(text.includes(`${base}/openapi/service-api-${locale}.json`), route);
|
||||
const bundle = JSON.parse(text.match(/```json\n([\s\S]*?)\n```/)?.[1] ?? "null");
|
||||
assert.ok(bundle, `${route}: missing operation bundle`);
|
||||
assert.deepEqual(bundle.paths[apiPath][method], operation, route);
|
||||
assert.deepEqual(bundle.security, operation.security ?? source.security ?? [], route);
|
||||
assert.deepEqual(bundle.servers, operation.servers ?? pathItem.servers ?? source.servers ?? [], route);
|
||||
if (bundle.security.length === 0) assert.match(text, /No authentication required/);
|
||||
else for (const alternative of bundle.security) for (const scheme of Object.keys(alternative)) {
|
||||
assert.deepEqual(bundle.components.securitySchemes[scheme], source.components.securitySchemes[scheme]);
|
||||
}
|
||||
function checkRefs(value) {
|
||||
if (!value || typeof value !== "object") return;
|
||||
if (value.$ref?.startsWith("#/")) {
|
||||
const keys = value.$ref.slice(2).split("/").map((key) => key.replaceAll("~1", "/").replaceAll("~0", "~"));
|
||||
const resolve = (document) => keys.reduce((node, key) => node?.[key], document);
|
||||
assert.ok(resolve(bundle), `${route}: unresolved ${value.$ref}`);
|
||||
assert.deepEqual(resolve(bundle), resolve(source));
|
||||
}
|
||||
for (const child of Object.values(value)) checkRefs(child);
|
||||
}
|
||||
checkRefs(bundle);
|
||||
}
|
||||
}
|
||||
}
|
||||
assert.equal(count, 177);
|
||||
});
|
||||
|
||||
test("Markdown links and images resolve inside the embedded output", async () => {
|
||||
const failures = [];
|
||||
for (const url of urls) {
|
||||
const text = await read(`${relative(url)}.md`);
|
||||
for (const link of links(unified().use(remarkParse).parse(text))) {
|
||||
if (!link.startsWith(base) && !/^\/(?:en|zh|ja|images|docs)\//.test(link)) continue;
|
||||
if (!link.startsWith(base)) { failures.push([url, link, "unscoped URL"]); continue; }
|
||||
const target = new URL(link);
|
||||
const file = decodeURIComponent(target.pathname.replace(/^\/docs\//, ""));
|
||||
try { await stat(path.join(output, file.includes(".") ? file : `${file}/index.html`)); }
|
||||
catch { failures.push([url, link, "missing target"]); }
|
||||
}
|
||||
}
|
||||
assert.deepEqual(failures, []);
|
||||
});
|
||||
Reference in New Issue
Block a user