fix(docs): generate complete agent-readable Markdown exports (#121)

Co-authored-by: dadachann <[email protected]>
2026-10-29 20:05:25 +08:00 · 2026-09-06 12:01:09 +08:00
parent 84e8119428
commit 070ef2cff9
8 changed files with 447 additions and 5 deletions
@@ -0,0 +1,58 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { canonicalLink, renderMdx, operationBundle, BASE } from "../scripts/generate-agent-docs.mjs";
+import { unified } from "unified";
+import remarkParse from "remark-parse";
+
+const routes = new Set(["zh/insight/guide", "zh/usage/models/readme"]);
+const document = "zh/insight/guide.mdx";
+test("canonical links preserve external/example URLs without trying to repair them", () => {
+  for (const url of ["http://127.0.0.1:5300。", "mailto:[email protected]", "https://space.langbot.app/cloud", "https://langbot.app/zh/roadmap"]) {
+    assert.equal(canonicalLink(url, document, routes), url);
+  }
+});
+test("canonical links handle base paths, Unicode, legacy hosts, relative routes and images", () => {
+  for (const url of ["/zh/usage/models/readme", "/docs/zh/usage/models/readme", "../usage/models/readme.html", "https://docs.langbot.app/zh/usage/models/readme.html", `${BASE}/zh/usage/models/readme.md`]) {
+    assert.equal(canonicalLink(`${url}?q=1#model`, document, routes), `${BASE}/zh/usage/models/readme.md?q=1#model`);
+  }
+  assert.equal(canonicalLink("../../images/测试.png", document, routes), `${BASE}/images/%E6%B5%8B%E8%AF%95.png`);
+  assert.equal(canonicalLink("#local", document, routes), "#local");
+});
+test("MDX exports preserve prose, wrapper content, literal images and code samples", () => {
+  const source = '---\ntitle: "Test"\ndescription: "Description"\n---\n\n<Warning>\nDanger **now**.\n</Warning>\n\n<Accordion title="More">\nMore text.\n</Accordion>\n\n<img src="/images/example.png" alt="Example" />\n\n[Models](/zh/usage/models/readme.html)\n\n```python\nprint("/images/foo", "/zh/usage/models/readme.html", "<Info>")\n```\n';
+  const result = renderMdx(source, document, routes).markdown;
+  assert.match(result, /Danger \*\*now\*\*/);
+  assert.match(result, /\*\*More\*\*/);
+  assert.match(result, /More text/);
+  assert.ok(result.includes(`![Example](${BASE}/images/example.png)`));
+  assert.ok(result.includes(`[Models](${BASE}/zh/usage/models/readme.md)`));
+  const tree = unified().use(remarkParse).parse(result);
+  assert.equal(tree.children.find((node) => node.type === "code").value, 'print("/images/foo", "/zh/usage/models/readme.html", "<Info>")');
+  assert.doesNotMatch(result, /<\/?(?:Warning|Accordion)|__img/);
+});
+test("semantic JSX preserves titles and card links; unknown components fail closed", () => {
+  const source = '---\ntitle: Test\n---\n<Callout title="Important" type="warning">\nRead this.\n</Callout>\n\n<Card title="Models" href="/zh/usage/models/readme">\nConfigure models.\n</Card>\n\n<Step title="Install">\nRun setup.\n</Step>\n\n<Tab title="Linux">\nUse shell.\n</Tab>\n';
+  const result = renderMdx(source, document, routes).markdown;
+  for (const title of ["Important", "Install", "Linux"]) assert.ok(result.includes(title));
+  assert.ok(result.includes(`[Models](${BASE}/zh/usage/models/readme.md)`));
+  assert.throws(() => renderMdx('---\ntitle: Test\n---\n<Unknown title="Lost" />', document, routes), /unsupported JSX/);
+});
+test("fixed redirect chains resolve with cycle detection", () => {
+  const redirects = [{ source: "/old", destination: "/older" }, { source: "/older", destination: "/zh/usage/models/readme" }];
+  assert.equal(canonicalLink("/old#section", document, routes, redirects), `${BASE}/zh/usage/models/readme.md#section`);
+  assert.throws(() => canonicalLink("/old", document, routes, [...redirects.slice(0, 1), { source: "/older", destination: "/old" }]), /Redirect cycle/);
+});
+test("legacy wildcard redirects resolve through multiple rules", () => {
+  const redirects = [{ source: "/deploy/:slug*", destination: "/zh/deploy/:slug*" }, { source: "/zh/deploy/:slug*", destination: "/zh/usage/:slug*" }];
+  assert.equal(canonicalLink("/deploy/models/readme#model", document, routes, redirects), `${BASE}/zh/usage/models/readme.md#model`);
+});
+test("OpenAPI closure preserves recursive refs, path parameters and security alternatives", () => {
+  const spec = { openapi: "3.0.3", info: { title: "Test", version: "1" }, security: [{ Key: [] }], components: { securitySchemes: { Key: { type: "apiKey", in: "header", name: "X-Key" } }, schemas: { Node: { type: "object", properties: { next: { $ref: "#/components/schemas/Node" } } } } }, paths: { "/items/{id}": { parameters: [{ name: "id", in: "path", required: true, schema: { type: "string" } }], get: { responses: { 200: { description: "OK", content: { "application/json": { schema: { $ref: "#/components/schemas/Node" } } } } } } } } };
+  const bundle = operationBundle(spec, "/items/{id}", "get");
+  assert.deepEqual(bundle.paths, spec.paths);
+  assert.deepEqual(bundle.components, spec.components);
+  assert.deepEqual(bundle.security, spec.security);
+  const bad = structuredClone(spec);
+  delete bad.components.schemas.Node;
+  assert.throws(() => operationBundle(bad, "/items/{id}", "get"), /Missing OpenAPI reference/);
+});
@@ -0,0 +1,107 @@
+import assert from "node:assert/strict";
+import { readFile, stat } from "node:fs/promises";
+import path from "node:path";
+import test from "node:test";
+import { unified } from "unified";
+import remarkParse from "remark-parse";
+
+const root = path.resolve(import.meta.dirname, "..");
+const output = process.env.AGENT_TEST_OUTPUT ?? path.join(root, "dist/public");
+const base = "https://langbot.app/docs";
+const locales = ["en", "zh", "ja"];
+const read = (file) => readFile(path.join(output, file), "utf8");
+const slug = (value) => value.replace(/\s+/g, "-").toLowerCase();
+const encode = (value) => value.split("/").map(encodeURIComponent).join("/");
+const sitemap = await read("sitemap.xml");
+const urls = [...sitemap.matchAll(/<loc>([^<]+)<\/loc>/g)].map((m) => m[1].replace(/\/$/, ""));
+const relative = (url) => decodeURIComponent(url.slice(base.length + 1));
+function links(node, result = []) {
+  if (["link", "image", "definition"].includes(node.type)) result.push(node.url);
+  for (const child of node.children ?? []) links(child, result);
+  return result;
+}
+
+test("every sitemap page has meaningful canonical Markdown and HTML discovery", async () => {
+  assert.equal(urls.length, 479);
+  for (const url of urls) {
+    const route = relative(url);
+    const text = await read(`${route}.md`);
+    assert.ok(text.length > 100, `${route}: empty Markdown`);
+    assert.ok(text.includes(url), `${route}: missing canonical URL`);
+    assert.doesNotMatch(text, /__img\d+/, route);
+    const html = await read(`${route}/index.html`);
+    assert.ok(html.includes(`rel="alternate" type="text/markdown" href="${url}.md"`), `${route}: no Markdown discovery`);
+  }
+});
+
+test("all index and full-text exports cover exactly the published pages", async () => {
+  for (const locale of [null, ...locales]) {
+    const prefix = locale ? `${locale}/` : "";
+    const index = await read(`${prefix}llms.txt`);
+    const full = await read(`${prefix}llms-full.txt`);
+    assert.equal(await read(`${prefix}llm.txt`), index);
+    const expected = urls.filter((url) => !locale || url.startsWith(`${base}/${locale}/`));
+    const actual = links(unified().use(remarkParse).parse(index)).filter((url) => url.endsWith(".md"));
+    assert.deepEqual(actual.sort(), expected.map((url) => `${url}.md`).sort());
+    for (const url of expected) assert.ok(full.includes((await read(`${relative(url)}.md`)).trim()), `${prefix}full export missing ${url}`);
+    for (const lang of locale ? [locale] : locales) assert.ok(index.includes(`${base}/openapi/service-api-${lang}.json`));
+    assert.doesNotMatch(index, /https:\/\/docs\.langbot\.app/);
+    assert.ok((await stat(path.join(output, `${prefix}llms-full.txt`))).size < 25 * 1024 * 1024);
+  }
+});
+
+test("every API operation preserves the exact OpenAPI contract and auth override", async () => {
+  let count = 0;
+  for (const locale of locales) {
+    const source = JSON.parse(await readFile(path.join(root, `openapi/service-api-${locale}.json`), "utf8"));
+    assert.deepEqual(JSON.parse(await read(`openapi/service-api-${locale}.json`)), source);
+    for (const [apiPath, pathItem] of Object.entries(source.paths)) {
+      for (const [method, operation] of Object.entries(pathItem)) {
+        if (!/^(get|put|post|delete|patch|options|head|trace)$/.test(method)) continue;
+        count++;
+        const route = `${locale}/api-reference/${slug(operation.tags?.[0] ?? "unknown")}/${slug(operation.summary)}`;
+        const text = await read(`${route}.md`);
+        assert.ok(text.includes(`${method.toUpperCase()} ${apiPath}`), route);
+        assert.ok(text.includes(`${base}/${encode(route)}`), route);
+        assert.ok(text.includes(`${base}/openapi/service-api-${locale}.json`), route);
+        const bundle = JSON.parse(text.match(/```json\n([\s\S]*?)\n```/)?.[1] ?? "null");
+        assert.ok(bundle, `${route}: missing operation bundle`);
+        assert.deepEqual(bundle.paths[apiPath][method], operation, route);
+        assert.deepEqual(bundle.security, operation.security ?? source.security ?? [], route);
+        assert.deepEqual(bundle.servers, operation.servers ?? pathItem.servers ?? source.servers ?? [], route);
+        if (bundle.security.length === 0) assert.match(text, /No authentication required/);
+        else for (const alternative of bundle.security) for (const scheme of Object.keys(alternative)) {
+          assert.deepEqual(bundle.components.securitySchemes[scheme], source.components.securitySchemes[scheme]);
+        }
+        function checkRefs(value) {
+          if (!value || typeof value !== "object") return;
+          if (value.$ref?.startsWith("#/")) {
+            const keys = value.$ref.slice(2).split("/").map((key) => key.replaceAll("~1", "/").replaceAll("~0", "~"));
+            const resolve = (document) => keys.reduce((node, key) => node?.[key], document);
+            assert.ok(resolve(bundle), `${route}: unresolved ${value.$ref}`);
+            assert.deepEqual(resolve(bundle), resolve(source));
+          }
+          for (const child of Object.values(value)) checkRefs(child);
+        }
+        checkRefs(bundle);
+      }
+    }
+  }
+  assert.equal(count, 177);
+});
+
+test("Markdown links and images resolve inside the embedded output", async () => {
+  const failures = [];
+  for (const url of urls) {
+    const text = await read(`${relative(url)}.md`);
+    for (const link of links(unified().use(remarkParse).parse(text))) {
+      if (!link.startsWith(base) && !/^\/(?:en|zh|ja|images|docs)\//.test(link)) continue;
+      if (!link.startsWith(base)) { failures.push([url, link, "unscoped URL"]); continue; }
+      const target = new URL(link);
+      const file = decodeURIComponent(target.pathname.replace(/^\/docs\//, ""));
+      try { await stat(path.join(output, file.includes(".") ? file : `${file}/index.html`)); }
+      catch { failures.push([url, link, "missing target"]); }
+    }
+  }
+  assert.deepEqual(failures, []);
+});