diff --git a/.github/workflows/sync-github-wiki.yml b/.github/workflows/sync-github-wiki.yml new file mode 100644 index 000000000..e710ef8b8 --- /dev/null +++ b/.github/workflows/sync-github-wiki.yml @@ -0,0 +1,87 @@ +name: Sync GitHub Wiki + +on: + push: + branches: [main] + paths: + - "en/**" + - "zh/**" + - "ja/**" + - "scripts/build-github-wiki.py" + - ".github/workflows/sync-github-wiki.yml" + pull_request: + paths: + - "en/**" + - "zh/**" + - "ja/**" + - "scripts/build-github-wiki.py" + - "tests/test_build_github_wiki.py" + - ".github/workflows/sync-github-wiki.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: sync-langbot-github-wiki + cancel-in-progress: false + +jobs: + build: + name: Build Wiki mirror + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Run converter tests + run: python -m unittest tests/test_build_github_wiki.py + - name: Build Wiki pages + run: python scripts/build-github-wiki.py --output dist/github-wiki + - name: Validate generated pages + run: | + test -f dist/github-wiki/Home.md + test -f dist/github-wiki/_Sidebar.md + test "$(find dist/github-wiki -maxdepth 1 -name '*.md' | wc -l)" -gt 250 + - name: Upload generated Wiki + uses: actions/upload-artifact@v4 + with: + name: github-wiki + path: dist/github-wiki + if-no-files-found: error + + sync: + name: Publish to LangBot Wiki + if: github.event_name != 'pull_request' + needs: build + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/download-artifact@v4 + with: + name: github-wiki + path: generated + - name: Clone target Wiki + env: + WIKI_PUSH_TOKEN: ${{ secrets.LANGBOT_WIKI_PUSH_TOKEN }} + run: | + test -n "$WIKI_PUSH_TOKEN" + git clone "https://x-access-token:${WIKI_PUSH_TOKEN}@github.com/langbot-app/LangBot.wiki.git" target + - name: Replace and publish Wiki contents + env: + SOURCE_SHA: ${{ github.sha }} + run: | + find target -mindepth 1 -maxdepth 1 ! -name .git -exec rm -rf {} + + cp -a generated/. target/ + cd target + git add -A + if git diff --cached --quiet; then + echo "GitHub Wiki is already up to date." + exit 0 + fi + git config user.name "langbot-wiki-sync[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git commit -m "docs: sync from langbot-wiki@${SOURCE_SHA::7}" + git push origin HEAD:master diff --git a/.gitignore b/.gitignore index 39a991799..3f6b4fdee 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,12 @@ -node_modules/ -.mintlify/ -.DS_Store -/.omc -.mintlify-last/ - -# Per-user Claude Code settings (skill symlinks under .claude/skills/ stay tracked) -.claude/settings.local.json \ No newline at end of file +node_modules/ +.mintlify/ +.DS_Store +/.omc +.mintlify-last/ + +# Per-user Claude Code settings (skill symlinks under .claude/skills/ stay tracked) +.claude/settings.local.json + +# Python build/test cache +__pycache__/ +*.py[cod] diff --git a/scripts/build-github-wiki.py b/scripts/build-github-wiki.py new file mode 100755 index 000000000..4387d5d2c --- /dev/null +++ b/scripts/build-github-wiki.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +"""Build a GitHub Wiki-compatible mirror from the Mintlify MDX sources.""" + +from __future__ import annotations + +import argparse +import re +import shutil +from pathlib import Path, PurePosixPath +from urllib.parse import urlsplit, urlunsplit + +LANGUAGES = {"zh": "简体中文", "en": "English", "ja": "日本語"} +DOC_EXTENSIONS = (".md", ".mdx", ".html") +CALLOUTS = {"Info": "NOTE", "Note": "NOTE", "Tip": "TIP", "Warning": "WARNING"} +MOVED_SECTIONS = {"platforms", "models", "pipelines", "knowledge", "mcp"} + + +def page_name(path: PurePosixPath) -> str: + return path.with_suffix("").as_posix().replace("/", "-") + + +def extract_frontmatter(text: str) -> tuple[dict[str, str], str]: + if not text.startswith("---\n"): + return {}, text + end = text.find("\n---\n", 4) + if end == -1: + return {}, text + metadata: dict[str, str] = {} + for line in text[4:end].splitlines(): + key, separator, value = line.partition(":") + if separator: + metadata[key.strip()] = value.strip().strip('"\'') + return metadata, text[end + 5 :] + + +def resolve_doc_target(source: PurePosixPath, target: str) -> str | None: + parsed = urlsplit(target) + if parsed.scheme or parsed.netloc or target.startswith("#"): + return None + raw_path = parsed.path + if not raw_path: + return None + resolved = PurePosixPath(raw_path.lstrip("/")) if raw_path.startswith("/") else source.parent / raw_path + parts: list[str] = [] + for part in resolved.parts: + if part in ("", "."): + continue + if part == "..": + if parts: + parts.pop() + else: + parts.append(part) + if not parts or parts[0] not in LANGUAGES: + return None + normalized = PurePosixPath(*parts) + suffix = normalized.suffix.lower() + if suffix in DOC_EXTENSIONS: + normalized = normalized.with_suffix("") + elif suffix: + return None + # Mintlify redirects these sections from their old deploy location. Apply the + # same redirect in the flat Wiki mirror so historical source links still work. + if len(normalized.parts) >= 3 and normalized.parts[1] == "deploy" and normalized.parts[2] in MOVED_SECTIONS: + normalized = PurePosixPath(normalized.parts[0], "usage", *normalized.parts[2:]) + return urlunsplit(("", "", page_name(normalized), parsed.query, parsed.fragment)) + + +def rewrite_url(source: PurePosixPath, target: str, *, image: bool = False) -> str: + parsed = urlsplit(target) + if parsed.scheme or parsed.netloc or target.startswith(("#", "mailto:")): + return target + if image: + path = parsed.path.lstrip("/") if parsed.path.startswith("/") else (source.parent / parsed.path).as_posix() + return urlunsplit(("https", "raw.githubusercontent.com", f"/langbot-app/langbot-wiki/main/{path}", parsed.query, parsed.fragment)) + return resolve_doc_target(source, target) or target + + +def convert_callouts(text: str) -> str: + pattern = re.compile( + r"<(Info|Note|Tip|Warning)(?:\s+title=(?:\"([^\"]*)\"|'([^']*)'))?\s*>(.*?)", + re.DOTALL, + ) + + def replace(match: re.Match[str]) -> str: + component = match.group(1) + title = match.group(2) or match.group(3) or "" + body = match.group(4).strip() + lines = [f"> [!{CALLOUTS[component]}]"] + if title: + lines.append(f"> **{title}**") + lines.extend(">" if not line else f"> {line}" for line in body.splitlines()) + return "\n".join(lines) + + previous = None + while previous != text: + previous = text + text = pattern.sub(replace, text) + return text + + +def convert_mdx(source: PurePosixPath, text: str) -> tuple[str, str]: + metadata, body = extract_frontmatter(text) + title = metadata.get("title") or source.stem.replace("-", " ").title() + body = convert_callouts(body) + body = re.sub( + r'', + lambda m: f"\n
\n{m.group(1) or m.group(2)}\n", + body, + ) + body = body.replace("", "\n
") + body = re.sub(r"", "", body) + body = re.sub( + r"(!\[[^\]]*\]\()([^)\s]+)([^)]*\))", + lambda m: m.group(1) + rewrite_url(source, m.group(2), image=True) + m.group(3), + body, + ) + body = re.sub( + r"(?]*?\bsrc=["\'])([^"\']+)(["\'])', + lambda m: m.group(1) + rewrite_url(source, m.group(2), image=True) + m.group(3), + body, + flags=re.IGNORECASE, + ) + body = body.replace("
", "
") + body = re.sub(r"\n{3,}", "\n\n", body).strip() + return title, f"# {title}\n\n{body}\n" + + +def discover_documents(root: Path) -> list[Path]: + return sorted(path for language in LANGUAGES for path in (root / language).rglob("*.mdx")) + + +def build(root: Path, output: Path) -> int: + documents = discover_documents(root) + if not documents: + raise SystemExit("No MDX documents found") + if output.exists(): + shutil.rmtree(output) + output.mkdir(parents=True) + index: dict[str, list[tuple[str, str]]] = {language: [] for language in LANGUAGES} + for path in documents: + source = PurePosixPath(path.relative_to(root).as_posix()) + title, converted = convert_mdx(source, path.read_text(encoding="utf-8")) + name = page_name(source) + (output / f"{name}.md").write_text(converted, encoding="utf-8") + index[source.parts[0]].append((name, title)) + home = [ + "# LangBot Documentation", + "", + "This GitHub Wiki is automatically synchronized from [langbot-app/langbot-wiki](https://github.com/langbot-app/langbot-wiki).", + "", + ] + sidebar = ["## LangBot Documentation", ""] + for language, label in LANGUAGES.items(): + pages = index[language] + home.extend((f"## {label}", "", f"[{pages[0][1]}]({pages[0][0]})", "")) + sidebar.extend((f"### {label}", "")) + sidebar.extend(f"- [{title}]({name})" for name, title in pages) + sidebar.append("") + (output / "Home.md").write_text("\n".join(home).rstrip() + "\n", encoding="utf-8") + (output / "_Sidebar.md").write_text("\n".join(sidebar).rstrip() + "\n", encoding="utf-8") + (output / "_Footer.md").write_text( + "Automatically synchronized from [langbot-app/langbot-wiki](https://github.com/langbot-app/langbot-wiki).\n", + encoding="utf-8", + ) + return len(documents) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1]) + parser.add_argument("--output", type=Path, default=Path("dist/github-wiki")) + args = parser.parse_args() + count = build(args.root.resolve(), args.output.resolve()) + print(f"Built {count} GitHub Wiki pages in {args.output}") + + +if __name__ == "__main__": + main() diff --git a/tests/test_build_github_wiki.py b/tests/test_build_github_wiki.py new file mode 100644 index 000000000..2aa6b52d6 --- /dev/null +++ b/tests/test_build_github_wiki.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +import importlib.util +import tempfile +import unittest +from pathlib import Path, PurePosixPath + +MODULE_PATH = Path(__file__).resolve().parents[1] / "scripts" / "build-github-wiki.py" +SPEC = importlib.util.spec_from_file_location("build_github_wiki", MODULE_PATH) +assert SPEC and SPEC.loader +wiki = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(wiki) + + +class BuildGitHubWikiTest(unittest.TestCase): + def test_converts_frontmatter_callout_links_and_images(self) -> None: + source = PurePosixPath("zh/insight/guide.mdx") + title, output = wiki.convert_mdx( + source, + '''--- +title: "开始使用" +description: "ignored" +--- + +阅读[模型文档](/zh/usage/models/readme)。 + + +![界面](/images/ui.png) +''', + ) + self.assertEqual(title, "开始使用") + self.assertTrue(output.startswith("# 开始使用\n")) + self.assertNotIn("description:", output) + self.assertIn("> [!TIP]", output) + self.assertIn("> **提示**", output) + self.assertIn("[模型文档](zh-usage-models-readme)", output) + self.assertIn( + "https://raw.githubusercontent.com/langbot-app/langbot-wiki/main/images/ui.png", + output, + ) + + def test_resolves_relative_document_links(self) -> None: + source = PurePosixPath("en/usage/mcp/readme.mdx") + self.assertEqual( + wiki.resolve_doc_target(source, "../models/readme#tools"), + "en-usage-models-readme#tools", + ) + self.assertEqual( + wiki.resolve_doc_target(source, "../../deploy/settings.html"), + "en-deploy-settings", + ) + self.assertEqual( + wiki.resolve_doc_target(source, "/en/deploy/pipelines/readme#request-variables"), + "en-usage-pipelines-readme#request-variables", + ) + + def test_build_creates_flat_pages_and_navigation(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + for language in wiki.LANGUAGES: + path = root / language / "guide.mdx" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(f"---\ntitle: {language} Guide\n---\nContent\n", encoding="utf-8") + output = root / "out" + count = wiki.build(root, output) + self.assertEqual(count, 3) + self.assertTrue((output / "zh-guide.md").is_file()) + self.assertIn("[zh Guide](zh-guide)", (output / "_Sidebar.md").read_text()) + self.assertTrue((output / "Home.md").is_file()) + + +if __name__ == "__main__": + unittest.main()