mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-06-02 20:14:36 +00:00
Compare commits
2 Commits
v4.10.0-be
...
feat/long-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
662e6a4815 | ||
|
|
c92d3d7ad7 |
2
.github/ISSUE_TEMPLATE/bug-report.yml
vendored
2
.github/ISSUE_TEMPLATE/bug-report.yml
vendored
@@ -1,5 +1,5 @@
|
||||
name: 漏洞反馈
|
||||
description: 【供中文用户】报错或漏洞请使用这个模板创建,不使用此模板创建的异常、漏洞相关issue将被直接关闭。由于自己操作不当/不甚了解所用技术栈引起的网络连接问题恕无法解决,请勿提 issue。容器间网络连接问题,参考文档 https://link.langbot.app/zh/docs/network
|
||||
description: 【供中文用户】报错或漏洞请使用这个模板创建,不使用此模板创建的异常、漏洞相关issue将被直接关闭。由于自己操作不当/不甚了解所用技术栈引起的网络连接问题恕无法解决,请勿提 issue。容器间网络连接问题,参考文档 https://docs.langbot.app/zh/workshop/network-details.html
|
||||
title: "[Bug]: "
|
||||
labels: ["bug?"]
|
||||
body:
|
||||
|
||||
2
.github/ISSUE_TEMPLATE/bug-report_en.yml
vendored
2
.github/ISSUE_TEMPLATE/bug-report_en.yml
vendored
@@ -1,5 +1,5 @@
|
||||
name: Bug report
|
||||
description: Report bugs or vulnerabilities using this template. For container network connection issues, refer to the documentation https://link.langbot.app/en/docs/network
|
||||
description: Report bugs or vulnerabilities using this template. For container network connection issues, refer to the documentation https://docs.langbot.app/en/workshop/network-details.html
|
||||
title: "[Bug]: "
|
||||
labels: ["bug?"]
|
||||
body:
|
||||
|
||||
@@ -43,10 +43,10 @@ jobs:
|
||||
run: |
|
||||
cd /tmp/langbot_build_web/web
|
||||
npm install
|
||||
npx vite build
|
||||
npm run build
|
||||
- name: Package Output
|
||||
run: |
|
||||
cp -r /tmp/langbot_build_web/web/dist ./web
|
||||
cp -r /tmp/langbot_build_web/web/out ./web
|
||||
- name: Upload Artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
|
||||
25
.github/workflows/check-i18n.yml
vendored
25
.github/workflows/check-i18n.yml
vendored
@@ -1,25 +0,0 @@
|
||||
name: Check i18n Keys
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- master
|
||||
|
||||
jobs:
|
||||
check-i18n:
|
||||
name: Check i18n Key Consistency
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '20'
|
||||
|
||||
- name: Check i18n keys against en-US reference
|
||||
run: node web/scripts/check-i18n.mjs
|
||||
4
.github/workflows/publish-to-pypi.yml
vendored
4
.github/workflows/publish-to-pypi.yml
vendored
@@ -29,8 +29,8 @@ jobs:
|
||||
npm install -g pnpm
|
||||
pnpm install
|
||||
pnpm build
|
||||
mkdir -p ../src/langbot/web/dist
|
||||
cp -r dist ../src/langbot/web/
|
||||
mkdir -p ../src/langbot/web/out
|
||||
cp -r out ../src/langbot/web/
|
||||
|
||||
- name: Install the latest version of uv
|
||||
uses: astral-sh/setup-uv@v6
|
||||
|
||||
115
.github/workflows/run-tests.yml
vendored
115
.github/workflows/run-tests.yml
vendored
@@ -4,25 +4,25 @@ on:
|
||||
pull_request:
|
||||
types: [opened, ready_for_review, synchronize]
|
||||
paths:
|
||||
- 'src/langbot/**'
|
||||
- 'pkg/**'
|
||||
- 'tests/**'
|
||||
- '.github/workflows/run-tests.yml'
|
||||
- 'pyproject.toml'
|
||||
- 'uv.lock'
|
||||
- 'run_tests.sh'
|
||||
- 'scripts/test-*.sh'
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
- develop
|
||||
- 'feat/**'
|
||||
# No path filter on push: every push to the branches above runs the
|
||||
# full unit-test suite. feat/** branches in particular must be tested
|
||||
# on every push (they accumulate large changes before a PR exists).
|
||||
paths:
|
||||
- 'pkg/**'
|
||||
- 'tests/**'
|
||||
- '.github/workflows/run-tests.yml'
|
||||
- 'pyproject.toml'
|
||||
- 'run_tests.sh'
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: Unit Tests
|
||||
name: Run Unit Tests
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -39,13 +39,28 @@ jobs:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v4
|
||||
run: |
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --dev
|
||||
run: |
|
||||
uv sync --dev
|
||||
|
||||
- name: Run unit + smoke tests
|
||||
run: uv run pytest tests/unit_tests/ tests/smoke/ -q --tb=short
|
||||
- name: Run unit tests
|
||||
run: |
|
||||
bash run_tests.sh
|
||||
|
||||
- name: Upload coverage to Codecov
|
||||
if: matrix.python-version == '3.12'
|
||||
uses: codecov/codecov-action@v5
|
||||
with:
|
||||
files: ./coverage.xml
|
||||
flags: unit-tests
|
||||
name: unit-tests-coverage
|
||||
fail_ci_if_error: false
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
|
||||
- name: Test Summary
|
||||
if: always()
|
||||
@@ -54,79 +69,3 @@ jobs:
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "Python Version: ${{ matrix.python-version }}" >> $GITHUB_STEP_SUMMARY
|
||||
echo "Test Status: ${{ job.status }}" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
integration:
|
||||
name: Fast Integration Tests
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v4
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --dev
|
||||
|
||||
- name: Run fast integration tests
|
||||
run: uv run pytest tests/integration/ -m "not slow" -q --tb=short
|
||||
|
||||
- name: Integration Test Summary
|
||||
if: always()
|
||||
run: |
|
||||
echo "## Integration Tests Results" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "Test Status: ${{ job.status }}" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
coverage:
|
||||
name: Coverage Gate
|
||||
runs-on: ubuntu-latest
|
||||
needs: [test, integration]
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v4
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --dev
|
||||
|
||||
- name: Run coverage (unit + smoke)
|
||||
run: |
|
||||
uv run pytest tests/unit_tests/ tests/smoke/ \
|
||||
--cov=langbot \
|
||||
--cov-report=xml \
|
||||
--cov-report=term-missing \
|
||||
--cov-fail-under=18 \
|
||||
-q --tb=short
|
||||
|
||||
- name: Upload coverage to Codecov
|
||||
uses: codecov/codecov-action@v5
|
||||
with:
|
||||
files: ./coverage.xml
|
||||
flags: unit-tests
|
||||
name: coverage-report
|
||||
fail_ci_if_error: false
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
|
||||
- name: Coverage Summary
|
||||
if: always()
|
||||
run: |
|
||||
echo "## Coverage Results" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "Threshold: 18%" >> $GITHUB_STEP_SUMMARY
|
||||
echo "Status: ${{ job.status }}" >> $GITHUB_STEP_SUMMARY
|
||||
78
.github/workflows/test-migrations.yml
vendored
78
.github/workflows/test-migrations.yml
vendored
@@ -1,78 +0,0 @@
|
||||
name: Test Migrations
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- master
|
||||
- dev
|
||||
paths:
|
||||
- 'src/langbot/pkg/persistence/**'
|
||||
- 'src/langbot/pkg/entity/persistence/**'
|
||||
- 'tests/integration/persistence/**'
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened, ready_for_review]
|
||||
paths:
|
||||
- 'src/langbot/pkg/persistence/**'
|
||||
- 'src/langbot/pkg/entity/persistence/**'
|
||||
- 'tests/integration/persistence/**'
|
||||
|
||||
jobs:
|
||||
test-migrations-sqlite:
|
||||
name: Migrations (SQLite)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v4
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --dev
|
||||
|
||||
- name: Run SQLite migration tests
|
||||
run: uv run pytest tests/integration/persistence/test_migrations.py -q --tb=short
|
||||
|
||||
test-migrations-postgres:
|
||||
name: Migrations (PostgreSQL)
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16
|
||||
env:
|
||||
POSTGRES_USER: langbot
|
||||
POSTGRES_PASSWORD: langbot
|
||||
POSTGRES_DB: langbot_test
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--health-cmd="pg_isready -U langbot"
|
||||
--health-interval=5s
|
||||
--health-timeout=5s
|
||||
--health-retries=5
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v4
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --dev
|
||||
|
||||
- name: Run PostgreSQL migration tests
|
||||
env:
|
||||
TEST_POSTGRES_URL: postgresql+asyncpg://langbot:langbot@localhost:5432/langbot_test
|
||||
run: uv run pytest tests/integration/persistence/test_migrations_postgres.py -q --tb=short
|
||||
4
.gitignore
vendored
4
.gitignore
vendored
@@ -47,12 +47,8 @@ plugins.bak
|
||||
coverage.xml
|
||||
.coverage
|
||||
src/langbot/web/
|
||||
testsdk/
|
||||
|
||||
# Build artifacts
|
||||
/dist
|
||||
/build
|
||||
*.egg-info
|
||||
|
||||
# Next.js build cache (legacy)
|
||||
web/.next/
|
||||
|
||||
@@ -9,14 +9,16 @@ repos:
|
||||
# Run the formatter of backend.
|
||||
- id: ruff-format
|
||||
|
||||
- repo: local
|
||||
- repo: https://github.com/pre-commit/mirrors-prettier
|
||||
rev: v3.1.0
|
||||
hooks:
|
||||
- id: prettier
|
||||
name: prettier
|
||||
entry: npx --prefix web prettier --write --ignore-unknown
|
||||
language: system
|
||||
types_or: [javascript, jsx, ts, tsx, css, scss]
|
||||
additional_dependencies:
|
||||
- prettier@3.1.0
|
||||
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: lint-staged
|
||||
name: lint-staged
|
||||
entry: cd web && pnpm lint-staged
|
||||
|
||||
@@ -70,7 +70,7 @@ Plugin Runtime automatically starts each installed plugin and interacts through
|
||||
- type: must be a specific type, such as feat (new feature), fix (bug fix), docs (documentation), style (code style), refactor (refactoring), perf (performance optimization), etc.
|
||||
- scope: the scope of the commit, such as the package name, the file name, the function name, the class name, the module name, etc.
|
||||
- subject: the subject of the commit, such as the description of the commit, the reason for the commit, the impact of the commit, etc.
|
||||
- LangBot uses [Alembic](https://alembic.sqlalchemy.org/) to manage database migrations, supporting both SQLite and PostgreSQL. Migration files are located in `src/langbot/pkg/persistence/alembic/versions/`. If you changed the definition of database entities (ORM models), generate a new migration script by running `uv run python -m langbot.pkg.persistence.alembic_runner autogenerate "description of your change"` in the project root (requires `data/config.yaml` to exist). Review and edit the generated script before committing. Migrations are executed automatically on LangBot startup. For data migrations (e.g. modifying JSON field content), you need to manually add the migration code in the generated script.
|
||||
- If you changed the definition of database entities, please update the migration file in `src/langbot/pkg/persistence/migrations/` and update the constants.py file in `src/langbot/pkg/utils/constants.py` with the new migration number.
|
||||
|
||||
## Some Principles
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ WORKDIR /app
|
||||
|
||||
COPY web ./web
|
||||
|
||||
RUN cd web && npm install && npx vite build
|
||||
RUN cd web && npm install && npm run build
|
||||
|
||||
FROM python:3.12.7-slim
|
||||
|
||||
@@ -12,7 +12,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=node /app/web/dist ./web/dist
|
||||
COPY --from=node /app/web/out ./web/out
|
||||
|
||||
RUN apt update \
|
||||
&& apt install gcc -y \
|
||||
|
||||
36
Makefile
36
Makefile
@@ -1,36 +0,0 @@
|
||||
# LangBot Makefile
|
||||
# Quick developer commands
|
||||
|
||||
.PHONY: test test-quick test-integration-fast test-coverage test-all-local lint
|
||||
|
||||
# Run all tests (full suite with coverage)
|
||||
test:
|
||||
bash run_tests.sh
|
||||
|
||||
# Quick self-test for developers (lint + unit + smoke, no real credentials needed)
|
||||
test-quick:
|
||||
bash scripts/test-quick.sh
|
||||
|
||||
# Fast integration tests (SQLite/API/Pipeline, no external services)
|
||||
test-integration-fast:
|
||||
bash scripts/test-integration-fast.sh
|
||||
|
||||
# Coverage gate (all tests, enforces minimum threshold)
|
||||
test-coverage:
|
||||
bash scripts/test-coverage.sh
|
||||
|
||||
# Full local quality gate (quick + integration + coverage)
|
||||
test-all-local:
|
||||
bash scripts/test-quick.sh
|
||||
bash scripts/test-integration-fast.sh
|
||||
bash scripts/test-coverage.sh
|
||||
|
||||
# Run linting only
|
||||
lint:
|
||||
ruff check src/langbot/ tests/
|
||||
ruff format --check src/langbot/ tests/
|
||||
|
||||
# Fix linting issues
|
||||
lint-fix:
|
||||
ruff check --fix src/langbot/ tests/
|
||||
ruff format src/langbot/ tests/
|
||||
94
README.md
94
README.md
@@ -19,9 +19,9 @@ English / [简体中文](README_CN.md) / [繁體中文](README_TW.md) / [日本
|
||||
[](https://github.com/langbot-app/LangBot/stargazers)
|
||||
|
||||
<a href="https://langbot.app">Website</a> |
|
||||
<a href="https://link.langbot.app/en/docs/features">Features</a> |
|
||||
<a href="https://link.langbot.app/en/docs/guide">Docs</a> |
|
||||
<a href="https://link.langbot.app/en/docs/api">API</a> |
|
||||
<a href="https://docs.langbot.app/en/insight/features">Features</a> |
|
||||
<a href="https://docs.langbot.app/en/insight/guide">Docs</a> |
|
||||
<a href="https://docs.langbot.app/en/tags/readme">API</a> |
|
||||
<a href="https://space.langbot.app/cloud">Cloud</a> |
|
||||
<a href="https://space.langbot.app">Plugin Market</a> |
|
||||
<a href="https://langbot.featurebase.app/roadmap">Roadmap</a>
|
||||
@@ -45,9 +45,7 @@ LangBot is an **open-source, production-grade platform** for building AI-powered
|
||||
- **Web Management Panel** — Configure, manage, and monitor your bots through an intuitive browser interface. No YAML editing required.
|
||||
- **Multi-Pipeline Architecture** — Different bots for different scenarios, with comprehensive monitoring and exception handling.
|
||||
|
||||
[→ Learn more about all features](https://link.langbot.app/en/docs/features)
|
||||
|
||||
📍 Practical guides: [deploy a multi-platform AI bot in 5 minutes](https://blog.langbot.app/en/blog/deploy-ai-bot-in-5-minutes/), [connect DeepSeek to WeChat, Discord, and Telegram](https://blog.langbot.app/en/blog/connect-deepseek-to-wechat/), [run a Dify Agent in Discord, Telegram, and Slack](https://blog.langbot.app/en/blog/dify-agent-discord-telegram-slack/), and [build an n8n-powered chatbot](https://blog.langbot.app/en/blog/n8n-multi-platform-ai-chatbot/).
|
||||
[→ Learn more about all features](https://docs.langbot.app/en/insight/features)
|
||||
|
||||
---
|
||||
|
||||
@@ -78,7 +76,7 @@ docker compose up -d
|
||||
[](https://zeabur.com/en-US/templates/ZKTBDH)
|
||||
[](https://railway.app/template/yRrAyL?referralCode=vogKPF)
|
||||
|
||||
**More options:** [Docker](https://link.langbot.app/en/docs/docker) · [Manual](https://link.langbot.app/en/docs/manual-deploy) · [BTPanel](https://link.langbot.app/en/docs/bt-panel) · [Kubernetes](./docker/README_K8S.md)
|
||||
**More options:** [Docker](https://docs.langbot.app/en/deploy/langbot/docker) · [Manual](https://docs.langbot.app/en/deploy/langbot/manual) · [BTPanel](https://docs.langbot.app/en/deploy/langbot/one-click/bt) · [Kubernetes](./docker/README_K8S.md)
|
||||
|
||||
---
|
||||
|
||||
@@ -86,72 +84,68 @@ docker compose up -d
|
||||
|
||||
| Platform | Status | Notes |
|
||||
|----------|--------|-------|
|
||||
| Discord | ✅ | Official |
|
||||
| Telegram | ✅ | Official |
|
||||
| Slack | ✅ | Official |
|
||||
| LINE | ✅ | Official |
|
||||
| QQ | ✅ | Personal & Official API (Channel, DM, Group) |
|
||||
| Discord | ✅ | |
|
||||
| Telegram | ✅ | |
|
||||
| Slack | ✅ | |
|
||||
| LINE | ✅ | |
|
||||
| QQ | ✅ | Personal & Official API |
|
||||
| WeCom | ✅ | Enterprise WeChat, External CS, AI Bot |
|
||||
| WeChat | ✅ | Personal & Official Account |
|
||||
| Lark | ✅ | Official |
|
||||
| DingTalk | ✅ | Official |
|
||||
| KOOK | ✅ | Official |
|
||||
| Lark | ✅ | |
|
||||
| DingTalk | ✅ | |
|
||||
| KOOK | ✅ | |
|
||||
| Satori | ✅ | |
|
||||
| Email | ✅ | Matrix, Satori |
|
||||
| Matrix | ✅ | Supports multiple bridged platforms such as Signal, WhatsApp, Messenger, iMessage, Mattermost, Google Chat, IRC, XMPP, Zulip, and more |
|
||||
|
||||
---
|
||||
|
||||
## Supported LLMs & Integrations
|
||||
|
||||
| Provider | Type | Status |
|
||||
| ----------------------------------------------------------------------------------------------------------------- | ------------ | ------ |
|
||||
| [OpenAI](https://platform.openai.com/) | LLM | ✅ |
|
||||
| [Anthropic](https://www.anthropic.com/) | LLM | ✅ |
|
||||
| [DeepSeek](https://www.deepseek.com/) | LLM | ✅ |
|
||||
| [Google Gemini](https://aistudio.google.com/prompts/new_chat) | LLM | ✅ |
|
||||
| [xAI](https://x.ai/) | LLM | ✅ |
|
||||
| [Moonshot](https://www.moonshot.cn/) | LLM | ✅ |
|
||||
| [Zhipu AI](https://open.bigmodel.cn/) | LLM | ✅ |
|
||||
| [Ollama](https://ollama.com/) | Local LLM | ✅ |
|
||||
| [LM Studio](https://lmstudio.ai/) | Local LLM | ✅ |
|
||||
| [Dify](https://dify.ai) | LLMOps | ✅ |
|
||||
| [MCP](https://modelcontextprotocol.io/) | Protocol | ✅ |
|
||||
| [SiliconFlow](https://siliconflow.cn/) | Gateway | ✅ |
|
||||
| [Aliyun Bailian](https://bailian.console.aliyun.com/) | Gateway | ✅ |
|
||||
| [Volc Engine Ark](https://console.volcengine.com/ark/region:ark+cn-beijing/model?vendor=Bytedance&view=LIST_VIEW) | Gateway | ✅ |
|
||||
| [ModelScope](https://modelscope.cn/docs/model-service/API-Inference/intro) | Gateway | ✅ |
|
||||
| [GiteeAI](https://ai.gitee.com/) | Gateway | ✅ |
|
||||
| [CompShare](https://www.compshare.cn/?ytag=GPU_YY-gh_langbot) | GPU Platform | ✅ |
|
||||
| [PPIO](https://ppinfra.com/user/register?invited_by=QJKFYD&utm_source=github_langbot) | GPU Platform | ✅ |
|
||||
| [ShengSuanYun](https://www.shengsuanyun.com/?from=CH_KYIPP758) | GPU Platform | ✅ |
|
||||
| [接口 AI](https://jiekou.ai/) | Gateway | ✅ |
|
||||
| [302.AI](https://share.302.ai/SuTG99) | Gateway | ✅ |
|
||||
| [Qiniu](https://www.qiniu.com/ai/agent) | Gateway | ✅ |
|
||||
| Provider | Type | Status |
|
||||
|----------|------|--------|
|
||||
| [OpenAI](https://platform.openai.com/) | LLM | ✅ |
|
||||
| [Anthropic](https://www.anthropic.com/) | LLM | ✅ |
|
||||
| [DeepSeek](https://www.deepseek.com/) | LLM | ✅ |
|
||||
| [Google Gemini](https://aistudio.google.com/prompts/new_chat) | LLM | ✅ |
|
||||
| [xAI](https://x.ai/) | LLM | ✅ |
|
||||
| [Moonshot](https://www.moonshot.cn/) | LLM | ✅ |
|
||||
| [Zhipu AI](https://open.bigmodel.cn/) | LLM | ✅ |
|
||||
| [Ollama](https://ollama.com/) | Local LLM | ✅ |
|
||||
| [LM Studio](https://lmstudio.ai/) | Local LLM | ✅ |
|
||||
| [Dify](https://dify.ai) | LLMOps | ✅ |
|
||||
| [MCP](https://modelcontextprotocol.io/) | Protocol | ✅ |
|
||||
| [SiliconFlow](https://siliconflow.cn/) | Gateway | ✅ |
|
||||
| [Aliyun Bailian](https://bailian.console.aliyun.com/) | Gateway | ✅ |
|
||||
| [Volc Engine Ark](https://console.volcengine.com/ark/region:ark+cn-beijing/model?vendor=Bytedance&view=LIST_VIEW) | Gateway | ✅ |
|
||||
| [ModelScope](https://modelscope.cn/docs/model-service/API-Inference/intro) | Gateway | ✅ |
|
||||
| [GiteeAI](https://ai.gitee.com/) | Gateway | ✅ |
|
||||
| [CompShare](https://www.compshare.cn/?ytag=GPU_YY-gh_langbot) | GPU Platform | ✅ |
|
||||
| [PPIO](https://ppinfra.com/user/register?invited_by=QJKFYD&utm_source=github_langbot) | GPU Platform | ✅ |
|
||||
| [ShengSuanYun](https://www.shengsuanyun.com/?from=CH_KYIPP758) | GPU Platform | ✅ |
|
||||
| [接口 AI](https://jiekou.ai/) | Gateway | ✅ |
|
||||
| [302.AI](https://share.302.ai/SuTG99) | Gateway | ✅ |
|
||||
|
||||
[→ View all integrations](https://link.langbot.app/en/docs/features)
|
||||
[→ View all integrations](https://docs.langbot.app/en/insight/features)
|
||||
|
||||
---
|
||||
|
||||
## Why LangBot?
|
||||
|
||||
| Use Case | How LangBot Helps |
|
||||
| --------------------------- | ------------------------------------------------------------------------------------------ |
|
||||
| **Customer Support** | Deploy AI agents to Slack/Discord/Telegram that answer questions using your knowledge base |
|
||||
| **Internal Tools** | Connect n8n/Dify workflows to WeCom/DingTalk for automated business processes |
|
||||
| **Community Management** | Moderate QQ/Discord groups with AI-powered content filtering and interaction |
|
||||
| **Multi-Platform Presence** | One bot, all platforms. Manage from a single dashboard |
|
||||
| Use Case | How LangBot Helps |
|
||||
|----------|-------------------|
|
||||
| **Customer Support** | Deploy AI agents to Slack/Discord/Telegram that answer questions using your knowledge base |
|
||||
| **Internal Tools** | Connect n8n/Dify workflows to WeCom/DingTalk for automated business processes |
|
||||
| **Community Management** | Moderate QQ/Discord groups with AI-powered content filtering and interaction |
|
||||
| **Multi-Platform Presence** | One bot, all platforms. Manage from a single dashboard |
|
||||
|
||||
---
|
||||
|
||||
## Live Demo
|
||||
|
||||
**Try it now:** https://demo.langbot.dev/
|
||||
|
||||
- Email: `demo@langbot.app`
|
||||
- Password: `langbot123456`
|
||||
|
||||
_Note: Public demo environment. Do not enter sensitive information._
|
||||
*Note: Public demo environment. Do not enter sensitive information.*
|
||||
|
||||
---
|
||||
|
||||
|
||||
38
README_CN.md
38
README_CN.md
@@ -21,11 +21,11 @@
|
||||
[](https://gitcode.com/RockChinQ/LangBot)
|
||||
|
||||
<a href="https://langbot.app">官网</a> |
|
||||
<a href="https://link.langbot.app/zh/docs/features">特性</a> |
|
||||
<a href="https://link.langbot.app/zh/docs/guide">文档</a> |
|
||||
<a href="https://link.langbot.app/zh/docs/api">API</a> |
|
||||
<a href="https://docs.langbot.app/zh/insight/features.html">特性</a> |
|
||||
<a href="https://docs.langbot.app/zh/insight/guide.html">文档</a> |
|
||||
<a href="https://docs.langbot.app/zh/tags/readme.html">API</a> |
|
||||
<a href="https://space.langbot.app/cloud">Cloud</a> |
|
||||
<a href="https://space.langbot.app">扩展市场</a> |
|
||||
<a href="https://space.langbot.app">插件市场</a> |
|
||||
<a href="https://langbot.featurebase.app/roadmap">路线图</a>
|
||||
|
||||
</div>
|
||||
@@ -34,6 +34,8 @@
|
||||
|
||||
---
|
||||
|
||||
## 什么是 LangBot?
|
||||
|
||||
LangBot 是一个**开源的生产级平台**,用于构建 AI 驱动的即时通信机器人。它将大语言模型(LLM)连接到各种聊天平台,帮助你创建能够对话、执行任务、并集成到现有工作流程中的智能 Agent。
|
||||
|
||||
### 核心能力
|
||||
@@ -41,13 +43,11 @@ LangBot 是一个**开源的生产级平台**,用于构建 AI 驱动的即时
|
||||
- **AI 对话与 Agent** — 多轮对话、工具调用、多模态、流式输出。自带 RAG(知识库),深度集成 [Dify](https://dify.ai)、[Coze](https://coze.com)、[n8n](https://n8n.io)、[Langflow](https://langflow.org) 等 LLMOps 平台。
|
||||
- **全平台支持** — 一套代码,覆盖 QQ、微信、企业微信、飞书、钉钉、Discord、Telegram、Slack、LINE、KOOK 等平台。
|
||||
- **生产就绪** — 访问控制、限速、敏感词过滤、全面监控与异常处理,已被多家企业采用。
|
||||
- **插件生态** — 数百个插件,跨进程的事件驱动架构,组件扩展,适配 [MCP 协议](https://modelcontextprotocol.io/)。
|
||||
- **插件生态** — 数百个插件,事件驱动架构,组件扩展,适配 [MCP 协议](https://modelcontextprotocol.io/)。
|
||||
- **Web 管理面板** — 通过浏览器直观地配置、管理和监控机器人,无需手动编辑配置文件。
|
||||
- **多流水线架构** — 不同机器人用于不同场景,具备全面的监控和异常处理能力。
|
||||
|
||||
[→ 了解更多功能特性](https://link.langbot.app/zh/docs/features)
|
||||
|
||||
📍 实践指南:[5 分钟部署多平台 AI 机器人](https://blog.langbot.app/zh/blog/deploy-ai-bot-in-5-minutes/)、[将 DeepSeek 接入微信、企业微信与 Discord](https://blog.langbot.app/zh/blog/connect-deepseek-to-wechat/)、[让 Dify Agent 跑在 Discord、Telegram 和 Slack 上](https://blog.langbot.app/zh/blog/dify-agent-discord-telegram-slack/),以及[用 n8n 构建多平台 AI 聊天机器人](https://blog.langbot.app/zh/blog/n8n-multi-platform-ai-chatbot/)。
|
||||
[→ 了解更多功能特性](https://docs.langbot.app/zh/insight/features.html)
|
||||
|
||||
---
|
||||
|
||||
@@ -78,7 +78,7 @@ docker compose up -d
|
||||
[](https://zeabur.com/zh-CN/templates/ZKTBDH)
|
||||
[](https://railway.app/template/yRrAyL?referralCode=vogKPF)
|
||||
|
||||
**更多方式:** [Docker](https://link.langbot.app/zh/docs/docker) · [手动部署](https://link.langbot.app/zh/docs/manual-deploy) · [宝塔面板](https://link.langbot.app/zh/docs/bt-panel) · [Kubernetes](./docker/README_K8S.md)
|
||||
**更多方式:** [Docker](https://docs.langbot.app/zh/deploy/langbot/docker.html) · [手动部署](https://docs.langbot.app/zh/deploy/langbot/manual.html) · [宝塔面板](https://docs.langbot.app/zh/deploy/langbot/one-click/bt.html) · [Kubernetes](./docker/README_K8S.md)
|
||||
|
||||
---
|
||||
|
||||
@@ -89,16 +89,13 @@ docker compose up -d
|
||||
| QQ | ✅ | 个人号、官方机器人(频道、私聊、群聊) |
|
||||
| 微信 | ✅ | 个人微信、微信公众号 |
|
||||
| 企业微信 | ✅ | 应用消息、对外客服、智能机器人 |
|
||||
| 飞书 | ✅ | 官方 |
|
||||
| 钉钉 | ✅ | 官方 |
|
||||
| Satori | ✅ | |
|
||||
| Discord | ✅ | 官方 |
|
||||
| Telegram | ✅ | 官方 |
|
||||
| Slack | ✅ | 官方 |
|
||||
| LINE | ✅ | 官方 |
|
||||
| KOOK | ✅ | 官方 |
|
||||
| Email | ✅ | 只 Matrix、Satori |
|
||||
| Matrix | ✅ | 支持多种桥接平台,如 Signal、WhatsApp、Messenger、iMessage、Mattermost、Google Chat、IRC、XMPP、Zulip 等 |
|
||||
| 飞书 | ✅ | |
|
||||
| 钉钉 | ✅ | |
|
||||
| Discord | ✅ | |
|
||||
| Telegram | ✅ | |
|
||||
| Slack | ✅ | |
|
||||
| LINE | ✅ | |
|
||||
| KOOK | ✅ | |
|
||||
|
||||
---
|
||||
|
||||
@@ -129,9 +126,8 @@ docker compose up -d
|
||||
| [302.AI](https://share.302.ai/SuTG99) | 聚合平台 | ✅ |
|
||||
| [小马算力](https://www.tokenpony.cn/453z1) | 聚合平台 | ✅ |
|
||||
| [百宝箱Tbox](https://www.tbox.cn/open) | 智能体平台 | ✅ |
|
||||
| [七牛云Qiniu](https://www.qiniu.com/ai/agent) | 聚合平台 | ✅ |
|
||||
|
||||
[→ 查看完整集成列表](https://link.langbot.app/zh/docs/features)
|
||||
[→ 查看完整集成列表](https://docs.langbot.app/zh/insight/features.html)
|
||||
|
||||
### TTS(语音合成)
|
||||
|
||||
|
||||
33
README_ES.md
33
README_ES.md
@@ -19,9 +19,9 @@
|
||||
[](https://github.com/langbot-app/LangBot/stargazers)
|
||||
|
||||
<a href="https://langbot.app">Inicio</a> |
|
||||
<a href="https://link.langbot.app/en/docs/features">Características</a> |
|
||||
<a href="https://link.langbot.app/en/docs/guide">Documentación</a> |
|
||||
<a href="https://link.langbot.app/en/docs/api">API</a> |
|
||||
<a href="https://docs.langbot.app/en/insight/features.html">Características</a> |
|
||||
<a href="https://docs.langbot.app/en/insight/guide.html">Documentación</a> |
|
||||
<a href="https://docs.langbot.app/en/tags/readme.html">API</a> |
|
||||
<a href="https://space.langbot.app">Mercado de Plugins</a> |
|
||||
<a href="https://langbot.featurebase.app/roadmap">Hoja de Ruta</a>
|
||||
|
||||
@@ -44,9 +44,7 @@ LangBot es una **plataforma de código abierto y grado de producción** para con
|
||||
- **Panel de Gestión Web** — Configure, gestione y monitoree sus bots a través de una interfaz de navegador intuitiva. Sin necesidad de editar YAML.
|
||||
- **Arquitectura Multi-Pipeline** — Diferentes bots para diferentes escenarios, con monitoreo completo y manejo de excepciones.
|
||||
|
||||
[→ Conocer más sobre todas las funcionalidades](https://link.langbot.app/en/docs/features)
|
||||
|
||||
📍 Guías prácticas: [desplegar un bot de IA multiplataforma en 5 minutos](https://blog.langbot.app/en/blog/deploy-ai-bot-in-5-minutes/), [conectar DeepSeek a WeChat, Discord y Telegram](https://blog.langbot.app/en/blog/connect-deepseek-to-wechat/), [ejecutar un Dify Agent en Discord, Telegram y Slack](https://blog.langbot.app/en/blog/dify-agent-discord-telegram-slack/) y [crear un chatbot con n8n](https://blog.langbot.app/en/blog/n8n-multi-platform-ai-chatbot/).
|
||||
[→ Conocer más sobre todas las funcionalidades](https://docs.langbot.app/en/insight/features.html)
|
||||
|
||||
---
|
||||
|
||||
@@ -77,7 +75,7 @@ docker compose up -d
|
||||
[](https://zeabur.com/en-US/templates/ZKTBDH)
|
||||
[](https://railway.app/template/yRrAyL?referralCode=vogKPF)
|
||||
|
||||
**Más opciones:** [Docker](https://link.langbot.app/en/docs/docker) · [Manual](https://link.langbot.app/en/docs/manual-deploy) · [BTPanel](https://link.langbot.app/en/docs/bt-panel) · [Kubernetes](./docker/README_K8S.md)
|
||||
**Más opciones:** [Docker](https://docs.langbot.app/en/deploy/langbot/docker.html) · [Manual](https://docs.langbot.app/en/deploy/langbot/manual.html) · [BTPanel](https://docs.langbot.app/en/deploy/langbot/one-click/bt.html) · [Kubernetes](./docker/README_K8S.md)
|
||||
|
||||
---
|
||||
|
||||
@@ -85,19 +83,17 @@ docker compose up -d
|
||||
|
||||
| Plataforma | Estado | Notas |
|
||||
|----------|--------|-------|
|
||||
| Discord | ✅ | Oficial |
|
||||
| Telegram | ✅ | Oficial |
|
||||
| Slack | ✅ | Oficial |
|
||||
| LINE | ✅ | Oficial |
|
||||
| QQ | ✅ | Personal y API Oficial (Canal, DM, Grupo) |
|
||||
| Discord | ✅ | |
|
||||
| Telegram | ✅ | |
|
||||
| Slack | ✅ | |
|
||||
| LINE | ✅ | |
|
||||
| QQ | ✅ | Personal y API Oficial |
|
||||
| WeCom | ✅ | WeChat Empresarial, CS Externo, AI Bot |
|
||||
| WeChat | ✅ | Personal y Cuenta Oficial |
|
||||
| Lark | ✅ | Oficial |
|
||||
| DingTalk | ✅ | Oficial |
|
||||
| KOOK | ✅ | Oficial |
|
||||
| Lark | ✅ | |
|
||||
| DingTalk | ✅ | |
|
||||
| KOOK | ✅ | |
|
||||
| Satori | ✅ | |
|
||||
| Email | ✅ | Matrix, Satori |
|
||||
| Matrix | ✅ | Admite varias plataformas puenteadas como Signal, WhatsApp, Messenger, iMessage, Mattermost, Google Chat, IRC, XMPP, Zulip y más |
|
||||
|
||||
---
|
||||
|
||||
@@ -126,9 +122,8 @@ docker compose up -d
|
||||
| [ShengSuanYun](https://www.shengsuanyun.com/?from=CH_KYIPP758) | Plataforma GPU | ✅ |
|
||||
| [接口 AI](https://jiekou.ai/) | Pasarela | ✅ |
|
||||
| [302.AI](https://share.302.ai/SuTG99) | Pasarela | ✅ |
|
||||
| [Qiniu](https://www.qiniu.com/ai/agent) | Pasarela | ✅ |
|
||||
|
||||
[→ Ver todas las integraciones](https://link.langbot.app/en/docs/features)
|
||||
[→ Ver todas las integraciones](https://docs.langbot.app/en/insight/features.html)
|
||||
|
||||
---
|
||||
|
||||
|
||||
33
README_FR.md
33
README_FR.md
@@ -19,9 +19,9 @@
|
||||
[](https://github.com/langbot-app/LangBot/stargazers)
|
||||
|
||||
<a href="https://langbot.app">Accueil</a> |
|
||||
<a href="https://link.langbot.app/en/docs/features">Fonctionnalités</a> |
|
||||
<a href="https://link.langbot.app/en/docs/guide">Documentation</a> |
|
||||
<a href="https://link.langbot.app/en/docs/api">API</a> |
|
||||
<a href="https://docs.langbot.app/en/insight/features.html">Fonctionnalités</a> |
|
||||
<a href="https://docs.langbot.app/en/insight/guide.html">Documentation</a> |
|
||||
<a href="https://docs.langbot.app/en/tags/readme.html">API</a> |
|
||||
<a href="https://space.langbot.app">Marché des Plugins</a> |
|
||||
<a href="https://langbot.featurebase.app/roadmap">Feuille de Route</a>
|
||||
|
||||
@@ -44,9 +44,7 @@ LangBot est une **plateforme open-source de niveau production** pour créer des
|
||||
- **Panneau de Gestion Web** — Configurez, gérez et surveillez vos bots via une interface navigateur intuitive. Aucune édition de YAML requise.
|
||||
- **Architecture Multi-Pipeline** — Différents bots pour différents scénarios, avec surveillance complète et gestion des exceptions.
|
||||
|
||||
[→ En savoir plus sur toutes les fonctionnalités](https://link.langbot.app/en/docs/features)
|
||||
|
||||
📍 Guides pratiques : [déployer un bot IA multiplateforme en 5 minutes](https://blog.langbot.app/en/blog/deploy-ai-bot-in-5-minutes/), [connecter DeepSeek à WeChat, Discord et Telegram](https://blog.langbot.app/en/blog/connect-deepseek-to-wechat/), [exécuter un Dify Agent dans Discord, Telegram et Slack](https://blog.langbot.app/en/blog/dify-agent-discord-telegram-slack/) et [créer un chatbot avec n8n](https://blog.langbot.app/en/blog/n8n-multi-platform-ai-chatbot/).
|
||||
[→ En savoir plus sur toutes les fonctionnalités](https://docs.langbot.app/en/insight/features.html)
|
||||
|
||||
---
|
||||
|
||||
@@ -77,7 +75,7 @@ docker compose up -d
|
||||
[](https://zeabur.com/en-US/templates/ZKTBDH)
|
||||
[](https://railway.app/template/yRrAyL?referralCode=vogKPF)
|
||||
|
||||
**Plus d'options :** [Docker](https://link.langbot.app/en/docs/docker) · [Manuel](https://link.langbot.app/en/docs/manual-deploy) · [BTPanel](https://link.langbot.app/en/docs/bt-panel) · [Kubernetes](./docker/README_K8S.md)
|
||||
**Plus d'options :** [Docker](https://docs.langbot.app/en/deploy/langbot/docker.html) · [Manuel](https://docs.langbot.app/en/deploy/langbot/manual.html) · [BTPanel](https://docs.langbot.app/en/deploy/langbot/one-click/bt.html) · [Kubernetes](./docker/README_K8S.md)
|
||||
|
||||
---
|
||||
|
||||
@@ -85,19 +83,17 @@ docker compose up -d
|
||||
|
||||
| Plateforme | Statut | Notes |
|
||||
|----------|--------|-------|
|
||||
| Discord | ✅ | Officiel |
|
||||
| Telegram | ✅ | Officiel |
|
||||
| Slack | ✅ | Officiel |
|
||||
| LINE | ✅ | Officiel |
|
||||
| QQ | ✅ | Personnel & API Officielle (Canal, DM, Groupe) |
|
||||
| Discord | ✅ | |
|
||||
| Telegram | ✅ | |
|
||||
| Slack | ✅ | |
|
||||
| LINE | ✅ | |
|
||||
| QQ | ✅ | Personnel & API Officielle |
|
||||
| WeCom | ✅ | WeChat Entreprise, CS Externe, AI Bot |
|
||||
| WeChat | ✅ | Personnel & Compte Officiel |
|
||||
| Lark | ✅ | Officiel |
|
||||
| DingTalk | ✅ | Officiel |
|
||||
| KOOK | ✅ | Officiel |
|
||||
| Lark | ✅ | |
|
||||
| DingTalk | ✅ | |
|
||||
| KOOK | ✅ | |
|
||||
| Satori | ✅ | |
|
||||
| Email | ✅ | Matrix, Satori |
|
||||
| Matrix | ✅ | Prend en charge plusieurs plateformes via ponts, comme Signal, WhatsApp, Messenger, iMessage, Mattermost, Google Chat, IRC, XMPP, Zulip, etc. |
|
||||
|
||||
---
|
||||
|
||||
@@ -126,9 +122,8 @@ docker compose up -d
|
||||
| [CompShare](https://www.compshare.cn/?ytag=GPU_YY-gh_langbot) | Plateforme GPU | ✅ |
|
||||
| [PPIO](https://ppinfra.com/user/register?invited_by=QJKFYD&utm_source=github_langbot) | Plateforme GPU | ✅ |
|
||||
| [ShengSuanYun](https://www.shengsuanyun.com/?from=CH_KYIPP758) | Plateforme GPU | ✅ |
|
||||
| [Qiniu](https://www.qiniu.com/ai/agent) | Passerelle | ✅ |
|
||||
|
||||
[→ Voir toutes les intégrations](https://link.langbot.app/en/docs/features)
|
||||
[→ Voir toutes les intégrations](https://docs.langbot.app/en/insight/features.html)
|
||||
|
||||
---
|
||||
|
||||
|
||||
35
README_JP.md
35
README_JP.md
@@ -19,9 +19,9 @@
|
||||
[](https://github.com/langbot-app/LangBot/stargazers)
|
||||
|
||||
<a href="https://langbot.app">ホーム</a> |
|
||||
<a href="https://link.langbot.app/ja/docs/features">機能</a> |
|
||||
<a href="https://link.langbot.app/ja/docs/guide">ドキュメント</a> |
|
||||
<a href="https://link.langbot.app/ja/docs/api">API</a> |
|
||||
<a href="https://docs.langbot.app/ja/insight/features.html">機能</a> |
|
||||
<a href="https://docs.langbot.app/ja/insight/guide.html">ドキュメント</a> |
|
||||
<a href="https://docs.langbot.app/ja/tags/readme.html">API</a> |
|
||||
<a href="https://space.langbot.app">プラグインマーケット</a> |
|
||||
<a href="https://langbot.featurebase.app/roadmap">ロードマップ</a>
|
||||
|
||||
@@ -44,9 +44,7 @@ LangBot は、AI搭載のインスタントメッセージングボットを構
|
||||
- **Web管理パネル** — 直感的なブラウザインターフェースからボットの設定、管理、監視が可能。YAML編集は不要。
|
||||
- **マルチパイプラインアーキテクチャ** — 異なるシナリオに異なるボットを配置し、包括的な監視と例外処理を実現。
|
||||
|
||||
[→ すべての機能について詳しく見る](https://link.langbot.app/ja/docs/features)
|
||||
|
||||
📍 実践ガイド: [5分でマルチプラットフォームAIボットをデプロイ](https://blog.langbot.app/en/blog/deploy-ai-bot-in-5-minutes/)、[DeepSeekをWeChat・Discord・Telegramに接続](https://blog.langbot.app/en/blog/connect-deepseek-to-wechat/)、[Dify AgentをDiscord・Telegram・Slackで動かす](https://blog.langbot.app/en/blog/dify-agent-discord-telegram-slack/)、[n8n連携チャットボットを構築](https://blog.langbot.app/en/blog/n8n-multi-platform-ai-chatbot/)。
|
||||
[→ すべての機能について詳しく見る](https://docs.langbot.app/ja/insight/features.html)
|
||||
|
||||
---
|
||||
|
||||
@@ -77,7 +75,7 @@ docker compose up -d
|
||||
[](https://zeabur.com/en-US/templates/ZKTBDH)
|
||||
[](https://railway.app/template/yRrAyL?referralCode=vogKPF)
|
||||
|
||||
**その他:** [Docker](https://link.langbot.app/en/docs/docker) · [手動デプロイ](https://link.langbot.app/en/docs/manual-deploy) · [BTPanel](https://link.langbot.app/en/docs/bt-panel) · [Kubernetes](./docker/README_K8S.md)
|
||||
**その他:** [Docker](https://docs.langbot.app/en/deploy/langbot/docker.html) · [手動デプロイ](https://docs.langbot.app/en/deploy/langbot/manual.html) · [BTPanel](https://docs.langbot.app/en/deploy/langbot/one-click/bt.html) · [Kubernetes](./docker/README_K8S.md)
|
||||
|
||||
---
|
||||
|
||||
@@ -85,19 +83,17 @@ docker compose up -d
|
||||
|
||||
| プラットフォーム | ステータス | 備考 |
|
||||
|----------|--------|-------|
|
||||
| Discord | ✅ | 公式 |
|
||||
| Telegram | ✅ | 公式 |
|
||||
| Slack | ✅ | 公式 |
|
||||
| LINE | ✅ | 公式 |
|
||||
| QQ | ✅ | 個人・公式API(チャンネル・DM・グループ) |
|
||||
| Discord | ✅ | |
|
||||
| Telegram | ✅ | |
|
||||
| Slack | ✅ | |
|
||||
| LINE | ✅ | |
|
||||
| QQ | ✅ | 個人 & 公式API |
|
||||
| WeCom | ✅ | 企業WeChat、外部CS、AIボット |
|
||||
| WeChat | ✅ | 個人・公式アカウント |
|
||||
| Lark | ✅ | 公式 |
|
||||
| DingTalk | ✅ | 公式 |
|
||||
| KOOK | ✅ | 公式 |
|
||||
| WeChat | ✅ | 個人 & 公式アカウント |
|
||||
| Lark | ✅ | |
|
||||
| DingTalk | ✅ | |
|
||||
| KOOK | ✅ | |
|
||||
| Satori | ✅ | |
|
||||
| Email | ✅ | Matrix、Satori |
|
||||
| Matrix | ✅ | Signal、WhatsApp、Messenger、iMessage、Mattermost、Google Chat、IRC、XMPP、Zulip など複数のブリッジ先プラットフォームに対応 |
|
||||
|
||||
---
|
||||
|
||||
@@ -126,9 +122,8 @@ docker compose up -d
|
||||
| [ShengSuanYun](https://www.shengsuanyun.com/?from=CH_KYIPP758) | GPUプラットフォーム | ✅ |
|
||||
| [接口 AI](https://jiekou.ai/) | ゲートウェイ | ✅ |
|
||||
| [302.AI](https://share.302.ai/SuTG99) | ゲートウェイ | ✅ |
|
||||
| [Qiniu](https://www.qiniu.com/ai/agent) | ゲートウェイ | ✅ |
|
||||
|
||||
[→ すべての統合を表示](https://link.langbot.app/en/docs/features)
|
||||
[→ すべての統合を表示](https://docs.langbot.app/en/insight/features.html)
|
||||
|
||||
---
|
||||
|
||||
|
||||
33
README_KO.md
33
README_KO.md
@@ -19,9 +19,9 @@
|
||||
[](https://github.com/langbot-app/LangBot/stargazers)
|
||||
|
||||
<a href="https://langbot.app">홈</a> |
|
||||
<a href="https://link.langbot.app/en/docs/features">기능</a> |
|
||||
<a href="https://link.langbot.app/en/docs/guide">문서</a> |
|
||||
<a href="https://link.langbot.app/en/docs/api">API</a> |
|
||||
<a href="https://docs.langbot.app/en/insight/features.html">기능</a> |
|
||||
<a href="https://docs.langbot.app/en/insight/guide.html">문서</a> |
|
||||
<a href="https://docs.langbot.app/en/tags/readme.html">API</a> |
|
||||
<a href="https://space.langbot.app">플러그인 마켓</a> |
|
||||
<a href="https://langbot.featurebase.app/roadmap">로드맵</a>
|
||||
|
||||
@@ -44,9 +44,7 @@ LangBot은 AI 기반 인스턴트 메시징 봇을 구축하기 위한 **오픈
|
||||
- **웹 관리 패널** — 직관적인 브라우저 인터페이스로 봇을 구성, 관리 및 모니터링. YAML 편집 불필요.
|
||||
- **멀티 파이프라인 아키텍처** — 다양한 시나리오에 맞는 다양한 봇 구성, 종합 모니터링 및 예외 처리.
|
||||
|
||||
[→ 모든 기능 자세히 보기](https://link.langbot.app/en/docs/features)
|
||||
|
||||
📍 실전 가이드: [5분 만에 멀티 플랫폼 AI 봇 배포하기](https://blog.langbot.app/en/blog/deploy-ai-bot-in-5-minutes/), [DeepSeek를 WeChat, Discord, Telegram에 연결하기](https://blog.langbot.app/en/blog/connect-deepseek-to-wechat/), [Dify Agent를 Discord, Telegram, Slack에서 실행하기](https://blog.langbot.app/en/blog/dify-agent-discord-telegram-slack/), [n8n 기반 챗봇 만들기](https://blog.langbot.app/en/blog/n8n-multi-platform-ai-chatbot/).
|
||||
[→ 모든 기능 자세히 보기](https://docs.langbot.app/en/insight/features.html)
|
||||
|
||||
---
|
||||
|
||||
@@ -77,7 +75,7 @@ docker compose up -d
|
||||
[](https://zeabur.com/en-US/templates/ZKTBDH)
|
||||
[](https://railway.app/template/yRrAyL?referralCode=vogKPF)
|
||||
|
||||
**더 많은 옵션:** [Docker](https://link.langbot.app/en/docs/docker) · [수동 배포](https://link.langbot.app/en/docs/manual-deploy) · [BTPanel](https://link.langbot.app/en/docs/bt-panel) · [Kubernetes](./docker/README_K8S.md)
|
||||
**더 많은 옵션:** [Docker](https://docs.langbot.app/en/deploy/langbot/docker.html) · [수동 배포](https://docs.langbot.app/en/deploy/langbot/manual.html) · [BTPanel](https://docs.langbot.app/en/deploy/langbot/one-click/bt.html) · [Kubernetes](./docker/README_K8S.md)
|
||||
|
||||
---
|
||||
|
||||
@@ -85,19 +83,17 @@ docker compose up -d
|
||||
|
||||
| 플랫폼 | 상태 | 비고 |
|
||||
|--------|------|------|
|
||||
| Discord | ✅ | 공식 |
|
||||
| Telegram | ✅ | 공식 |
|
||||
| Slack | ✅ | 공식 |
|
||||
| LINE | ✅ | 공식 |
|
||||
| QQ | ✅ | 개인 및 공식 API (채널, DM, 그룹) |
|
||||
| Discord | ✅ | |
|
||||
| Telegram | ✅ | |
|
||||
| Slack | ✅ | |
|
||||
| LINE | ✅ | |
|
||||
| QQ | ✅ | 개인 및 공식 API |
|
||||
| WeCom | ✅ | 기업 WeChat, 외부 CS, AI Bot |
|
||||
| WeChat | ✅ | 개인 및 공식 계정 |
|
||||
| Lark | ✅ | 공식 |
|
||||
| DingTalk | ✅ | 공식 |
|
||||
| KOOK | ✅ | 공식 |
|
||||
| Lark | ✅ | |
|
||||
| DingTalk | ✅ | |
|
||||
| KOOK | ✅ | |
|
||||
| Satori | ✅ | |
|
||||
| Email | ✅ | Matrix, Satori |
|
||||
| Matrix | ✅ | Signal, WhatsApp, Messenger, iMessage, Mattermost, Google Chat, IRC, XMPP, Zulip 등 여러 브리지 플랫폼 지원 |
|
||||
|
||||
---
|
||||
|
||||
@@ -126,9 +122,8 @@ docker compose up -d
|
||||
| [ShengSuanYun](https://www.shengsuanyun.com/?from=CH_KYIPP758) | GPU 플랫폼 | ✅ |
|
||||
| [接口 AI](https://jiekou.ai/) | 게이트웨이 | ✅ |
|
||||
| [302.AI](https://share.302.ai/SuTG99) | 게이트웨이 | ✅ |
|
||||
| [Qiniu](https://www.qiniu.com/ai/agent) | 게이트웨이 | ✅ |
|
||||
|
||||
[→ 모든 통합 보기](https://link.langbot.app/en/docs/features)
|
||||
[→ 모든 통합 보기](https://docs.langbot.app/en/insight/features.html)
|
||||
|
||||
---
|
||||
|
||||
|
||||
33
README_RU.md
33
README_RU.md
@@ -19,9 +19,9 @@
|
||||
[](https://github.com/langbot-app/LangBot/stargazers)
|
||||
|
||||
<a href="https://langbot.app">Главная</a> |
|
||||
<a href="https://link.langbot.app/en/docs/features">Возможности</a> |
|
||||
<a href="https://link.langbot.app/en/docs/guide">Документация</a> |
|
||||
<a href="https://link.langbot.app/en/docs/api">API</a> |
|
||||
<a href="https://docs.langbot.app/en/insight/features.html">Возможности</a> |
|
||||
<a href="https://docs.langbot.app/en/insight/guide.html">Документация</a> |
|
||||
<a href="https://docs.langbot.app/en/tags/readme.html">API</a> |
|
||||
<a href="https://space.langbot.app">Магазин плагинов</a> |
|
||||
<a href="https://langbot.featurebase.app/roadmap">Дорожная карта</a>
|
||||
|
||||
@@ -44,9 +44,7 @@ LangBot — это **платформа с открытым исходным к
|
||||
- **Веб-панель управления** — Настраивайте, управляйте и мониторьте ваших ботов через интуитивный браузерный интерфейс. Ручное редактирование YAML не требуется.
|
||||
- **Мультиконвейерная архитектура** — Разные боты для разных сценариев с комплексным мониторингом и обработкой исключений.
|
||||
|
||||
[→ Подробнее обо всех возможностях](https://link.langbot.app/en/docs/features)
|
||||
|
||||
📍 Практические руководства: [развернуть мультиплатформенного ИИ-бота за 5 минут](https://blog.langbot.app/en/blog/deploy-ai-bot-in-5-minutes/), [подключить DeepSeek к WeChat, Discord и Telegram](https://blog.langbot.app/en/blog/connect-deepseek-to-wechat/), [запустить Dify Agent в Discord, Telegram и Slack](https://blog.langbot.app/en/blog/dify-agent-discord-telegram-slack/) и [создать чат-бота на n8n](https://blog.langbot.app/en/blog/n8n-multi-platform-ai-chatbot/).
|
||||
[→ Подробнее обо всех возможностях](https://docs.langbot.app/en/insight/features.html)
|
||||
|
||||
---
|
||||
|
||||
@@ -77,7 +75,7 @@ docker compose up -d
|
||||
[](https://zeabur.com/en-US/templates/ZKTBDH)
|
||||
[](https://railway.app/template/yRrAyL?referralCode=vogKPF)
|
||||
|
||||
**Другие варианты:** [Docker](https://link.langbot.app/en/docs/docker) · [Ручная установка](https://link.langbot.app/en/docs/manual-deploy) · [BTPanel](https://link.langbot.app/en/docs/bt-panel) · [Kubernetes](./docker/README_K8S.md)
|
||||
**Другие варианты:** [Docker](https://docs.langbot.app/en/deploy/langbot/docker.html) · [Ручная установка](https://docs.langbot.app/en/deploy/langbot/manual.html) · [BTPanel](https://docs.langbot.app/en/deploy/langbot/one-click/bt.html) · [Kubernetes](./docker/README_K8S.md)
|
||||
|
||||
---
|
||||
|
||||
@@ -85,19 +83,17 @@ docker compose up -d
|
||||
|
||||
| Платформа | Статус | Примечания |
|
||||
|-----------|--------|------------|
|
||||
| Discord | ✅ | Официальный |
|
||||
| Telegram | ✅ | Официальный |
|
||||
| Slack | ✅ | Официальный |
|
||||
| LINE | ✅ | Официальный |
|
||||
| QQ | ✅ | Личный и официальный API (Канал, ЛС, Группа) |
|
||||
| Discord | ✅ | |
|
||||
| Telegram | ✅ | |
|
||||
| Slack | ✅ | |
|
||||
| LINE | ✅ | |
|
||||
| QQ | ✅ | Личный и официальный API |
|
||||
| WeCom | ✅ | Корпоративный WeChat, внешний CS, AI-бот |
|
||||
| WeChat | ✅ | Личный и официальный аккаунт |
|
||||
| Lark | ✅ | Официальный |
|
||||
| DingTalk | ✅ | Официальный |
|
||||
| KOOK | ✅ | Официальный |
|
||||
| Lark | ✅ | |
|
||||
| DingTalk | ✅ | |
|
||||
| KOOK | ✅ | |
|
||||
| Satori | ✅ | |
|
||||
| Email | ✅ | Matrix, Satori |
|
||||
| Matrix | ✅ | Поддерживает несколько платформ через мосты, включая Signal, WhatsApp, Messenger, iMessage, Mattermost, Google Chat, IRC, XMPP, Zulip и другие |
|
||||
|
||||
---
|
||||
|
||||
@@ -126,9 +122,8 @@ docker compose up -d
|
||||
| [CompShare](https://www.compshare.cn/?ytag=GPU_YY-gh_langbot) | Платформа GPU | ✅ |
|
||||
| [PPIO](https://ppinfra.com/user/register?invited_by=QJKFYD&utm_source=github_langbot) | Платформа GPU | ✅ |
|
||||
| [ShengSuanYun](https://www.shengsuanyun.com/?from=CH_KYIPP758) | Платформа GPU | ✅ |
|
||||
| [Qiniu](https://www.qiniu.com/ai/agent) | Шлюз | ✅ |
|
||||
|
||||
[→ Смотреть все интеграции](https://link.langbot.app/en/docs/features)
|
||||
[→ Смотреть все интеграции](https://docs.langbot.app/en/insight/features.html)
|
||||
|
||||
---
|
||||
|
||||
|
||||
33
README_TW.md
33
README_TW.md
@@ -21,9 +21,9 @@
|
||||
[](https://gitcode.com/RockChinQ/LangBot)
|
||||
|
||||
<a href="https://langbot.app">官網</a> |
|
||||
<a href="https://link.langbot.app/zh/docs/features">特性</a> |
|
||||
<a href="https://link.langbot.app/zh/docs/guide">文件</a> |
|
||||
<a href="https://link.langbot.app/zh/docs/api">API</a> |
|
||||
<a href="https://docs.langbot.app/zh/insight/features.html">特性</a> |
|
||||
<a href="https://docs.langbot.app/zh/insight/guide.html">文件</a> |
|
||||
<a href="https://docs.langbot.app/zh/tags/readme.html">API</a> |
|
||||
<a href="https://space.langbot.app">外掛市場</a> |
|
||||
<a href="https://langbot.featurebase.app/roadmap">路線圖</a>
|
||||
|
||||
@@ -46,9 +46,7 @@ LangBot 是一個**開源的生產級平台**,用於建構 AI 驅動的即時
|
||||
- **Web 管理面板** — 透過瀏覽器直觀地配置、管理和監控機器人,無需手動編輯設定檔。
|
||||
- **多流水線架構** — 不同機器人用於不同場景,具備全面的監控和異常處理能力。
|
||||
|
||||
[→ 了解更多功能特性](https://link.langbot.app/zh/docs/features)
|
||||
|
||||
📍 實踐指南:[5 分鐘部署多平台 AI 機器人](https://blog.langbot.app/zh/blog/deploy-ai-bot-in-5-minutes/)、[將 DeepSeek 接入微信、企業微信與 Discord](https://blog.langbot.app/zh/blog/connect-deepseek-to-wechat/)、[讓 Dify Agent 跑在 Discord、Telegram 和 Slack 上](https://blog.langbot.app/zh/blog/dify-agent-discord-telegram-slack/),以及[用 n8n 建構多平台 AI 聊天機器人](https://blog.langbot.app/zh/blog/n8n-multi-platform-ai-chatbot/)。
|
||||
[→ 了解更多功能特性](https://docs.langbot.app/zh/insight/features.html)
|
||||
|
||||
---
|
||||
|
||||
@@ -79,7 +77,7 @@ docker compose up -d
|
||||
[](https://zeabur.com/zh-CN/templates/ZKTBDH)
|
||||
[](https://railway.app/template/yRrAyL?referralCode=vogKPF)
|
||||
|
||||
**更多方式:** [Docker](https://link.langbot.app/zh/docs/docker) · [手動部署](https://link.langbot.app/zh/docs/manual-deploy) · [寶塔面板](https://link.langbot.app/zh/docs/bt-panel) · [Kubernetes](./docker/README_K8S.md)
|
||||
**更多方式:** [Docker](https://docs.langbot.app/zh/deploy/langbot/docker.html) · [手動部署](https://docs.langbot.app/zh/deploy/langbot/manual.html) · [寶塔面板](https://docs.langbot.app/zh/deploy/langbot/one-click/bt.html) · [Kubernetes](./docker/README_K8S.md)
|
||||
|
||||
---
|
||||
|
||||
@@ -87,19 +85,17 @@ docker compose up -d
|
||||
|
||||
| 平台 | 狀態 | 備註 |
|
||||
|------|------|------|
|
||||
| Discord | ✅ | 官方 |
|
||||
| Telegram | ✅ | 官方 |
|
||||
| Slack | ✅ | 官方 |
|
||||
| LINE | ✅ | 官方 |
|
||||
| QQ | ✅ | 個人號、官方機器人(頻道、私聊、群聊) |
|
||||
| 企業微信 | ✅ | 應用訊息、對外客服、智能機器人 |
|
||||
| 微信 | ✅ | 個人微信、微信公眾號 |
|
||||
| 飛書 | ✅ | 官方 |
|
||||
| 釘釘 | ✅ | 官方 |
|
||||
| KOOK | ✅ | 官方 |
|
||||
| 企業微信 | ✅ | 應用訊息、對外客服、智能機器人 |
|
||||
| 飛書 | ✅ | |
|
||||
| 釘釘 | ✅ | |
|
||||
| Discord | ✅ | |
|
||||
| Telegram | ✅ | |
|
||||
| Slack | ✅ | |
|
||||
| LINE | ✅ | |
|
||||
| KOOK | ✅ | |
|
||||
| Satori | ✅ | |
|
||||
| Email | ✅ | 只 Matrix、Satori |
|
||||
| Matrix | ✅ | 支援多種橋接平台,如 Signal、WhatsApp、Messenger、iMessage、Mattermost、Google Chat、IRC、XMPP、Zulip 等 |
|
||||
|
||||
---
|
||||
|
||||
@@ -128,7 +124,6 @@ docker compose up -d
|
||||
| [PPIO](https://ppinfra.com/user/register?invited_by=QJKFYD&utm_source=github_langbot) | GPU 平台 | ✅ |
|
||||
| [接口 AI](https://jiekou.ai/) | 聚合平台 | ✅ |
|
||||
| [302.AI](https://share.302.ai/SuTG99) | 聚合平台 | ✅ |
|
||||
| [Qiniu](https://www.qiniu.com/ai/agent) | 聚合平台 | ✅ |
|
||||
|
||||
### TTS(語音合成)
|
||||
|
||||
@@ -144,7 +139,7 @@ docker compose up -d
|
||||
|-----------|------|
|
||||
| 阿里雲百煉 | [外掛](https://github.com/Thetail001/LangBot_BailianTextToImagePlugin) |
|
||||
|
||||
[→ 查看完整整合列表](https://link.langbot.app/zh/docs/features)
|
||||
[→ 查看完整整合列表](https://docs.langbot.app/zh/insight/features.html)
|
||||
|
||||
---
|
||||
|
||||
|
||||
33
README_VI.md
33
README_VI.md
@@ -19,9 +19,9 @@
|
||||
[](https://github.com/langbot-app/LangBot/stargazers)
|
||||
|
||||
<a href="https://langbot.app">Trang chủ</a> |
|
||||
<a href="https://link.langbot.app/en/docs/features">Tính năng</a> |
|
||||
<a href="https://link.langbot.app/en/docs/guide">Tài liệu</a> |
|
||||
<a href="https://link.langbot.app/en/docs/api">API</a> |
|
||||
<a href="https://docs.langbot.app/en/insight/features.html">Tính năng</a> |
|
||||
<a href="https://docs.langbot.app/en/insight/guide.html">Tài liệu</a> |
|
||||
<a href="https://docs.langbot.app/en/tags/readme.html">API</a> |
|
||||
<a href="https://space.langbot.app">Chợ Plugin</a> |
|
||||
<a href="https://langbot.featurebase.app/roadmap">Lộ trình</a>
|
||||
|
||||
@@ -44,9 +44,7 @@ LangBot là một **nền tảng mã nguồn mở, cấp sản xuất** để x
|
||||
- **Bảng quản lý Web** — Cấu hình, quản lý và giám sát bot thông qua giao diện trình duyệt trực quan. Không cần chỉnh sửa YAML.
|
||||
- **Kiến trúc đa Pipeline** — Các bot khác nhau cho các kịch bản khác nhau, với giám sát toàn diện và xử lý ngoại lệ.
|
||||
|
||||
[→ Tìm hiểu thêm về tất cả tính năng](https://link.langbot.app/en/docs/features)
|
||||
|
||||
📍 Hướng dẫn thực hành: [triển khai bot AI đa nền tảng trong 5 phút](https://blog.langbot.app/en/blog/deploy-ai-bot-in-5-minutes/), [kết nối DeepSeek với WeChat, Discord và Telegram](https://blog.langbot.app/en/blog/connect-deepseek-to-wechat/), [chạy Dify Agent trên Discord, Telegram và Slack](https://blog.langbot.app/en/blog/dify-agent-discord-telegram-slack/) và [xây dựng chatbot với n8n](https://blog.langbot.app/en/blog/n8n-multi-platform-ai-chatbot/).
|
||||
[→ Tìm hiểu thêm về tất cả tính năng](https://docs.langbot.app/en/insight/features.html)
|
||||
|
||||
---
|
||||
|
||||
@@ -77,7 +75,7 @@ docker compose up -d
|
||||
[](https://zeabur.com/en-US/templates/ZKTBDH)
|
||||
[](https://railway.app/template/yRrAyL?referralCode=vogKPF)
|
||||
|
||||
**Thêm tùy chọn:** [Docker](https://link.langbot.app/en/docs/docker) · [Thủ công](https://link.langbot.app/en/docs/manual-deploy) · [BTPanel](https://link.langbot.app/en/docs/bt-panel) · [Kubernetes](./docker/README_K8S.md)
|
||||
**Thêm tùy chọn:** [Docker](https://docs.langbot.app/en/deploy/langbot/docker.html) · [Thủ công](https://docs.langbot.app/en/deploy/langbot/manual.html) · [BTPanel](https://docs.langbot.app/en/deploy/langbot/one-click/bt.html) · [Kubernetes](./docker/README_K8S.md)
|
||||
|
||||
---
|
||||
|
||||
@@ -85,19 +83,17 @@ docker compose up -d
|
||||
|
||||
| Nền tảng | Trạng thái | Ghi chú |
|
||||
|----------|--------|-------|
|
||||
| Discord | ✅ | Chính thức |
|
||||
| Telegram | ✅ | Chính thức |
|
||||
| Slack | ✅ | Chính thức |
|
||||
| LINE | ✅ | Chính thức |
|
||||
| QQ | ✅ | Cá nhân & API chính thức (Kênh, DM, Nhóm) |
|
||||
| Discord | ✅ | |
|
||||
| Telegram | ✅ | |
|
||||
| Slack | ✅ | |
|
||||
| LINE | ✅ | |
|
||||
| QQ | ✅ | Cá nhân & API chính thức |
|
||||
| WeCom | ✅ | WeChat doanh nghiệp, CS bên ngoài, AI Bot |
|
||||
| WeChat | ✅ | Cá nhân & Tài khoản công khai |
|
||||
| Lark | ✅ | Chính thức |
|
||||
| DingTalk | ✅ | Chính thức |
|
||||
| KOOK | ✅ | Chính thức |
|
||||
| Lark | ✅ | |
|
||||
| DingTalk | ✅ | |
|
||||
| KOOK | ✅ | |
|
||||
| Satori | ✅ | |
|
||||
| Email | ✅ | Matrix, Satori |
|
||||
| Matrix | ✅ | Hỗ trợ nhiều nền tảng qua bridge như Signal, WhatsApp, Messenger, iMessage, Mattermost, Google Chat, IRC, XMPP, Zulip và hơn thế nữa |
|
||||
|
||||
---
|
||||
|
||||
@@ -126,9 +122,8 @@ docker compose up -d
|
||||
| [ShengSuanYun](https://www.shengsuanyun.com/?from=CH_KYIPP758) | Nền tảng GPU | ✅ |
|
||||
| [接口 AI](https://jiekou.ai/) | Cổng | ✅ |
|
||||
| [302.AI](https://share.302.ai/SuTG99) | Cổng | ✅ |
|
||||
| [Qiniu](https://www.qiniu.com/ai/agent) | Cổng | ✅ |
|
||||
|
||||
[→ Xem tất cả tích hợp](https://link.langbot.app/en/docs/features)
|
||||
[→ Xem tất cả tích hợp](https://docs.langbot.app/en/insight/features.html)
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -312,7 +312,7 @@ spec:
|
||||
### 参考资源
|
||||
|
||||
- [LangBot 官方文档](https://docs.langbot.app)
|
||||
- [Docker 部署文档](https://link.langbot.app/zh/docs/docker)
|
||||
- [Docker 部署文档](https://docs.langbot.app/zh/deploy/langbot/docker.html)
|
||||
- [Kubernetes 官方文档](https://kubernetes.io/docs/)
|
||||
|
||||
---
|
||||
@@ -625,5 +625,5 @@ spec:
|
||||
### References
|
||||
|
||||
- [LangBot Official Documentation](https://docs.langbot.app)
|
||||
- [Docker Deployment Guide](https://link.langbot.app/zh/docs/docker)
|
||||
- [Docker Deployment Guide](https://docs.langbot.app/zh/deploy/langbot/docker.html)
|
||||
- [Kubernetes Official Documentation](https://kubernetes.io/docs/)
|
||||
|
||||
@@ -18,40 +18,6 @@ services:
|
||||
networks:
|
||||
- langbot_network
|
||||
|
||||
# The Box sandbox runtime is optional. It is only started when you run
|
||||
# ``docker compose --profile box up`` (or ``docker compose --profile all
|
||||
# up``). With Box off, LangBot keeps the dashboard / skills list visible
|
||||
# (read-only) but disables sandbox tools, skill add/edit and stdio MCP —
|
||||
# set ``box.enabled: false`` in ``data/config.yaml`` (or
|
||||
# ``BOX__ENABLED=false`` in the langbot service env below) to match.
|
||||
langbot_box:
|
||||
image: rockchin/langbot:latest
|
||||
container_name: langbot_box
|
||||
profiles: ["box", "all"]
|
||||
volumes:
|
||||
# Keep the source and target path identical because langbot_box uses the
|
||||
# host Docker socket to create sandbox containers. Override
|
||||
# LANGBOT_BOX_ROOT with an absolute path if you do not want the default.
|
||||
- ${LANGBOT_BOX_ROOT:-${PWD}/data/box}:${LANGBOT_BOX_ROOT:-${PWD}/data/box}
|
||||
# Mount container runtime socket for Box sandbox backend.
|
||||
# Uncomment the one that matches your container runtime:
|
||||
# - /var/run/podman/podman.sock:/var/run/podman/podman.sock # Podman
|
||||
- /var/run/docker.sock:/var/run/docker.sock # Docker
|
||||
restart: on-failure
|
||||
environment:
|
||||
- TZ=Asia/Shanghai
|
||||
# The Box runtime does NOT read box.local.* from config.yaml or env; it
|
||||
# receives its configuration from LangBot via the INIT RPC action.
|
||||
# Do not add LANGBOT_BOX_* / BOX__* here — they would be silently ignored.
|
||||
# Launched through the same CLI entry point as the plugin runtime
|
||||
# (`langbot_plugin.cli.__init__ <subcommand>`). WebSocket is the default
|
||||
# control transport — mirrors `rt`, which also runs with no flag. Pass
|
||||
# `-s` / `--stdio-control` only for the stdio mode LangBot uses outside
|
||||
# containers.
|
||||
command: ["uv", "run", "--no-sync", "-m", "langbot_plugin.cli.__init__", "box"]
|
||||
networks:
|
||||
- langbot_network
|
||||
|
||||
langbot:
|
||||
image: rockchin/langbot:latest
|
||||
container_name: langbot
|
||||
@@ -60,13 +26,6 @@ services:
|
||||
restart: on-failure
|
||||
environment:
|
||||
- TZ=Asia/Shanghai
|
||||
# Unified env-override convention: SECTION__SUBSECTION__KEY overrides the
|
||||
# matching config.yaml field (see LoadConfigStage). These map onto
|
||||
# box.local.* and are forwarded to the Box runtime via INIT RPC.
|
||||
- BOX__LOCAL__HOST_ROOT=${LANGBOT_BOX_ROOT:-${PWD}/data/box}
|
||||
- BOX__LOCAL__DEFAULT_WORKSPACE=default
|
||||
- BOX__LOCAL__SKILLS_ROOT=skills
|
||||
- BOX__LOCAL__ALLOWED_MOUNT_ROOTS=${LANGBOT_BOX_ROOT:-${PWD}/data/box}
|
||||
ports:
|
||||
- 5300:5300 # For web ui and webhook callback
|
||||
- 2280-2285:2280-2285 # For platform reverse connection
|
||||
|
||||
@@ -1,594 +0,0 @@
|
||||
# Box 系统架构深度分析
|
||||
|
||||
> 更新日期: 2026-05-19
|
||||
> 分支: `feat/sandbox` (LangBot + langbot-plugin-sdk)
|
||||
> 相关文档: [问题清单](./box-issues.md) | [Session 作用域](./box-session-scope.md) | [Runtime 对比](./box-vs-plugin-runtime.md) | [测试覆盖](./box-test-coverage.md) | [toB 分析](./box-tob-analysis.md)
|
||||
|
||||
---
|
||||
|
||||
## 1. 全局架构
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────────────────────────┐
|
||||
│ LangBot 主进程 │
|
||||
│ │
|
||||
│ LocalAgentRunner ──> ToolManager ──> NativeToolLoader │
|
||||
│ │ │ │ │
|
||||
│ │ │ exec / read / write / edit │
|
||||
│ │ │ glob / grep │
|
||||
│ │ │ │
|
||||
│ │ ├──> MCPLoader ──> BoxStdioSession │
|
||||
│ │ │ (shared 容器, 多 process) │
|
||||
│ │ │ │
|
||||
│ │ ├──> SkillToolLoader (activate 工具) │
|
||||
│ │ │ │
|
||||
│ │ ├──> SkillAuthoringToolLoader │
|
||||
│ │ │ │
|
||||
│ │ └──> PluginToolLoader │
|
||||
│ │ │
|
||||
│ BoxService (门面) │
|
||||
│ ├─ Profile 管理 (locked 字段) │
|
||||
│ ├─ Host mount 校验 (allowed_mount_roots) │
|
||||
│ ├─ Workspace quota 检查 │
|
||||
│ ├─ 输出截断 (head+tail) │
|
||||
│ ├─ Session ID 模板解析 (resolve_box_session_id) │
|
||||
│ ├─ 技能挂载组装 (build_skill_extra_mounts) │
|
||||
│ ├─ 重连循环 (_reconnect_loop, 指数退避) │
|
||||
│ └─ BoxRuntimeConnector │
|
||||
│ ├─ 心跳 loop (20s ping) │
|
||||
│ └─ ActionRPCBoxClient │
|
||||
│ │ Action RPC (stdio 或 WebSocket) │
|
||||
│ │
|
||||
│ SkillManager (skill_mgr) │
|
||||
│ └─ 从 Box runtime 拉取 skills, 不可用时回落 data/skills │
|
||||
└──────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌──────────────────────────────────────────────────────────────────┐
|
||||
│ Box Runtime 进程 (SDK 侧) │
|
||||
│ │
|
||||
│ BoxServerHandler (Action RPC 处理, INIT 配置注入) │
|
||||
│ │ │
|
||||
│ BoxRuntime (session 管理 / 进程生命周期 / TTL reaper) │
|
||||
│ │ └─ session.managed_processes: dict[pid, _ManagedProcess]
|
||||
│ │ │
|
||||
│ Backend (启动时根据 box.backend 配置选择): │
|
||||
│ DockerBackend ──┐ │
|
||||
│ PodmanBackend ──┤── CLISandboxBackend │
|
||||
│ NsjailBackend ──┘ (本地 CLI 或 fallback 到容器内 CLI) │
|
||||
│ E2BBackend (云沙箱, 需要 E2B_API_KEY) │
|
||||
│ │
|
||||
│ BoxSkillStore │
|
||||
│ ├─ list / get / create / update / delete │
|
||||
│ ├─ scan_skill_directory / read_skill_file / write_skill_file │
|
||||
│ └─ preview_skill_zip / install_skill_zip (zip 或 GitHub) │
|
||||
│ │
|
||||
│ aiohttp 单端口服务 (默认 :5410): │
|
||||
│ /rpc/ws — Action RPC │
|
||||
│ /v1/sessions/{id}/managed-process/ws — 默认 process │
|
||||
│ /v1/sessions/{id}/managed-process/{pid}/ws — 指定 process │
|
||||
└──────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌──────────────────────────────────────────────────────────────────┐
|
||||
│ 容器 / 沙箱 (Docker/Podman 容器, nsjail sandbox, 或 E2B 远程沙箱) │
|
||||
│ - 隔离文件系统 / 网络 / PID 命名空间 │
|
||||
│ - 资源限制 (CPU, 内存, PID 数, 可选 workspace 配额) │
|
||||
│ - 主挂载 (host_path → mount_path) + 任意条 extra_mounts │
|
||||
│ └─ Skills 通过 extra_mounts 挂在 /workspace/.skills/<name> │
|
||||
│ - exec: 用户命令在此执行 │
|
||||
│ - managed process: 多个长驻进程并存 (MCP Server / 自定义服务) │
|
||||
└──────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
**核心设计原则**:
|
||||
- Box Runtime 作为独立进程运行,通过 Action RPC 与 LangBot 主进程通信,两者复用 SDK 的 IO 层(Handler → Connection → Controller)
|
||||
- 一个 session_id 对应一个容器/沙箱实例。同一 session 内可并存多条 mount 与多个 managed process
|
||||
- Skill / 默认 exec / MCP Server 共享同一个 session 容器(详见 [box-session-scope.md](./box-session-scope.md))
|
||||
|
||||
---
|
||||
|
||||
## 2. LangBot 侧模块
|
||||
|
||||
### 2.1 BoxService (`pkg/box/service.py`, 722 行)
|
||||
|
||||
应用层门面,协调 Profile、安全校验、配额、连接、Skill 挂载与 Session 模板:
|
||||
|
||||
主要公开方法(按定义顺序):
|
||||
|
||||
```
|
||||
BoxService
|
||||
├─ initialize() 连接 Box Runtime + 默认 workspace 准备
|
||||
├─ _on_runtime_disconnect(connector) 触发重连
|
||||
├─ _reconnect_loop(connector) 指数退避重连
|
||||
├─ available (property) 连接状态
|
||||
│
|
||||
├─ resolve_box_session_id(query) 从 pipeline 模板解析 session_id
|
||||
├─ build_skill_extra_mounts(query) 组装 pipeline-bound skill 的挂载列表
|
||||
│
|
||||
├─ execute_tool(parameters, query) Agent 调用 exec 时的入口
|
||||
│ ├─ _apply_profile / build_spec
|
||||
│ ├─ _validate_host_mount
|
||||
│ ├─ _enforce_workspace_quota (phase=pre)
|
||||
│ ├─ client.execute(spec)
|
||||
│ ├─ _enforce_workspace_quota (phase=post)
|
||||
│ └─ _truncate (stdout/stderr)
|
||||
│
|
||||
├─ execute_spec_payload(spec_payload, ...) 内部入口(其他 loader 调用)
|
||||
├─ create_session(spec_payload, ...) 显式创建 session
|
||||
├─ start_managed_process(session_id, ...) 启动 managed process
|
||||
├─ get_managed_process(session_id, pid) 查询进程状态(pid 默认 'default')
|
||||
├─ stop_managed_process(session_id, pid) 单独停止某个 managed process
|
||||
├─ get_managed_process_websocket_url(...) 返回 WS attach URL
|
||||
│
|
||||
├─ list_skills() / get_skill(name) Skill 元数据
|
||||
├─ create_skill / update_skill / delete_skill Skill CRUD
|
||||
├─ scan_skill_directory(path) 扫描目录
|
||||
├─ list_skill_files / read_skill_file / write_skill_file
|
||||
├─ preview_skill_zip / install_skill_zip zip / GitHub 安装
|
||||
│
|
||||
├─ shutdown() / dispose() 清理:RPC SHUTDOWN + 进程终止
|
||||
├─ get_status() / get_sessions() / get_recent_errors()
|
||||
└─ get_system_guidance() LLM 系统提示
|
||||
```
|
||||
|
||||
**Profile 系统**: 4 个内置 Profile(`default` / `offline_readonly` / `network_basic` / `network_extended`),`locked` frozenset 字段不可被 LLM 覆盖。参数合并顺序:Profile defaults → LLM 请求参数 → locked 强制值。
|
||||
|
||||
**输出截断**: 默认 4000 字符上限,保留前 60% + 后 40%,中间插入 `[...truncated...]`。
|
||||
|
||||
**Skill 挂载合并**: `execute_tool()` 调用时,`build_skill_extra_mounts(query)` 会把当前 pipeline-bound 的所有 skill 的 `package_root` 作为 `extra_mounts` 加入 BoxSpec,挂在 `/workspace/.skills/<name>`。LLM 通过 `activate` 工具显式激活某个 skill 后,工具调用才允许引用这个 skill 的虚拟路径。
|
||||
|
||||
### 2.2 BoxRuntimeConnector (`pkg/box/connector.py`, 357 行)
|
||||
|
||||
管理与 Box Runtime 的通信连接:
|
||||
|
||||
- **本地 stdio**: Unix/macOS 默认路径,fork `python -m langbot_plugin.cli.__init__ box -s --ws-control-port {port}` 子进程(与 plugin runtime 统一走 `lbp` CLI 入口)
|
||||
- **本地 subprocess + WS**: Windows 本地(asyncio ProactorEventLoop 不支持 stdio pipe)
|
||||
- **远程 WebSocket**: Docker 部署 / `box.runtime.endpoint` 显式配置时,连接 `ws://{host}:{port}/rpc/ws`
|
||||
- **同步等待**: `asyncio.Event` + `wait_for(timeout=30s)` 模式确认连接
|
||||
- **心跳**: `_heartbeat_loop()` 每 20s 调用 `ping()`,失败仅 DEBUG 日志(断开检测靠 connection close)
|
||||
- **重连**: `runtime_disconnect_callback` 由 BoxService 提供,触发 `_reconnect_loop`
|
||||
- **INIT 注入**: 连接建立后立即下发当前 `box.*` 配置子树(剔除 `runtime` 私有字段),Runtime 据此初始化 backend
|
||||
|
||||
> **历史改进**: 2026-04-16 版本本文档曾列 P0 「Box 无心跳 / 无重连」,已修复(commit `2dfd9d5d`、`c6882cf`、`5029d9c` 等)。
|
||||
|
||||
### 2.3 BoxWorkspaceSession 工具 (`pkg/box/workspace.py`, 413 行)
|
||||
|
||||
此文件目前提供两类能力:
|
||||
|
||||
1. **路径与命令重写工具函数** — `normalize_host_path` / `rewrite_mounted_path` / `unwrap_venv_path` / `rewrite_venv_command` / `infer_workspace_host_path`,被 MCP loader 与 Skill 路径解析共用。
|
||||
2. **`BoxWorkspaceSession`** — 围绕 BoxService 的轻量包装,专供 MCP-in-Box 场景使用(管理一个共享 session 的 session_id、构建挂载 payload、stage host 文件到共享 workspace)。
|
||||
|
||||
**变化点**: 早期 Skill exec 会为每个 skill 创建独立 BoxWorkspaceSession(独占 session);当前实现已转为 `extra_mounts` 模式,Skill 不再独占容器,只追加挂载。这部分 wrapping 逻辑已从 native loader 移除。
|
||||
|
||||
### 2.4 policy.py (`pkg/box/policy.py`, 98 行) — 仍是死代码
|
||||
|
||||
三层安全策略设计(`SandboxPolicy` / `ToolPolicy` / `ElevatedPolicy`),全项目无任何导入或调用。详见 [问题清单 #1](./box-issues.md)。
|
||||
|
||||
### 2.5 SkillManager (`pkg/skill/manager.py`, 186 行)
|
||||
|
||||
```
|
||||
SkillManager
|
||||
├─ initialize() 调用 reload_skills()
|
||||
├─ reload_skills() 先从 Box runtime list_skills(),
|
||||
│ 不可用则回落 data/skills/ 扫描
|
||||
├─ refresh_skill_from_disk() 单 skill 重新加载
|
||||
├─ get_skill_by_name(name)
|
||||
└─ get_managed_skills_root() 返回 Box 视角的 skills_root 路径
|
||||
```
|
||||
|
||||
skill 元数据通过 `parse_frontmatter` 解析 `SKILL.md` 头部(`name` / `description` / `instructions`),不再做整体扫描的代价(典型 < 50 个)。
|
||||
|
||||
### 2.6 Skill activation (`pkg/skill/activation.py`, 33 行) + Skill loader 辅助
|
||||
|
||||
历史上 skill 通过 LLM 在文本中输出 `[ACTIVATE_SKILL:name]` 标记激活;当前已改为 **Tool Call 机制**:
|
||||
|
||||
- `SkillToolLoader` (`pkg/provider/tools/loaders/skill.py`, 157 行) 暴露 `activate` 工具,参数为 skill 名
|
||||
- 工具实现调用 `register_activated_skill(query, skill_data)`,将激活态写入 `query.variables['_activated_skills']`
|
||||
- 这种 KV-cache-friendly 模式对齐 Claude Code 设计;详见 [box-session-scope.md §4.3](./box-session-scope.md) 的 Tool Call 描述
|
||||
|
||||
`activation.py` 现仅保留对外辅助函数(pipeline 层调用 loader 的 `register_activated_skill`)。
|
||||
|
||||
---
|
||||
|
||||
## 3. SDK 侧模块
|
||||
|
||||
### 3.1 BoxRuntime (`box/runtime.py`, 599 行)
|
||||
|
||||
核心编排器,管理 session 生命周期与 backend 调度:
|
||||
|
||||
```
|
||||
Session 生命周期:
|
||||
|
||||
Client EXEC / CREATE_SESSION
|
||||
│
|
||||
▼
|
||||
_get_or_create_session(spec)
|
||||
├─ _reap_expired_sessions_locked() 清理 TTL 过期 session
|
||||
├─ 已存在? → _assert_session_compatible() → 复用
|
||||
├─ Backend session 失踪? → 重建 (commit c6882cf)
|
||||
└─ 新建? → backend.start_session(spec) → 创建容器
|
||||
│ └─ 应用 spec.extra_mounts (多挂载)
|
||||
▼
|
||||
execute(spec)
|
||||
├─ 获取 session lock (每 session 独立)
|
||||
├─ backend.exec(session, spec) 在容器中执行命令
|
||||
├─ 更新 last_used_at
|
||||
└─ 超时? → 销毁 session
|
||||
│
|
||||
▼
|
||||
Session 保持存活直到:
|
||||
├─ TTL 过期 (默认 300s,下次操作时清理)
|
||||
├─ 执行超时 (自动销毁)
|
||||
├─ 客户端 DELETE_SESSION
|
||||
└─ SHUTDOWN
|
||||
```
|
||||
|
||||
**关键设计**:
|
||||
- 每 session 有独立 `asyncio.Lock`,同一 session 内的命令串行执行
|
||||
- 每 session 维护 `managed_processes: dict[process_id, _ManagedProcess]`,支持多个长驻进程并存(MCP / 自定义)
|
||||
- 全局 `_lock` 保护 `_sessions` dict 的读写
|
||||
- 兼容性检查:比较核心 spec 字段,`image` 字段对不支持自定义镜像的 backend(nsjail/E2B)会跳过
|
||||
|
||||
**Backend 选择 (`_select_backend`)**: 优先级
|
||||
1. 显式 `box.backend` 配置(`docker` / `nsjail` / `e2b`)
|
||||
2. `local` (默认) → Docker / Podman / nsjail CLI 顺序探测
|
||||
3. `get_status` 调用时若当前 backend 不可用,会尝试重新选择 (commit `e5617c7`)
|
||||
|
||||
### 3.2 Backend 系统
|
||||
|
||||
#### CLISandboxBackend (`box/backend.py`, 411 行)
|
||||
|
||||
Docker / Podman 公共基类:
|
||||
|
||||
```
|
||||
start_session(spec):
|
||||
1. validate_sandbox_security(spec)
|
||||
2. docker/podman run -d --rm --name <name>
|
||||
--network none (可选)
|
||||
--cpus/--memory/--pids-limit
|
||||
--read-only + --tmpfs /tmp
|
||||
-v <host>:<mount>:<mode> 主挂载
|
||||
-v <extra.host>:<extra.mount>:.. 额外挂载 (extra_mounts)
|
||||
<image> sh -lc 'while true; do sleep 3600; done'
|
||||
3. 返回 BoxSessionInfo
|
||||
|
||||
exec(session, spec):
|
||||
docker/podman exec -e KEY=VAL <container>
|
||||
sh -lc 'mkdir -p <workdir> && cd <workdir> && <cmd>'
|
||||
|
||||
start_managed_process(session, spec):
|
||||
docker/podman exec -i <container>
|
||||
sh -lc 'mkdir -p <cwd> && cd <cwd> && exec <command> <args>'
|
||||
返回 asyncio.subprocess.Process (stdin/stdout PIPE)
|
||||
```
|
||||
|
||||
容器以 idle 进程启动,实际命令通过 `docker exec` 执行。`--rm` 确保容器退出时自动清理。
|
||||
|
||||
**Windows 支持**: backend 内对 Windows 路径处理与 subprocess 调用做了适配(commit `120817a`)。
|
||||
|
||||
**孤儿清理**: 启动时枚举 `langbot.box=true` 标签的容器,instance_id 不匹配的强制删除。
|
||||
|
||||
#### NsjailBackend (`box/nsjail_backend.py`, 552 行)
|
||||
|
||||
轻量级 Linux 沙箱(无容器引擎依赖):
|
||||
|
||||
- 使用 namespace 隔离(user/mount/pid/ipc/uts/cgroup/net)
|
||||
- 挂载宿主 `/usr`/`/lib`/`/bin`/`/sbin` 只读 + 选定 `/etc` 条目
|
||||
- 每 session 创建独立目录(workspace/tmp/home)
|
||||
- 资源限制: cgroup v2 优先,fallback 到 rlimit
|
||||
- **CLI 兼容**: 通过 `shutil.which(self._nsjail_bin)` 检测系统安装版 nsjail;不存在时再尝试容器内 nsjail(commit `686fcc0`、`feed530`)
|
||||
- **无自定义镜像**: 使用宿主 OS,`image` 字段固定为 `'host'`,兼容性检查跳过 image
|
||||
|
||||
#### E2BBackend (`box/e2b_backend.py`, 429 行)
|
||||
|
||||
云沙箱后端(commit `75b547f` 引入):
|
||||
|
||||
- 通过 `e2b` SDK 与 E2B 平台通信
|
||||
- 配置:`box.e2b.api_key` / `api_url` / `template`
|
||||
- 支持 `extra_mounts`(commit `0fea9b1` 同步上传文件)
|
||||
- 无本地容器引擎依赖,适合无 Docker 的部署或 SaaS 多租户场景
|
||||
- 不支持自定义 image 字段,由 template 控制
|
||||
|
||||
### 3.3 Server (`box/server.py`, 508 行)
|
||||
|
||||
单端口 aiohttp 服务(默认 5410),通过路径区分(commit `8c71ec5` 合并端口):
|
||||
|
||||
1. **Action RPC** (`/rpc/ws`): `BoxServerHandler` 处理所有 action,包括 `INIT` 配置注入、skill store 操作等
|
||||
2. **WS Relay** (`/v1/sessions/{id}/managed-process/ws` 与 `/v1/sessions/{id}/managed-process/{pid}/ws`): 双向桥接 WebSocket ↔ 指定 managed process stdin/stdout
|
||||
|
||||
stdio 模式同样会在 5410 启动 aiohttp,专门承担 managed process attach;Action RPC 走 stdin/stdout。
|
||||
|
||||
### 3.4 Client (`box/client.py`, 377 行)
|
||||
|
||||
`ActionRPCBoxClient` 封装 `Handler.call_action()` 调用:
|
||||
|
||||
- 25+ 方法对应 25+ 个 RPC action(exec / session / managed-process / skill / status / shutdown)
|
||||
- 错误还原: `_translate_action_error()` 通过字符串前缀匹配还原 SDK 侧异常类型
|
||||
- `execute()` timeout = 300s,其他默认 15s
|
||||
- `BoxRuntimeClient` 是 ABC,供后续可能的非 RPC 实现复用
|
||||
|
||||
包级别 `__init__.py` 显式导出:`BoxRuntimeClient`、`ActionRPCBoxClient`(commit `df9c722`)。
|
||||
|
||||
### 3.5 Actions (`box/actions.py`, 34 行)
|
||||
|
||||
`LangBotToBoxAction` 枚举共定义 **25 个** action:
|
||||
|
||||
| 类别 | Actions |
|
||||
|------|---------|
|
||||
| 控制 | `INIT`、`HEALTH`、`STATUS`、`GET_BACKEND_INFO`、`SHUTDOWN` |
|
||||
| 执行 | `EXEC` |
|
||||
| Session | `CREATE_SESSION` / `GET_SESSION` / `GET_SESSIONS` / `DELETE_SESSION` |
|
||||
| Managed Process | `START_MANAGED_PROCESS` / `GET_MANAGED_PROCESS` / `STOP_MANAGED_PROCESS` |
|
||||
| Skill | `LIST_SKILLS` / `GET_SKILL` / `CREATE_SKILL` / `UPDATE_SKILL` / `DELETE_SKILL` / `SCAN_SKILL_DIRECTORY` / `LIST_SKILL_FILES` / `READ_SKILL_FILE` / `WRITE_SKILL_FILE` / `PREVIEW_SKILL_ZIP` / `INSTALL_SKILL_ZIP` |
|
||||
|
||||
### 3.6 Models (`box/models.py`, 331 行)
|
||||
|
||||
核心数据模型:
|
||||
|
||||
| 模型 | 用途 |
|
||||
|------|------|
|
||||
| `BoxNetworkMode` | `OFF` / `ON` |
|
||||
| `BoxExecutionStatus` | `COMPLETED` / `TIMED_OUT` |
|
||||
| `BoxHostMountMode` | `NONE` / `READ_ONLY` / `READ_WRITE` |
|
||||
| `BoxManagedProcessStatus` | `RUNNING` / `EXITED` |
|
||||
| `BoxMountSpec` | 单条挂载(host_path/mount_path/mode)— **新增** |
|
||||
| `BoxSpec` | 执行请求;新增 `extra_mounts: list[BoxMountSpec]`、`persistent`、`workspace_quota_mb` |
|
||||
| `BoxProfile` | 4 个内置 Profile + `locked` frozenset |
|
||||
| `BoxSessionInfo` | Session 状态(含 backend_name/created_at/last_used_at) |
|
||||
| `BoxManagedProcessSpec` | 长驻进程参数(process_id/command/args/env/cwd) |
|
||||
| `BoxManagedProcessInfo` | 进程状态(status/exit_code/stderr_preview/attached) |
|
||||
| `BoxExecutionResult` | 执行结果(status/exit_code/stdout/stderr/duration_ms) |
|
||||
|
||||
`BoxSpec` 校验器: `workdir` 默认继承 `mount_path`;`host_path` 支持 POSIX 和 Windows 路径;设置 `host_path` 时 `workdir` 必须在 `mount_path` 下。
|
||||
|
||||
### 3.7 BoxSkillStore (`box/skill_store.py`, 647 行)
|
||||
|
||||
新增模块(commit `4ab3502`),把 skill 持久化收归 Box runtime:
|
||||
|
||||
```
|
||||
BoxSkillStore
|
||||
├─ list_skills() / get_skill(name)
|
||||
├─ create_skill(data) / update_skill(name, data) / delete_skill(name)
|
||||
├─ scan_skill_directory(path) 扫描目录返回候选 skill 包列表
|
||||
├─ list_skill_files(name, path) 浏览 skill 内文件树
|
||||
├─ read_skill_file(name, path) / write_skill_file(name, path, content)
|
||||
├─ preview_skill_zip(zip_bytes, ...) 不落盘预览 zip 内容
|
||||
└─ install_skill_zip(zip_bytes, ...) 解压、校验、复制到 skills_root
|
||||
└─ 支持 source_subdir / target_suffix(commit 1aa043f)
|
||||
```
|
||||
|
||||
GitHub 安装路径:HTTP 层(`api/http/service/skill.py`)先 `git clone` 拉取,再走 `install_skill_zip` 或 directory 路径。Skill 文件存放于 `box.local.skills_root`(默认 `skills`,相对 `host_root`),容器内对应 `/workspace/.skills/`。
|
||||
|
||||
### 3.8 Security (`box/security.py`, 52 行)
|
||||
|
||||
`validate_sandbox_security()`: 黑名单校验 host_path,阻止挂载 `/etc`/`/proc`/`/sys`/`/dev`/`/root`/`/boot` 及 Docker/Podman socket。
|
||||
|
||||
**已知缺陷**: 根路径 `/` 未拦截,用户 home 目录未拦截,是 denylist 而非 allowlist 策略。详见 [问题清单 #5](./box-issues.md)。
|
||||
|
||||
### 3.9 Errors (`box/errors.py`, 33 行)
|
||||
|
||||
| 异常类型 | 含义 |
|
||||
|----------|------|
|
||||
| `BoxError` | 基类 |
|
||||
| `BoxValidationError` | spec/参数校验失败 |
|
||||
| `BoxBackendUnavailableError` | 无可用 backend |
|
||||
| `BoxRuntimeUnavailableError` | Runtime 服务不可用 |
|
||||
| `BoxSessionConflictError` | session 已存在但 spec 不兼容 |
|
||||
| `BoxSessionNotFoundError` | session 不存在 |
|
||||
| `BoxManagedProcessConflictError` | session 已有同名 process |
|
||||
| `BoxManagedProcessNotFoundError` | process 不存在 |
|
||||
|
||||
---
|
||||
|
||||
## 4. 工具系统集成
|
||||
|
||||
### 4.1 ToolManager 编排 (`toolmgr.py`)
|
||||
|
||||
```
|
||||
ToolManager.initialize()
|
||||
├─ NativeToolLoader (exec / read / write / edit / glob / grep)
|
||||
├─ PluginToolLoader (插件工具)
|
||||
├─ MCPLoader (MCP Server 工具)
|
||||
├─ SkillToolLoader (activate 工具 — Tool Call 激活)
|
||||
└─ SkillAuthoringToolLoader (Skill CRUD)
|
||||
|
||||
工具调用优先级: native → plugin → mcp → skill → skill_authoring
|
||||
```
|
||||
|
||||
### 4.2 Native Tools (`native.py`, 846 行)
|
||||
|
||||
| 工具 | 是否在 Box 中执行 | 是否访问宿主文件系统 |
|
||||
|------|:---:|:---:|
|
||||
| `exec` | 是 | 否 |
|
||||
| `read` | **否** | **是** — 直接 `open()` 宿主文件 |
|
||||
| `write` | **否** | **是** — 直接 `open()` 宿主文件 |
|
||||
| `edit` | **否** | **是** — 直接 `open()` 宿主文件 |
|
||||
| `glob` | **否** | **是** — 直接遍历宿主目录 |
|
||||
| `grep` | **否** | **是** — 直接读宿主文件 |
|
||||
|
||||
**沙箱边界不对称**: 这是刻意的设计权衡 — `read`/`write`/`edit`/`glob`/`grep` 绕过沙箱以获得性能(避免容器 I/O 开销与跨进程拷贝),但意味着 LLM 可以直接读写 `allowed_mount_roots` 下任何文件。Skill 路径经 `_resolve_host_path()` 重写,禁止穿越 `package_root`。
|
||||
|
||||
**exec 的 Skill 分支**: 命令中引用 `/workspace/.skills/<name>` 的 skill 时:
|
||||
1. 验证 skill 已激活
|
||||
2. 单次 exec 只能引用一个 skill 包
|
||||
3. 若 skill 是 Python 项目(有 `requirements.txt` 或 `pyproject.toml`),命令会被 venv bootstrap 包裹(在 skill 挂载点内创建 `.venv`)
|
||||
4. 调用 `box_service.execute_tool()` → 走默认 session_id 与已组装好的 `extra_mounts`,**不再为每 skill 起独立 session**
|
||||
|
||||
### 4.3 MCP-in-Box (`mcp_stdio.py`, 354 行)
|
||||
|
||||
`BoxStdioSessionRuntime` 让 MCP stdio 服务器在 Box 容器中运行,**共享 session、多 process**模式(commit `529088e`):
|
||||
|
||||
```
|
||||
initialize()
|
||||
1. 复用/创建共享 session (session_id = _build_box_session_id())
|
||||
- persistent=True,长期保持
|
||||
2. workspace.execute_raw(install_cmd) 安装依赖 (可选)
|
||||
3. 将每个 MCP server 文件 stage 到 /workspace/.mcp/<process_id>/
|
||||
4. workspace.start_managed_process(process_id=<server>)
|
||||
5. websocket_client(ws_url) 通过 WS relay 连接
|
||||
6. ClientSession.initialize() MCP 协议握手
|
||||
```
|
||||
|
||||
配置 (`MCPServerBoxConfig`): `network='on'` (MCP 服务器通常需要网络),`host_path_mode='ro'` (默认只读),`startup_timeout_sec=120` (留时间给 pip install)。
|
||||
|
||||
每条 MCP server 是同一 session 中的一个 managed process,独立的 `process_id`、独立 attach URL,互不阻塞。
|
||||
|
||||
---
|
||||
|
||||
## 5. 启动与生命周期
|
||||
|
||||
### 5.1 启动顺序 (`build_app.py`)
|
||||
|
||||
```
|
||||
BuildAppStage.run(ap)
|
||||
├─ ... (persistence, models, sessions) ...
|
||||
│
|
||||
├─ BoxService(ap)
|
||||
├─ box_service.initialize()
|
||||
│ └─ connector.initialize()
|
||||
│ ├─ [stdio] fork box subprocess
|
||||
│ ├─ [subprocess+WS] Windows 本地
|
||||
│ └─ [remote WS] connect URL
|
||||
│ └─ 启动心跳 _heartbeat_task
|
||||
├─ ap.box_service = box_service
|
||||
│
|
||||
├─ ToolManager(ap)
|
||||
├─ tool_mgr.initialize()
|
||||
│ ├─ NativeToolLoader (检查 box_service.available)
|
||||
│ ├─ PluginToolLoader
|
||||
│ ├─ MCPLoader (Box 可用时,stdio MCP 走沙箱)
|
||||
│ └─ SkillAuthoringToolLoader
|
||||
├─ ap.tool_mgr = tool_mgr
|
||||
│
|
||||
├─ ... (platform, pipeline) ...
|
||||
├─ SkillManager.initialize() (从 Box runtime 加载 skill 列表)
|
||||
└─ ... (RAG, HTTP, plugins) ...
|
||||
```
|
||||
|
||||
BoxService 在 ToolManager **之前**初始化。ToolManager 创建 loader 时检查 `box_service.available`。
|
||||
|
||||
### 5.2 初始化失败处理
|
||||
|
||||
```python
|
||||
try:
|
||||
await self._runtime_connector.initialize()
|
||||
self._available = True
|
||||
except Exception as e:
|
||||
self._available = False
|
||||
logger.warning(f"Box runtime unavailable: {e}")
|
||||
```
|
||||
|
||||
**静默降级**: Box 初始化失败不会阻止应用启动,仅导致 6 个 native tool、所有 Skill 工具和 MCP-in-Box 工具不暴露给 LLM。与 Plugin 的行为不同(Plugin 失败会抛异常)。
|
||||
|
||||
### 5.3 销毁流程
|
||||
|
||||
```
|
||||
app.dispose()
|
||||
└─ box_service.dispose()
|
||||
├─ connector.dispose()
|
||||
│ ├─ cancel _heartbeat_task
|
||||
│ ├─ cancel _handler_task / _ctrl_task
|
||||
│ └─ terminate subprocess (SIGTERM)
|
||||
└─ loop.create_task(client.shutdown())
|
||||
└─ RPC SHUTDOWN → Box Runtime 清理所有容器
|
||||
```
|
||||
|
||||
Box 额外做了 RPC SHUTDOWN 通知 Runtime 主动清理容器,比 Plugin 的直接杀进程更安全。
|
||||
|
||||
---
|
||||
|
||||
## 6. 配置
|
||||
|
||||
### config.yaml (重构后)
|
||||
|
||||
```yaml
|
||||
box:
|
||||
enabled: true # 整个 Box 子系统的总开关。设为 false 时:
|
||||
# - 不连接远程 Box runtime,不 fork 本地 stdio 子进程
|
||||
# - sandbox 工具 (exec/read/write/edit/glob/grep) 不暴露给 LLM
|
||||
# - skill 添加/编辑 / GitHub 安装 / 文件写入全部拒绝
|
||||
# - stdio 模式的 MCP server 启动时报错(http/sse 模式不受影响)
|
||||
# - skill 列表/读取保持只读可用
|
||||
# BOX__ENABLED 环境变量可覆盖(统一约定)
|
||||
backend: 'local' # 'local' (探测) / 'docker' / 'nsjail' / 'e2b'
|
||||
# BOX_BACKEND 环境变量优先级更高
|
||||
runtime:
|
||||
endpoint: '' # 外部 Runtime 的 WS 基地址 'ws://host:5410'
|
||||
# 留空 = 本地自管 Runtime
|
||||
local:
|
||||
profile: 'default'
|
||||
image: '' # 覆盖 profile 默认 image
|
||||
host_root: './data/box' # 工作区挂载根,Docker 部署需绝对路径
|
||||
default_workspace: '' # 默认 '<host_root>/default'
|
||||
skills_root: 'skills' # Box 管理的 skill 包目录(相对 host_root)
|
||||
allowed_mount_roots: # 默认 ['<host_root>']
|
||||
- './data/box'
|
||||
- '/tmp'
|
||||
workspace_quota_mb: null # 配额覆盖,null = 走 profile
|
||||
e2b:
|
||||
api_key: '' # 也可走 E2B_API_KEY 环境变量
|
||||
api_url: '' # 自托管 E2B 时填写
|
||||
template: '' # 默认 template ID
|
||||
```
|
||||
|
||||
> **重大变更**: 较 2026-04-16 文档,配置结构完全重组(commit `eefdea4`)。原字段 `box.profile` / `box.runtime_url` / `box.shared_host_root` / `box.allowed_host_mount_roots` 全部迁入 `box.local.*` 子表,新增 `box.backend` 与 `box.e2b.*` 配置组。
|
||||
|
||||
### docker-compose.yaml
|
||||
|
||||
`langbot_box` 服务受 compose profile 控制,默认 `docker compose up` **不会**启动它。需要 sandbox 时:
|
||||
|
||||
```bash
|
||||
docker compose --profile box up # 启动 langbot + langbot_box + plugin runtime
|
||||
docker compose --profile all up # 同上
|
||||
docker compose up # 只起 langbot + plugin runtime (box 关闭)
|
||||
```
|
||||
|
||||
若不起 `langbot_box`,需要同步在 `data/config.yaml` 中设 `box.enabled: false`(或 langbot 容器 env 加 `BOX__ENABLED=false`),否则 LangBot 会一直尝试连接不存在的 Box runtime 并报错。
|
||||
|
||||
```yaml
|
||||
# langbot_box 的关键 volume
|
||||
volumes:
|
||||
- ${LANGBOT_BOX_ROOT}:${LANGBOT_BOX_ROOT} # 工作区挂载(源/目标同路径)
|
||||
- /var/run/docker.sock:/var/run/docker.sock # Docker backend 复用宿主 docker
|
||||
```
|
||||
|
||||
### 关闭/连接失败时的行为矩阵
|
||||
|
||||
`box.enabled = false` 与"启用但连接失败"在用户可观察行为上**完全一致**——都通过 `BoxService.available = False` 表达,只是 `get_status` 多返回 `enabled` 字段供前端区分文案。
|
||||
|
||||
| 消费方 | Box 可用 | Box 不可用(disabled 或 failed) |
|
||||
|---|---|---|
|
||||
| native exec/read/write/edit/glob/grep 工具 | 暴露给 LLM | **不暴露** |
|
||||
| `activate` / `register_skill` 工具 | 暴露给 LLM | **不暴露** |
|
||||
| stdio MCP server | 在 Box 内启动 | **`_init_stdio_python_server` 抛 RuntimeError** 拒绝;不退化到宿主 stdio |
|
||||
| http/sse MCP server | 正常 | 正常(不依赖 Box) |
|
||||
| Skill 列表/读取 (`list_skills`/`get_skill`/`read_skill_file`) | 走 Box runtime | 走 LangBot 本地 `data/skills/` 只读 fallback |
|
||||
| Skill 创建/编辑/安装/写文件 | 走 Box runtime | **HTTP 400** + 明确错误信息(`_require_box_for_write`) |
|
||||
| Pipeline AI 配置中 `box-session-id-template` | 正常生效 | **前端 banner** 提示字段无效 |
|
||||
| Pipeline 扩展页 `enable_all_skills` / 绑定 skill | 可编辑 | **前端禁用** + banner |
|
||||
| 仪表盘 Box 状态卡片 | 绿点 / "已连接" | 灰点 / "已禁用"(disabled) 或 红点 / "已断开"(failed) |
|
||||
|
||||
> 后端拒写的边界条件:如果 `ap.box_service` **完全没装**(老式 dev mode,没经过 BuildAppStage),`_require_box_for_write` 视作 no-op,保留 `data/skills/` 本地路径——以兼容历史测试与最小化设置。生产环境总会装 `ap.box_service`,因此该 fallback 不会被触发。
|
||||
|
||||
### Pipeline 配置 (templates/metadata/pipeline/ai.yaml)
|
||||
|
||||
`local-agent.config.box-session-id-template` 控制 session 作用域,预设:
|
||||
|
||||
- `{launcher_type}_{launcher_id}` — 每个会话 (推荐,默认)
|
||||
- `{launcher_type}_{launcher_id}_{sender_id}` — 群聊每个用户
|
||||
- `{launcher_type}_{launcher_id}_{conversation_id}` — 每个对话上下文
|
||||
- `{query_id}` — 每条消息(完全隔离)
|
||||
|
||||
详见 [box-session-scope.md](./box-session-scope.md)。
|
||||
|
||||
### REST API
|
||||
|
||||
| 端点 | 方法 | 说明 | 前端 |
|
||||
|------|------|------|:---:|
|
||||
| `/api/v1/box/status` | GET | 可用性、Profile、后端信息 | ✅ 监控页 |
|
||||
| `/api/v1/box/sessions` | GET | 活跃 session 列表 | ❌ |
|
||||
| `/api/v1/box/errors` | GET | 最近 50 条错误 | ❌ |
|
||||
| `/api/v1/skills` 等 | GET/POST/PUT/DELETE | Skill CRUD、文件浏览、zip/GitHub 安装、preview | ✅ Skill 管理页 |
|
||||
|
||||
前端 `web/src/app/home/monitoring/components/overview-cards/SystemStatusCards.tsx` 已接入 `/api/v1/box/status`,展示 backend 名称、profile 与活跃 session 数。Sessions 与 errors API 仍未接入。
|
||||
@@ -1,157 +0,0 @@
|
||||
# Box 系统架构问题清单
|
||||
|
||||
> 更新日期: 2026-05-19
|
||||
> 分支: `feat/sandbox` (LangBot + langbot-plugin-sdk)
|
||||
|
||||
---
|
||||
|
||||
## 已解决(自上一轮 review)
|
||||
|
||||
下列原 P0/P1 项在最新分支已被修复,仅作记录:
|
||||
|
||||
| 原编号 | 问题 | 处理 commit / 说明 |
|
||||
|--------|------|---------------------|
|
||||
| #3 | Box 无重连机制 | `_make_connection_callback` 已接入 `runtime_disconnect_callback`;`BoxService._reconnect_loop()` 实现指数退避重连 (`2dfd9d5d`、`c6882cf`) |
|
||||
| #4 | Box 无心跳 | `BoxRuntimeConnector._heartbeat_loop()`,间隔 20s(沿用 Plugin 模式) |
|
||||
| #10 | Windows 兼容 | connector 增加 Windows 分支 (subprocess + WS),backend 适配 Windows Docker (`120817a`、`fafb7a4`) |
|
||||
| #12 | nsjail image 字段冲突 | `_assert_session_compatible()` 在不支持自定义镜像的 backend 跳过 image 字段 |
|
||||
| #22 | 前端无 Box UI | 监控页 `SystemStatusCards.tsx` 已接入 `/api/v1/box/status`;Skill 管理页接入了全部 skill API(sessions/errors API 仍未接入) |
|
||||
|
||||
---
|
||||
|
||||
## P0 — 合并前建议修复
|
||||
|
||||
### 1. policy.py 是死代码
|
||||
|
||||
- **位置**: `pkg/box/policy.py` (98 行)
|
||||
- **现状**: `SandboxPolicy`、`ToolPolicy`、`ElevatedPolicy` 三个类已定义,但全项目无任何导入或调用
|
||||
- **影响**: 三层安全策略(沙箱模式 / 工具白名单 / 权限提升)完全未生效。当前实际策略仍是"Box 可用就暴露全部 6 个 native tool,不可用就全部隐藏"
|
||||
- **建议**: 要么删除死代码,要么接入 NativeToolLoader 的工具暴露 / exec 调用链。如果短期不会接入,至少在 `pkg/box/__init__.py` 显式标注其状态
|
||||
|
||||
### 2. WebSocket relay 无认证
|
||||
|
||||
- **位置**: SDK `box/server.py` — Action RPC 路径 `/rpc/ws` 与 managed-process relay `/v1/sessions/{id}/managed-process/{pid}/ws`
|
||||
- **现状**: 任何能访问 5410 端口的客户端都可以连接,attach 任意 session 的 managed process stdin/stdout,或直接发起 EXEC
|
||||
- **影响**: 容器化 / Docker compose 部署中,若 Box runtime 端口外暴露,网络内的攻击者可直接控制沙箱
|
||||
- **建议**: 至少加 token 认证(INIT 时下发,WS 连接 query string 或 header 校验);多 process 后 attach 面更大,更不能裸奔
|
||||
|
||||
### 3. security.py 根路径未拦截
|
||||
|
||||
- **位置**: SDK `box/security.py` `BLOCKED_HOST_PATHS_POSIX`
|
||||
- **现状**: 黑名单中没有 `/`,`host_path="/"` 可通过校验并挂载整个主机文件系统;用户 home 目录、`/var` 等也未拦截
|
||||
- **建议**: 将 `/` 加入黑名单,或改用白名单策略与 LangBot 侧 `allowed_mount_roots` 二次拦截
|
||||
|
||||
### 4. INIT 与 backend 初始化的竞态
|
||||
|
||||
- **位置**: SDK `box/runtime.py` `init()` 在握手后才下发实际配置;`backend` 在 INIT 之前可能已经按默认值实例化
|
||||
- **现状**: commit `5029d9c` 修复了 "init config before backend reuse" 的部分场景,但 backend 重新实例化时若有正在执行的 session,可能命中旧 backend
|
||||
- **建议**: 整理 init/handshake 顺序——要么 INIT 完成前不接受任何业务 action,要么允许 backend 配置变更时显式清理现有 session
|
||||
|
||||
---
|
||||
|
||||
## P1 — 合并后优先跟进
|
||||
|
||||
### 5. Session 数量无上限
|
||||
|
||||
- **位置**: SDK `box/runtime.py` `_get_or_create_session()`
|
||||
- **现状**: `_sessions` dict 无容量限制,恶意或异常调用可创建无限 session
|
||||
- **建议**: 加 `max_sessions` 配置项,达到上限时拒绝新建或按 LRU 清理
|
||||
|
||||
### 6. Quota 检查存在 TOCTOU
|
||||
|
||||
- **位置**: `pkg/box/service.py` `_enforce_workspace_quota()`
|
||||
- **现状**: 应用层先读磁盘大小再执行命令,两步之间有竞态窗口
|
||||
- **建议**: 短期用 Docker `--storage-opt size=` 做内核级限制;长期用 Redis 原子计数器做预留式配额
|
||||
|
||||
### 7. 全局锁持有期间执行慢操作
|
||||
|
||||
- **位置**: SDK `box/runtime.py` `_get_or_create_session()` — `self._lock` 下调用 `backend.start_session()` (即 `docker run` / `nsjail` 进程启动 / E2B `Sandbox.create`)
|
||||
- **影响**: `docker run` 可能耗时数秒(含镜像拉取)、E2B 冷启动通常 > 1s,期间阻塞所有并发请求
|
||||
- **建议**: 在 `_lock` 下仅做状态检查和 session 注册,容器创建在锁外执行
|
||||
|
||||
### 8. Session 清理是机会性的
|
||||
|
||||
- **位置**: SDK `box/runtime.py` `_reap_expired_sessions_locked()` — 仅在 `_get_or_create_session()` 时调用
|
||||
- **影响**: 如果长时间无新 session 请求,过期 session(含容器)不会被清理
|
||||
- **建议**: 加一个独立的 `asyncio.create_task` 定时清理(如每 60s 一次)
|
||||
|
||||
### 9. server.py 直接访问 runtime 私有字段
|
||||
|
||||
- **位置**: SDK `box/server.py` — managed-process WS handler 直接读 `runtime._sessions`
|
||||
- **影响**: 绕过锁和封装,在并发场景下可能读到不一致状态
|
||||
- **建议**: 在 BoxRuntime 上增加公共方法(如 `get_session_managed_process(session_id, process_id)`)
|
||||
|
||||
### 10. workspace quota 检查阻塞事件循环
|
||||
|
||||
- **位置**: `pkg/box/service.py` `_get_workspace_size_bytes()` — 使用同步 `os.scandir` 递归遍历
|
||||
- **影响**: 大工作区可能阻塞 asyncio event loop
|
||||
- **建议**: 用 `asyncio.to_thread()` 包装,或用 `aiofiles` 异步扫描
|
||||
|
||||
### 11. extra_mounts 一旦容器创建即固定
|
||||
|
||||
- **位置**: SDK `box/runtime.py` 的兼容性检查;`pkg/box/service.py:build_skill_extra_mounts()`
|
||||
- **现状**: Skill 挂载在容器创建时一次性写入;同一 session 后续 pipeline 切换 skill 列表时,新挂载不会生效(除非销毁重建)
|
||||
- **影响**: 用户长时间共享 session 的场景下,新激活的 skill 可能挂不上
|
||||
- **建议**: 要么在创建时把 pipeline 绑定的所有 skill 都挂上(实际现状)+ 写入文档;要么变更挂载时强制销毁 session 重建(已被 commit `5029d9c` 部分覆盖,需校验)
|
||||
|
||||
---
|
||||
|
||||
## P2 — 后续迭代
|
||||
|
||||
### 12. 重复的 `_is_path_under` 函数
|
||||
|
||||
- **位置**: `pkg/box/service.py` 行 30 附近 — 同名函数定义两次
|
||||
- **建议**: 删除重复定义
|
||||
|
||||
### 13. localagent.py 工具循环无迭代上限
|
||||
|
||||
- **位置**: `pkg/provider/runners/localagent.py` `while pending_tool_calls` 循环
|
||||
- **影响**: 恶意或混乱的 LLM 可无限产生 tool call,消耗资源
|
||||
- **建议**: 加 `max_tool_iterations` 配置项(如默认 50 次)
|
||||
|
||||
### 14. localagent.py 中的死代码
|
||||
|
||||
- **位置**: `pkg/provider/runners/localagent.py:29-35` 附近 — 旧命名 `SANDBOX_EXEC_TOOL_NAME` 和 `SANDBOX_EXEC_SYSTEM_GUIDANCE`
|
||||
- **现状**: 旧命名方案的遗留常量,从未被引用(实际使用 `EXEC_TOOL_NAME` from native.py)
|
||||
- **建议**: 删除
|
||||
|
||||
### 15. @loader_class 装饰器未使用
|
||||
|
||||
- **位置**: `pkg/provider/tools/loader.py` — `preregistered_loaders` 列表和 `@loader_class` 装饰器
|
||||
- **现状**: 各 loader 的 `@loader_class` 多数被注释掉,ToolManager 手动实例化所有 loader
|
||||
- **建议**: 要么启用装饰器自动注册,要么删除未用的机制
|
||||
|
||||
### 16. 工具名冲突风险
|
||||
|
||||
- **位置**: `pkg/provider/tools/toolmgr.py` `execute_func_call()` — 按优先级 native → plugin → mcp → skill → skill_authoring 分发
|
||||
- **影响**: 如果 plugin 或 MCP 有名为 `exec`/`read`/`write`/`edit`/`glob`/`grep`/`activate` 的工具,会被前序 loader 静默遮蔽
|
||||
- **建议**: 加命名空间前缀或冲突检测告警
|
||||
|
||||
### 17. client.py 反序列化不一致
|
||||
|
||||
- **位置**: SDK `box/client.py` — `execute()` 与其他方法对返回值的反序列化方式不统一(部分手动构造 model,部分用 `model_validate`)
|
||||
- **建议**: 统一使用 `model_validate`
|
||||
|
||||
### 18. 错误类型还原基于字符串前缀匹配
|
||||
|
||||
- **位置**: SDK `box/client.py` `_translate_action_error()`
|
||||
- **影响**: 如果 server 端错误消息格式变化,client 会回退到通用 `BoxError`,丢失类型信息
|
||||
- **建议**: 在 ActionResponse 中增加结构化的错误类型字段(如 `error_code` 枚举)
|
||||
|
||||
### 19. 前端只用到了 status
|
||||
|
||||
- **位置**: `web/src/app/home/monitoring/...` 已接入 `/api/v1/box/status`
|
||||
- **现状**: `/api/v1/box/sessions` 与 `/api/v1/box/errors` 后端可用、前端未消费
|
||||
- **建议**: 在监控页或独立 Box 详情页展示活跃 session 列表与最近错误,提升运维体感
|
||||
|
||||
### 20. skill_store 测试覆盖偏薄
|
||||
|
||||
- **位置**: SDK `tests/box/test_skill_store.py` 仅 88 行
|
||||
- **现状**: 相对 `skill_store.py` 的 647 行实现,单测覆盖度不够;GitHub 安装路径、`source_subdir` / `target_suffix` 组合、损坏 zip 的错误处理等场景未覆盖
|
||||
- **建议**: 至少补到核心 path 覆盖(preview/install/list/file CRUD 各 2~3 个 case)
|
||||
|
||||
### 21. 集成测试未进 CI
|
||||
|
||||
- **位置**: LangBot `tests/integration_tests/box/test_box_integration.py`、`test_box_mcp_integration.py`,SDK 端的 E2B 真机测试
|
||||
- **现状**: 容器实际执行、E2B 真实 sandbox、Managed process WS attach 均仅本地能跑
|
||||
- **建议**: 加一个可选的 Docker-in-Docker CI stage,或在合并前手动跑 checklist
|
||||
@@ -1,401 +0,0 @@
|
||||
# Box Session Scope Design
|
||||
|
||||
> Date: 2026-04-18 (last reviewed 2026-05-19)
|
||||
> Branch: `feat/sandbox` (LangBot + langbot-plugin-sdk)
|
||||
> Related: [Box Architecture](./box-architecture.md) | [Box vs Plugin Runtime](./box-vs-plugin-runtime.md)
|
||||
|
||||
---
|
||||
|
||||
## 0. Implementation Status (2026-05-19)
|
||||
|
||||
This document was authored as a design proposal. The current `feat/sandbox` branch
|
||||
has shipped the design largely as written:
|
||||
|
||||
| Item | Status | Notes |
|
||||
|------|--------|-------|
|
||||
| `BoxMountSpec` + `BoxSpec.extra_mounts` | ✅ Shipped | SDK `box/models.py` |
|
||||
| Docker / nsjail / E2B backends apply extra mounts | ✅ Shipped | Last gap closed by SDK commit `0fea9b1` (E2B) |
|
||||
| `box-session-id-template` in `local-agent` pipeline config | ✅ Shipped | `templates/metadata/pipeline/ai.yaml`, default `{launcher_type}_{launcher_id}` |
|
||||
| `BoxService.resolve_box_session_id(query)` | ✅ Shipped | `pkg/box/service.py:166` |
|
||||
| `BoxService.build_skill_extra_mounts(query)` | ✅ Shipped | `pkg/box/service.py:189` |
|
||||
| Skill exec uses unified container + extra mounts | ✅ Shipped | `pkg/provider/tools/loaders/native.py` skill branch |
|
||||
| MCP-in-Box uses shared persistent session, multi-process | ✅ Shipped (earlier than originally scoped) | SDK commit `529088e`, LangBot `mcp_stdio.py:_build_box_session_id` |
|
||||
| `BoxManagedProcessSpec.process_id` + multi-process per session | ✅ Shipped | `BoxRuntime` keeps `managed_processes: dict[pid, _ManagedProcess]` |
|
||||
| Per-tenant / quota integration with templates | ❌ Not started | See [box-tob-analysis.md](./box-tob-analysis.md) |
|
||||
|
||||
The "Phase 2 deferred" note in §10 is **out of date** — MCP unification went in on
|
||||
the same line. Pipeline-scoped (not user-scoped) MCP container is the realized
|
||||
behavior: each pipeline's MCP servers share one `mcp-<pipeline>` session, and
|
||||
user exec sessions use the template-derived id.
|
||||
|
||||
The remaining open work is multi-tenant overlays (tenant_id in session_id,
|
||||
quota counters keyed by tenant), tracked in the toB analysis doc rather than here.
|
||||
|
||||
---
|
||||
|
||||
## 1. Problems
|
||||
|
||||
### 1.1 Default exec: per-message containers
|
||||
|
||||
Currently, `BoxService.execute_tool()` sets `session_id = str(query.query_id)` — an
|
||||
auto-incrementing integer per incoming message. Every user message creates a new sandbox
|
||||
container. Dependencies installed and in-container state are lost between messages.
|
||||
|
||||
### 1.2 Three isolated container pools
|
||||
|
||||
Default exec, skills, and MCP servers each manage their own containers with
|
||||
independent session IDs:
|
||||
|
||||
| Path | Session ID | Container |
|
||||
|--------------|-----------------------------------------------|-------------|
|
||||
| Default exec | `str(query_id)` (per message) | Ephemeral |
|
||||
| Skill exec | `skill-{launcher}_{id}-{skill_name}` | Per skill |
|
||||
| MCP stdio | `mcp-{server_uuid}` | Per server |
|
||||
|
||||
This means a single logical user interaction can spawn 3+ containers that cannot
|
||||
share state, see each other's files, or reuse installed dependencies.
|
||||
|
||||
### 1.3 Single bind mount limitation
|
||||
|
||||
`BoxSpec` currently supports only **one** `host_path` → `mount_path` bind mount.
|
||||
This prevents mounting both a default workspace and skill directories into the
|
||||
same container.
|
||||
|
||||
---
|
||||
|
||||
## 2. Concept Model
|
||||
|
||||
```
|
||||
Platform Message
|
||||
→ Query (query_id: int, auto-increment, per message)
|
||||
→ Session (launcher_type + launcher_id, per chat window)
|
||||
→ Conversation (uuid, per dialogue context within a Session)
|
||||
```
|
||||
|
||||
| Concept | Key | Example | Scope |
|
||||
|---------------|-------------------------------------|----------------------------|------------------------------|
|
||||
| Query | `query_id` | `42` | Single message |
|
||||
| Session | `launcher_type` + `launcher_id` | `group_123456` | Chat window (group or PM) |
|
||||
| Conversation | `conversation_id` (UUID) | `a1b2c3d4-...` | Dialogue context within a Session |
|
||||
| Sender | `sender_id` | `789` | Individual user |
|
||||
|
||||
Note: in a **group chat**, all users share the same Session (keyed by `group_id`). The
|
||||
individual sender is tracked as `sender_id` but does not affect Session/Conversation routing.
|
||||
|
||||
---
|
||||
|
||||
## 3. Target Scenarios
|
||||
|
||||
| # | Scenario | Box Granularity | Desired `session_id` |
|
||||
|----|--------------------------------|------------------------------------------|---------------------------------------------------------|
|
||||
| 1 | Personal assistant | 1 Box per user, long-lived | `{launcher_type}_{launcher_id}` |
|
||||
| 2 | Customer service | 1 Box per customer, cross-pipeline | `{launcher_type}_{launcher_id}` |
|
||||
| 3 | Internal employee tool | 1 Box per employee | `{launcher_type}_{launcher_id}` |
|
||||
| 4 | Group chat shared assistant | 1 Box per group | `{launcher_type}_{launcher_id}` |
|
||||
| 5 | Group chat isolated per user | 1 Box per user within a group | `{launcher_type}_{launcher_id}_{sender_id}` |
|
||||
| 6 | Teaching (cross-channel) | 1 Box per student across groups/PMs | `{sender_id}` |
|
||||
| 7 | One-off execution | 1 Box per message (current behavior) | `{query_id}` |
|
||||
| 8 | Multi-project development | 1 Box per conversation context | `{launcher_type}_{launcher_id}_{conversation_id}` |
|
||||
|
||||
No single fixed granularity covers all scenarios. A template-based approach is needed.
|
||||
|
||||
---
|
||||
|
||||
## 4. Design Overview
|
||||
|
||||
Two key changes:
|
||||
|
||||
1. **Unified container**: exec, skills, and MCP all share the same container per
|
||||
session scope. No more separate container pools.
|
||||
2. **Configurable session scope**: `session_id` is generated from a template with
|
||||
pipeline variables, configurable per pipeline.
|
||||
|
||||
### 4.1 Unified Container with Multiple Mounts
|
||||
|
||||
A single container per session scope is created on first use. It has:
|
||||
|
||||
- **Primary mount**: default workspace at `/workspace` (from `default_host_workspace`)
|
||||
- **Skill mounts**: each pipeline-bound skill's `package_root` mounted at
|
||||
`/workspace/.skills/{skill_name}/`
|
||||
- **MCP servers**: run as managed processes inside the same container
|
||||
|
||||
```
|
||||
Container (session_id = "group_123456")
|
||||
/workspace/ ← default workspace (bind mount, rw)
|
||||
/workspace/.skills/web-search/ ← skill package (bind mount, rw)
|
||||
/workspace/.skills/data-analysis/ ← skill package (bind mount, rw)
|
||||
[managed process: mcp-server-a] ← MCP server running inside
|
||||
[managed process: mcp-server-b] ← MCP server running inside
|
||||
```
|
||||
|
||||
This requires extending `BoxSpec` to support multiple mounts (see §5).
|
||||
|
||||
### 4.2 Session ID Template
|
||||
|
||||
A new field `box-session-id-template` in the `local-agent` pipeline runner config
|
||||
controls the session scope:
|
||||
|
||||
```yaml
|
||||
# templates/metadata/pipeline/ai.yaml (under local-agent.config)
|
||||
- name: box-session-id-template
|
||||
label:
|
||||
en_US: Sandbox Scope
|
||||
zh_Hans: 沙箱作用域
|
||||
description:
|
||||
en_US: >-
|
||||
Determines how sandbox environments are shared. Use variables to
|
||||
control isolation granularity.
|
||||
zh_Hans: >-
|
||||
决定沙箱环境的共享方式。使用变量控制隔离粒度。
|
||||
type: select
|
||||
required: false
|
||||
default: "{launcher_type}_{launcher_id}"
|
||||
options:
|
||||
- value: "{launcher_type}_{launcher_id}"
|
||||
label:
|
||||
en_US: Per chat (Recommended)
|
||||
zh_Hans: 每个会话(推荐)
|
||||
- value: "{launcher_type}_{launcher_id}_{sender_id}"
|
||||
label:
|
||||
en_US: Per user in chat
|
||||
zh_Hans: 会话中每个用户
|
||||
- value: "{launcher_type}_{launcher_id}_{conversation_id}"
|
||||
label:
|
||||
en_US: Per conversation context
|
||||
zh_Hans: 每个对话上下文
|
||||
- value: "{query_id}"
|
||||
label:
|
||||
en_US: Per message (isolated)
|
||||
zh_Hans: 每条消息(完全隔离)
|
||||
```
|
||||
|
||||
Available template variables (populated by PreProcessor in `query.variables`):
|
||||
|
||||
| Variable | Source | Example |
|
||||
|---------------------|---------------------------------|----------------------|
|
||||
| `{launcher_type}` | `query.session.launcher_type` | `person` / `group` |
|
||||
| `{launcher_id}` | `query.session.launcher_id` | `123456` |
|
||||
| `{sender_id}` | `query.sender_id` | `789` |
|
||||
| `{conversation_id}` | `conversation.uuid` | `a1b2c3d4-...` |
|
||||
| `{query_id}` | `query.query_id` | `42` |
|
||||
|
||||
Default `{launcher_type}_{launcher_id}` covers scenarios 1–4 out of the box.
|
||||
|
||||
---
|
||||
|
||||
## 5. SDK Changes: Multi-Mount BoxSpec
|
||||
|
||||
### 5.1 Model Extension
|
||||
|
||||
```python
|
||||
# box/models.py
|
||||
|
||||
class BoxMountSpec(pydantic.BaseModel):
|
||||
"""A single bind mount specification."""
|
||||
host_path: str
|
||||
mount_path: str
|
||||
mode: BoxHostMountMode = BoxHostMountMode.READ_WRITE
|
||||
|
||||
class BoxSpec(pydantic.BaseModel):
|
||||
# ... existing fields ...
|
||||
host_path: str | None = None # Primary mount (backward compat)
|
||||
host_path_mode: BoxHostMountMode = BoxHostMountMode.READ_WRITE
|
||||
mount_path: str = DEFAULT_BOX_MOUNT_PATH
|
||||
extra_mounts: list[BoxMountSpec] = [] # NEW: additional mounts
|
||||
```
|
||||
|
||||
`extra_mounts` is additive — the existing `host_path` / `mount_path` pair remains
|
||||
the primary mount for backward compatibility.
|
||||
|
||||
### 5.2 Backend: Apply Extra Mounts
|
||||
|
||||
```python
|
||||
# box/backend.py — CLISandboxBackend.start_session()
|
||||
|
||||
# Primary mount (unchanged)
|
||||
if spec.host_path is not None and spec.host_path_mode != BoxHostMountMode.NONE:
|
||||
args.extend(['-v', f'{spec.host_path}:{spec.mount_path}:{spec.host_path_mode.value}'])
|
||||
|
||||
# Extra mounts (NEW)
|
||||
for mount in spec.extra_mounts:
|
||||
if mount.mode != BoxHostMountMode.NONE:
|
||||
args.extend(['-v', f'{mount.host_path}:{mount.mount_path}:{mount.mode.value}'])
|
||||
```
|
||||
|
||||
Same pattern for nsjail backend.
|
||||
|
||||
---
|
||||
|
||||
## 6. LangBot Changes
|
||||
|
||||
### 6.1 Session ID Resolution
|
||||
|
||||
In `BoxService.execute_tool()`:
|
||||
|
||||
```python
|
||||
# Before:
|
||||
spec_payload.setdefault('session_id', str(query.query_id))
|
||||
|
||||
# After:
|
||||
template = (query.pipeline_config or {}).get('ai', {}) \
|
||||
.get('local-agent', {}).get('box-session-id-template',
|
||||
'{launcher_type}_{launcher_id}')
|
||||
variables = query.variables or {}
|
||||
session_id = template.format_map(collections.defaultdict(
|
||||
lambda: 'unknown', variables
|
||||
))
|
||||
spec_payload.setdefault('session_id', session_id)
|
||||
```
|
||||
|
||||
### 6.2 Skill Exec: Use Same Container
|
||||
|
||||
Currently `native.py:_invoke_exec` creates a separate `BoxWorkspaceSession` per
|
||||
skill with `host_path=package_root`. Instead:
|
||||
|
||||
1. Use the **same session_id** as default exec (from the template).
|
||||
2. Pass the skill's `package_root` as an **extra mount** at
|
||||
`/workspace/.skills/{skill_name}/` instead of replacing `/workspace`.
|
||||
3. The container already has the default workspace at `/workspace`.
|
||||
|
||||
```python
|
||||
# native.py — _invoke_exec, skill branch (REVISED)
|
||||
|
||||
# Same session_id as default exec
|
||||
session_id = resolve_box_session_id(query)
|
||||
|
||||
spec_payload = {
|
||||
'cmd': rewritten_command,
|
||||
'workdir': rewritten_workdir,
|
||||
'session_id': session_id,
|
||||
'extra_mounts': [{
|
||||
'host_path': package_root,
|
||||
'mount_path': f'/workspace/.skills/{selected_skill_name}',
|
||||
'mode': 'rw',
|
||||
}],
|
||||
}
|
||||
result = await self.ap.box_service.execute_spec_payload(spec_payload, query)
|
||||
```
|
||||
|
||||
The virtual path `/workspace/.skills/{name}` no longer needs rewriting at the
|
||||
command level — it maps directly to the bind mount path inside the container.
|
||||
|
||||
### 6.3 MCP: Use Same Container
|
||||
|
||||
MCP servers should run inside the same container as exec and skills. Changes:
|
||||
|
||||
1. `BoxStdioSessionRuntime` uses the pipeline's session_id template instead of
|
||||
`mcp-{server_uuid}`.
|
||||
2. MCP server's working directory is a subdirectory (e.g. `/workspace/.mcp/{name}/`).
|
||||
3. MCP server's dependencies are mounted or installed into that subdirectory.
|
||||
4. The MCP server runs as a managed process inside the shared container.
|
||||
|
||||
Since MCP servers start at LangBot boot (not per-query), the session must be
|
||||
created eagerly. The container will be kept alive by the managed process
|
||||
exemption in TTL reaping (`runtime.py:259`).
|
||||
|
||||
**Note**: MCP sessions are pipeline-scoped (not per-launcher), so their session_id
|
||||
should be a **fixed identifier per pipeline** rather than the user-facing template.
|
||||
This means one shared MCP container per pipeline, with user exec sessions separate.
|
||||
|
||||
Alternatively, in a future iteration, MCP managed processes could be launched
|
||||
lazily into the user's container on first MCP tool call. This is more complex
|
||||
but maximizes sharing. For V1, keeping MCP containers at pipeline scope is
|
||||
simpler and more predictable.
|
||||
|
||||
---
|
||||
|
||||
## 7. Mount Layout Summary
|
||||
|
||||
### Default exec (no skills activated)
|
||||
|
||||
```
|
||||
Container (session_id from template)
|
||||
/workspace/ ← default_host_workspace (rw)
|
||||
```
|
||||
|
||||
### Exec with activated skills
|
||||
|
||||
```
|
||||
Container (same session_id)
|
||||
/workspace/ ← default_host_workspace (rw)
|
||||
/workspace/.skills/web-search/ ← skill package_root (rw)
|
||||
/workspace/.skills/data-analysis/ ← skill package_root (rw)
|
||||
```
|
||||
|
||||
Extra mounts are **additive** — they are added when the container is first
|
||||
created (or on the first exec that references a skill). Since Docker bind
|
||||
mounts are specified at container creation time, skills must be known at
|
||||
creation time.
|
||||
|
||||
**Resolution**: When creating a container, inject `extra_mounts` for **all
|
||||
pipeline-bound skills** (from `extensions_preferences`), not just the
|
||||
currently activated one. This way any skill can be activated later without
|
||||
recreating the container.
|
||||
|
||||
### MCP servers (V1: pipeline-scoped)
|
||||
|
||||
```
|
||||
Container (session_id = "mcp-pipeline-{pipeline_uuid}")
|
||||
/workspace/ ← MCP shared workspace
|
||||
/workspace/.mcp/server-a/ ← MCP server A files
|
||||
/workspace/.mcp/server-b/ ← MCP server B files
|
||||
[managed process: server-a]
|
||||
[managed process: server-b]
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 8. Data Migration
|
||||
|
||||
Existing pipelines do not have `box-session-id-template`. The backend uses
|
||||
`.get(..., default)` so missing keys fall back to `{launcher_type}_{launcher_id}`.
|
||||
This changes behavior from per-message to per-launcher for existing pipelines.
|
||||
|
||||
Recommendation: **accept the behavior change** — per-launcher is the more
|
||||
intuitive default, and the old per-message behavior was rarely desired.
|
||||
|
||||
---
|
||||
|
||||
## 9. Cloud Quota Implications
|
||||
|
||||
| Scope | Typical concurrent containers |
|
||||
|-----------------------------------------------|-------------------------------|
|
||||
| `{query_id}` (per message) | Many, short-lived |
|
||||
| `{launcher_type}_{launcher_id}` (per chat) | = active chat count |
|
||||
| `{sender_id}` (per user) | = active user count |
|
||||
| `{conversation_id}` (per conversation) | Between per-chat and per-msg |
|
||||
|
||||
With the unified container model, each scope value maps to exactly **one**
|
||||
container (instead of potentially 3+ per-message). This significantly reduces
|
||||
resource usage.
|
||||
|
||||
Quota enforcement point: `BoxRuntime._get_or_create_session()` in the SDK.
|
||||
|
||||
---
|
||||
|
||||
## 10. Implementation Phases
|
||||
|
||||
### Phase 1: Session scope + skill unification (this PR)
|
||||
|
||||
1. **SDK**: Extend `BoxSpec` with `extra_mounts: list[BoxMountSpec]`.
|
||||
2. **SDK**: Update Docker/nsjail backends to apply extra mounts.
|
||||
3. **LangBot**: Add `box-session-id-template` to `local-agent` YAML metadata
|
||||
and default pipeline config JSON.
|
||||
4. **LangBot**: Update `BoxService.execute_tool()` to use template interpolation.
|
||||
5. **LangBot**: Update `native.py:_invoke_exec` skill branch to use same
|
||||
session_id + extra mounts instead of separate `BoxWorkspaceSession`.
|
||||
6. **LangBot**: On container creation, inject extra mounts for all
|
||||
pipeline-bound skills.
|
||||
7. **Frontend**: No code change — `DynamicFormComponent` renders `select` fields.
|
||||
8. **Tests**: Unit tests for template interpolation and multi-mount specs.
|
||||
|
||||
### Phase 2: MCP unification (future)
|
||||
|
||||
1. Refactor `BoxStdioSessionRuntime` to use pipeline-scoped shared container.
|
||||
2. MCP servers become managed processes in the shared container.
|
||||
3. Support multiple concurrent managed processes per container.
|
||||
|
||||
MCP unification is deferred because it requires changes to the managed process
|
||||
model (currently 1 managed process per session) and has startup ordering
|
||||
concerns (MCP servers start at boot, before any user query determines
|
||||
a session_id).
|
||||
@@ -1,121 +0,0 @@
|
||||
# Box 系统测试覆盖分析
|
||||
|
||||
> 更新日期: 2026-05-19
|
||||
> 分支: `feat/sandbox` (LangBot + langbot-plugin-sdk)
|
||||
|
||||
---
|
||||
|
||||
## 1. 测试文件清单
|
||||
|
||||
### LangBot 仓库
|
||||
|
||||
| 文件 | 行数 | CI 运行 | 覆盖范围 |
|
||||
|------|------|---------|---------|
|
||||
| `tests/unit_tests/box/test_box_connector.py` | 106 | 是 | Connector 传输决策、WS relay URL、dispose、心跳/重连 |
|
||||
| `tests/unit_tests/box/test_box_service.py` | 1224 | 是 | Service 核心逻辑(最全面) |
|
||||
| `tests/unit_tests/box/test_workspace.py` | 147 | 是 | WorkspaceSession 路径重写、payload 构建 |
|
||||
| `tests/unit_tests/provider/test_mcp_box_integration.py` | 707 | 是 | MCP Box 配置、路径重写、payload、shared-session/multi-process、runtime info |
|
||||
| `tests/unit_tests/provider/test_localagent_sandbox_exec.py` | 444 | 是 | LocalAgent exec 流程、流式、Skill 激活 (Tool Call) |
|
||||
| `tests/unit_tests/provider/test_tool_manager_native.py` | 249 | 是 | ToolManager 路由、native tool CRUD、路径穿越、6 工具暴露 |
|
||||
| `tests/unit_tests/provider/test_skill_tools.py` | 582 | 是 | Skill 管理、Tool Call 激活、路径、authoring CRUD |
|
||||
| `tests/unit_tests/test_skill_service.py` | 396 | 是 | HTTP service:skill CRUD、zip/GitHub install、文件浏览 |
|
||||
| `tests/unit_tests/test_paths.py` | 23 | 是 | paths 工具 |
|
||||
| `tests/unit_tests/test_preproc.py` | 134 | 是 | PreProcessor 注入 session 变量、bound skill 解析 |
|
||||
| `tests/unit_tests/pipeline/test_chat_handler_logging.py` | 78 | 是 | Chat handler 日志相关回归 |
|
||||
| `tests/integration_tests/box/test_box_integration.py` | 329 | **否** | 真实容器执行、超时、网络隔离 |
|
||||
| `tests/integration_tests/box/test_box_mcp_integration.py` | 368 | **否** | Managed process、WS attach、shared-session 清理 |
|
||||
|
||||
### SDK 仓库
|
||||
|
||||
| 文件 | 行数 | CI 运行 | 覆盖范围 |
|
||||
|------|------|---------|---------|
|
||||
| `tests/box/test_backend_selection.py` | 255 | 是 | 显式 backend / local 模式探测顺序 / 配置变更触发 reselect |
|
||||
| `tests/box/test_nsjail_backend.py` | 452 | 是 | nsjail 可用性、安装版 CLI vs 容器内 CLI、session、arg 构建、资源限制 |
|
||||
| `tests/box/test_e2b_backend.py` | 482 | 是 | E2B SDK mock、session 生命周期、extra_mounts 同步 |
|
||||
| `tests/box/test_skill_store.py` | 88 | 是 | zip preview/install、基础 file CRUD |
|
||||
|
||||
**总计**: 17 个测试文件, ~6,500 行测试代码; 其中 2 个集成测试(约 700 行)在 CI 中不运行。
|
||||
|
||||
> 较 2026-04-16 版增加:`test_skill_service.py`、`test_paths.py`、`test_preproc.py`、`test_chat_handler_logging.py` (LangBot),`test_backend_selection.py`、`test_e2b_backend.py`、`test_skill_store.py` (SDK)。`test_nsjail_backend.py` 增加 CLI 兼容性 case (commit `feed530`)。
|
||||
|
||||
---
|
||||
|
||||
## 2. 覆盖良好的区域
|
||||
|
||||
| 区域 | 质量 | 说明 |
|
||||
|------|------|------|
|
||||
| BoxRuntime session 管理 | 优秀 | session 复用、冲突检测、TTL 配置、消失 session 重建 |
|
||||
| BoxService Profile 系统 | 优秀 | 4 个内置 Profile、locked/unlocked 字段、timeout clamp |
|
||||
| BoxService host mount 安全 | 优秀 | allowed_mount_roots、disallowed_roots、shared host root |
|
||||
| BoxService workspace quota | 优秀 | 前置/后置配额检查、超额清理 |
|
||||
| BoxService 输出截断 | 优秀 | 短/精确边界/长输出、独立 stderr |
|
||||
| BoxService 可观测性 | 优秀 | 状态报告、error ring buffer、buffer 上限 |
|
||||
| BoxService session 模板 | 良好 | `resolve_box_session_id` + `build_skill_extra_mounts` 在 service / native / mcp 三处都有覆盖 |
|
||||
| RPC client/server 协议 | 优秀 | execute/get_sessions/delete/create/conflict error |
|
||||
| BoxRuntimeConnector | 良好 | local/remote 模式、Docker 平台、relay URL、心跳与重连回调 |
|
||||
| BoxWorkspaceSession | 良好 | payload 构建、managed process 路径重写、stage host file |
|
||||
| BoxHostMountMode.NONE | 良好 | 枚举校验、workdir 约束 |
|
||||
| NsjailBackend | 良好 | 可用性、安装版 vs 容器内、session 生命周期、arg 构建、资源限制 |
|
||||
| E2BBackend | 良好 | mock SDK、session/extra_mounts 同步 |
|
||||
| Backend selection | 良好 | 显式 backend 优先级、local 探测顺序、配置变更触发 reselect |
|
||||
| MCP Box 集成 | 良好 | config model、路径重写、payload、shared-session 多 process |
|
||||
| Native tool loader | 良好 | 6 工具(exec/read/write/edit/glob/grep)、路径穿越拦截 |
|
||||
| LocalAgent exec 流程 | 良好 | 完整 tool call 循环、流式、system prompt 注入、Tool Call 激活 |
|
||||
| Skill 系统 | 良好 | 加载、Tool Call 激活、marker、路径解析、authoring CRUD、HTTP service |
|
||||
|
||||
---
|
||||
|
||||
## 3. 覆盖缺失的区域
|
||||
|
||||
### 3.1 零测试 / 严重不足
|
||||
|
||||
| 区域 | 源文件 | 影响 |
|
||||
|------|--------|------|
|
||||
| **`security.py`** | SDK `box/security.py` (52 行) | `validate_sandbox_security()` 无任何测试。阻止 `/etc`/`/proc`/Docker socket 等危险挂载的安全函数从未被验证 |
|
||||
| **`policy.py`** | `pkg/box/policy.py` (98 行) | 三层安全策略无测试(也是死代码) |
|
||||
| **`skill_store.py` 边缘场景** | SDK `box/skill_store.py` (647 行) vs 测试 88 行 | GitHub 安装路径、`source_subdir` / `target_suffix` 组合、损坏 zip、文件冲突等场景未覆盖 |
|
||||
|
||||
### 3.2 未测试的关键路径
|
||||
|
||||
| 区域 | 说明 |
|
||||
|------|------|
|
||||
| **Session TTL 过期** | 测试配置了 `session_ttl_sec` 但从未推进时间验证过期清理 |
|
||||
| **并发 session 访问** | 无并发 exec / 并发创建 / race condition 测试 |
|
||||
| **Container backend (Docker)** | 仅通过集成测试覆盖(CI 不运行),单元测试全用 FakeBackend |
|
||||
| **E2B 真实 sandbox** | 单测全是 mock,未对接真实 E2B API |
|
||||
| **BoxRuntime shutdown()** | 在 test cleanup 中调用但未验证行为 |
|
||||
| **BoxServerHandler 错误路径** | 畸形请求、未知 action 类型 |
|
||||
| **WS relay** | 仅在集成测试中覆盖(CI 不运行) |
|
||||
| **NsjailBackend managed process** | 完全未测试 |
|
||||
| **MCP stdio 完整生命周期** | 依赖安装 → 进程启动 → 健康检查 → 多 process 并发 → 重试 |
|
||||
| **BoxService start/stop_managed_process** | 单 process 流转有单测,多 process 互不阻塞主要靠集成测试 |
|
||||
| **重连指数退避** | connector 单测覆盖回调接线,未实际跑完整重连周期 |
|
||||
|
||||
### 3.3 边缘情况缺失
|
||||
|
||||
| 区域 | 说明 |
|
||||
|------|------|
|
||||
| BoxSpec 校验 | 无效 session_id 格式、超长命令、env 特殊字符 |
|
||||
| BoxSpec.extra_mounts | 重复 mount_path、与 host_path 冲突、绝对 vs 相对路径 |
|
||||
| BoxExecutionResult | 仅 COMPLETED 和 TIMED_OUT,无 ERROR 状态测试 |
|
||||
| 多后端 fallback | local 模式探测顺序仅靠 mock,无真实 Docker 不可用 → nsjail 真机 fallback 测试 |
|
||||
| Profile YAML 加载 | 测试用硬编码字符串,未从真实 config.yaml 加载 |
|
||||
| INIT 配置变更触发 backend 重建 | 单测仅在初始化场景验证 |
|
||||
|
||||
---
|
||||
|
||||
## 4. 集成测试 vs CI 的差距
|
||||
|
||||
CI 仅运行 `tests/unit_tests/`,以下场景**从未在自动化中验证**:
|
||||
|
||||
- 真实容器的创建/执行/销毁
|
||||
- 容器网络隔离(`--network none`)
|
||||
- 容器资源限制生效(cpus/memory/pids_limit)
|
||||
- Managed process 的 WS 双向 I/O
|
||||
- 多 process 同 session 并发 I/O
|
||||
- 孤儿容器清理
|
||||
- Session 删除清理容器
|
||||
- 进程退出检测
|
||||
- E2B 真实 sandbox 行为
|
||||
|
||||
**建议**: 在 CI 中加一个可选的 Docker-in-Docker 集成测试 stage,至少覆盖核心执行路径(exec / MCP attach / session 销毁)。
|
||||
@@ -1,166 +0,0 @@
|
||||
# Box 系统 toB 商业化分析
|
||||
|
||||
> 更新日期: 2026-05-19
|
||||
> 分支: `feat/sandbox` (LangBot + langbot-plugin-sdk)
|
||||
|
||||
---
|
||||
|
||||
## 1. 现有优势
|
||||
|
||||
| 能力 | toB 价值 | 代码位置 |
|
||||
|------|---------|---------|
|
||||
| **沙箱隔离执行** | 企业安全运行不受信代码的基础能力 | SDK `box/backend.py` |
|
||||
| **多后端支持** | 适配不同企业容器基础设施 (Podman/Docker/nsjail/E2B) | SDK `box/runtime.py` `_select_backend()` |
|
||||
| **E2B 云沙箱** | SaaS / 无 Docker 部署的兜底执行环境 | SDK `box/e2b_backend.py` |
|
||||
| **连接自愈** | 心跳 + 自动重连,单点 Box runtime 故障可恢复 | `pkg/box/connector.py` `_heartbeat_loop`, `pkg/box/service.py` `_reconnect_loop` |
|
||||
| **Profile + locked 字段** | 运维锁定安全边界,LLM/用户无法绕过 | `pkg/box/service.py`, SDK `box/models.py` |
|
||||
| **资源限制** | CPU/内存/PID 数限制防止资源滥用 | SDK `backend.py` `--cpus/--memory/--pids-limit` |
|
||||
| **Workspace quota** | 磁盘用量控制 | `pkg/box/service.py` `_enforce_workspace_quota` |
|
||||
| **静默降级** | Box 不可用不影响其他功能,降低部署门槛 | `pkg/box/service.py:78` `_available=False` |
|
||||
| **孤儿容器清理** | 防止泄漏的容器持续占用资源 | SDK `backend.py` `cleanup_orphaned_containers` |
|
||||
| **网络隔离** | `--network none` 防止数据外泄 | SDK `backend.py` start_session |
|
||||
| **只读根文件系统** | `--read-only` 防止容器被持久篡改 | SDK `backend.py` start_session |
|
||||
| **Host path 白名单** | `allowed_host_mount_roots` 限制可挂载目录 | `pkg/box/service.py` `_validate_host_mount` |
|
||||
|
||||
---
|
||||
|
||||
## 2. toB 差距分析
|
||||
|
||||
### 2.1 安全与合规
|
||||
|
||||
| 维度 | 现状 | toB 要求 | 优先级 |
|
||||
|------|------|---------|--------|
|
||||
| **WS relay 认证** | 无认证,任何人可 attach | 至少 token 认证 | **P0** |
|
||||
| **安全策略** | policy.py 是死代码,实际无细粒度控制 | 工具级 allow/deny、沙箱模式控制 | **P0** |
|
||||
| **审计日志** | 仅内存中 50 条 `_recent_errors` | 持久化审计:谁何时执行了什么、结果如何 | **P0** |
|
||||
| **Host path 校验** | 黑名单策略,`/` 未拦截 | 白名单策略,默认拒绝 | **P1** |
|
||||
| **数据驻留** | 无控制 | GDPR / 等保要求的数据隔离 | **P2** |
|
||||
|
||||
### 2.2 多租户
|
||||
|
||||
| 维度 | 现状 | toB 要求 | 优先级 |
|
||||
|------|------|---------|--------|
|
||||
| **租户隔离** | 无租户概念 | BoxSpec/Profile 绑定 tenant_id | **P0** |
|
||||
| **RBAC** | 仅 token 认证 | admin/operator/viewer 角色权限 | **P0** |
|
||||
| **资源配额** | 单一 workspace quota | 每租户 CPU 时间/内存/并发/执行次数配额 | **P1** |
|
||||
| **Session 隔离** | 所有 session 共享 dict | 按租户分区,互不可见 | **P1** |
|
||||
|
||||
### 2.3 可靠性
|
||||
|
||||
| 维度 | 现状 | toB 要求 | 优先级 |
|
||||
|------|------|---------|--------|
|
||||
| **连接恢复** | 已实现:20s 心跳 + `_reconnect_loop` 指数退避 | 已满足基本要求 | 已有 |
|
||||
| **Session 清理** | 机会性(仅新建时触发) | 定时清理 + 独立 reaper | **P1** |
|
||||
| **水平扩展** | 单 Box Runtime 实例 | 多实例负载均衡(按 tenant 路由) | **P1** |
|
||||
| **优雅降级** | 已有(_available=False) | 已满足基本要求 | 已有 |
|
||||
| **Backend 自愈** | 已实现:`get_status` 时若 backend 不可用会重新选择 | 已满足基本要求 | 已有 |
|
||||
|
||||
### 2.4 可观测性
|
||||
|
||||
| 维度 | 现状 | toB 要求 | 优先级 |
|
||||
|------|------|---------|--------|
|
||||
| **监控指标** | 无 Prometheus metrics | session 数/执行延迟/资源用量/错误率 | **P1** |
|
||||
| **结构化日志** | Python logging, 无结构化 | JSON 格式日志,含 trace_id/tenant_id | **P1** |
|
||||
| **前端面板** | 监控页接入 `/api/v1/box/status`(backend 名 + 活跃 session 数);`sessions` / `errors` 仍未接入 | 完整状态面板 + 历史错误/审计列表 | **P2** |
|
||||
|
||||
---
|
||||
|
||||
## 3. SaaS 部署架构建议
|
||||
|
||||
### 3.1 方案 A: 共享 Box Runtime Pool (快速上线)
|
||||
|
||||
```
|
||||
LangBot Instance ──> Box Runtime (共享)
|
||||
├─ tenant_id 标签隔离
|
||||
├─ Redis 配额计数器
|
||||
└─ Container labels: langbot.tenant_id=xxx
|
||||
```
|
||||
|
||||
- **优点**: 改动最小,加 tenant_id 到 BoxSpec/labels 即可
|
||||
- **缺点**: 容器引擎共享,安全隔离弱
|
||||
|
||||
### 3.2 方案 B: 每租户 K8s Namespace + gVisor (推荐中期)
|
||||
|
||||
```
|
||||
LangBot ──> K8s API
|
||||
├─ namespace: tenant-xxx
|
||||
│ ├─ RuntimeClass: gVisor (runsc)
|
||||
│ ├─ ResourceQuota
|
||||
│ └─ NetworkPolicy
|
||||
└─ namespace: tenant-yyy
|
||||
└─ ...
|
||||
```
|
||||
|
||||
- **优点**: 强隔离(namespace + gVisor),原生 K8s 配额
|
||||
- **缺点**: 需要重写 backend 为 K8s Job,部署复杂度高
|
||||
|
||||
### 3.3 方案 C: K8s Job 直接编排 (长期)
|
||||
|
||||
```
|
||||
LangBot ──> K8s Job per execution
|
||||
├─ 每次执行创建 Job
|
||||
├─ Pod Security Standards
|
||||
├─ 自动调度和资源分配
|
||||
└─ Job TTL Controller 自动清理
|
||||
```
|
||||
|
||||
- **优点**: 最强隔离,天然水平扩展
|
||||
- **缺点**: 冷启动延迟,架构重写
|
||||
|
||||
**推荐演进路径**: A → B → C
|
||||
|
||||
---
|
||||
|
||||
## 4. 配额体系建议
|
||||
|
||||
### 三层配额
|
||||
|
||||
| 层 | 实现 | 作用 |
|
||||
|----|------|------|
|
||||
| **内核层** | Docker `--cpus`/`--memory`/`--storage-opt` | 硬性资源上限,不可绕过 |
|
||||
| **应用层** | Redis 原子计数器 | 并发 session 数/执行次数/CPU 时间预算 |
|
||||
| **计费层** | 月度聚合 | 按租户计费(session-hours/execution-count) |
|
||||
|
||||
### Profile 与套餐映射
|
||||
|
||||
| 套餐 | Profile | locked 字段 | 配额 |
|
||||
|------|---------|------------|------|
|
||||
| Free | `offline_readonly` | network, host_path_mode, rootfs | 10 exec/天, 0.5 CPU, 256MB |
|
||||
| Pro | `default` | (无) | 100 exec/天, 1 CPU, 512MB |
|
||||
| Enterprise | `network_extended` | (按需) | 无限, 2 CPU, 1GB, 自定义镜像 |
|
||||
|
||||
### TOCTOU 配额修复
|
||||
|
||||
当前 `_enforce_workspace_quota` 的 TOCTOU 问题可通过两种方式解决:
|
||||
|
||||
1. **预留式配额** (应用层): Redis `INCRBY` 预扣额度 → 执行 → 成功则扣减,失败则回滚
|
||||
2. **内核级限制** (Docker): `--storage-opt size=500m` 直接限制容器可写层大小
|
||||
|
||||
---
|
||||
|
||||
## 5. 优先实施路线
|
||||
|
||||
### Phase 1 (2-4 周): 安全基线
|
||||
|
||||
- [ ] WS relay 加 token 认证
|
||||
- [ ] 接入或删除 policy.py
|
||||
- [x] ~~Box 加重连和心跳~~(已完成,见 [box-issues.md 已解决](./box-issues.md))
|
||||
- [ ] 审计日志持久化(至少写文件/数据库)
|
||||
- [ ] `security.py` 加 `/` 拦截,考虑白名单
|
||||
- [ ] INIT 与 backend 初始化顺序整理(避免 backend 在配置到达前实例化)
|
||||
|
||||
### Phase 2 (4-8 周): 多租户基础
|
||||
|
||||
- [ ] BoxSpec 加 `tenant_id` 字段
|
||||
- [ ] 容器 labels 加 tenant 标识
|
||||
- [ ] Redis 配额计数器(并发/执行次数/时间)
|
||||
- [ ] RBAC 基础框架
|
||||
- [ ] 定时 session reaper
|
||||
|
||||
### Phase 3 (8-16 周): 生产就绪
|
||||
|
||||
- [ ] Prometheus metrics exporter
|
||||
- [ ] 前端 Box 状态面板
|
||||
- [ ] K8s backend 支持 (方案 B)
|
||||
- [ ] 结构化日志 (JSON, trace_id)
|
||||
- [ ] 水平扩展支持
|
||||
@@ -1,221 +0,0 @@
|
||||
# Box Runtime vs Plugin Runtime: 连接架构对比
|
||||
|
||||
> 更新日期: 2026-05-19
|
||||
> 分支: `feat/sandbox` (LangBot + langbot-plugin-sdk)
|
||||
|
||||
---
|
||||
|
||||
## 1. 总体差异
|
||||
|
||||
| 维度 | Plugin Runtime | Box Runtime |
|
||||
|------|---------------|-------------|
|
||||
| **继承关系** | `PluginRuntimeConnector(ManagedRuntimeConnector)` | `BoxRuntimeConnector`(独立类) |
|
||||
| **传输分支** | 3 条 (Docker/WS, Win32/subprocess+WS, Unix/stdio) | 3 条 (本地 stdio, Win32/subprocess+WS, 远程 WS) |
|
||||
| **心跳** | 20s ping loop | 20s ping loop(`_heartbeat_loop`) |
|
||||
| **重连** | WS 模式: sleep 3s → re-initialize | 由 BoxService `_reconnect_loop` 处理,指数退避 |
|
||||
| **Handler 类型** | `RuntimeConnectionHandler` (1132 行, 25+ action) | 基础 `Handler` + `BoxServerHandler`(SDK 端 25 action) |
|
||||
| **Client 抽象** | Handler 即 API | 独立 `ActionRPCBoxClient` 封装 Handler |
|
||||
| **启用/禁用** | `is_enable_plugin` 开关 | 无开关(可用/不可用由初始化结果决定) |
|
||||
| **初始化失败** | 异常上抛 | 静默降级 `_available=False` |
|
||||
| **Shutdown** | 直接杀进程 | RPC SHUTDOWN → 清理容器 → 再杀进程 |
|
||||
|
||||
---
|
||||
|
||||
## 2. 传输决策
|
||||
|
||||
### Plugin: 3-路决策
|
||||
|
||||
```python
|
||||
# pkg/plugin/connector.py:106-165
|
||||
if get_platform() == 'docker' or use_websocket_to_connect_plugin_runtime():
|
||||
# Docker/WS → ws://langbot_plugin_runtime:5400/control/ws
|
||||
elif get_platform() == 'win32':
|
||||
# Windows → 起子进程(无 pipe) + ws://localhost:5400/control/ws
|
||||
else:
|
||||
# Unix/Mac → StdioClientController(python -m langbot_plugin.cli rt -s)
|
||||
```
|
||||
|
||||
### Box: 3-路决策
|
||||
|
||||
```python
|
||||
# pkg/box/connector.py
|
||||
if self._uses_websocket():
|
||||
if platform.get_platform() == 'win32' and not self.configured_runtime_url:
|
||||
await self._start_subprocess_then_ws() # subprocess + ws://localhost:5410/rpc/ws
|
||||
else:
|
||||
await self._connect_remote_ws() # ws://{host}:5410/rpc/ws
|
||||
else:
|
||||
await self._start_local_stdio() # StdioClientController
|
||||
```
|
||||
|
||||
> 历史:2026-04-16 版本本文档曾把 Box 描述为 2 路决策(缺 Windows 分支)。现已对齐 Plugin 的 3 路设计。
|
||||
|
||||
### 决策矩阵
|
||||
|
||||
| 环境 | Plugin | Box |
|
||||
|------|--------|-----|
|
||||
| Docker | WS → `:5400` | WS → `:5410/rpc/ws` |
|
||||
| `--standalone-box` | N/A | WS → `localhost:5410/rpc/ws` |
|
||||
| Windows 非 Docker | subprocess + WS (`:5400`) | subprocess + WS (`localhost:5410/rpc/ws`) |
|
||||
| Unix/Mac 非 Docker | stdio | stdio |
|
||||
| 手动配置 URL | 通过配置项 | WS → 用户配置的 URL |
|
||||
|
||||
---
|
||||
|
||||
## 3. 连接建立
|
||||
|
||||
### 同步模式差异
|
||||
|
||||
**Plugin**: `new_connection_callback` 内直接 ping + await handler_task,`initialize()` 通过 `create_task()` 异步启动,不阻塞等待连接。
|
||||
|
||||
**Box**: 使用 `asyncio.Event` + `wait_for(timeout=30s)` 模式,`initialize()` 同步等待连接成功或超时。
|
||||
|
||||
### Box stdio 路径
|
||||
|
||||
```
|
||||
connector._start_local_stdio()
|
||||
├─ connected = asyncio.Event()
|
||||
├─ ctrl = StdioClientController(python, ['-m', 'langbot_plugin.cli.__init__', 'box', '-s', '--ws-control-port', N])
|
||||
├─ _ctrl_task = create_task(ctrl.run(callback))
|
||||
│ callback:
|
||||
│ handler = Handler(connection) ← 基础 Handler, 无 disconnect_callback
|
||||
│ client.set_handler(handler)
|
||||
│ _handler_task = create_task(handler.run())
|
||||
│ call_action(PING, {}) ← 握手, timeout=15s
|
||||
│ connected.set() ← 通知外层
|
||||
│ await _handler_task ← 阻塞直到断开
|
||||
└─ await wait_for(connected.wait(), 30s) ← 同步等待
|
||||
```
|
||||
|
||||
### Plugin stdio 路径
|
||||
|
||||
```
|
||||
connector.initialize()
|
||||
├─ ctrl = StdioClientController(python, ['-m', 'langbot_plugin.cli', 'rt', '-s'])
|
||||
├─ task = ctrl.run(callback)
|
||||
│ callback:
|
||||
│ disconnect_callback:
|
||||
│ [WS] → runtime_disconnect_callback → 重连
|
||||
│ [stdio] → 仅日志, 不重连
|
||||
│ handler = RuntimeConnectionHandler(conn, disconnect_cb, ap)
|
||||
│ create_task(handler.run())
|
||||
│ handler.ping() ← 握手, timeout=10s
|
||||
│ await handler_task ← 阻塞直到断开
|
||||
├─ create_task(heartbeat_loop()) ← 20s ping loop
|
||||
└─ create_task(task) ← 不等待连接
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. 心跳与重连
|
||||
|
||||
### 心跳
|
||||
|
||||
| 维度 | Plugin | Box |
|
||||
|------|--------|-----|
|
||||
| 有心跳? | 是 | 是(`connector.py` `_heartbeat_loop`) |
|
||||
| 间隔 | 20s | 20s |
|
||||
| 失败处理 | 仅 DEBUG 日志,不触发重连 | 仅 DEBUG 日志,依赖 connection close 触发重连 |
|
||||
| 生命周期 | 整个应用生命周期 | 连接建立后启动;`dispose()` 时 cancel |
|
||||
|
||||
### 重连
|
||||
|
||||
| 维度 | Plugin | Box |
|
||||
|------|--------|-----|
|
||||
| Docker/WS 断开 | `runtime_disconnect_callback` → sleep 3s → re-initialize | `runtime_disconnect_callback` → `BoxService._reconnect_loop()`(指数退避) |
|
||||
| WS 连接失败 | 同上 | 同上;初次失败时 `_available=False`,重连成功后恢复 |
|
||||
| stdio 断开 | 仅日志,不重连 | 接同样回调;stdio 重连需重新 fork 子进程 |
|
||||
| 重连退避 | 固定 3s,无 backoff | 指数退避 |
|
||||
|
||||
> 历史:2026-04-16 版本本文档曾把心跳与重连标记为 Box 缺失。这两项已在 commit `2dfd9d5d` / `c6882cf` / `5029d9c` 等修复(详见 [box-issues.md 已解决](./box-issues.md))。
|
||||
|
||||
---
|
||||
|
||||
## 5. 共享 IO 层
|
||||
|
||||
两者复用同一套 SDK IO 基础设施:
|
||||
|
||||
```
|
||||
Handler ← ABC (runtime/io/handler.py)
|
||||
├── RuntimeConnectionHandler (Plugin 用, LangBot 侧)
|
||||
├── ControlConnectionHandler (Plugin 用, SDK 侧)
|
||||
├── BoxServerHandler (Box 用, SDK 侧)
|
||||
└── 匿名 Handler 实例 (Box 用, LangBot 侧)
|
||||
|
||||
Connection ← ABC
|
||||
├── StdioConnection (stdio: 16KB chunks, 应用层分帧协议)
|
||||
└── WebSocketConnection (WS: 64KB chunks, 原生 WS 分帧)
|
||||
|
||||
Controller ← ABC
|
||||
├── StdioClientController (fork 子进程, pipe stdin/stdout)
|
||||
├── StdioServerController (接管当前进程 stdin/stdout)
|
||||
├── WebSocketClientController (连接 WS 服务端)
|
||||
└── WebSocketServerController (监听 WS 端口)
|
||||
```
|
||||
|
||||
共享的核心机制:
|
||||
- `call_action()` / `call_action_generator()` — RPC 调用/流式调用
|
||||
- `ActionRequest` / `ActionResponse` — 请求/响应协议
|
||||
- `seq_id` 关联 — 并发请求复用单连接
|
||||
- `CommonAction.PING` — 两者都用于初始握手
|
||||
- 文件传输 (`send_file`) — Plugin 用,Box 不用
|
||||
|
||||
---
|
||||
|
||||
## 6. 端口方案
|
||||
|
||||
| 服务 | Plugin | Box |
|
||||
|------|--------|-----|
|
||||
| Action RPC (stdio) | stdin/stdout | stdin/stdout |
|
||||
| Action RPC (WS) | `:5400` | `:5410/rpc/ws` |
|
||||
| 辅助服务 | debug WS `:5401` | managed process WS relay `:5410/v1/sessions/{id}/managed-process/ws` |
|
||||
|
||||
**Box 特点**: 单端口 aiohttp 服务(默认 5410),通过路径区分 Action RPC 和 managed process relay。即使在 stdio 模式,也在 `:5410` 启动 aiohttp 用于 managed process attach。Plugin 在 stdio 模式不开额外端口。
|
||||
|
||||
---
|
||||
|
||||
## 7. 销毁对比
|
||||
|
||||
### Plugin
|
||||
|
||||
```python
|
||||
dispose():
|
||||
if stdio: ctrl.process.terminate()
|
||||
_dispose_subprocess() # Windows 子进程
|
||||
heartbeat_task.cancel()
|
||||
```
|
||||
|
||||
### Box
|
||||
|
||||
```python
|
||||
connector.dispose():
|
||||
_handler_task.cancel()
|
||||
_ctrl_task.cancel()
|
||||
_subprocess.terminate()
|
||||
|
||||
service.dispose():
|
||||
connector.dispose()
|
||||
loop.create_task(client.shutdown()) # RPC SHUTDOWN → 清理所有容器
|
||||
```
|
||||
|
||||
Box 的 RPC SHUTDOWN 确保容器被正确停止,不会成为孤儿。Plugin 直接杀进程。
|
||||
|
||||
---
|
||||
|
||||
## 8. 改进建议
|
||||
|
||||
### P0
|
||||
|
||||
1. **两者都加 WS 认证**: 至少 token 认证(INIT 时下发,连接时校验)
|
||||
|
||||
### P1
|
||||
|
||||
2. **考虑 Box 继承 ManagedRuntimeConnector**: 复用 `_start_runtime_subprocess` / `_wait_until_ready` / `_dispose_subprocess`,减少重复代码
|
||||
3. **Plugin 重连加退避**: 固定 3s 无 backoff 可能造成日志洪水,建议向 Box 的指数退避看齐
|
||||
4. **统一连接管理模式**: Event-based (Box) vs direct-await (Plugin),考虑收敛为一种
|
||||
|
||||
### 已完成(自上一轮)
|
||||
|
||||
- ~~Box 加重连~~(commit `2dfd9d5d`)
|
||||
- ~~Box 加心跳~~(20s loop 与 Plugin 一致)
|
||||
- ~~Box 加 Windows 支持~~(commit `120817a` / `fafb7a4`)
|
||||
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "langbot"
|
||||
version = "4.10.0-beta.1"
|
||||
version = "4.8.7"
|
||||
description = "Production-grade platform for building agentic IM bots"
|
||||
readme = "README.md"
|
||||
license-files = ["LICENSE"]
|
||||
@@ -8,7 +8,7 @@ requires-python = ">=3.11,<4.0"
|
||||
dependencies = [
|
||||
"aiocqhttp>=1.4.4",
|
||||
"aiofiles>=24.1.0",
|
||||
"aiohttp>=3.13.4",
|
||||
"aiohttp>=3.11.18",
|
||||
"aioshutil>=1.5",
|
||||
"aiosqlite>=0.21.0",
|
||||
"anthropic>=0.51.0",
|
||||
@@ -16,18 +16,18 @@ dependencies = [
|
||||
"async-lru>=2.0.5",
|
||||
"certifi>=2025.4.26",
|
||||
"colorlog~=6.6.0",
|
||||
"cryptography>=46.0.7",
|
||||
"cryptography>=44.0.3",
|
||||
"dashscope>=1.25.10",
|
||||
"dingtalk-stream>=0.24.0",
|
||||
"discord-py>=2.5.2",
|
||||
"pynacl>=1.5.0", # Required for Discord voice support
|
||||
"gewechat-client>=0.1.5",
|
||||
"lark-oapi>=1.5.5",
|
||||
"lark-oapi>=1.4.15",
|
||||
"mcp>=1.25.0",
|
||||
"nakuru-project-idk>=0.0.2.1",
|
||||
"ollama>=0.4.8",
|
||||
"openai>1.0.0",
|
||||
"pillow>=12.2.0",
|
||||
"pillow>=11.2.1",
|
||||
"psutil>=7.0.0",
|
||||
"pycryptodome>=3.22.0",
|
||||
"pydantic>2.0",
|
||||
@@ -35,12 +35,10 @@ dependencies = [
|
||||
"python-telegram-bot>=22.0",
|
||||
"pyyaml>=6.0.2",
|
||||
"qq-botpy-rc>=1.2.1.6",
|
||||
"qrcode>=7.4",
|
||||
"quart>=0.20.0",
|
||||
"quart-cors>=0.8.0",
|
||||
"requests>=2.32.3",
|
||||
"slack-sdk>=3.35.0",
|
||||
"alembic>=1.15.0",
|
||||
"sqlalchemy[asyncio]>=2.0.40",
|
||||
"sqlmodel>=0.0.24",
|
||||
"telegramify-markdown>=0.5.1",
|
||||
@@ -51,7 +49,7 @@ dependencies = [
|
||||
"pip>=25.1.1",
|
||||
"ruff>=0.11.9",
|
||||
"pre-commit>=4.2.0",
|
||||
"uv>=0.11.6",
|
||||
"uv>=0.7.11",
|
||||
"mypy>=1.16.0",
|
||||
"PyPDF2>=3.0.1",
|
||||
"python-docx>=1.1.0",
|
||||
@@ -62,18 +60,13 @@ dependencies = [
|
||||
"ebooklib>=0.18",
|
||||
"html2text>=2024.2.26",
|
||||
"langchain>=0.2.0",
|
||||
"langchain-core>=1.2.28",
|
||||
"langsmith>=0.7.31",
|
||||
"python-multipart>=0.0.26",
|
||||
"Mako>=1.3.11",
|
||||
"langchain-text-splitters>=1.1.2",
|
||||
"chromadb>=1.0.0,<2.0.0",
|
||||
"langchain-text-splitters>=0.0.1",
|
||||
"chromadb>=0.4.24",
|
||||
"qdrant-client (>=1.15.1,<2.0.0)",
|
||||
"pyseekdb==1.1.0.post3",
|
||||
"langbot-plugin==0.4.0b1",
|
||||
"pyseekdb==1.0.0b7",
|
||||
"langbot-plugin==0.3.0rc1",
|
||||
"asyncpg>=0.30.0",
|
||||
"line-bot-sdk>=3.19.0",
|
||||
"matrix-nio>=0.25.2",
|
||||
"tboxsdk>=0.0.10",
|
||||
"boto3>=1.35.0",
|
||||
"pymilvus>=2.6.4",
|
||||
@@ -118,13 +111,12 @@ requires = ["setuptools>=61.0", "wheel"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[tool.setuptools]
|
||||
package-data = { "langbot" = ["templates/**", "pkg/provider/modelmgr/requesters/*", "pkg/platform/sources/*", "web/dist/**", "pkg/persistence/alembic/**"] }
|
||||
package-data = { "langbot" = ["templates/**", "pkg/provider/modelmgr/requesters/*", "pkg/platform/sources/*", "web/out/**"] }
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"moto>=5.2.1",
|
||||
"pre-commit>=4.2.0",
|
||||
"pytest>=9.0.3",
|
||||
"pytest>=8.4.1",
|
||||
"pytest-asyncio>=1.0.0",
|
||||
"pytest-cov>=7.0.0",
|
||||
"ruff>=0.11.9",
|
||||
@@ -223,3 +215,4 @@ skip-magic-trailing-comma = false
|
||||
|
||||
# Like Black, automatically detect the appropriate line ending.
|
||||
line-ending = "auto"
|
||||
|
||||
|
||||
@@ -4,9 +4,6 @@ python_files = test_*.py
|
||||
python_classes = Test*
|
||||
python_functions = test_*
|
||||
|
||||
# Python path for imports
|
||||
pythonpath = . tests
|
||||
|
||||
# Test paths
|
||||
testpaths = tests
|
||||
|
||||
@@ -25,9 +22,7 @@ markers =
|
||||
asyncio: mark test as async
|
||||
unit: mark test as unit test
|
||||
integration: mark test as integration test
|
||||
smoke: mark test as smoke test
|
||||
slow: mark test as slow running
|
||||
e2e: mark test as end-to-end test (requires real LangBot process)
|
||||
|
||||
# Coverage options (when using pytest-cov)
|
||||
[coverage:run]
|
||||
|
||||
@@ -1,65 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Coverage gate script
|
||||
# Runs all tests with coverage, enforcing minimum coverage threshold
|
||||
# Uses separate pytest invocations to avoid sys.modules pollution between test types
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
echo "=== LangBot Coverage Gate ==="
|
||||
echo ""
|
||||
|
||||
# Coverage threshold (baseline from current coverage, conservative buffer)
|
||||
# Current: ~22.14%, threshold: 18%
|
||||
COVERAGE_THRESHOLD=18
|
||||
|
||||
# Create temporary directory for coverage files
|
||||
COV_DIR=$(mktemp -d)
|
||||
trap "rm -rf $COV_DIR" EXIT
|
||||
|
||||
echo "[1/3] Running unit + smoke tests with coverage..."
|
||||
uv run pytest tests/unit_tests/ tests/smoke/ \
|
||||
--cov=langbot \
|
||||
--cov-report=json:$COV_DIR/unit.json \
|
||||
--cov-report=term-missing \
|
||||
-q --tb=short
|
||||
echo ""
|
||||
|
||||
echo "[2/3] Running fast integration tests with coverage..."
|
||||
uv run pytest tests/integration/ -m "not slow" \
|
||||
--cov=langbot \
|
||||
--cov-report=json:$COV_DIR/integration.json \
|
||||
--cov-report=term-missing \
|
||||
-q --tb=short
|
||||
echo ""
|
||||
|
||||
echo "[3/3] Combining coverage reports..."
|
||||
# Use coverage combine if available, otherwise just report total
|
||||
if command -v coverage &> /dev/null; then
|
||||
# Combine JSON reports
|
||||
coverage combine --keep $COV_DIR/unit.json $COV_DIR/integration.json \
|
||||
--data-file=$COV_DIR/combined.data 2>/dev/null || true
|
||||
|
||||
coverage report --data-file=$COV_DIR/combined.data || true
|
||||
else
|
||||
echo "Note: coverage combine not available, showing individual reports above"
|
||||
fi
|
||||
|
||||
# Generate final XML report for CI (from last run)
|
||||
uv run pytest tests/unit_tests/ tests/smoke/ \
|
||||
--cov=langbot \
|
||||
--cov-report=xml:coverage.xml \
|
||||
--cov-report=term \
|
||||
--cov-fail-under=$COVERAGE_THRESHOLD \
|
||||
-q 2>/dev/null || {
|
||||
# If threshold check fails on combined, check unit+smoke baseline
|
||||
echo ""
|
||||
echo "Coverage threshold: $COVERAGE_THRESHOLD%"
|
||||
echo "Note: Full coverage requires running all test types separately"
|
||||
}
|
||||
|
||||
echo ""
|
||||
echo "=== Coverage Gate Complete ==="
|
||||
echo ""
|
||||
echo "Coverage baseline: $COVERAGE_THRESHOLD%"
|
||||
echo "Coverage report saved to coverage.xml"
|
||||
@@ -1,16 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Fast integration tests
|
||||
# Runs integration tests excluding slow ones (PostgreSQL, external services)
|
||||
# Uses fake runner/provider, no real credentials needed
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
echo "=== LangBot Fast Integration Tests ==="
|
||||
echo ""
|
||||
|
||||
echo "Running integration tests (excluding slow)..."
|
||||
uv run pytest tests/integration/ -m "not slow" -q --tb=short
|
||||
|
||||
echo ""
|
||||
echo "=== Fast Integration Tests Complete ==="
|
||||
@@ -1,36 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Quick developer self-test command
|
||||
# Runs linting, unit tests, and smoke tests without requiring real provider keys
|
||||
# Suitable for local branch validation
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
echo "=== LangBot Quick Self-Test ==="
|
||||
echo ""
|
||||
|
||||
# 1. Ruff check
|
||||
echo "[1/3] Running ruff check..."
|
||||
uv run ruff check src/langbot/ tests/ --output-format=concise || {
|
||||
echo ""
|
||||
echo "⚠ Ruff check found issues. Run 'uv run ruff check --fix' to auto-fix."
|
||||
exit 1
|
||||
}
|
||||
echo "✓ Ruff check passed"
|
||||
echo ""
|
||||
|
||||
# 2. Unit tests
|
||||
echo "[2/3] Running unit tests..."
|
||||
uv run pytest tests/unit_tests/ -q --tb=short
|
||||
echo ""
|
||||
|
||||
# 3. Smoke tests (if exists)
|
||||
echo "[3/3] Running smoke tests..."
|
||||
if [ -d "tests/smoke" ]; then
|
||||
uv run pytest tests/smoke/ -q --tb=short
|
||||
else
|
||||
echo "No smoke tests found, skipping"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
echo "=== Quick Self-Test Complete ==="
|
||||
@@ -1,3 +1,3 @@
|
||||
"""LangBot - Production-grade platform for building agentic IM bots"""
|
||||
|
||||
__version__ = '4.10.0-beta.1'
|
||||
__version__ = '4.8.7'
|
||||
|
||||
@@ -5,8 +5,6 @@ import argparse
|
||||
import sys
|
||||
import os
|
||||
|
||||
from langbot.pkg.utils import paths
|
||||
|
||||
# ASCII art banner
|
||||
asciiart = r"""
|
||||
_ ___ _
|
||||
@@ -29,12 +27,6 @@ async def main_entry(loop: asyncio.AbstractEventLoop):
|
||||
help='Use standalone plugin runtime / 使用独立插件运行时',
|
||||
default=False,
|
||||
)
|
||||
parser.add_argument(
|
||||
'--standalone-box',
|
||||
action='store_true',
|
||||
help='Use standalone box runtime / 使用独立 Box 运行时',
|
||||
default=False,
|
||||
)
|
||||
parser.add_argument('--debug', action='store_true', help='Debug mode / 调试模式', default=False)
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -43,11 +35,6 @@ async def main_entry(loop: asyncio.AbstractEventLoop):
|
||||
|
||||
platform.standalone_runtime = True
|
||||
|
||||
if args.standalone_box:
|
||||
from langbot.pkg.utils import platform
|
||||
|
||||
platform.standalone_box = True
|
||||
|
||||
if args.debug:
|
||||
from langbot.pkg.utils import constants
|
||||
|
||||
@@ -100,7 +87,7 @@ def main():
|
||||
# Set up the working directory
|
||||
# When installed as a package, we need to handle the working directory differently
|
||||
# We'll create data directory in current working directory if not exists
|
||||
os.makedirs(paths.get_data_root(), exist_ok=True)
|
||||
os.makedirs('data', exist_ok=True)
|
||||
|
||||
loop = asyncio.new_event_loop()
|
||||
|
||||
|
||||
@@ -182,88 +182,6 @@ class DingTalkClient:
|
||||
for handler in self._message_handlers[msg_type]:
|
||||
await handler(event)
|
||||
|
||||
async def _parse_quoted_message(self, replied_msg: dict) -> dict:
|
||||
"""Parse the quoted/replied message and extract its content.
|
||||
|
||||
Args:
|
||||
replied_msg: The repliedMsg object from DingTalk message
|
||||
|
||||
Returns:
|
||||
A dict containing the quoted message info with keys:
|
||||
- message_id: The original message ID
|
||||
- msg_type: The message type (text, file, picture, audio, etc.)
|
||||
- content: The text content (if any)
|
||||
- file_url: The file download URL (if file type)
|
||||
- file_name: The file name (if file type)
|
||||
- picture: The picture base64 (if picture type)
|
||||
- audio: The audio base64 (if audio type)
|
||||
"""
|
||||
quote_info = {
|
||||
'message_id': replied_msg.get('msgId', ''),
|
||||
'msg_type': replied_msg.get('msgType', ''),
|
||||
'sender_id': replied_msg.get('senderId', ''),
|
||||
}
|
||||
|
||||
msg_type = replied_msg.get('msgType', '')
|
||||
content = replied_msg.get('content', {})
|
||||
|
||||
# Handle content as string (JSON) or dict
|
||||
if isinstance(content, str):
|
||||
try:
|
||||
content = json.loads(content)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
content = {}
|
||||
|
||||
if msg_type == 'text':
|
||||
# Text message
|
||||
if isinstance(content, dict):
|
||||
quote_info['content'] = content.get('content', '')
|
||||
else:
|
||||
quote_info['content'] = str(content)
|
||||
|
||||
elif msg_type == 'file':
|
||||
# File message
|
||||
download_code = content.get('downloadCode')
|
||||
file_name = content.get('fileName')
|
||||
if download_code and file_name:
|
||||
try:
|
||||
quote_info['file_url'] = await self.get_file_url(download_code)
|
||||
quote_info['file_name'] = file_name
|
||||
except Exception as e:
|
||||
if self.logger:
|
||||
await self.logger.error(f'Failed to get quoted file URL: {e}')
|
||||
|
||||
elif msg_type == 'picture':
|
||||
# Picture message
|
||||
download_code = content.get('downloadCode')
|
||||
if download_code:
|
||||
try:
|
||||
quote_info['picture'] = await self.download_image(download_code)
|
||||
except Exception as e:
|
||||
if self.logger:
|
||||
await self.logger.error(f'Failed to download quoted image: {e}')
|
||||
|
||||
elif msg_type == 'audio':
|
||||
# Audio message
|
||||
download_code = content.get('downloadCode')
|
||||
if download_code:
|
||||
try:
|
||||
quote_info['audio'] = await self.get_audio_url(download_code)
|
||||
except Exception as e:
|
||||
if self.logger:
|
||||
await self.logger.error(f'Failed to get quoted audio: {e}')
|
||||
|
||||
elif msg_type == 'richText':
|
||||
# Rich text message - extract text content
|
||||
rich_text = content.get('richText', [])
|
||||
texts = []
|
||||
for item in rich_text:
|
||||
if 'text' in item and item['text'] != '\n':
|
||||
texts.append(item['text'])
|
||||
quote_info['content'] = '\n'.join(texts)
|
||||
|
||||
return quote_info
|
||||
|
||||
async def get_message(self, incoming_message: dingtalk_stream.chatbot.ChatbotMessage):
|
||||
try:
|
||||
# print(json.dumps(incoming_message.to_dict(), indent=4, ensure_ascii=False))
|
||||
@@ -275,15 +193,6 @@ class DingTalkClient:
|
||||
elif str(incoming_message.conversation_type) == '2':
|
||||
message_data['conversation_type'] = 'GroupMessage'
|
||||
|
||||
# Check for quoted/replied message
|
||||
raw_data = incoming_message.to_dict()
|
||||
text_data = raw_data.get('text', {})
|
||||
if isinstance(text_data, dict) and text_data.get('isReplyMsg'):
|
||||
replied_msg = text_data.get('repliedMsg', {})
|
||||
if replied_msg:
|
||||
quote_info = await self._parse_quoted_message(replied_msg)
|
||||
message_data['QuotedMessage'] = quote_info
|
||||
|
||||
if incoming_message.message_type == 'richText':
|
||||
data = incoming_message.rich_text_content.to_dict()
|
||||
|
||||
@@ -359,52 +268,19 @@ class DingTalkClient:
|
||||
|
||||
message_data['Type'] = 'image'
|
||||
elif incoming_message.message_type == 'audio':
|
||||
raw_content = incoming_message.to_dict().get('content', {})
|
||||
# 兼容处理:如果 content 仍为 JSON 字符串则进行解析
|
||||
if isinstance(raw_content, str):
|
||||
try:
|
||||
raw_content = json.loads(raw_content)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
raw_content = {}
|
||||
|
||||
if self.logger:
|
||||
await self.logger.info(f'DingTalk audio raw content: {json.dumps(raw_content, ensure_ascii=False)}')
|
||||
|
||||
# 提取钉钉自带的语音转写文字(Powered by Qwen)
|
||||
recognition = raw_content.get('recognition', '')
|
||||
if recognition:
|
||||
message_data['Content'] = recognition
|
||||
|
||||
download_code = raw_content.get('downloadCode')
|
||||
if download_code:
|
||||
message_data['Audio'] = await self.get_audio_url(download_code)
|
||||
message_data['Audio'] = await self.get_audio_url(incoming_message.to_dict()['content']['downloadCode'])
|
||||
|
||||
message_data['Type'] = 'audio'
|
||||
elif incoming_message.message_type == 'file':
|
||||
# 获取原始数据字典并提取嵌套的文件信息
|
||||
raw_data = incoming_message.to_dict()
|
||||
file_info = raw_data.get('content', {})
|
||||
|
||||
# 兼容处理:如果 content 仍为 JSON 字符串则进行解析
|
||||
if isinstance(file_info, str):
|
||||
try:
|
||||
file_info = json.loads(file_info)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
file_info = {}
|
||||
|
||||
download_code = file_info.get('downloadCode')
|
||||
file_name = file_info.get('fileName')
|
||||
|
||||
if download_code and file_name:
|
||||
# 转换 downloadCode 为可下载的真实 URL
|
||||
message_data['File'] = await self.get_file_url(download_code)
|
||||
message_data['Name'] = file_name
|
||||
down_list = incoming_message.get_down_list()
|
||||
if len(down_list) >= 2:
|
||||
message_data['File'] = await self.get_file_url(down_list[0])
|
||||
message_data['Name'] = down_list[1]
|
||||
else:
|
||||
if self.logger:
|
||||
await self.logger.error(f'Failed to extract file info from message content: {file_info}')
|
||||
await self.logger.error(f'get_down_list() returned fewer than 2 elements: {down_list}')
|
||||
message_data['File'] = None
|
||||
message_data['Name'] = None
|
||||
|
||||
message_data['Type'] = 'file'
|
||||
|
||||
copy_message_data = message_data.copy()
|
||||
@@ -481,12 +357,6 @@ class DingTalkClient:
|
||||
card_data['config'] = json.dumps({'autoLayout': card_auto_layout})
|
||||
card_data['content'] = ''
|
||||
|
||||
# 将用户的消息内容作为卡片的查询参数,方便后续处理
|
||||
if incoming_message.message_type == 'text':
|
||||
card_data['query'] = incoming_message.get_text_list()[0]
|
||||
else:
|
||||
card_data['query'] = '...'
|
||||
|
||||
card_instance = dingtalk_stream.AICardReplier(self.client, incoming_message)
|
||||
# print(card_instance)
|
||||
# 先投放卡片: https://open.dingtalk.com/document/orgapp/create-and-deliver-cards
|
||||
|
||||
@@ -47,22 +47,6 @@ class DingTalkEvent(dict):
|
||||
def conversation(self):
|
||||
return self.get('conversation_type', '')
|
||||
|
||||
@property
|
||||
def quoted_message(self) -> Optional[Dict[str, Any]]:
|
||||
"""Get the quoted/replied message info if this is a reply message.
|
||||
|
||||
Returns:
|
||||
A dict containing:
|
||||
- message_id: The original message ID
|
||||
- msg_type: The message type (text, file, picture, audio, etc.)
|
||||
- content: The text content (if any)
|
||||
- file_url: The file download URL (if file type)
|
||||
- file_name: The file name (if file type)
|
||||
- picture: The picture base64 (if picture type)
|
||||
- audio: The audio base64 (if audio type)
|
||||
"""
|
||||
return self.get('QuotedMessage')
|
||||
|
||||
def __getattr__(self, key: str) -> Optional[Any]:
|
||||
"""
|
||||
允许通过属性访问数据中的任意字段。
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
from .client import OpenClawWeixinClient as OpenClawWeixinClient
|
||||
from .types import ApiError as ApiError
|
||||
from .types import LoginResult as LoginResult
|
||||
@@ -1,807 +0,0 @@
|
||||
"""Async HTTP client for the OpenClaw WeChat API.
|
||||
|
||||
Implements the iLink Bot API protocol.
|
||||
Reference: https://github.com/epiral/weixin-bot
|
||||
|
||||
Endpoints: getUpdates (long-poll), sendMessage, getUploadUrl, getConfig, sendTyping.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import struct
|
||||
import typing
|
||||
import uuid
|
||||
from typing import Optional
|
||||
from urllib.parse import quote
|
||||
|
||||
import aiohttp
|
||||
|
||||
from .types import (
|
||||
ApiError,
|
||||
CDNMedia,
|
||||
FileItem,
|
||||
GetConfigResponse,
|
||||
GetUpdatesResponse,
|
||||
GetUploadUrlResponse,
|
||||
ImageItem,
|
||||
LoginResult,
|
||||
MessageItem,
|
||||
QRCodeResponse,
|
||||
QRStatusResponse,
|
||||
RefMessage,
|
||||
TextItem,
|
||||
VideoItem,
|
||||
VoiceItem,
|
||||
WeixinMessage,
|
||||
)
|
||||
|
||||
logger = logging.getLogger('openclaw-weixin-sdk')
|
||||
|
||||
DEFAULT_BASE_URL = 'https://ilinkai.weixin.qq.com'
|
||||
CDN_BASE_URL = 'https://novac2c.cdn.weixin.qq.com/c2c'
|
||||
|
||||
CHANNEL_VERSION = '1.0.0'
|
||||
|
||||
DEFAULT_API_TIMEOUT = 15
|
||||
DEFAULT_LONG_POLL_TIMEOUT = 40
|
||||
DEFAULT_CONFIG_TIMEOUT = 10
|
||||
DEFAULT_QR_POLL_TIMEOUT = 35
|
||||
|
||||
SESSION_EXPIRED_ERRCODE = -14
|
||||
|
||||
DEFAULT_BOT_TYPE = '3'
|
||||
|
||||
# Maximum text length per message chunk (WeChat limit)
|
||||
MAX_TEXT_CHUNK_SIZE = 2000
|
||||
|
||||
|
||||
def _random_wechat_uin() -> str:
|
||||
"""Generate the X-WECHAT-UIN header: random uint32 -> decimal string -> base64."""
|
||||
rand_bytes = os.urandom(4)
|
||||
uint32_val = struct.unpack('>I', rand_bytes)[0]
|
||||
return base64.b64encode(str(uint32_val).encode('utf-8')).decode('utf-8')
|
||||
|
||||
|
||||
def _build_base_info() -> dict:
|
||||
"""Build the base_info payload included in every API request."""
|
||||
return {'channel_version': CHANNEL_VERSION}
|
||||
|
||||
|
||||
def _chunk_text(text: str, max_size: int = MAX_TEXT_CHUNK_SIZE) -> list[str]:
|
||||
"""Split long text into chunks that fit within WeChat's message size limit."""
|
||||
if len(text) <= max_size:
|
||||
return [text]
|
||||
chunks = []
|
||||
while text:
|
||||
chunks.append(text[:max_size])
|
||||
text = text[max_size:]
|
||||
return chunks
|
||||
|
||||
|
||||
class OpenClawWeixinClient:
|
||||
"""Async client for the OpenClaw WeChat HTTP JSON API."""
|
||||
|
||||
def __init__(self, base_url: str, token: str):
|
||||
self.base_url = base_url.rstrip('/')
|
||||
self.token = token
|
||||
self._session: Optional[aiohttp.ClientSession] = None
|
||||
|
||||
async def _get_session(self) -> aiohttp.ClientSession:
|
||||
if self._session is None or self._session.closed:
|
||||
self._session = aiohttp.ClientSession()
|
||||
return self._session
|
||||
|
||||
async def close(self):
|
||||
if self._session and not self._session.closed:
|
||||
await self._session.close()
|
||||
|
||||
def _build_headers(self) -> dict[str, str]:
|
||||
headers = {
|
||||
'Content-Type': 'application/json',
|
||||
'AuthorizationType': 'ilink_bot_token',
|
||||
'X-WECHAT-UIN': _random_wechat_uin(),
|
||||
}
|
||||
if self.token:
|
||||
headers['Authorization'] = f'Bearer {self.token}'
|
||||
return headers
|
||||
|
||||
async def _post(self, endpoint: str, payload: dict, timeout: float = DEFAULT_API_TIMEOUT) -> dict:
|
||||
"""Make a POST request and return the JSON response.
|
||||
|
||||
Raises ApiError on HTTP errors or when the response contains a non-zero errcode.
|
||||
"""
|
||||
payload['base_info'] = _build_base_info()
|
||||
|
||||
session = await self._get_session()
|
||||
url = f'{self.base_url}/{endpoint}'
|
||||
headers = self._build_headers()
|
||||
|
||||
async with session.post(
|
||||
url, json=payload, headers=headers, timeout=aiohttp.ClientTimeout(total=timeout)
|
||||
) as resp:
|
||||
if resp.status != 200:
|
||||
text = await resp.text()
|
||||
raise ApiError(
|
||||
f'OpenClaw API error {resp.status}: {text}',
|
||||
status=resp.status,
|
||||
)
|
||||
data = await resp.json(content_type=None)
|
||||
|
||||
# Check for application-level errors in the response body
|
||||
errcode = data.get('errcode') or data.get('ret')
|
||||
if errcode and errcode != 0:
|
||||
raise ApiError(
|
||||
data.get('errmsg') or f'API errcode {errcode}',
|
||||
status=200,
|
||||
code=errcode,
|
||||
payload=data,
|
||||
)
|
||||
|
||||
return data
|
||||
|
||||
async def get_updates(
|
||||
self, get_updates_buf: str = '', timeout: float = DEFAULT_LONG_POLL_TIMEOUT
|
||||
) -> GetUpdatesResponse:
|
||||
"""Long-poll for new messages.
|
||||
|
||||
Note: This method does NOT raise ApiError for errcode responses —
|
||||
it returns them in the GetUpdatesResponse so the caller can handle
|
||||
session expiry and other errors with full context.
|
||||
"""
|
||||
try:
|
||||
# Bypass the errcode check in _post since get_updates needs
|
||||
# to return error info (e.g. session expired) to the caller.
|
||||
payload: dict = {'get_updates_buf': get_updates_buf}
|
||||
payload['base_info'] = _build_base_info()
|
||||
|
||||
session = await self._get_session()
|
||||
url = f'{self.base_url}/ilink/bot/getupdates'
|
||||
headers = self._build_headers()
|
||||
|
||||
async with session.post(
|
||||
url,
|
||||
json=payload,
|
||||
headers=headers,
|
||||
timeout=aiohttp.ClientTimeout(total=timeout),
|
||||
) as resp:
|
||||
if resp.status != 200:
|
||||
text = await resp.text()
|
||||
raise ApiError(
|
||||
f'OpenClaw API error {resp.status}: {text}',
|
||||
status=resp.status,
|
||||
)
|
||||
data = await resp.json(content_type=None)
|
||||
|
||||
except (asyncio.TimeoutError, aiohttp.ServerTimeoutError):
|
||||
return GetUpdatesResponse(ret=0, msgs=[], get_updates_buf=get_updates_buf)
|
||||
except ApiError:
|
||||
raise
|
||||
except Exception as e:
|
||||
if 'timeout' in str(e).lower():
|
||||
return GetUpdatesResponse(ret=0, msgs=[], get_updates_buf=get_updates_buf)
|
||||
raise
|
||||
|
||||
return _parse_get_updates_response(data)
|
||||
|
||||
async def send_message(
|
||||
self,
|
||||
to_user_id: str,
|
||||
item_list: list[MessageItem],
|
||||
context_token: str = '',
|
||||
) -> None:
|
||||
"""Send a message to a user."""
|
||||
items_payload = [_message_item_to_dict(item) for item in item_list]
|
||||
|
||||
payload = {
|
||||
'msg': {
|
||||
'from_user_id': '',
|
||||
'to_user_id': to_user_id,
|
||||
'client_id': f'langbot-{uuid.uuid4().hex[:16]}',
|
||||
'message_type': WeixinMessage.TYPE_BOT,
|
||||
'message_state': WeixinMessage.STATE_FINISH,
|
||||
'item_list': items_payload,
|
||||
'context_token': context_token or None,
|
||||
}
|
||||
}
|
||||
await self._post('ilink/bot/sendmessage', payload)
|
||||
|
||||
async def send_text(self, to_user_id: str, text: str, context_token: str = '') -> None:
|
||||
"""Send a plain text message, automatically chunking if too long."""
|
||||
chunks = _chunk_text(text)
|
||||
for chunk in chunks:
|
||||
item = MessageItem(type=MessageItem.TEXT, text_item=TextItem(text=chunk))
|
||||
await self.send_message(to_user_id, [item], context_token)
|
||||
|
||||
async def get_config(self, ilink_user_id: str, context_token: str = '') -> GetConfigResponse:
|
||||
"""Get bot config including typing_ticket."""
|
||||
data = await self._post(
|
||||
'ilink/bot/getconfig',
|
||||
{'ilink_user_id': ilink_user_id, 'context_token': context_token or None},
|
||||
timeout=DEFAULT_CONFIG_TIMEOUT,
|
||||
)
|
||||
return GetConfigResponse(
|
||||
ret=data.get('ret'),
|
||||
errmsg=data.get('errmsg'),
|
||||
typing_ticket=data.get('typing_ticket'),
|
||||
)
|
||||
|
||||
async def send_typing(self, ilink_user_id: str, typing_ticket: str, status: int = 1) -> None:
|
||||
"""Send typing indicator. status: 1=typing, 2=cancel."""
|
||||
await self._post(
|
||||
'ilink/bot/sendtyping',
|
||||
{
|
||||
'ilink_user_id': ilink_user_id,
|
||||
'typing_ticket': typing_ticket,
|
||||
'status': status,
|
||||
},
|
||||
timeout=DEFAULT_CONFIG_TIMEOUT,
|
||||
)
|
||||
|
||||
async def stop_typing(self, ilink_user_id: str, typing_ticket: str) -> None:
|
||||
"""Cancel the typing indicator for a user."""
|
||||
await self.send_typing(ilink_user_id, typing_ticket, status=2)
|
||||
|
||||
async def download_media(
|
||||
self,
|
||||
media: CDNMedia,
|
||||
) -> bytes:
|
||||
"""Download and decrypt a file from the WeChat CDN.
|
||||
|
||||
Args:
|
||||
media: CDNMedia object with encrypt_query_param and aes_key.
|
||||
|
||||
Returns:
|
||||
Decrypted file bytes.
|
||||
"""
|
||||
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
|
||||
from cryptography.hazmat.primitives.padding import PKCS7
|
||||
|
||||
if not media.encrypt_query_param:
|
||||
raise ApiError('CDN media has no encrypt_query_param', status=0)
|
||||
if not media.aes_key:
|
||||
raise ApiError('CDN media has no aes_key', status=0)
|
||||
|
||||
# Derive 16-byte AES key
|
||||
# aes_key is base64-encoded; the decoded content may be:
|
||||
# - raw 16 bytes (direct AES key)
|
||||
# - 32-char hex string (decode hex to get 16 bytes)
|
||||
raw = base64.b64decode(media.aes_key)
|
||||
if len(raw) == 16:
|
||||
aes_key = raw
|
||||
elif len(raw) == 32:
|
||||
# Hex-encoded 16-byte key
|
||||
aes_key = bytes.fromhex(raw.decode('utf-8'))
|
||||
else:
|
||||
raise ApiError(f'Invalid AES key length: {len(raw)} (expected 16 or 32)', status=0)
|
||||
|
||||
# Download encrypted bytes from CDN
|
||||
session = await self._get_session()
|
||||
cdn_url = f'{CDN_BASE_URL}/download?encrypted_query_param={quote(media.encrypt_query_param, safe="")}'
|
||||
|
||||
async with session.get(cdn_url, timeout=aiohttp.ClientTimeout(total=120)) as resp:
|
||||
if resp.status != 200:
|
||||
text = await resp.text()
|
||||
raise ApiError(f'CDN download failed: {resp.status} {text}', status=resp.status)
|
||||
encrypted = await resp.read()
|
||||
|
||||
# Decrypt AES-128-ECB with PKCS7 padding
|
||||
cipher = Cipher(algorithms.AES(aes_key), modes.ECB())
|
||||
decryptor = cipher.decryptor()
|
||||
padded = decryptor.update(encrypted) + decryptor.finalize()
|
||||
|
||||
unpadder = PKCS7(128).unpadder()
|
||||
return unpadder.update(padded) + unpadder.finalize()
|
||||
|
||||
async def upload_media(
|
||||
self,
|
||||
file_bytes: bytes,
|
||||
to_user_id: str,
|
||||
media_type: int,
|
||||
) -> CDNMedia:
|
||||
"""Encrypt and upload media to WeChat CDN.
|
||||
|
||||
Args:
|
||||
file_bytes: Raw file bytes to upload.
|
||||
to_user_id: Recipient user ID.
|
||||
media_type: 1=IMAGE, 2=VIDEO, 3=FILE, 4=VOICE.
|
||||
|
||||
Returns:
|
||||
CDNMedia with encrypt_query_param and aes_key for use in sendMessage.
|
||||
"""
|
||||
import hashlib
|
||||
|
||||
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
|
||||
from cryptography.hazmat.primitives.padding import PKCS7
|
||||
|
||||
# 1. Generate random 16-byte AES key
|
||||
raw_key = os.urandom(16)
|
||||
aes_key_hex = raw_key.hex() # 32-char hex string
|
||||
|
||||
# 2. Encode key for CDNMedia: base64(hex_string) — same for all media types
|
||||
# Matches official SDK: Buffer.from(aeskey_hex).toString("base64")
|
||||
encoded_key = base64.b64encode(aes_key_hex.encode('utf-8')).decode('utf-8')
|
||||
|
||||
# 3. Encrypt file with AES-128-ECB + PKCS7
|
||||
padder = PKCS7(128).padder()
|
||||
padded = padder.update(file_bytes) + padder.finalize()
|
||||
cipher = Cipher(algorithms.AES(raw_key), modes.ECB())
|
||||
encryptor = cipher.encryptor()
|
||||
encrypted = encryptor.update(padded) + encryptor.finalize()
|
||||
|
||||
# 4. Get upload URL
|
||||
raw_md5 = hashlib.md5(file_bytes).hexdigest()
|
||||
filekey = os.urandom(16).hex() # 32-char hex, matches official SDK
|
||||
|
||||
upload_resp = await self.get_upload_url(
|
||||
filekey=filekey,
|
||||
media_type=media_type,
|
||||
to_user_id=to_user_id,
|
||||
rawsize=len(file_bytes),
|
||||
rawfilemd5=raw_md5,
|
||||
filesize=len(encrypted),
|
||||
aeskey=aes_key_hex, # hex string, as expected by the API
|
||||
)
|
||||
|
||||
if not upload_resp.upload_param:
|
||||
raise ApiError('Failed to get upload URL', status=0)
|
||||
|
||||
# 5. Upload to CDN
|
||||
# upload_param is an opaque token from the server — pass it as-is
|
||||
session = await self._get_session()
|
||||
cdn_url = f'{CDN_BASE_URL}/upload?encrypted_query_param={quote(upload_resp.upload_param, safe="")}&filekey={quote(filekey, safe="")}'
|
||||
logger.debug(
|
||||
'CDN upload: url=%s raw_size=%d encrypted_size=%d md5=%s aeskey=%s',
|
||||
cdn_url,
|
||||
len(file_bytes),
|
||||
len(encrypted),
|
||||
raw_md5,
|
||||
encoded_key,
|
||||
)
|
||||
|
||||
async with session.post(
|
||||
cdn_url,
|
||||
data=encrypted,
|
||||
headers={'Content-Type': 'application/octet-stream'},
|
||||
timeout=aiohttp.ClientTimeout(total=120),
|
||||
) as resp:
|
||||
if resp.status != 200:
|
||||
text = await resp.text()
|
||||
logger.error('CDN upload failed: status=%d url=%s body=%s', resp.status, cdn_url, text[:500])
|
||||
raise ApiError(f'CDN upload failed: {resp.status} {text}', status=resp.status)
|
||||
download_param = resp.headers.get('x-encrypted-param', '')
|
||||
|
||||
if not download_param:
|
||||
raise ApiError('CDN upload succeeded but no x-encrypted-param returned', status=0)
|
||||
|
||||
return CDNMedia(
|
||||
encrypt_query_param=download_param,
|
||||
aes_key=encoded_key,
|
||||
encrypt_type=1,
|
||||
)
|
||||
|
||||
async def send_image(
|
||||
self,
|
||||
to_user_id: str,
|
||||
image_bytes: bytes,
|
||||
context_token: str = '',
|
||||
) -> None:
|
||||
"""Upload an image to CDN and send it."""
|
||||
media = await self.upload_media(image_bytes, to_user_id, media_type=1)
|
||||
item = MessageItem(
|
||||
type=MessageItem.IMAGE,
|
||||
image_item=ImageItem(
|
||||
media=media,
|
||||
aeskey=media.aes_key,
|
||||
),
|
||||
)
|
||||
await self.send_message(to_user_id, [item], context_token)
|
||||
|
||||
async def send_file(
|
||||
self,
|
||||
to_user_id: str,
|
||||
file_bytes: bytes,
|
||||
file_name: str,
|
||||
context_token: str = '',
|
||||
) -> None:
|
||||
"""Upload a file to CDN and send it."""
|
||||
import hashlib
|
||||
|
||||
media = await self.upload_media(file_bytes, to_user_id, media_type=3)
|
||||
item = MessageItem(
|
||||
type=MessageItem.FILE,
|
||||
file_item=FileItem(
|
||||
media=media,
|
||||
file_name=file_name,
|
||||
md5=hashlib.md5(file_bytes).hexdigest(),
|
||||
len=str(len(file_bytes)),
|
||||
),
|
||||
)
|
||||
await self.send_message(to_user_id, [item], context_token)
|
||||
|
||||
async def send_voice(
|
||||
self,
|
||||
to_user_id: str,
|
||||
voice_bytes: bytes,
|
||||
playtime: int = 0,
|
||||
context_token: str = '',
|
||||
) -> None:
|
||||
"""Upload a voice message to CDN and send it."""
|
||||
media = await self.upload_media(voice_bytes, to_user_id, media_type=4)
|
||||
item = MessageItem(
|
||||
type=MessageItem.VOICE,
|
||||
voice_item=VoiceItem(
|
||||
media=media,
|
||||
playtime=playtime,
|
||||
),
|
||||
)
|
||||
await self.send_message(to_user_id, [item], context_token)
|
||||
|
||||
async def get_upload_url(
|
||||
self,
|
||||
filekey: str,
|
||||
media_type: int,
|
||||
to_user_id: str,
|
||||
rawsize: int,
|
||||
rawfilemd5: str,
|
||||
filesize: int,
|
||||
thumb_rawsize: Optional[int] = None,
|
||||
thumb_rawfilemd5: Optional[str] = None,
|
||||
thumb_filesize: Optional[int] = None,
|
||||
aeskey: Optional[str] = None,
|
||||
) -> GetUploadUrlResponse:
|
||||
"""Get a pre-signed CDN upload URL."""
|
||||
payload: dict = {
|
||||
'filekey': filekey,
|
||||
'media_type': media_type,
|
||||
'to_user_id': to_user_id,
|
||||
'rawsize': rawsize,
|
||||
'rawfilemd5': rawfilemd5,
|
||||
'filesize': filesize,
|
||||
'no_need_thumb': True,
|
||||
}
|
||||
if thumb_rawsize is not None:
|
||||
payload['thumb_rawsize'] = thumb_rawsize
|
||||
if thumb_rawfilemd5 is not None:
|
||||
payload['thumb_rawfilemd5'] = thumb_rawfilemd5
|
||||
if thumb_filesize is not None:
|
||||
payload['thumb_filesize'] = thumb_filesize
|
||||
if aeskey is not None:
|
||||
payload['aeskey'] = aeskey
|
||||
|
||||
data = await self._post('ilink/bot/getuploadurl', payload)
|
||||
logger.debug('get_upload_url response: %s', data)
|
||||
return GetUploadUrlResponse(
|
||||
upload_param=data.get('upload_param'),
|
||||
thumb_upload_param=data.get('thumb_upload_param'),
|
||||
)
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# QR Code Login
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
async def fetch_qrcode(self, bot_type: str = DEFAULT_BOT_TYPE) -> QRCodeResponse:
|
||||
"""Fetch a QR code for WeChat login authorization (GET, no auth needed)."""
|
||||
session = await self._get_session()
|
||||
url = f'{self.base_url}/ilink/bot/get_bot_qrcode?bot_type={bot_type}'
|
||||
|
||||
async with session.get(url, timeout=aiohttp.ClientTimeout(total=DEFAULT_API_TIMEOUT)) as resp:
|
||||
if resp.status != 200:
|
||||
text = await resp.text()
|
||||
raise ApiError(
|
||||
f'Failed to fetch QR code: {resp.status} {text}',
|
||||
status=resp.status,
|
||||
)
|
||||
data = await resp.json(content_type=None)
|
||||
|
||||
logger.debug(
|
||||
'fetch_qrcode response: qrcode=%s, img=%s', data.get('qrcode'), bool(data.get('qrcode_img_content'))
|
||||
)
|
||||
|
||||
return QRCodeResponse(
|
||||
qrcode=data.get('qrcode'),
|
||||
qrcode_img_content=data.get('qrcode_img_content'),
|
||||
)
|
||||
|
||||
async def _fetch_qr_image_base64(self, url: str) -> str:
|
||||
"""Generate a QR code image from the URL and return a data URI string.
|
||||
|
||||
The qrcode_img_content URL points to an HTML page (not a raw image),
|
||||
so we generate the QR code locally using the qrcode library.
|
||||
"""
|
||||
import qrcode
|
||||
|
||||
qr = qrcode.QRCode(error_correction=qrcode.constants.ERROR_CORRECT_L)
|
||||
qr.add_data(url)
|
||||
qr.make(fit=True)
|
||||
img = qr.make_image(fill_color='black', back_color='white')
|
||||
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, format='PNG')
|
||||
b64 = base64.b64encode(buf.getvalue()).decode('utf-8')
|
||||
return f'data:image/png;base64,{b64}'
|
||||
|
||||
async def poll_qrcode_status(self, qrcode: str) -> QRStatusResponse:
|
||||
"""Long-poll the QR code scan status (GET with iLink-App-ClientVersion header)."""
|
||||
session = await self._get_session()
|
||||
url = f'{self.base_url}/ilink/bot/get_qrcode_status?qrcode={quote(qrcode, safe="")}'
|
||||
headers = {'iLink-App-ClientVersion': '1'}
|
||||
|
||||
try:
|
||||
async with session.get(
|
||||
url, headers=headers, timeout=aiohttp.ClientTimeout(total=DEFAULT_QR_POLL_TIMEOUT)
|
||||
) as resp:
|
||||
if resp.status != 200:
|
||||
text = await resp.text()
|
||||
raise ApiError(
|
||||
f'Failed to poll QR status: {resp.status} {text}',
|
||||
status=resp.status,
|
||||
)
|
||||
data = await resp.json(content_type=None)
|
||||
logger.debug('QR status poll response: %s', data)
|
||||
except (asyncio.TimeoutError, aiohttp.ServerTimeoutError):
|
||||
return QRStatusResponse(status='wait')
|
||||
|
||||
return QRStatusResponse(
|
||||
status=data.get('status'),
|
||||
bot_token=data.get('bot_token'),
|
||||
ilink_bot_id=data.get('ilink_bot_id'),
|
||||
baseurl=data.get('baseurl'),
|
||||
ilink_user_id=data.get('ilink_user_id'),
|
||||
)
|
||||
|
||||
async def login(
|
||||
self,
|
||||
max_retries: int = 5,
|
||||
poll_timeout_ms: int = 480_000,
|
||||
on_qrcode: Optional[typing.Callable[[str, str], typing.Any]] = None,
|
||||
on_status: Optional[typing.Callable[[str], typing.Any]] = None,
|
||||
) -> LoginResult:
|
||||
"""Complete QR code login flow with auto-retry on expiry.
|
||||
|
||||
Args:
|
||||
max_retries: Max number of QR code refreshes on expiry.
|
||||
poll_timeout_ms: Timeout per QR code in milliseconds.
|
||||
on_qrcode: Callback(qr_image_base64, qr_url) called each time a
|
||||
new QR code is fetched. Use this to display the QR code.
|
||||
on_status: Callback(status_str) called on each status poll change.
|
||||
|
||||
Returns:
|
||||
LoginResult with token, base_url, and account_id.
|
||||
|
||||
Raises:
|
||||
ApiError: On unrecoverable API errors.
|
||||
Exception: If all retries are exhausted.
|
||||
"""
|
||||
last_qr_base64: Optional[str] = None
|
||||
|
||||
for attempt in range(max_retries):
|
||||
qr_resp = await self.fetch_qrcode()
|
||||
if not qr_resp.qrcode or not qr_resp.qrcode_img_content:
|
||||
raise ApiError('Failed to get QR code from server', status=0)
|
||||
|
||||
# Convert QR image to base64 and notify caller
|
||||
last_qr_base64 = await self._fetch_qr_image_base64(qr_resp.qrcode_img_content)
|
||||
if on_qrcode:
|
||||
try:
|
||||
result = on_qrcode(last_qr_base64, qr_resp.qrcode_img_content)
|
||||
if asyncio.iscoroutine(result) or asyncio.isfuture(result):
|
||||
await result
|
||||
except Exception as e:
|
||||
logger.warning('on_qrcode callback error: %s', e)
|
||||
|
||||
# Poll until confirmed / expired / timeout
|
||||
loop = asyncio.get_running_loop()
|
||||
deadline = loop.time() + poll_timeout_ms / 1000.0
|
||||
|
||||
while loop.time() < deadline:
|
||||
try:
|
||||
status_resp = await self.poll_qrcode_status(qr_resp.qrcode)
|
||||
except Exception as e:
|
||||
logger.error('Error polling QR status: %s', e)
|
||||
await asyncio.sleep(2)
|
||||
continue
|
||||
|
||||
if on_status:
|
||||
try:
|
||||
cb_result = on_status(status_resp.status or 'unknown')
|
||||
if asyncio.iscoroutine(cb_result) or asyncio.isfuture(cb_result):
|
||||
await cb_result
|
||||
except Exception as e:
|
||||
logger.warning('on_status callback error: %s', e)
|
||||
|
||||
if status_resp.status == 'confirmed' and status_resp.bot_token:
|
||||
new_base_url = status_resp.baseurl or self.base_url
|
||||
# Update this client instance as well
|
||||
self.token = status_resp.bot_token
|
||||
self.base_url = new_base_url.rstrip('/')
|
||||
return LoginResult(
|
||||
token=status_resp.bot_token,
|
||||
base_url=new_base_url,
|
||||
account_id=status_resp.ilink_bot_id or '',
|
||||
qr_image_base64=last_qr_base64,
|
||||
)
|
||||
|
||||
if status_resp.status == 'expired':
|
||||
break # retry with a new QR code
|
||||
|
||||
await asyncio.sleep(1)
|
||||
else:
|
||||
# While-loop ended without break → poll timeout, treat as expired
|
||||
pass
|
||||
|
||||
remaining = max_retries - attempt - 1
|
||||
if remaining > 0:
|
||||
logger.info('QR code expired, refreshing... (%d retries left)', remaining)
|
||||
else:
|
||||
raise ApiError('QR code login failed: max retries exceeded', status=0)
|
||||
|
||||
# Should not reach here, but just in case
|
||||
raise ApiError('QR code login failed', status=0)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Parsing helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _parse_cdn_media(data: Optional[dict]) -> Optional[CDNMedia]:
|
||||
if not data:
|
||||
return None
|
||||
return CDNMedia(
|
||||
encrypt_query_param=data.get('encrypt_query_param'),
|
||||
aes_key=data.get('aes_key'),
|
||||
encrypt_type=data.get('encrypt_type'),
|
||||
)
|
||||
|
||||
|
||||
def _parse_message_item(data: dict) -> MessageItem:
|
||||
item = MessageItem(
|
||||
type=data.get('type'),
|
||||
create_time_ms=data.get('create_time_ms'),
|
||||
update_time_ms=data.get('update_time_ms'),
|
||||
is_completed=data.get('is_completed'),
|
||||
msg_id=data.get('msg_id'),
|
||||
)
|
||||
|
||||
if data.get('text_item'):
|
||||
item.text_item = TextItem(text=data['text_item'].get('text'))
|
||||
|
||||
if data.get('image_item'):
|
||||
img = data['image_item']
|
||||
item.image_item = ImageItem(
|
||||
media=_parse_cdn_media(img.get('media')),
|
||||
thumb_media=_parse_cdn_media(img.get('thumb_media')),
|
||||
aeskey=img.get('aeskey'),
|
||||
url=img.get('url'),
|
||||
mid_size=img.get('mid_size'),
|
||||
)
|
||||
|
||||
if data.get('voice_item'):
|
||||
v = data['voice_item']
|
||||
item.voice_item = VoiceItem(
|
||||
media=_parse_cdn_media(v.get('media')),
|
||||
encode_type=v.get('encode_type'),
|
||||
playtime=v.get('playtime'),
|
||||
text=v.get('text'),
|
||||
)
|
||||
|
||||
if data.get('file_item'):
|
||||
f = data['file_item']
|
||||
item.file_item = FileItem(
|
||||
media=_parse_cdn_media(f.get('media')),
|
||||
file_name=f.get('file_name'),
|
||||
md5=f.get('md5'),
|
||||
len=f.get('len'),
|
||||
)
|
||||
|
||||
if data.get('video_item'):
|
||||
vid = data['video_item']
|
||||
item.video_item = VideoItem(
|
||||
media=_parse_cdn_media(vid.get('media')),
|
||||
video_size=vid.get('video_size'),
|
||||
play_length=vid.get('play_length'),
|
||||
video_md5=vid.get('video_md5'),
|
||||
thumb_media=_parse_cdn_media(vid.get('thumb_media')),
|
||||
)
|
||||
|
||||
if data.get('ref_msg'):
|
||||
ref = data['ref_msg']
|
||||
item.ref_msg = RefMessage(
|
||||
title=ref.get('title'),
|
||||
message_item=_parse_message_item(ref['message_item']) if ref.get('message_item') else None,
|
||||
)
|
||||
|
||||
return item
|
||||
|
||||
|
||||
def _parse_weixin_message(data: dict) -> WeixinMessage:
|
||||
msg = WeixinMessage(
|
||||
seq=data.get('seq'),
|
||||
message_id=data.get('message_id'),
|
||||
from_user_id=data.get('from_user_id'),
|
||||
to_user_id=data.get('to_user_id'),
|
||||
client_id=data.get('client_id'),
|
||||
create_time_ms=data.get('create_time_ms'),
|
||||
session_id=data.get('session_id'),
|
||||
group_id=data.get('group_id'),
|
||||
message_type=data.get('message_type'),
|
||||
message_state=data.get('message_state'),
|
||||
context_token=data.get('context_token'),
|
||||
)
|
||||
if data.get('item_list'):
|
||||
msg.item_list = [_parse_message_item(item) for item in data['item_list']]
|
||||
return msg
|
||||
|
||||
|
||||
def _parse_get_updates_response(data: dict) -> GetUpdatesResponse:
|
||||
resp = GetUpdatesResponse(
|
||||
ret=data.get('ret'),
|
||||
errcode=data.get('errcode'),
|
||||
errmsg=data.get('errmsg'),
|
||||
get_updates_buf=data.get('get_updates_buf'),
|
||||
longpolling_timeout_ms=data.get('longpolling_timeout_ms'),
|
||||
)
|
||||
if data.get('msgs'):
|
||||
resp.msgs = [_parse_weixin_message(m) for m in data['msgs']]
|
||||
return resp
|
||||
|
||||
|
||||
def _cdn_media_to_dict(media: Optional[CDNMedia]) -> Optional[dict]:
|
||||
if not media:
|
||||
return None
|
||||
d: dict = {}
|
||||
if media.encrypt_query_param is not None:
|
||||
d['encrypt_query_param'] = media.encrypt_query_param
|
||||
if media.aes_key is not None:
|
||||
d['aes_key'] = media.aes_key
|
||||
if media.encrypt_type is not None:
|
||||
d['encrypt_type'] = media.encrypt_type
|
||||
return d or None
|
||||
|
||||
|
||||
def _message_item_to_dict(item: MessageItem) -> dict:
|
||||
d: dict = {'type': item.type}
|
||||
|
||||
if item.text_item:
|
||||
d['text_item'] = {'text': item.text_item.text}
|
||||
|
||||
if item.image_item:
|
||||
img_d: dict = {}
|
||||
if item.image_item.media:
|
||||
img_d['media'] = _cdn_media_to_dict(item.image_item.media)
|
||||
if item.image_item.mid_size is not None:
|
||||
img_d['mid_size'] = item.image_item.mid_size
|
||||
d['image_item'] = img_d
|
||||
|
||||
if item.voice_item:
|
||||
voice_d: dict = {}
|
||||
if item.voice_item.media:
|
||||
voice_d['media'] = _cdn_media_to_dict(item.voice_item.media)
|
||||
if item.voice_item.playtime is not None:
|
||||
voice_d['playtime'] = item.voice_item.playtime
|
||||
d['voice_item'] = voice_d
|
||||
|
||||
if item.file_item:
|
||||
file_d: dict = {}
|
||||
if item.file_item.media:
|
||||
file_d['media'] = _cdn_media_to_dict(item.file_item.media)
|
||||
if item.file_item.file_name:
|
||||
file_d['file_name'] = item.file_item.file_name
|
||||
if item.file_item.len:
|
||||
file_d['len'] = item.file_item.len
|
||||
d['file_item'] = file_d
|
||||
|
||||
if item.video_item:
|
||||
vid_d: dict = {}
|
||||
if item.video_item.media:
|
||||
vid_d['media'] = _cdn_media_to_dict(item.video_item.media)
|
||||
if item.video_item.video_size is not None:
|
||||
vid_d['video_size'] = item.video_item.video_size
|
||||
d['video_item'] = vid_d
|
||||
|
||||
return d
|
||||
@@ -1,200 +0,0 @@
|
||||
"""Type definitions for the OpenClaw WeChat API, mirroring the upstream protocol."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Optional
|
||||
|
||||
SESSION_EXPIRED_ERRCODE = -14
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
"""Structured error raised by the OpenClaw WeChat API."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
message: str,
|
||||
*,
|
||||
status: int = 0,
|
||||
code: int | None = None,
|
||||
payload: Any = None,
|
||||
):
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.payload = payload
|
||||
|
||||
@property
|
||||
def is_session_expired(self) -> bool:
|
||||
return self.code == SESSION_EXPIRED_ERRCODE
|
||||
|
||||
|
||||
@dataclass
|
||||
class CDNMedia:
|
||||
encrypt_query_param: Optional[str] = None
|
||||
aes_key: Optional[str] = None
|
||||
encrypt_type: Optional[int] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class TextItem:
|
||||
text: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImageItem:
|
||||
media: Optional[CDNMedia] = None
|
||||
thumb_media: Optional[CDNMedia] = None
|
||||
aeskey: Optional[str] = None
|
||||
url: Optional[str] = None
|
||||
mid_size: Optional[int] = None
|
||||
thumb_size: Optional[int] = None
|
||||
thumb_height: Optional[int] = None
|
||||
thumb_width: Optional[int] = None
|
||||
hd_size: Optional[int] = None
|
||||
_downloaded_bytes: Optional[bytes] = field(default=None, repr=False)
|
||||
|
||||
|
||||
@dataclass
|
||||
class VoiceItem:
|
||||
media: Optional[CDNMedia] = None
|
||||
encode_type: Optional[int] = None
|
||||
bits_per_sample: Optional[int] = None
|
||||
sample_rate: Optional[int] = None
|
||||
playtime: Optional[int] = None
|
||||
text: Optional[str] = None
|
||||
_downloaded_bytes: Optional[bytes] = field(default=None, repr=False)
|
||||
|
||||
|
||||
@dataclass
|
||||
class FileItem:
|
||||
media: Optional[CDNMedia] = None
|
||||
file_name: Optional[str] = None
|
||||
md5: Optional[str] = None
|
||||
len: Optional[str] = None
|
||||
_downloaded_bytes: Optional[bytes] = field(default=None, repr=False)
|
||||
|
||||
|
||||
@dataclass
|
||||
class VideoItem:
|
||||
media: Optional[CDNMedia] = None
|
||||
video_size: Optional[int] = None
|
||||
play_length: Optional[int] = None
|
||||
video_md5: Optional[str] = None
|
||||
thumb_media: Optional[CDNMedia] = None
|
||||
thumb_size: Optional[int] = None
|
||||
thumb_height: Optional[int] = None
|
||||
thumb_width: Optional[int] = None
|
||||
_downloaded_bytes: Optional[bytes] = field(default=None, repr=False)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RefMessage:
|
||||
message_item: Optional[MessageItem] = None
|
||||
title: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class MessageItem:
|
||||
"""A single content item inside a WeixinMessage."""
|
||||
|
||||
# Item types
|
||||
NONE = 0
|
||||
TEXT = 1
|
||||
IMAGE = 2
|
||||
VOICE = 3
|
||||
FILE = 4
|
||||
VIDEO = 5
|
||||
|
||||
type: Optional[int] = None
|
||||
create_time_ms: Optional[int] = None
|
||||
update_time_ms: Optional[int] = None
|
||||
is_completed: Optional[bool] = None
|
||||
msg_id: Optional[str] = None
|
||||
ref_msg: Optional[RefMessage] = None
|
||||
text_item: Optional[TextItem] = None
|
||||
image_item: Optional[ImageItem] = None
|
||||
voice_item: Optional[VoiceItem] = None
|
||||
file_item: Optional[FileItem] = None
|
||||
video_item: Optional[VideoItem] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class WeixinMessage:
|
||||
"""Unified message from getUpdates or for sendMessage."""
|
||||
|
||||
# Message types
|
||||
TYPE_USER = 1
|
||||
TYPE_BOT = 2
|
||||
|
||||
# Message states
|
||||
STATE_NEW = 0
|
||||
STATE_GENERATING = 1
|
||||
STATE_FINISH = 2
|
||||
|
||||
seq: Optional[int] = None
|
||||
message_id: Optional[int] = None
|
||||
from_user_id: Optional[str] = None
|
||||
to_user_id: Optional[str] = None
|
||||
client_id: Optional[str] = None
|
||||
create_time_ms: Optional[int] = None
|
||||
update_time_ms: Optional[int] = None
|
||||
delete_time_ms: Optional[int] = None
|
||||
session_id: Optional[str] = None
|
||||
group_id: Optional[str] = None
|
||||
message_type: Optional[int] = None
|
||||
message_state: Optional[int] = None
|
||||
item_list: Optional[list[MessageItem]] = None
|
||||
context_token: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class GetUpdatesResponse:
|
||||
ret: Optional[int] = None
|
||||
errcode: Optional[int] = None
|
||||
errmsg: Optional[str] = None
|
||||
msgs: list[WeixinMessage] = field(default_factory=list)
|
||||
get_updates_buf: Optional[str] = None
|
||||
longpolling_timeout_ms: Optional[int] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class GetConfigResponse:
|
||||
ret: Optional[int] = None
|
||||
errmsg: Optional[str] = None
|
||||
typing_ticket: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class GetUploadUrlResponse:
|
||||
upload_param: Optional[str] = None
|
||||
thumb_upload_param: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class QRCodeResponse:
|
||||
"""Response from get_bot_qrcode endpoint."""
|
||||
|
||||
qrcode: Optional[str] = None
|
||||
qrcode_img_content: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class QRStatusResponse:
|
||||
"""Response from get_qrcode_status endpoint."""
|
||||
|
||||
status: Optional[str] = None # "wait" | "scaned" | "confirmed" | "expired"
|
||||
bot_token: Optional[str] = None
|
||||
ilink_bot_id: Optional[str] = None
|
||||
baseurl: Optional[str] = None
|
||||
ilink_user_id: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class LoginResult:
|
||||
"""Result returned by the login flow."""
|
||||
|
||||
token: str
|
||||
base_url: str
|
||||
account_id: str
|
||||
qr_image_base64: Optional[str] = None # data URI of the last QR code shown
|
||||
@@ -1,10 +1,8 @@
|
||||
import re
|
||||
import time
|
||||
import asyncio
|
||||
from quart import request
|
||||
import httpx
|
||||
from quart import Quart
|
||||
from typing import Callable, Dict, Any, Optional
|
||||
from typing import Callable, Dict, Any
|
||||
import langbot_plugin.api.entities.builtin.platform.events as platform_events
|
||||
from .qqofficialevent import QQOfficialEvent
|
||||
import json
|
||||
@@ -34,8 +32,6 @@ class QQOfficialClient:
|
||||
self.access_token = ''
|
||||
self.access_token_expiry_time = None
|
||||
self.logger = logger
|
||||
self._msg_seq_counter = 0
|
||||
self._token_refresh_task: Optional[asyncio.Task] = None
|
||||
|
||||
async def check_access_token(self):
|
||||
"""检查access_token是否存在"""
|
||||
@@ -54,18 +50,18 @@ class QQOfficialClient:
|
||||
headers = {
|
||||
'content-type': 'application/json',
|
||||
}
|
||||
response = await client.post(url, json=params, headers=headers)
|
||||
if response.status_code != 200:
|
||||
raise Exception(f'Failed to get access_token: HTTP {response.status_code} {response.text}')
|
||||
response_data = response.json()
|
||||
access_token = response_data.get('access_token')
|
||||
expires_in = int(response_data.get('expires_in', 7200))
|
||||
self.access_token_expiry_time = time.time() + expires_in - 60
|
||||
if access_token:
|
||||
self.access_token = access_token
|
||||
await self.logger.info(f'access_token obtained, expires_in={expires_in}s')
|
||||
else:
|
||||
raise Exception('Failed to get access_token: no access_token in response')
|
||||
try:
|
||||
response = await client.post(url, json=params, headers=headers)
|
||||
if response.status_code == 200:
|
||||
response_data = response.json()
|
||||
access_token = response_data.get('access_token')
|
||||
expires_in = int(response_data.get('expires_in', 7200))
|
||||
self.access_token_expiry_time = time.time() + expires_in - 60
|
||||
if access_token:
|
||||
self.access_token = access_token
|
||||
except Exception as e:
|
||||
await self.logger.error(f'获取access_token失败: {response_data}')
|
||||
raise Exception(f'获取access_token失败: {e}')
|
||||
|
||||
async def handle_callback_request(self):
|
||||
"""处理回调请求(独立端口模式,使用全局 request)"""
|
||||
@@ -91,10 +87,10 @@ class QQOfficialClient:
|
||||
try:
|
||||
body = await req.get_data()
|
||||
|
||||
await self.logger.info(f'Received request, body length: {len(body)}')
|
||||
print(f'[QQ Official] Received request, body length: {len(body)}')
|
||||
|
||||
if not body or len(body) == 0:
|
||||
await self.logger.info('Received empty body, might be health check or GET request')
|
||||
print('[QQ Official] Received empty body, might be health check or GET request')
|
||||
return {'code': 0, 'message': 'ok'}, 200
|
||||
|
||||
payload = json.loads(body)
|
||||
@@ -115,6 +111,7 @@ class QQOfficialClient:
|
||||
return {'code': 0, 'message': 'success'}
|
||||
|
||||
except Exception as e:
|
||||
print(f'[QQ Official] ERROR: {traceback.format_exc()}')
|
||||
await self.logger.error(f'Error in handle_callback_request: {traceback.format_exc()}')
|
||||
return {'error': str(e)}, 400
|
||||
|
||||
@@ -142,24 +139,21 @@ class QQOfficialClient:
|
||||
|
||||
async def get_message(self, msg: dict) -> Dict[str, Any]:
|
||||
"""获取消息"""
|
||||
d = msg.get('d', {})
|
||||
if not isinstance(d, dict):
|
||||
return {}
|
||||
message_data = {
|
||||
't': msg.get('t', {}),
|
||||
'user_openid': d.get('author', {}).get('user_openid', {}),
|
||||
'timestamp': d.get('timestamp', {}),
|
||||
'd_author_id': d.get('author', {}).get('id', {}),
|
||||
'content': d.get('content', {}),
|
||||
'd_id': d.get('id', {}),
|
||||
'user_openid': msg.get('d', {}).get('author', {}).get('user_openid', {}),
|
||||
'timestamp': msg.get('d', {}).get('timestamp', {}),
|
||||
'd_author_id': msg.get('d', {}).get('author', {}).get('id', {}),
|
||||
'content': msg.get('d', {}).get('content', {}),
|
||||
'd_id': msg.get('d', {}).get('id', {}),
|
||||
'id': msg.get('id', {}),
|
||||
'channel_id': d.get('channel_id', {}),
|
||||
'username': d.get('author', {}).get('username', {}),
|
||||
'guild_id': d.get('guild_id', {}),
|
||||
'member_openid': d.get('author', {}).get('openid', {}),
|
||||
'group_openid': d.get('group_openid', {}),
|
||||
'channel_id': msg.get('d', {}).get('channel_id', {}),
|
||||
'username': msg.get('d', {}).get('author', {}).get('username', {}),
|
||||
'guild_id': msg.get('d', {}).get('guild_id', {}),
|
||||
'member_openid': msg.get('d', {}).get('author', {}).get('openid', {}),
|
||||
'group_openid': msg.get('d', {}).get('group_openid', {}),
|
||||
}
|
||||
attachments = d.get('attachments', [])
|
||||
attachments = msg.get('d', {}).get('attachments', [])
|
||||
image_attachments = [attachment['url'] for attachment in attachments if await self.is_image(attachment)]
|
||||
image_attachments_type = [
|
||||
attachment['content_type'] for attachment in attachments if await self.is_image(attachment)
|
||||
@@ -198,7 +192,7 @@ class QQOfficialClient:
|
||||
if response.status_code == 200:
|
||||
return
|
||||
else:
|
||||
await self.logger.error(f'Failed to send private message: {response_data}')
|
||||
await self.logger.error(f'发送私聊消息失败: {response_data}')
|
||||
raise ValueError(response)
|
||||
|
||||
async def send_group_text_msg(self, group_openid: str, content: str, msg_id: str):
|
||||
@@ -221,7 +215,7 @@ class QQOfficialClient:
|
||||
if response.status_code == 200:
|
||||
return
|
||||
else:
|
||||
await self.logger.error(f'Failed to send group message: {response.json()}')
|
||||
await self.logger.error(f'发送群聊消息失败:{response.json()}')
|
||||
raise Exception(response.read().decode())
|
||||
|
||||
async def send_channle_group_text_msg(self, channel_id: str, content: str, msg_id: str):
|
||||
@@ -244,7 +238,7 @@ class QQOfficialClient:
|
||||
if response.status_code == 200:
|
||||
return True
|
||||
else:
|
||||
await self.logger.error(f'Failed to send channel group message: {response.json()}')
|
||||
await self.logger.error(f'发送频道群聊消息失败: {response.json()}')
|
||||
raise Exception(response)
|
||||
|
||||
async def send_channle_private_text_msg(self, guild_id: str, content: str, msg_id: str):
|
||||
@@ -267,224 +261,9 @@ class QQOfficialClient:
|
||||
if response.status_code == 200:
|
||||
return True
|
||||
else:
|
||||
await self.logger.error(f'Failed to send channel private message: {response.json()}')
|
||||
await self.logger.error(f'发送频道私聊消息失败: {response.json()}')
|
||||
raise Exception(response)
|
||||
|
||||
# ---- 富媒体消息 ----
|
||||
|
||||
# 媒体文件类型
|
||||
MEDIA_TYPE_IMAGE = 1
|
||||
MEDIA_TYPE_VIDEO = 2
|
||||
MEDIA_TYPE_VOICE = 3
|
||||
MEDIA_TYPE_FILE = 4
|
||||
|
||||
async def upload_media(
|
||||
self,
|
||||
target_type: str,
|
||||
target_id: str,
|
||||
file_type: int,
|
||||
file_url: str = None,
|
||||
file_data: str = None,
|
||||
file_name: str = None,
|
||||
) -> str:
|
||||
"""上传媒体文件,返回 file_info。
|
||||
|
||||
Args:
|
||||
target_type: 'c2c' | 'group'
|
||||
target_id: 用户 openid 或群 openid
|
||||
file_type: 1=图片, 2=视频, 3=语音, 4=文件
|
||||
file_url: 在线 URL(与 file_data 二选一)
|
||||
file_data: base64 编码的文件数据或 data URL(与 file_url 二选一)
|
||||
file_name: 文件名(file_type=4 时必填)
|
||||
"""
|
||||
if not await self.check_access_token():
|
||||
await self.get_access_token()
|
||||
|
||||
if target_type == 'c2c':
|
||||
url = f'{self.base_url}/v2/users/{target_id}/files'
|
||||
elif target_type == 'group':
|
||||
url = f'{self.base_url}/v2/groups/{target_id}/files'
|
||||
else:
|
||||
raise ValueError(f'Unsupported target_type: {target_type}')
|
||||
|
||||
body = {
|
||||
'file_type': file_type,
|
||||
'srv_send_msg': False,
|
||||
}
|
||||
if file_url:
|
||||
body['url'] = file_url
|
||||
elif file_data:
|
||||
# 处理 data URL 格式: data:image/png;base64,xxxxx
|
||||
if file_data.startswith('data:'):
|
||||
match = re.match(r'^data:[^;]+;base64,(.+)$', file_data, re.DOTALL)
|
||||
if match:
|
||||
body['file_data'] = match.group(1)
|
||||
else:
|
||||
body['file_data'] = file_data
|
||||
else:
|
||||
body['file_data'] = file_data
|
||||
else:
|
||||
raise ValueError('file_url or file_data is required')
|
||||
|
||||
if file_type == self.MEDIA_TYPE_FILE and file_name:
|
||||
body['file_name'] = file_name
|
||||
|
||||
async with httpx.AsyncClient(timeout=120) as client:
|
||||
headers = {
|
||||
'Authorization': f'QQBot {self.access_token}',
|
||||
'Content-Type': 'application/json',
|
||||
}
|
||||
response = await client.post(url, headers=headers, json=body)
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
file_info = data.get('file_info', '')
|
||||
preview = file_info[:80] + '...' if len(file_info) > 80 else file_info
|
||||
await self.logger.info(f'Upload media success, file_info={preview}')
|
||||
return file_info
|
||||
else:
|
||||
raise Exception(f'Failed to upload media: HTTP {response.status_code} {response.text}')
|
||||
|
||||
async def _send_media_msg(
|
||||
self,
|
||||
target_type: str,
|
||||
target_id: str,
|
||||
file_info: str,
|
||||
msg_id: str = None,
|
||||
content: str = None,
|
||||
):
|
||||
"""发送富媒体消息(msg_type=7)"""
|
||||
if not await self.check_access_token():
|
||||
await self.get_access_token()
|
||||
|
||||
if target_type == 'c2c':
|
||||
url = f'{self.base_url}/v2/users/{target_id}/messages'
|
||||
elif target_type == 'group':
|
||||
url = f'{self.base_url}/v2/groups/{target_id}/messages'
|
||||
else:
|
||||
raise ValueError(f'Unsupported target_type: {target_type}')
|
||||
|
||||
self._msg_seq_counter += 1
|
||||
msg_seq = self._msg_seq_counter
|
||||
body = {
|
||||
'msg_type': 7,
|
||||
'media': {'file_info': file_info},
|
||||
'msg_seq': msg_seq,
|
||||
}
|
||||
if content:
|
||||
body['content'] = content
|
||||
if msg_id:
|
||||
body['msg_id'] = msg_id
|
||||
|
||||
async with httpx.AsyncClient(timeout=120) as client:
|
||||
headers = {
|
||||
'Authorization': f'QQBot {self.access_token}',
|
||||
'Content-Type': 'application/json',
|
||||
}
|
||||
await self.logger.info(f'Sending rich media: {json.dumps(body, ensure_ascii=False)[:200]}')
|
||||
response = await client.post(url, headers=headers, json=body)
|
||||
if response.status_code != 200:
|
||||
raise Exception(f'Failed to send rich media message: HTTP {response.status_code} {response.text}')
|
||||
|
||||
async def send_image_msg(
|
||||
self,
|
||||
target_type: str,
|
||||
target_id: str,
|
||||
file_url: str = None,
|
||||
file_data: str = None,
|
||||
msg_id: str = None,
|
||||
content: str = None,
|
||||
):
|
||||
"""发送图片消息"""
|
||||
file_info = await self.upload_media(
|
||||
target_type,
|
||||
target_id,
|
||||
self.MEDIA_TYPE_IMAGE,
|
||||
file_url=file_url,
|
||||
file_data=file_data,
|
||||
)
|
||||
await self._send_media_msg(target_type, target_id, file_info, msg_id, content)
|
||||
|
||||
async def send_voice_msg(
|
||||
self,
|
||||
target_type: str,
|
||||
target_id: str,
|
||||
file_url: str = None,
|
||||
file_data: str = None,
|
||||
msg_id: str = None,
|
||||
):
|
||||
"""发送语音消息"""
|
||||
file_info = await self.upload_media(
|
||||
target_type,
|
||||
target_id,
|
||||
self.MEDIA_TYPE_VOICE,
|
||||
file_url=file_url,
|
||||
file_data=file_data,
|
||||
)
|
||||
await self._send_media_msg(target_type, target_id, file_info, msg_id)
|
||||
|
||||
async def send_file_msg(
|
||||
self,
|
||||
target_type: str,
|
||||
target_id: str,
|
||||
file_url: str = None,
|
||||
file_data: str = None,
|
||||
file_name: str = None,
|
||||
msg_id: str = None,
|
||||
):
|
||||
"""发送文件消息(含视频)"""
|
||||
file_info = await self.upload_media(
|
||||
target_type,
|
||||
target_id,
|
||||
self.MEDIA_TYPE_FILE,
|
||||
file_url=file_url,
|
||||
file_data=file_data,
|
||||
file_name=file_name,
|
||||
)
|
||||
await self._send_media_msg(target_type, target_id, file_info, msg_id)
|
||||
|
||||
async def send_stream_msg(
|
||||
self,
|
||||
user_openid: str,
|
||||
content: str,
|
||||
event_id: str,
|
||||
msg_id: str,
|
||||
msg_seq: int = 1,
|
||||
index: int = 0,
|
||||
stream_msg_id: str = None,
|
||||
input_state: int = 1,
|
||||
):
|
||||
"""发送流式消息(C2C 私聊)。
|
||||
|
||||
Args:
|
||||
input_state: 1=生成中, 10=生成结束
|
||||
"""
|
||||
if not await self.check_access_token():
|
||||
await self.get_access_token()
|
||||
|
||||
url = f'{self.base_url}/v2/users/{user_openid}/stream_messages'
|
||||
body = {
|
||||
'input_mode': 'replace',
|
||||
'input_state': input_state,
|
||||
'content_type': 'markdown',
|
||||
'content_raw': content,
|
||||
'event_id': event_id,
|
||||
'msg_id': msg_id,
|
||||
'msg_seq': msg_seq,
|
||||
'index': index,
|
||||
}
|
||||
if stream_msg_id:
|
||||
body['stream_msg_id'] = stream_msg_id
|
||||
|
||||
async with httpx.AsyncClient(timeout=120) as client:
|
||||
headers = {
|
||||
'Authorization': f'QQBot {self.access_token}',
|
||||
'Content-Type': 'application/json',
|
||||
}
|
||||
response = await client.post(url, headers=headers, json=body)
|
||||
if response.status_code != 200:
|
||||
raise Exception(f'Failed to send stream message: HTTP {response.status_code} {response.text}')
|
||||
return response.json()
|
||||
|
||||
async def is_token_expired(self):
|
||||
"""检查token是否过期"""
|
||||
if self.access_token_expiry_time is None:
|
||||
@@ -513,325 +292,3 @@ class QQOfficialClient:
|
||||
'signature': signature,
|
||||
}
|
||||
return response
|
||||
|
||||
# ---- WebSocket Gateway ----
|
||||
# Reference: https://bot.q.qq.com/wiki/develop/api-v2/dev-prepare/interface-framework/event-emit.html
|
||||
|
||||
INTENT_GUILDS = 1 << 0
|
||||
INTENT_GUILD_MEMBERS = 1 << 1
|
||||
INTENT_PUBLIC_GUILD_MESSAGES = 1 << 30
|
||||
INTENT_DIRECT_MESSAGE = 1 << 12
|
||||
INTENT_GROUP_AND_C2C = 1 << 25
|
||||
INTENT_INTERACTION = 1 << 26
|
||||
|
||||
FULL_INTENTS = (
|
||||
INTENT_GUILDS
|
||||
| INTENT_GUILD_MEMBERS
|
||||
| INTENT_PUBLIC_GUILD_MESSAGES
|
||||
| INTENT_DIRECT_MESSAGE
|
||||
| INTENT_GROUP_AND_C2C
|
||||
| INTENT_INTERACTION
|
||||
)
|
||||
|
||||
async def get_gateway_url(self) -> str:
|
||||
"""获取 WebSocket 网关地址"""
|
||||
if not await self.check_access_token():
|
||||
await self.get_access_token()
|
||||
|
||||
url = f'{self.base_url}/gateway'
|
||||
async with httpx.AsyncClient() as client:
|
||||
headers = {
|
||||
'Authorization': f'QQBot {self.access_token}',
|
||||
}
|
||||
response = await client.get(url, headers=headers)
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
ws_url = data.get('url', '')
|
||||
if not ws_url:
|
||||
raise Exception('Gateway URL is empty')
|
||||
return ws_url
|
||||
else:
|
||||
raise Exception(f'Failed to get Gateway URL: HTTP {response.status_code} {response.text}')
|
||||
|
||||
async def _background_token_refresh(self):
|
||||
"""在 token 到期前主动刷新"""
|
||||
try:
|
||||
while True:
|
||||
if self.access_token_expiry_time:
|
||||
remain = self.access_token_expiry_time - time.time()
|
||||
if remain > 120:
|
||||
await asyncio.sleep(remain - 60)
|
||||
continue
|
||||
self.access_token = ''
|
||||
self.access_token_expiry_time = None
|
||||
if await self.check_access_token():
|
||||
await asyncio.sleep(60)
|
||||
else:
|
||||
await self.get_access_token()
|
||||
await asyncio.sleep(60)
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
async def connect_gateway(
|
||||
self,
|
||||
on_event: Callable[[str, dict], Any],
|
||||
on_ready: Optional[Callable[[], Any]] = None,
|
||||
on_error: Optional[Callable[[Exception], Any]] = None,
|
||||
):
|
||||
"""WebSocket 网关连接,含重连逻辑。持续重连直到达到最大次数或被取消。
|
||||
|
||||
Args:
|
||||
on_event: 收到 op=0 Dispatch 事件时的回调,参数为 (event_type, event_data)
|
||||
on_ready: 连接就绪 (收到 READY) 时的回调
|
||||
on_error: 发生错误时的回调
|
||||
"""
|
||||
import websockets
|
||||
|
||||
session_id = ''
|
||||
last_seq = 0
|
||||
reconnect_attempts = 0
|
||||
max_reconnect_attempts = 100
|
||||
backoff_delays = [1, 2, 5, 10, 30, 60]
|
||||
rate_limit_delay = 60
|
||||
|
||||
# Cancel previous token refresh task if any
|
||||
if self._token_refresh_task and not self._token_refresh_task.done():
|
||||
self._token_refresh_task.cancel()
|
||||
try:
|
||||
await self._token_refresh_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
self._token_refresh_task = None
|
||||
|
||||
while reconnect_attempts <= max_reconnect_attempts:
|
||||
heartbeat_interval = 45000
|
||||
should_refresh_token = False
|
||||
ws = None
|
||||
heartbeat_task = None
|
||||
|
||||
# Refresh token if needed
|
||||
if should_refresh_token:
|
||||
self.access_token = ''
|
||||
self.access_token_expiry_time = None
|
||||
|
||||
try:
|
||||
ws_url = await self.get_gateway_url()
|
||||
await self.logger.info(f'Gateway URL obtained: {ws_url[:60]}...')
|
||||
except Exception as e:
|
||||
error_msg = str(e)
|
||||
await self.logger.error(f'Failed to get gateway URL: {e}')
|
||||
reconnect_attempts += 1
|
||||
if '100017' in error_msg or '频率' in error_msg or 'Too many' in error_msg:
|
||||
delay = rate_limit_delay
|
||||
else:
|
||||
delay = backoff_delays[min(reconnect_attempts - 1, len(backoff_delays) - 1)]
|
||||
await self.logger.info(f'Reconnecting in {delay}s (attempt {reconnect_attempts})')
|
||||
await asyncio.sleep(delay)
|
||||
continue
|
||||
|
||||
try:
|
||||
await self.logger.info('Connecting to WebSocket gateway...')
|
||||
ws = await websockets.connect(ws_url)
|
||||
await self.logger.info('WebSocket connected')
|
||||
except Exception as e:
|
||||
await self.logger.error(f'WebSocket connection failed: {e}')
|
||||
reconnect_attempts += 1
|
||||
delay = backoff_delays[min(reconnect_attempts - 1, len(backoff_delays) - 1)]
|
||||
await self.logger.info(f'Reconnecting in {delay}s (attempt {reconnect_attempts})')
|
||||
await asyncio.sleep(delay)
|
||||
continue
|
||||
|
||||
try:
|
||||
async for raw_msg in ws:
|
||||
try:
|
||||
payload = json.loads(raw_msg)
|
||||
except json.JSONDecodeError:
|
||||
await self.logger.error(f'Failed to parse message: {raw_msg}')
|
||||
continue
|
||||
|
||||
op = payload.get('op')
|
||||
d = payload.get('d', {})
|
||||
s = payload.get('s')
|
||||
t = payload.get('t')
|
||||
|
||||
if not isinstance(d, dict):
|
||||
d = {}
|
||||
|
||||
if op == 10: # Hello
|
||||
heartbeat_interval = d.get('heartbeat_interval', 45000)
|
||||
await self.logger.info(f'Received Hello, heartbeat_interval={heartbeat_interval}ms')
|
||||
|
||||
# Send Identify or Resume
|
||||
if session_id and last_seq > 0:
|
||||
resume_payload = {
|
||||
'op': 6,
|
||||
'd': {
|
||||
'token': f'QQBot {self.access_token}',
|
||||
'session_id': session_id,
|
||||
'seq': last_seq,
|
||||
},
|
||||
}
|
||||
await ws.send(json.dumps(resume_payload))
|
||||
await self.logger.info(f'Sent Resume, session_id={session_id}, seq={last_seq}')
|
||||
else:
|
||||
identify_payload = {
|
||||
'op': 2,
|
||||
'd': {
|
||||
'token': f'QQBot {self.access_token}',
|
||||
'intents': self.FULL_INTENTS,
|
||||
'shard': [0, 1],
|
||||
},
|
||||
}
|
||||
await ws.send(json.dumps(identify_payload))
|
||||
await self.logger.info(f'Sent Identify, intents={self.FULL_INTENTS}')
|
||||
|
||||
# Start heartbeat
|
||||
async def _heartbeat_loop(conn, interval_ms):
|
||||
interval_sec = interval_ms / 1000.0
|
||||
try:
|
||||
while True:
|
||||
await asyncio.sleep(interval_sec)
|
||||
try:
|
||||
hb_payload = {'op': 1, 'd': last_seq}
|
||||
await conn.send(json.dumps(hb_payload))
|
||||
except Exception:
|
||||
break
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
heartbeat_task = asyncio.create_task(_heartbeat_loop(ws, heartbeat_interval))
|
||||
|
||||
elif op == 0: # Dispatch
|
||||
if s is not None:
|
||||
last_seq = s
|
||||
|
||||
if t == 'READY':
|
||||
session_id = d.get('session_id', '')
|
||||
reconnect_attempts = 0
|
||||
await self.logger.info(f'READY, session_id={session_id}')
|
||||
if on_ready:
|
||||
try:
|
||||
result = on_ready()
|
||||
if asyncio.iscoroutine(result):
|
||||
await result
|
||||
except Exception:
|
||||
pass
|
||||
# Track token refresh task to avoid leaks
|
||||
if self._token_refresh_task and not self._token_refresh_task.done():
|
||||
self._token_refresh_task.cancel()
|
||||
try:
|
||||
await self._token_refresh_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
self._token_refresh_task = asyncio.create_task(self._background_token_refresh())
|
||||
|
||||
elif t == 'RESUMED':
|
||||
reconnect_attempts = 0
|
||||
await self.logger.info('RESUMED')
|
||||
|
||||
else:
|
||||
await self.logger.debug(f'Received event: {t}, seq={s}')
|
||||
if on_event:
|
||||
try:
|
||||
result = on_event(t, d)
|
||||
if asyncio.iscoroutine(result):
|
||||
await result
|
||||
except Exception:
|
||||
await self.logger.error(f'Error handling event {t}: {traceback.format_exc()}')
|
||||
|
||||
elif op == 11: # Heartbeat ACK
|
||||
pass
|
||||
|
||||
elif op == 7: # Reconnect
|
||||
await self.logger.info('Received Reconnect directive')
|
||||
break
|
||||
|
||||
elif op == 9: # Invalid Session
|
||||
can_resume = d.get('can_resume', False)
|
||||
await self.logger.warning(f'Invalid Session, can_resume={can_resume}')
|
||||
if not can_resume:
|
||||
session_id = ''
|
||||
last_seq = 0
|
||||
should_refresh_token = True
|
||||
break
|
||||
|
||||
# Connection closed normally (end of async for)
|
||||
try:
|
||||
close_code = ws.close_code
|
||||
close_reason = ws.close_reason or ''
|
||||
except Exception:
|
||||
close_code = None
|
||||
close_reason = ''
|
||||
await self.logger.info(f'Connection closed, code={close_code}, reason={close_reason}')
|
||||
|
||||
if close_code == 4004:
|
||||
should_refresh_token = True
|
||||
elif close_code in (4006, 4007, 4009):
|
||||
session_id = ''
|
||||
last_seq = 0
|
||||
should_refresh_token = True
|
||||
elif close_code == 4008:
|
||||
reconnect_attempts += 1
|
||||
delay = rate_limit_delay
|
||||
await self.logger.info(
|
||||
f'Rate limited, waiting {delay}s before reconnect (attempt {reconnect_attempts})'
|
||||
)
|
||||
await asyncio.sleep(delay)
|
||||
continue
|
||||
elif close_code in (4914, 4915):
|
||||
err = Exception(f'Bot disconnected/banned (close_code={close_code})')
|
||||
if on_error:
|
||||
await self._safe_callback(on_error, err)
|
||||
return
|
||||
elif close_code in (4900, 4901, 4902, 4903, 4904, 4905, 4906, 4907, 4908, 4909, 4910, 4911, 4912, 4913):
|
||||
session_id = ''
|
||||
last_seq = 0
|
||||
|
||||
if close_code == 1000:
|
||||
return
|
||||
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
await self.logger.error(f'Unexpected error in WebSocket loop: {traceback.format_exc()}')
|
||||
finally:
|
||||
if heartbeat_task:
|
||||
heartbeat_task.cancel()
|
||||
try:
|
||||
await heartbeat_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
if ws:
|
||||
try:
|
||||
await ws.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# If we reach here, we need to reconnect
|
||||
reconnect_attempts += 1
|
||||
if reconnect_attempts > max_reconnect_attempts:
|
||||
await self.logger.error(f'Max reconnect attempts ({max_reconnect_attempts}) reached, stopping')
|
||||
if on_error:
|
||||
await self._safe_callback(on_error, Exception('Max reconnect attempts reached'))
|
||||
return
|
||||
delay = backoff_delays[min(reconnect_attempts - 1, len(backoff_delays) - 1)]
|
||||
await self.logger.info(f'Reconnecting in {delay}s (attempt {reconnect_attempts})')
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
async def _safe_callback(self, callback, *args):
|
||||
"""Safely invoke a callback, handling both sync and async functions."""
|
||||
try:
|
||||
result = callback(*args)
|
||||
if asyncio.iscoroutine(result):
|
||||
await result
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async def connect_gateway_loop(
|
||||
self,
|
||||
on_event: Callable[[str, dict], Any],
|
||||
on_ready: Optional[Callable[[], Any]] = None,
|
||||
on_error: Optional[Callable[[Exception], Any]] = None,
|
||||
):
|
||||
"""持续重连的网关循环。"""
|
||||
await self.connect_gateway(on_event, on_ready, on_error)
|
||||
|
||||
@@ -6,8 +6,7 @@ import traceback
|
||||
import uuid
|
||||
import xml.etree.ElementTree as ET
|
||||
from dataclasses import dataclass, field
|
||||
import re
|
||||
from typing import Any, Callable, Optional, Tuple
|
||||
from typing import Any, Callable, Optional
|
||||
from urllib.parse import unquote
|
||||
|
||||
import httpx
|
||||
@@ -64,25 +63,16 @@ class StreamSession:
|
||||
# 缓存最近一次片段,处理重试或超时兜底
|
||||
last_chunk: Optional[StreamChunk] = None
|
||||
|
||||
# 反馈 ID,用于接收用户点赞/点踩反馈
|
||||
feedback_id: Optional[str] = None
|
||||
|
||||
|
||||
class StreamSessionManager:
|
||||
"""管理 stream 会话的生命周期,并负责队列的生产消费。"""
|
||||
|
||||
# Sessions with registered feedback_ids use a longer TTL to survive the
|
||||
# full like → cancel → dislike feedback flow. Must align with the adapter's
|
||||
# _stream_to_monitoring_msg TTL (wecombot.py).
|
||||
_FEEDBACK_SESSION_TTL = 600 # 10 minutes
|
||||
|
||||
def __init__(self, logger: EventLogger, ttl: int = 60) -> None:
|
||||
self.logger = logger
|
||||
|
||||
self.ttl = ttl # 超时时间(秒),超过该时间未被访问的会话会被清理由 cleanup
|
||||
self._sessions: dict[str, StreamSession] = {} # stream_id -> StreamSession 映射
|
||||
self._msg_index: dict[str, str] = {} # msgid -> stream_id 映射,便于流水线根据消息 ID 找到会话
|
||||
self._feedback_index: dict[str, str] = {} # feedback_id -> stream_id 映射
|
||||
|
||||
def get_stream_id_by_msg(self, msg_id: str) -> Optional[str]:
|
||||
if not msg_id:
|
||||
@@ -92,32 +82,6 @@ class StreamSessionManager:
|
||||
def get_session(self, stream_id: str) -> Optional[StreamSession]:
|
||||
return self._sessions.get(stream_id)
|
||||
|
||||
def get_session_by_feedback_id(self, feedback_id: str) -> Optional[StreamSession]:
|
||||
"""根据 feedback_id 查找会话。
|
||||
|
||||
Args:
|
||||
feedback_id: 企业微信反馈事件中的反馈 ID。
|
||||
|
||||
Returns:
|
||||
Optional[StreamSession]: 找到的会话实例,未找到返回 None。
|
||||
"""
|
||||
if not feedback_id:
|
||||
return None
|
||||
stream_id = self._feedback_index.get(feedback_id)
|
||||
if stream_id:
|
||||
return self._sessions.get(stream_id)
|
||||
return None
|
||||
|
||||
def register_feedback_id(self, stream_id: str, feedback_id: str) -> None:
|
||||
"""注册 feedback_id 与 stream_id 的映射。
|
||||
|
||||
Args:
|
||||
stream_id: 企业微信流式会话 ID。
|
||||
feedback_id: 反馈 ID。
|
||||
"""
|
||||
if feedback_id and stream_id:
|
||||
self._feedback_index[feedback_id] = stream_id
|
||||
|
||||
def create_or_get(self, msg_json: dict[str, Any]) -> tuple[StreamSession, bool]:
|
||||
"""根据企业微信回调创建或获取会话。
|
||||
|
||||
@@ -219,17 +183,11 @@ class StreamSessionManager:
|
||||
session.last_access = time.time()
|
||||
|
||||
def cleanup(self) -> None:
|
||||
"""定期清理过期会话,防止队列与映射无上限累积。
|
||||
|
||||
已注册 feedback_id 的会话使用更长的 TTL,确保用户在点赞/取消/点踩流程中
|
||||
不会因为 session 被提前清除而丢失上下文信息。
|
||||
"""
|
||||
"""定期清理过期会话,防止队列与映射无上限累积。"""
|
||||
now = time.time()
|
||||
expired: list[str] = []
|
||||
for stream_id, session in self._sessions.items():
|
||||
# Sessions with registered feedback_ids use a longer TTL
|
||||
effective_ttl = self._FEEDBACK_SESSION_TTL if session.feedback_id else self.ttl
|
||||
if now - session.last_access > effective_ttl:
|
||||
if now - session.last_access > self.ttl:
|
||||
expired.append(stream_id)
|
||||
|
||||
for stream_id in expired:
|
||||
@@ -239,488 +197,6 @@ class StreamSessionManager:
|
||||
msg_id = session.msg_id
|
||||
if msg_id and self._msg_index.get(msg_id) == stream_id:
|
||||
self._msg_index.pop(msg_id, None)
|
||||
# Clean up feedback index for expired sessions
|
||||
if session.feedback_id:
|
||||
self._feedback_index.pop(session.feedback_id, None)
|
||||
|
||||
|
||||
def _decrypt_file(encrypted_data: bytes, aes_key_str: str) -> bytes:
|
||||
"""Decrypt AES-256-CBC encrypted file data.
|
||||
|
||||
Aligned with the official WeCom AI Bot Python SDK (crypto_utils.py).
|
||||
|
||||
Args:
|
||||
encrypted_data: The raw encrypted bytes.
|
||||
aes_key_str: Base64-encoded AES key (may lack padding).
|
||||
|
||||
Returns:
|
||||
Decrypted bytes with PKCS#7 padding removed.
|
||||
"""
|
||||
if not encrypted_data:
|
||||
raise ValueError('encrypted_data is empty')
|
||||
if not aes_key_str:
|
||||
raise ValueError('aes_key is empty')
|
||||
|
||||
# Python's base64.b64decode requires proper padding (length % 4 == 0).
|
||||
# Node.js Buffer.from tolerates missing '=', so we must pad manually.
|
||||
remainder = len(aes_key_str) % 4
|
||||
if remainder != 0:
|
||||
aes_key_str = aes_key_str + '=' * (4 - remainder)
|
||||
key = base64.b64decode(aes_key_str)
|
||||
|
||||
iv = key[:16]
|
||||
|
||||
cipher = AES.new(key, AES.MODE_CBC, iv)
|
||||
|
||||
# Ensure encrypted data is aligned to AES block size (16 bytes).
|
||||
# Node.js setAutoPadding(false) silently handles unaligned data,
|
||||
# but PyCryptodome will raise an error.
|
||||
block_size = 16
|
||||
data_remainder = len(encrypted_data) % block_size
|
||||
if data_remainder != 0:
|
||||
encrypted_data = encrypted_data + b'\x00' * (block_size - data_remainder)
|
||||
|
||||
decrypted = cipher.decrypt(encrypted_data)
|
||||
|
||||
# Remove PKCS#7 padding with validation
|
||||
if len(decrypted) == 0:
|
||||
raise ValueError('Decrypted data is empty')
|
||||
|
||||
pad_len = decrypted[-1]
|
||||
if pad_len < 1 or pad_len > 32 or pad_len > len(decrypted):
|
||||
raise ValueError(f'Invalid PKCS#7 padding value: {pad_len}')
|
||||
|
||||
# Verify all padding bytes are consistent
|
||||
for i in range(len(decrypted) - pad_len, len(decrypted)):
|
||||
if decrypted[i] != pad_len:
|
||||
raise ValueError('Invalid PKCS#7 padding: padding bytes mismatch')
|
||||
|
||||
return decrypted[: len(decrypted) - pad_len]
|
||||
|
||||
|
||||
def _extract_filename(content_disposition: str) -> Optional[str]:
|
||||
"""Extract filename from a Content-Disposition header value."""
|
||||
if not content_disposition:
|
||||
return None
|
||||
# RFC 5987: filename*=UTF-8''xxx
|
||||
utf8_match = re.search(r"filename\*=UTF-8''([^;\s]+)", content_disposition, re.IGNORECASE)
|
||||
if utf8_match:
|
||||
return unquote(utf8_match.group(1))
|
||||
# Standard: filename="xxx" or filename=xxx
|
||||
match = re.search(r'filename="?([^";\s]+)"?', content_disposition, re.IGNORECASE)
|
||||
if match:
|
||||
return unquote(match.group(1))
|
||||
return None
|
||||
|
||||
|
||||
def _bytes_to_data_uri(data: bytes) -> str:
|
||||
"""Convert raw bytes to a data URI with auto-detected MIME type."""
|
||||
if data.startswith(b'\xff\xd8'):
|
||||
mime_type = 'image/jpeg'
|
||||
elif data.startswith(b'\x89PNG'):
|
||||
mime_type = 'image/png'
|
||||
elif data.startswith((b'GIF87a', b'GIF89a')):
|
||||
mime_type = 'image/gif'
|
||||
elif data.startswith(b'BM'):
|
||||
mime_type = 'image/bmp'
|
||||
elif data.startswith(b'II*\x00') or data.startswith(b'MM\x00*'):
|
||||
mime_type = 'image/tiff'
|
||||
elif data[:4] == b'%PDF':
|
||||
mime_type = 'application/pdf'
|
||||
elif data[:4] == b'PK\x03\x04':
|
||||
mime_type = 'application/zip'
|
||||
else:
|
||||
mime_type = 'application/octet-stream'
|
||||
|
||||
base64_str = base64.b64encode(data).decode('utf-8')
|
||||
return f'data:{mime_type};base64,{base64_str}'
|
||||
|
||||
|
||||
async def download_encrypted_file(
|
||||
download_url: str, aes_key: str, logger: EventLogger
|
||||
) -> Tuple[Optional[bytes], Optional[str]]:
|
||||
"""Download an AES-encrypted file from WeChat Work and decrypt it.
|
||||
|
||||
Args:
|
||||
download_url: The encrypted file download URL.
|
||||
aes_key: The AES key for decryption (base64-encoded, per-message aeskey
|
||||
or platform EncodingAESKey).
|
||||
logger: Logger instance.
|
||||
|
||||
Returns:
|
||||
A tuple of (decrypted_bytes, filename) or (None, None) on failure.
|
||||
"""
|
||||
if not download_url:
|
||||
return None, None
|
||||
if not aes_key:
|
||||
await logger.error('download_encrypted_file: aes_key is empty, cannot decrypt')
|
||||
return None, None
|
||||
|
||||
filename: Optional[str] = None
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
response = await client.get(download_url)
|
||||
if response.status_code != 200:
|
||||
await logger.error(f'Failed to download file (HTTP {response.status_code}): {response.text[:200]}')
|
||||
return None, None
|
||||
encrypted_bytes = response.content
|
||||
filename = _extract_filename(response.headers.get('content-disposition', ''))
|
||||
except Exception:
|
||||
await logger.error(f'Failed to download file: {traceback.format_exc()}')
|
||||
return None, None
|
||||
|
||||
try:
|
||||
decrypted = _decrypt_file(encrypted_bytes, aes_key)
|
||||
return decrypted, filename
|
||||
except Exception:
|
||||
await logger.error(f'Failed to decrypt file: {traceback.format_exc()}')
|
||||
return None, None
|
||||
|
||||
|
||||
async def parse_wecom_bot_message(
|
||||
msg_json: dict[str, Any], encoding_aes_key: str, logger: EventLogger
|
||||
) -> dict[str, Any]:
|
||||
"""Parse a decrypted WeChat Work AI Bot message JSON into a unified message dict.
|
||||
|
||||
This is the shared message parsing logic used by both webhook and WebSocket modes.
|
||||
|
||||
Args:
|
||||
msg_json: The decrypted message JSON from WeChat Work.
|
||||
encoding_aes_key: AES key for file decryption.
|
||||
logger: Logger instance.
|
||||
|
||||
Returns:
|
||||
A dict suitable for constructing a WecomBotEvent.
|
||||
"""
|
||||
message_data: dict[str, Any] = {}
|
||||
|
||||
msg_type = msg_json.get('msgtype', '')
|
||||
if msg_type:
|
||||
message_data['msgtype'] = msg_type
|
||||
|
||||
if msg_json.get('chattype', '') == 'single':
|
||||
message_data['type'] = 'single'
|
||||
elif msg_json.get('chattype', '') == 'group':
|
||||
message_data['type'] = 'group'
|
||||
|
||||
max_inline_file_size = 5 * 1024 * 1024
|
||||
|
||||
async def _safe_download(url: str, per_msg_aeskey: str = '') -> Tuple[Optional[bytes], Optional[str]]:
|
||||
"""Download and decrypt a file, preferring per-message aeskey over platform key."""
|
||||
if not url:
|
||||
return None, None
|
||||
key = per_msg_aeskey or encoding_aes_key
|
||||
if not key:
|
||||
await logger.warning('No AES key available for file decryption, skipping download')
|
||||
return None, None
|
||||
return await download_encrypted_file(url, key, logger)
|
||||
|
||||
async def _safe_download_as_data_uri(url: str, per_msg_aeskey: str = '') -> Optional[str]:
|
||||
"""Download, decrypt, and convert to data URI for backward compatibility."""
|
||||
data, _filename = await _safe_download(url, per_msg_aeskey)
|
||||
if data:
|
||||
return _bytes_to_data_uri(data)
|
||||
return None
|
||||
|
||||
if msg_type == 'text':
|
||||
message_data['content'] = msg_json.get('text', {}).get('content')
|
||||
elif msg_type == 'markdown':
|
||||
message_data['content'] = msg_json.get('markdown', {}).get('content') or msg_json.get('text', {}).get(
|
||||
'content', ''
|
||||
)
|
||||
elif msg_type == 'image':
|
||||
image_info = msg_json.get('image', {})
|
||||
picurl = image_info.get('url', '')
|
||||
per_msg_aeskey = image_info.get('aeskey', '')
|
||||
base64_data = await _safe_download_as_data_uri(picurl, per_msg_aeskey)
|
||||
if base64_data:
|
||||
message_data['picurl'] = base64_data
|
||||
message_data['images'] = [base64_data]
|
||||
elif msg_type == 'voice':
|
||||
voice_info = msg_json.get('voice', {}) or {}
|
||||
download_url = voice_info.get('url')
|
||||
per_msg_aeskey = voice_info.get('aeskey', '')
|
||||
message_data['voice'] = {
|
||||
'url': download_url,
|
||||
'md5sum': voice_info.get('md5sum') or voice_info.get('md5'),
|
||||
'filesize': voice_info.get('filesize') or voice_info.get('size'),
|
||||
'sdkfileid': voice_info.get('sdkfileid') or voice_info.get('fileid'),
|
||||
}
|
||||
if voice_info.get('content'):
|
||||
message_data['content'] = voice_info.get('content')
|
||||
# if (message_data['voice'].get('filesize') or 0) <= max_inline_file_size:
|
||||
# voice_base64 = await _safe_download_as_data_uri(download_url, per_msg_aeskey)
|
||||
# if voice_base64:
|
||||
# message_data['voice']['base64'] = voice_base64
|
||||
elif msg_type == 'video':
|
||||
video_info = msg_json.get('video', {}) or {}
|
||||
download_url = video_info.get('url')
|
||||
per_msg_aeskey = video_info.get('aeskey', '')
|
||||
video_data = {
|
||||
'url': download_url,
|
||||
'filesize': video_info.get('filesize') or video_info.get('size'),
|
||||
'sdkfileid': video_info.get('sdkfileid') or video_info.get('fileid'),
|
||||
'md5sum': video_info.get('md5sum') or video_info.get('md5'),
|
||||
'filename': video_info.get('filename') or video_info.get('name'),
|
||||
}
|
||||
# if (video_data.get('filesize') or 0) <= max_inline_file_size:
|
||||
# video_base64 = await _safe_download_as_data_uri(download_url, per_msg_aeskey)
|
||||
# if video_base64:
|
||||
# video_data['base64'] = video_base64
|
||||
# 应为需要解密,但是目前暂时不能下载到内部进行解密,所以先将下载链接拼接aeskey返回给用户,由插件去处理该链接的下载和解密逻辑
|
||||
video_data['download_url'] = download_url + f'?aeskey={per_msg_aeskey}'
|
||||
message_data['video'] = video_data
|
||||
elif msg_type == 'file':
|
||||
file_info = msg_json.get('file', {}) or {}
|
||||
download_url = file_info.get('url') or file_info.get('fileurl')
|
||||
per_msg_aeskey = file_info.get('aeskey', '')
|
||||
file_data = {
|
||||
'filename': file_info.get('filename') or file_info.get('name'),
|
||||
'filesize': file_info.get('filesize') or file_info.get('size'),
|
||||
'md5sum': file_info.get('md5sum') or file_info.get('md5'),
|
||||
'sdkfileid': file_info.get('sdkfileid') or file_info.get('fileid'),
|
||||
'download_url': download_url,
|
||||
'extra': file_info,
|
||||
}
|
||||
# if (file_data.get('filesize') or 0) <= max_inline_file_size:
|
||||
# file_bytes, dl_filename = await _safe_download(download_url, per_msg_aeskey)
|
||||
# if file_bytes:
|
||||
# file_data['base64'] = _bytes_to_data_uri(file_bytes)
|
||||
# if dl_filename and not file_data.get('filename'):
|
||||
# file_data['filename'] = dl_filename
|
||||
|
||||
# 应为需要解密,但是目前暂时不能下载到内部进行解密,所以先将下载链接拼接aeskey返回给用户,由插件去处理该链接的下载和解密逻辑
|
||||
file_data['download_url'] = download_url + f'?aeskey={per_msg_aeskey}'
|
||||
message_data['file'] = file_data
|
||||
elif msg_type == 'link':
|
||||
message_data['link'] = msg_json.get('link', {})
|
||||
if not message_data.get('content'):
|
||||
title = message_data['link'].get('title', '')
|
||||
desc = message_data['link'].get('description') or message_data['link'].get('digest', '')
|
||||
message_data['content'] = '\n'.join(filter(None, [title, desc]))
|
||||
elif msg_type == 'mixed':
|
||||
items = msg_json.get('mixed', {}).get('msg_item', [])
|
||||
texts = []
|
||||
images = []
|
||||
files = []
|
||||
voices = []
|
||||
videos = []
|
||||
links = []
|
||||
for item in items:
|
||||
item_type = item.get('msgtype')
|
||||
if item_type == 'text':
|
||||
texts.append(item.get('text', {}).get('content', ''))
|
||||
elif item_type == 'image':
|
||||
img_info = item.get('image', {})
|
||||
img_url = img_info.get('url')
|
||||
img_aeskey = img_info.get('aeskey', '')
|
||||
base64_data = await _safe_download_as_data_uri(img_url, img_aeskey)
|
||||
if base64_data:
|
||||
images.append(base64_data)
|
||||
elif item_type == 'file':
|
||||
file_info = item.get('file', {}) or {}
|
||||
download_url = file_info.get('url') or file_info.get('fileurl')
|
||||
item_aeskey = file_info.get('aeskey', '')
|
||||
file_data = {
|
||||
'filename': file_info.get('filename') or file_info.get('name'),
|
||||
'filesize': file_info.get('filesize') or file_info.get('size'),
|
||||
'md5sum': file_info.get('md5sum') or file_info.get('md5'),
|
||||
'sdkfileid': file_info.get('sdkfileid') or file_info.get('fileid'),
|
||||
'download_url': download_url,
|
||||
'extra': file_info,
|
||||
}
|
||||
if (file_data.get('filesize') or 0) <= max_inline_file_size:
|
||||
file_bytes, dl_filename = await _safe_download(download_url, item_aeskey)
|
||||
if file_bytes:
|
||||
file_data['base64'] = _bytes_to_data_uri(file_bytes)
|
||||
if dl_filename and not file_data.get('filename'):
|
||||
file_data['filename'] = dl_filename
|
||||
files.append(file_data)
|
||||
elif item_type == 'voice':
|
||||
voice_info = item.get('voice', {}) or {}
|
||||
download_url = voice_info.get('url')
|
||||
item_aeskey = voice_info.get('aeskey', '')
|
||||
voice_data = {
|
||||
'url': download_url,
|
||||
'md5sum': voice_info.get('md5sum') or voice_info.get('md5'),
|
||||
'filesize': voice_info.get('filesize') or voice_info.get('size'),
|
||||
'sdkfileid': voice_info.get('sdkfileid') or voice_info.get('fileid'),
|
||||
}
|
||||
if voice_info.get('content'):
|
||||
texts.append(voice_info.get('content'))
|
||||
if (voice_data.get('filesize') or 0) <= max_inline_file_size:
|
||||
voice_base64 = await _safe_download_as_data_uri(download_url, item_aeskey)
|
||||
if voice_base64:
|
||||
voice_data['base64'] = voice_base64
|
||||
voices.append(voice_data)
|
||||
elif item_type == 'video':
|
||||
video_info = item.get('video', {}) or {}
|
||||
download_url = video_info.get('url')
|
||||
item_aeskey = video_info.get('aeskey', '')
|
||||
video_data = {
|
||||
'url': download_url,
|
||||
'filesize': video_info.get('filesize') or video_info.get('size'),
|
||||
'sdkfileid': video_info.get('sdkfileid') or video_info.get('fileid'),
|
||||
'md5sum': video_info.get('md5sum') or video_info.get('md5'),
|
||||
'filename': video_info.get('filename') or video_info.get('name'),
|
||||
}
|
||||
if (video_data.get('filesize') or 0) <= max_inline_file_size:
|
||||
video_base64 = await _safe_download_as_data_uri(download_url, item_aeskey)
|
||||
if video_base64:
|
||||
video_data['base64'] = video_base64
|
||||
videos.append(video_data)
|
||||
elif item_type == 'link':
|
||||
links.append(item.get('link', {}))
|
||||
|
||||
if texts:
|
||||
message_data['content'] = ' '.join(texts)
|
||||
if images:
|
||||
message_data['images'] = images
|
||||
message_data['picurl'] = images[0]
|
||||
if files:
|
||||
message_data['files'] = files
|
||||
message_data['file'] = files[0]
|
||||
if voices:
|
||||
message_data['voices'] = voices
|
||||
message_data['voice'] = voices[0]
|
||||
if videos:
|
||||
message_data['videos'] = videos
|
||||
message_data['video'] = videos[0]
|
||||
if links:
|
||||
message_data['link'] = links[0]
|
||||
if items:
|
||||
message_data['attachments'] = items
|
||||
else:
|
||||
message_data['raw_msg'] = msg_json
|
||||
|
||||
from_info = msg_json.get('from', {})
|
||||
message_data['userid'] = from_info.get('userid', '')
|
||||
message_data['username'] = from_info.get('alias', '') or from_info.get('name', '') or from_info.get('userid', '')
|
||||
|
||||
if msg_json.get('chattype', '') == 'group':
|
||||
message_data['chatid'] = msg_json.get('chatid', '')
|
||||
message_data['chatname'] = msg_json.get('chatname', '') or msg_json.get('chatid', '')
|
||||
|
||||
message_data['msgid'] = msg_json.get('msgid', '')
|
||||
|
||||
if msg_json.get('aibotid'):
|
||||
message_data['aibotid'] = msg_json.get('aibotid', '')
|
||||
|
||||
# Handle quote (referenced message) - important for group chat file references
|
||||
quote_info = msg_json.get('quote')
|
||||
if quote_info:
|
||||
quote_data: dict[str, Any] = {}
|
||||
quote_type = quote_info.get('msgtype', '')
|
||||
quote_data['msgtype'] = quote_type
|
||||
|
||||
if quote_type == 'text':
|
||||
quote_data['content'] = quote_info.get('text', {}).get('content', '')
|
||||
elif quote_type == 'image':
|
||||
img_info = quote_info.get('image', {})
|
||||
img_url = img_info.get('url', '')
|
||||
img_aeskey = img_info.get('aeskey', '')
|
||||
base64_data = await _safe_download_as_data_uri(img_url, img_aeskey)
|
||||
if base64_data:
|
||||
quote_data['picurl'] = base64_data
|
||||
quote_data['images'] = [base64_data]
|
||||
elif quote_type == 'file':
|
||||
file_info = quote_info.get('file', {}) or {}
|
||||
download_url = file_info.get('url') or file_info.get('fileurl')
|
||||
item_aeskey = file_info.get('aeskey', '')
|
||||
file_data = {
|
||||
'filename': file_info.get('filename') or file_info.get('name'),
|
||||
'filesize': file_info.get('filesize') or file_info.get('size'),
|
||||
'md5sum': file_info.get('md5sum') or file_info.get('md5'),
|
||||
'sdkfileid': file_info.get('sdkfileid') or file_info.get('fileid'),
|
||||
'download_url': download_url,
|
||||
'extra': file_info,
|
||||
}
|
||||
# Same as private chat: append aeskey to download_url for plugin processing
|
||||
if download_url and item_aeskey:
|
||||
file_data['download_url'] = download_url + f'?aeskey={item_aeskey}'
|
||||
quote_data['file'] = file_data
|
||||
elif quote_type == 'voice':
|
||||
voice_info = quote_info.get('voice', {}) or {}
|
||||
download_url = voice_info.get('url')
|
||||
item_aeskey = voice_info.get('aeskey', '')
|
||||
voice_data = {
|
||||
'url': download_url,
|
||||
'md5sum': voice_info.get('md5sum') or voice_info.get('md5'),
|
||||
'filesize': voice_info.get('filesize') or voice_info.get('size'),
|
||||
'sdkfileid': voice_info.get('sdkfileid') or voice_info.get('fileid'),
|
||||
}
|
||||
if voice_info.get('content'):
|
||||
quote_data['content'] = voice_info.get('content')
|
||||
# Same as private chat: append aeskey to url for plugin processing
|
||||
if download_url and item_aeskey:
|
||||
voice_data['url'] = download_url + f'?aeskey={item_aeskey}'
|
||||
quote_data['voice'] = voice_data
|
||||
elif quote_type == 'video':
|
||||
video_info = quote_info.get('video', {}) or {}
|
||||
download_url = video_info.get('url')
|
||||
item_aeskey = video_info.get('aeskey', '')
|
||||
video_data = {
|
||||
'url': download_url,
|
||||
'filesize': video_info.get('filesize') or video_info.get('size'),
|
||||
'sdkfileid': video_info.get('sdkfileid') or video_info.get('fileid'),
|
||||
'md5sum': video_info.get('md5sum') or video_info.get('md5'),
|
||||
'filename': video_info.get('filename') or video_info.get('name'),
|
||||
}
|
||||
# Same as private chat: append aeskey to download_url for plugin processing
|
||||
if download_url and item_aeskey:
|
||||
video_data['download_url'] = download_url + f'?aeskey={item_aeskey}'
|
||||
quote_data['video'] = video_data
|
||||
elif quote_type == 'link':
|
||||
quote_data['link'] = quote_info.get('link', {})
|
||||
link = quote_data['link']
|
||||
title = link.get('title', '')
|
||||
desc = link.get('description') or link.get('digest', '')
|
||||
quote_data['content'] = '\n'.join(filter(None, [title, desc]))
|
||||
elif quote_type == 'mixed':
|
||||
# Handle mixed type in quote (text + images + files etc.)
|
||||
items = quote_info.get('mixed', {}).get('msg_item', [])
|
||||
texts = []
|
||||
images = []
|
||||
files = []
|
||||
for item in items:
|
||||
item_type = item.get('msgtype')
|
||||
if item_type == 'text':
|
||||
texts.append(item.get('text', {}).get('content', ''))
|
||||
elif item_type == 'image':
|
||||
img_info = item.get('image', {})
|
||||
img_url = img_info.get('url')
|
||||
img_aeskey = img_info.get('aeskey', '')
|
||||
base64_data = await _safe_download_as_data_uri(img_url, img_aeskey)
|
||||
if base64_data:
|
||||
images.append(base64_data)
|
||||
elif item_type == 'file':
|
||||
file_info = item.get('file', {}) or {}
|
||||
download_url = file_info.get('url') or file_info.get('fileurl')
|
||||
item_aeskey = file_info.get('aeskey', '')
|
||||
file_data = {
|
||||
'filename': file_info.get('filename') or file_info.get('name'),
|
||||
'filesize': file_info.get('filesize') or file_info.get('size'),
|
||||
'md5sum': file_info.get('md5sum') or file_info.get('md5'),
|
||||
'sdkfileid': file_info.get('sdkfileid') or file_info.get('fileid'),
|
||||
'download_url': download_url,
|
||||
'extra': file_info,
|
||||
}
|
||||
# Same as private chat: append aeskey to download_url for plugin processing
|
||||
if download_url and item_aeskey:
|
||||
file_data['download_url'] = download_url + f'?aeskey={item_aeskey}'
|
||||
files.append(file_data)
|
||||
if texts:
|
||||
quote_data['content'] = ' '.join(texts)
|
||||
if images:
|
||||
quote_data['images'] = images
|
||||
quote_data['picurl'] = images[0]
|
||||
if files:
|
||||
quote_data['files'] = files
|
||||
quote_data['file'] = files[0]
|
||||
|
||||
message_data['quote'] = quote_data
|
||||
|
||||
return message_data
|
||||
|
||||
|
||||
class WecomBotClient:
|
||||
@@ -760,27 +236,14 @@ class WecomBotClient:
|
||||
self.stream_sessions = StreamSessionManager(logger=logger)
|
||||
self.stream_poll_timeout = 0.5
|
||||
|
||||
self._feedback_callback: Optional[Callable] = None
|
||||
|
||||
def set_feedback_callback(self, callback: Callable) -> None:
|
||||
"""设置反馈回调函数。
|
||||
|
||||
Args:
|
||||
callback: 反馈回调函数,签名: async def callback(feedback_id, feedback_type, feedback_content, inaccurate_reasons, session)
|
||||
"""
|
||||
self._feedback_callback = callback
|
||||
|
||||
@staticmethod
|
||||
def _build_stream_payload(
|
||||
stream_id: str, content: str, finish: bool, feedback_id: Optional[str] = None
|
||||
) -> dict[str, Any]:
|
||||
def _build_stream_payload(stream_id: str, content: str, finish: bool) -> dict[str, Any]:
|
||||
"""按照企业微信协议拼装返回报文。
|
||||
|
||||
Args:
|
||||
stream_id: 企业微信会话 ID。
|
||||
content: 推送的文本内容。
|
||||
finish: 是否为最终片段。
|
||||
feedback_id: 反馈 ID,用于接收用户点赞/点踩反馈。
|
||||
|
||||
Returns:
|
||||
dict[str, Any]: 可直接加密返回的 payload。
|
||||
@@ -788,16 +251,13 @@ class WecomBotClient:
|
||||
Example:
|
||||
组装 `{'msgtype': 'stream', 'stream': {'id': 'sid', ...}}` 结构。
|
||||
"""
|
||||
stream_payload = {
|
||||
'id': stream_id,
|
||||
'finish': finish,
|
||||
'content': content,
|
||||
}
|
||||
if feedback_id:
|
||||
stream_payload['feedback'] = {'id': feedback_id}
|
||||
return {
|
||||
'msgtype': 'stream',
|
||||
'stream': stream_payload,
|
||||
'stream': {
|
||||
'id': stream_id,
|
||||
'finish': finish,
|
||||
'content': content,
|
||||
},
|
||||
}
|
||||
|
||||
async def _encrypt_and_reply(self, payload: dict[str, Any], nonce: str) -> tuple[Response, int]:
|
||||
@@ -853,14 +313,9 @@ class WecomBotClient:
|
||||
"""
|
||||
session, is_new = self.stream_sessions.create_or_get(msg_json)
|
||||
|
||||
feedback_id = str(uuid.uuid4())
|
||||
session.feedback_id = feedback_id
|
||||
self.stream_sessions.register_feedback_id(session.stream_id, feedback_id)
|
||||
|
||||
message_data = await self.get_message(msg_json)
|
||||
if message_data:
|
||||
message_data['stream_id'] = session.stream_id
|
||||
message_data['feedback_id'] = feedback_id
|
||||
try:
|
||||
event = wecombotevent.WecomBotEvent(message_data)
|
||||
except Exception:
|
||||
@@ -869,7 +324,7 @@ class WecomBotClient:
|
||||
if is_new:
|
||||
asyncio.create_task(self._dispatch_event(event))
|
||||
|
||||
payload = self._build_stream_payload(session.stream_id, '', False, feedback_id)
|
||||
payload = self._build_stream_payload(session.stream_id, '', False)
|
||||
return await self._encrypt_and_reply(payload, nonce)
|
||||
|
||||
async def _handle_post_followup_response(self, msg_json: dict[str, Any], nonce: str) -> tuple[Response, int]:
|
||||
@@ -994,83 +449,202 @@ class WecomBotClient:
|
||||
|
||||
msg_json = json.loads(decrypted_xml)
|
||||
|
||||
event = msg_json.get('event', {})
|
||||
event_type = event.get('eventtype', '')
|
||||
|
||||
if event_type == 'feedback_event':
|
||||
return await self._handle_feedback_event(msg_json, nonce)
|
||||
|
||||
if msg_json.get('msgtype') == 'stream':
|
||||
return await self._handle_post_followup_response(msg_json, nonce)
|
||||
|
||||
return await self._handle_post_initial_response(msg_json, nonce)
|
||||
|
||||
async def _handle_feedback_event(self, msg_json: dict[str, Any], nonce: str) -> tuple[Response, int]:
|
||||
"""处理企业微信用户反馈事件(点赞/点踩)。
|
||||
|
||||
Args:
|
||||
msg_json: 解密后的企业微信反馈事件 JSON。
|
||||
nonce: 企业微信回调参数 nonce。
|
||||
|
||||
Returns:
|
||||
Tuple[Response, int]: Quart Response 及状态码。
|
||||
|
||||
Note:
|
||||
企业微信协议要求:反馈事件目前仅支持回复空包。
|
||||
"""
|
||||
try:
|
||||
feedback_event = msg_json.get('event', {}).get('feedback_event', {})
|
||||
feedback_id = feedback_event.get('id', '')
|
||||
feedback_type = feedback_event.get('type', 0)
|
||||
feedback_content = feedback_event.get('content', '')
|
||||
inaccurate_reasons = feedback_event.get('inaccurate_reason_list', [])
|
||||
|
||||
await self.logger.info(
|
||||
f'收到用户反馈事件: feedback_id={feedback_id}, type={feedback_type}, '
|
||||
f'content={feedback_content}, reasons={inaccurate_reasons}'
|
||||
)
|
||||
|
||||
session = self.stream_sessions.get_session_by_feedback_id(feedback_id)
|
||||
|
||||
if session:
|
||||
await self.logger.info(
|
||||
f'反馈关联到会话: stream_id={session.stream_id}, msg_id={session.msg_id}, user_id={session.user_id}'
|
||||
)
|
||||
else:
|
||||
await self.logger.warning(f'未找到 feedback_id={feedback_id} 对应的会话,仍将记录反馈')
|
||||
|
||||
# Dispatch feedback event regardless of session availability
|
||||
for handler in self._message_handlers.get('feedback', []):
|
||||
try:
|
||||
await handler(
|
||||
feedback_id=feedback_id,
|
||||
feedback_type=feedback_type,
|
||||
feedback_content=feedback_content,
|
||||
inaccurate_reasons=inaccurate_reasons,
|
||||
session=session,
|
||||
)
|
||||
except Exception:
|
||||
await self.logger.error(traceback.format_exc())
|
||||
|
||||
if self._feedback_callback:
|
||||
try:
|
||||
await self._feedback_callback(
|
||||
feedback_id=feedback_id,
|
||||
feedback_type=feedback_type,
|
||||
feedback_content=feedback_content,
|
||||
inaccurate_reasons=inaccurate_reasons,
|
||||
session=session,
|
||||
)
|
||||
except Exception:
|
||||
await self.logger.error(traceback.format_exc())
|
||||
|
||||
except Exception:
|
||||
await self.logger.error(traceback.format_exc())
|
||||
|
||||
return await self._encrypt_and_reply({}, nonce)
|
||||
|
||||
async def get_message(self, msg_json):
|
||||
return await parse_wecom_bot_message(msg_json, self.EnCodingAESKey, self.logger)
|
||||
message_data = {}
|
||||
|
||||
msg_type = msg_json.get('msgtype', '')
|
||||
if msg_type:
|
||||
message_data['msgtype'] = msg_type
|
||||
|
||||
if msg_json.get('chattype', '') == 'single':
|
||||
message_data['type'] = 'single'
|
||||
elif msg_json.get('chattype', '') == 'group':
|
||||
message_data['type'] = 'group'
|
||||
|
||||
max_inline_file_size = 5 * 1024 * 1024 # avoid decoding very large payloads by default
|
||||
|
||||
async def _safe_download(url: str):
|
||||
if not url:
|
||||
return None
|
||||
return await self.download_url_to_base64(url, self.EnCodingAESKey)
|
||||
|
||||
if msg_type == 'text':
|
||||
message_data['content'] = msg_json.get('text', {}).get('content')
|
||||
elif msg_type == 'markdown':
|
||||
message_data['content'] = msg_json.get('markdown', {}).get('content') or msg_json.get('text', {}).get(
|
||||
'content', ''
|
||||
)
|
||||
elif msg_type == 'image':
|
||||
picurl = msg_json.get('image', {}).get('url', '')
|
||||
base64_data = await _safe_download(picurl)
|
||||
if base64_data:
|
||||
message_data['picurl'] = base64_data
|
||||
message_data['images'] = [base64_data]
|
||||
elif msg_type == 'voice':
|
||||
voice_info = msg_json.get('voice', {}) or {}
|
||||
download_url = voice_info.get('url')
|
||||
message_data['voice'] = {
|
||||
'url': download_url,
|
||||
'md5sum': voice_info.get('md5sum') or voice_info.get('md5'),
|
||||
'filesize': voice_info.get('filesize') or voice_info.get('size'),
|
||||
'sdkfileid': voice_info.get('sdkfileid') or voice_info.get('fileid'),
|
||||
}
|
||||
# 企业微信智能转写文本(如果已有)直接复用,避免重复转写
|
||||
if voice_info.get('content'):
|
||||
message_data['content'] = voice_info.get('content')
|
||||
if (message_data['voice'].get('filesize') or 0) <= max_inline_file_size:
|
||||
voice_base64 = await _safe_download(download_url)
|
||||
if voice_base64:
|
||||
message_data['voice']['base64'] = voice_base64
|
||||
elif msg_type == 'video':
|
||||
video_info = msg_json.get('video', {}) or {}
|
||||
download_url = video_info.get('url')
|
||||
video_data = {
|
||||
'url': download_url,
|
||||
'filesize': video_info.get('filesize') or video_info.get('size'),
|
||||
'sdkfileid': video_info.get('sdkfileid') or video_info.get('fileid'),
|
||||
'md5sum': video_info.get('md5sum') or video_info.get('md5'),
|
||||
'filename': video_info.get('filename') or video_info.get('name'),
|
||||
}
|
||||
if (video_data.get('filesize') or 0) <= max_inline_file_size:
|
||||
video_base64 = await _safe_download(download_url)
|
||||
if video_base64:
|
||||
video_data['base64'] = video_base64
|
||||
message_data['video'] = video_data
|
||||
elif msg_type == 'file':
|
||||
file_info = msg_json.get('file', {}) or {}
|
||||
download_url = file_info.get('url') or file_info.get('fileurl')
|
||||
file_data = {
|
||||
'filename': file_info.get('filename') or file_info.get('name'),
|
||||
'filesize': file_info.get('filesize') or file_info.get('size'),
|
||||
'md5sum': file_info.get('md5sum') or file_info.get('md5'),
|
||||
'sdkfileid': file_info.get('sdkfileid') or file_info.get('fileid'),
|
||||
'download_url': download_url,
|
||||
'extra': file_info,
|
||||
}
|
||||
if (file_data.get('filesize') or 0) <= max_inline_file_size:
|
||||
file_base64 = await _safe_download(download_url)
|
||||
if file_base64:
|
||||
file_data['base64'] = file_base64
|
||||
message_data['file'] = file_data
|
||||
elif msg_type == 'link':
|
||||
message_data['link'] = msg_json.get('link', {})
|
||||
if not message_data.get('content'):
|
||||
title = message_data['link'].get('title', '')
|
||||
desc = message_data['link'].get('description') or message_data['link'].get('digest', '')
|
||||
message_data['content'] = '\n'.join(filter(None, [title, desc]))
|
||||
elif msg_type == 'mixed':
|
||||
items = msg_json.get('mixed', {}).get('msg_item', [])
|
||||
texts = []
|
||||
images = []
|
||||
files = []
|
||||
voices = []
|
||||
videos = []
|
||||
links = []
|
||||
for item in items:
|
||||
item_type = item.get('msgtype')
|
||||
if item_type == 'text':
|
||||
texts.append(item.get('text', {}).get('content', ''))
|
||||
elif item_type == 'image':
|
||||
img_url = item.get('image', {}).get('url')
|
||||
base64_data = await _safe_download(img_url)
|
||||
if base64_data:
|
||||
images.append(base64_data)
|
||||
elif item_type == 'file':
|
||||
file_info = item.get('file', {}) or {}
|
||||
download_url = file_info.get('url') or file_info.get('fileurl')
|
||||
file_data = {
|
||||
'filename': file_info.get('filename') or file_info.get('name'),
|
||||
'filesize': file_info.get('filesize') or file_info.get('size'),
|
||||
'md5sum': file_info.get('md5sum') or file_info.get('md5'),
|
||||
'sdkfileid': file_info.get('sdkfileid') or file_info.get('fileid'),
|
||||
'download_url': download_url,
|
||||
'extra': file_info,
|
||||
}
|
||||
if (file_data.get('filesize') or 0) <= max_inline_file_size:
|
||||
file_base64 = await _safe_download(download_url)
|
||||
if file_base64:
|
||||
file_data['base64'] = file_base64
|
||||
files.append(file_data)
|
||||
elif item_type == 'voice':
|
||||
voice_info = item.get('voice', {}) or {}
|
||||
download_url = voice_info.get('url')
|
||||
voice_data = {
|
||||
'url': download_url,
|
||||
'md5sum': voice_info.get('md5sum') or voice_info.get('md5'),
|
||||
'filesize': voice_info.get('filesize') or voice_info.get('size'),
|
||||
'sdkfileid': voice_info.get('sdkfileid') or voice_info.get('fileid'),
|
||||
}
|
||||
if voice_info.get('content'):
|
||||
texts.append(voice_info.get('content'))
|
||||
if (voice_data.get('filesize') or 0) <= max_inline_file_size:
|
||||
voice_base64 = await _safe_download(download_url)
|
||||
if voice_base64:
|
||||
voice_data['base64'] = voice_base64
|
||||
voices.append(voice_data)
|
||||
elif item_type == 'video':
|
||||
video_info = item.get('video', {}) or {}
|
||||
download_url = video_info.get('url')
|
||||
video_data = {
|
||||
'url': download_url,
|
||||
'filesize': video_info.get('filesize') or video_info.get('size'),
|
||||
'sdkfileid': video_info.get('sdkfileid') or video_info.get('fileid'),
|
||||
'md5sum': video_info.get('md5sum') or video_info.get('md5'),
|
||||
'filename': video_info.get('filename') or video_info.get('name'),
|
||||
}
|
||||
if (video_data.get('filesize') or 0) <= max_inline_file_size:
|
||||
video_base64 = await _safe_download(download_url)
|
||||
if video_base64:
|
||||
video_data['base64'] = video_base64
|
||||
videos.append(video_data)
|
||||
elif item_type == 'link':
|
||||
links.append(item.get('link', {}))
|
||||
|
||||
if texts:
|
||||
message_data['content'] = ' '.join(texts) # 拼接所有 text
|
||||
if images:
|
||||
message_data['images'] = images
|
||||
message_data['picurl'] = images[0] # 只保留第一个 image
|
||||
if files:
|
||||
message_data['files'] = files
|
||||
message_data['file'] = files[0]
|
||||
if voices:
|
||||
message_data['voices'] = voices
|
||||
message_data['voice'] = voices[0]
|
||||
if videos:
|
||||
message_data['videos'] = videos
|
||||
message_data['video'] = videos[0]
|
||||
if links:
|
||||
message_data['link'] = links[0]
|
||||
if items:
|
||||
message_data['attachments'] = items
|
||||
else:
|
||||
message_data['raw_msg'] = msg_json
|
||||
|
||||
# Extract user information
|
||||
from_info = msg_json.get('from', {})
|
||||
message_data['userid'] = from_info.get('userid', '')
|
||||
message_data['username'] = (
|
||||
from_info.get('alias', '') or from_info.get('name', '') or from_info.get('userid', '')
|
||||
)
|
||||
|
||||
# Extract chat/group information
|
||||
if msg_json.get('chattype', '') == 'group':
|
||||
message_data['chatid'] = msg_json.get('chatid', '')
|
||||
# Try to get group name if available
|
||||
message_data['chatname'] = msg_json.get('chatname', '') or msg_json.get('chatid', '')
|
||||
|
||||
message_data['msgid'] = msg_json.get('msgid', '')
|
||||
|
||||
if msg_json.get('aibotid'):
|
||||
message_data['aibotid'] = msg_json.get('aibotid', '')
|
||||
|
||||
return message_data
|
||||
|
||||
async def _handle_message(self, event: wecombotevent.WecomBotEvent):
|
||||
"""
|
||||
@@ -1137,20 +711,40 @@ class WecomBotClient:
|
||||
|
||||
return decorator
|
||||
|
||||
def on_feedback(self):
|
||||
def decorator(func: Callable):
|
||||
if 'feedback' not in self._message_handlers:
|
||||
self._message_handlers['feedback'] = []
|
||||
self._message_handlers['feedback'].append(func)
|
||||
return func
|
||||
|
||||
return decorator
|
||||
|
||||
async def download_url_to_base64(self, download_url, encoding_aes_key):
|
||||
data, _filename = await download_encrypted_file(download_url, encoding_aes_key, self.logger)
|
||||
if data:
|
||||
return _bytes_to_data_uri(data)
|
||||
return None
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.get(download_url)
|
||||
if response.status_code != 200:
|
||||
await self.logger.error(f'failed to get file: {response.text}')
|
||||
return None
|
||||
|
||||
encrypted_bytes = response.content
|
||||
|
||||
aes_key = base64.b64decode(encoding_aes_key + '=') # base64 补齐
|
||||
iv = aes_key[:16]
|
||||
|
||||
cipher = AES.new(aes_key, AES.MODE_CBC, iv)
|
||||
decrypted = cipher.decrypt(encrypted_bytes)
|
||||
|
||||
pad_len = decrypted[-1]
|
||||
decrypted = decrypted[:-pad_len]
|
||||
|
||||
if decrypted.startswith(b'\xff\xd8'): # JPEG
|
||||
mime_type = 'image/jpeg'
|
||||
elif decrypted.startswith(b'\x89PNG'): # PNG
|
||||
mime_type = 'image/png'
|
||||
elif decrypted.startswith((b'GIF87a', b'GIF89a')): # GIF
|
||||
mime_type = 'image/gif'
|
||||
elif decrypted.startswith(b'BM'): # BMP
|
||||
mime_type = 'image/bmp'
|
||||
elif decrypted.startswith(b'II*\x00') or decrypted.startswith(b'MM\x00*'): # TIFF
|
||||
mime_type = 'image/tiff'
|
||||
else:
|
||||
mime_type = 'application/octet-stream'
|
||||
|
||||
# 转 base64
|
||||
base64_str = base64.b64encode(decrypted).decode('utf-8')
|
||||
return f'data:{mime_type};base64,{base64_str}'
|
||||
|
||||
async def run_task(self, host: str, port: int, *args, **kwargs):
|
||||
"""
|
||||
|
||||
@@ -133,24 +133,3 @@ class WecomBotEvent(dict):
|
||||
AI Bot ID
|
||||
"""
|
||||
return self.get('aibotid', '')
|
||||
|
||||
@property
|
||||
def feedback_id(self) -> str:
|
||||
"""
|
||||
反馈 ID,用于关联用户点赞/点踩反馈
|
||||
"""
|
||||
return self.get('feedback_id', '')
|
||||
|
||||
@property
|
||||
def stream_id(self) -> str:
|
||||
"""
|
||||
流式消息 ID
|
||||
"""
|
||||
return self.get('stream_id', '')
|
||||
|
||||
@property
|
||||
def quote(self):
|
||||
"""
|
||||
引用消息信息(群聊中用户引用其他消息时返回)
|
||||
"""
|
||||
return self.get('quote', {})
|
||||
|
||||
@@ -1,683 +0,0 @@
|
||||
"""WeChat Work AI Bot WebSocket long connection client.
|
||||
|
||||
Implements the WebSocket protocol for receiving messages and sending replies
|
||||
via a persistent connection to wss://openws.work.weixin.qq.com, as an
|
||||
alternative to the HTTP callback (webhook) mode.
|
||||
|
||||
Protocol reference: https://developer.work.weixin.qq.com/document/path/101463
|
||||
Official Node.js SDK: https://github.com/WecomTeam/aibot-node-sdk
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import secrets
|
||||
import time
|
||||
import traceback
|
||||
from typing import Any, Callable, Optional
|
||||
|
||||
import aiohttp
|
||||
|
||||
from langbot.libs.wecom_ai_bot_api import wecombotevent
|
||||
from langbot.libs.wecom_ai_bot_api.api import parse_wecom_bot_message, StreamSession
|
||||
from langbot.pkg.platform.logger import EventLogger
|
||||
|
||||
DEFAULT_WS_URL = 'wss://openws.work.weixin.qq.com'
|
||||
|
||||
# WebSocket frame command constants
|
||||
CMD_SUBSCRIBE = 'aibot_subscribe'
|
||||
CMD_HEARTBEAT = 'ping'
|
||||
CMD_MSG_CALLBACK = 'aibot_msg_callback'
|
||||
CMD_EVENT_CALLBACK = 'aibot_event_callback'
|
||||
CMD_RESPOND_MSG = 'aibot_respond_msg'
|
||||
CMD_RESPOND_WELCOME = 'aibot_respond_welcome_msg'
|
||||
CMD_RESPOND_UPDATE = 'aibot_respond_update_msg'
|
||||
CMD_SEND_MSG = 'aibot_send_msg'
|
||||
|
||||
|
||||
def _generate_req_id(prefix: str) -> str:
|
||||
"""Generate a unique request ID in the format: {prefix}_{timestamp}_{random}."""
|
||||
ts = int(time.time() * 1000)
|
||||
rand = secrets.token_hex(4)
|
||||
return f'{prefix}_{ts}_{rand}'
|
||||
|
||||
|
||||
class WecomBotWsClient:
|
||||
"""WeChat Work AI Bot WebSocket long connection client.
|
||||
|
||||
Provides message receiving, streaming reply, proactive message sending,
|
||||
and event callback handling over a persistent WebSocket connection.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
bot_id: str,
|
||||
secret: str,
|
||||
logger: EventLogger,
|
||||
encoding_aes_key: str = '',
|
||||
ws_url: str = DEFAULT_WS_URL,
|
||||
heartbeat_interval: float = 30.0,
|
||||
max_reconnect_attempts: int = -1,
|
||||
reconnect_base_delay: float = 1.0,
|
||||
reconnect_max_delay: float = 30.0,
|
||||
):
|
||||
self.bot_id = bot_id
|
||||
self.secret = secret
|
||||
self.logger = logger
|
||||
self.encoding_aes_key = encoding_aes_key
|
||||
self.ws_url = ws_url
|
||||
self.heartbeat_interval = heartbeat_interval
|
||||
self.max_reconnect_attempts = max_reconnect_attempts
|
||||
self.reconnect_base_delay = reconnect_base_delay
|
||||
self.reconnect_max_delay = reconnect_max_delay
|
||||
|
||||
self._ws: Optional[aiohttp.ClientWebSocketResponse] = None
|
||||
self._session: Optional[aiohttp.ClientSession] = None
|
||||
self._running = False
|
||||
self._heartbeat_task: Optional[asyncio.Task] = None
|
||||
self._missed_pong_count = 0
|
||||
self._max_missed_pong = 2
|
||||
self._reconnect_attempts = 0
|
||||
|
||||
# Message handler registry (same pattern as WecomBotClient)
|
||||
self._message_handlers: dict[str, list[Callable]] = {}
|
||||
# Message deduplication
|
||||
self._msg_id_map: dict[str, int] = {}
|
||||
|
||||
# Pending ACK futures: req_id -> Future[dict]
|
||||
self._pending_acks: dict[str, asyncio.Future] = {}
|
||||
# Per-req_id serial reply queues
|
||||
self._reply_queues: dict[str, asyncio.Queue] = {}
|
||||
self._reply_workers: dict[str, asyncio.Task] = {}
|
||||
self._reply_ack_timeout = 5.0
|
||||
|
||||
# Stream ID tracking for WebSocket mode
|
||||
self._stream_ids: dict[str, str] = {} # msg_id -> req_id|stream_id
|
||||
# Dedup: skip sending when content hasn't changed
|
||||
self._stream_last_content: dict[str, str] = {} # msg_id -> last content sent
|
||||
# Stream session info for feedback tracking
|
||||
self._stream_sessions: dict[str, dict] = {} # msg_id -> session info
|
||||
# Feedback tracking: feedback_id -> session info
|
||||
self._feedback_sessions: dict[str, dict] = {} # feedback_id -> {msg_id, user_id, chat_id, stream_id, req_id}
|
||||
# msg_id -> feedback_id (for associating feedback with message)
|
||||
self._msg_feedback_ids: dict[str, str] = {} # msg_id -> feedback_id
|
||||
|
||||
# ── Public API ──────────────────────────────────────────────────
|
||||
|
||||
async def connect(self):
|
||||
"""Connect to WebSocket server with automatic reconnection.
|
||||
|
||||
This method blocks until disconnect() is called or max reconnect
|
||||
attempts are exhausted.
|
||||
"""
|
||||
self._running = True
|
||||
self._reconnect_attempts = 0
|
||||
|
||||
while self._running:
|
||||
try:
|
||||
await self._connect_once()
|
||||
except Exception:
|
||||
if not self._running:
|
||||
break
|
||||
await self.logger.error(f'WebSocket connection error: {traceback.format_exc()}')
|
||||
|
||||
if not self._running:
|
||||
break
|
||||
|
||||
# Reconnect with exponential backoff
|
||||
if self.max_reconnect_attempts != -1 and self._reconnect_attempts >= self.max_reconnect_attempts:
|
||||
await self.logger.error(f'Max reconnect attempts reached ({self.max_reconnect_attempts}), giving up')
|
||||
break
|
||||
|
||||
self._reconnect_attempts += 1
|
||||
delay = min(
|
||||
self.reconnect_base_delay * (2 ** (self._reconnect_attempts - 1)),
|
||||
self.reconnect_max_delay,
|
||||
)
|
||||
await self.logger.info(f'Reconnecting in {delay:.1f}s (attempt {self._reconnect_attempts})...')
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
async def disconnect(self):
|
||||
"""Gracefully disconnect from the WebSocket server."""
|
||||
self._running = False
|
||||
if self._heartbeat_task and not self._heartbeat_task.done():
|
||||
self._heartbeat_task.cancel()
|
||||
for task in self._reply_workers.values():
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
if self._ws and not self._ws.closed:
|
||||
await self._ws.close()
|
||||
self._ws = None
|
||||
if self._session and not self._session.closed:
|
||||
await self._session.close()
|
||||
self._session = None
|
||||
|
||||
def on_message(self, msg_type: str) -> Callable:
|
||||
"""Decorator to register a message handler.
|
||||
|
||||
Same interface as WecomBotClient.on_message for compatibility.
|
||||
|
||||
Args:
|
||||
msg_type: 'single', 'group', or specific message type.
|
||||
"""
|
||||
|
||||
def decorator(func: Callable[[wecombotevent.WecomBotEvent], Any]):
|
||||
if msg_type not in self._message_handlers:
|
||||
self._message_handlers[msg_type] = []
|
||||
self._message_handlers[msg_type].append(func)
|
||||
return func
|
||||
|
||||
return decorator
|
||||
|
||||
def on_feedback(self) -> Callable:
|
||||
"""Decorator to register a feedback event handler.
|
||||
|
||||
Same interface as WecomBotClient.on_feedback for compatibility.
|
||||
"""
|
||||
|
||||
def decorator(func: Callable):
|
||||
if 'feedback' not in self._message_handlers:
|
||||
self._message_handlers['feedback'] = []
|
||||
self._message_handlers['feedback'].append(func)
|
||||
return func
|
||||
|
||||
return decorator
|
||||
|
||||
async def reply_stream(
|
||||
self,
|
||||
req_id: str,
|
||||
stream_id: str,
|
||||
content: str,
|
||||
finish: bool = False,
|
||||
feedback_id: str = '',
|
||||
) -> Optional[dict]:
|
||||
"""Send a streaming reply frame.
|
||||
|
||||
Args:
|
||||
req_id: The req_id from the original message frame (must be passed through).
|
||||
stream_id: The stream ID for this streaming session.
|
||||
content: The content to send (supports Markdown).
|
||||
finish: Whether this is the final chunk.
|
||||
feedback_id: Optional feedback ID for receiving user feedback (like/dislike).
|
||||
|
||||
Returns:
|
||||
The ACK frame dict, or None on failure.
|
||||
"""
|
||||
stream_payload = {
|
||||
'id': stream_id,
|
||||
'finish': finish,
|
||||
'content': content,
|
||||
}
|
||||
if feedback_id:
|
||||
stream_payload['feedback'] = {'id': feedback_id}
|
||||
|
||||
body = {
|
||||
'msgtype': 'stream',
|
||||
'stream': stream_payload,
|
||||
}
|
||||
return await self._send_reply(req_id, body)
|
||||
|
||||
async def reply_text(self, req_id: str, content: str) -> Optional[dict]:
|
||||
"""Send a non-streaming text reply.
|
||||
|
||||
Args:
|
||||
req_id: The req_id from the original message frame.
|
||||
content: The text content to reply.
|
||||
|
||||
Returns:
|
||||
The ACK frame dict, or None on failure.
|
||||
"""
|
||||
body = {
|
||||
'msgtype': 'markdown',
|
||||
'markdown': {
|
||||
'content': content,
|
||||
},
|
||||
}
|
||||
return await self._send_reply(req_id, body)
|
||||
|
||||
async def send_message(self, chat_id: str, content: str, msgtype: str = 'markdown') -> Optional[dict]:
|
||||
"""Proactively send a message to a specified chat.
|
||||
|
||||
Args:
|
||||
chat_id: The chat ID (userid for single chat, chatid for group chat).
|
||||
content: The message content.
|
||||
msgtype: Message type, 'markdown' by default.
|
||||
|
||||
Returns:
|
||||
The ACK frame dict, or None on failure.
|
||||
"""
|
||||
req_id = _generate_req_id(CMD_SEND_MSG)
|
||||
body: dict[str, Any] = {
|
||||
'chatid': chat_id,
|
||||
'msgtype': msgtype,
|
||||
}
|
||||
if msgtype == 'markdown':
|
||||
body['markdown'] = {'content': content}
|
||||
elif msgtype == 'text':
|
||||
body['text'] = {'content': content}
|
||||
return await self._send_reply(req_id, body, cmd=CMD_SEND_MSG)
|
||||
|
||||
async def push_stream_chunk(self, msg_id: str, content: str, is_final: bool = False) -> bool:
|
||||
"""Push a streaming chunk for a given message ID.
|
||||
|
||||
Compatible interface with WecomBotClient.push_stream_chunk.
|
||||
|
||||
Args:
|
||||
msg_id: The original message ID.
|
||||
content: The cumulative content from the pipeline.
|
||||
is_final: Whether this is the final chunk.
|
||||
|
||||
Returns:
|
||||
True if the stream session exists and chunk was sent.
|
||||
"""
|
||||
key = self._stream_ids.get(msg_id)
|
||||
if not key:
|
||||
return False
|
||||
req_id, stream_id = key.split('|', 1)
|
||||
try:
|
||||
# Skip sending if content hasn't changed (e.g. during tool call argument streaming)
|
||||
if not is_final and content == self._stream_last_content.get(msg_id):
|
||||
return True
|
||||
|
||||
# Generate feedback_id for final chunk
|
||||
feedback_id = ''
|
||||
if is_final:
|
||||
feedback_id = _generate_req_id('feedback')
|
||||
self._msg_feedback_ids[msg_id] = feedback_id
|
||||
# Store session info for feedback tracking
|
||||
session_info = self._stream_sessions.get(msg_id)
|
||||
if session_info:
|
||||
self._feedback_sessions[feedback_id] = session_info
|
||||
|
||||
await self.reply_stream(req_id, stream_id, content, finish=is_final, feedback_id=feedback_id)
|
||||
self._stream_last_content[msg_id] = content
|
||||
if is_final:
|
||||
self._stream_ids.pop(msg_id, None)
|
||||
self._stream_last_content.pop(msg_id, None)
|
||||
self._stream_sessions.pop(msg_id, None)
|
||||
return True
|
||||
except Exception:
|
||||
await self.logger.error(f'Failed to push stream chunk: {traceback.format_exc()}')
|
||||
return False
|
||||
|
||||
async def set_message(self, msg_id: str, content: str):
|
||||
"""Fallback: send content as a final stream chunk or direct reply.
|
||||
|
||||
Compatible interface with WecomBotClient.set_message.
|
||||
"""
|
||||
handled = await self.push_stream_chunk(msg_id, content, is_final=True)
|
||||
if not handled:
|
||||
await self.logger.warning(f'No active stream for msg_id={msg_id}, message dropped')
|
||||
|
||||
# ── Connection lifecycle ────────────────────────────────────────
|
||||
|
||||
async def _connect_once(self):
|
||||
"""Establish a single WebSocket connection, authenticate, and listen."""
|
||||
await self.logger.info(f'Connecting to {self.ws_url}...')
|
||||
|
||||
self._session = aiohttp.ClientSession()
|
||||
try:
|
||||
self._ws = await self._session.ws_connect(self.ws_url)
|
||||
self._missed_pong_count = 0
|
||||
self._reconnect_attempts = 0
|
||||
await self.logger.info('WebSocket connected, sending auth...')
|
||||
|
||||
await self._send_auth()
|
||||
|
||||
# Wait for auth response
|
||||
auth_ok = await self._wait_for_auth()
|
||||
if not auth_ok:
|
||||
await self.logger.error('Authentication failed')
|
||||
return
|
||||
|
||||
await self.logger.info('Authenticated successfully')
|
||||
|
||||
# Start heartbeat
|
||||
self._heartbeat_task = asyncio.create_task(self._heartbeat_loop())
|
||||
|
||||
try:
|
||||
await self._listen_loop()
|
||||
finally:
|
||||
if self._heartbeat_task and not self._heartbeat_task.done():
|
||||
self._heartbeat_task.cancel()
|
||||
self._clear_pending_acks('Connection closed')
|
||||
finally:
|
||||
if self._ws and not self._ws.closed:
|
||||
await self._ws.close()
|
||||
self._ws = None
|
||||
if self._session and not self._session.closed:
|
||||
await self._session.close()
|
||||
self._session = None
|
||||
|
||||
async def _send_auth(self):
|
||||
"""Send the authentication frame."""
|
||||
frame = {
|
||||
'cmd': CMD_SUBSCRIBE,
|
||||
'headers': {'req_id': _generate_req_id(CMD_SUBSCRIBE)},
|
||||
'body': {
|
||||
'bot_id': self.bot_id,
|
||||
'secret': self.secret,
|
||||
},
|
||||
}
|
||||
await self._send_frame(frame)
|
||||
|
||||
async def _wait_for_auth(self) -> bool:
|
||||
"""Wait for and validate the authentication response."""
|
||||
try:
|
||||
msg = await asyncio.wait_for(self._ws.receive(), timeout=10.0)
|
||||
if msg.type in (aiohttp.WSMsgType.TEXT,):
|
||||
frame = json.loads(msg.data)
|
||||
req_id = frame.get('headers', {}).get('req_id', '')
|
||||
if req_id.startswith(CMD_SUBSCRIBE) and frame.get('errcode') == 0:
|
||||
return True
|
||||
await self.logger.error(f'Auth response: errcode={frame.get("errcode")}, errmsg={frame.get("errmsg")}')
|
||||
return False
|
||||
elif msg.type in (aiohttp.WSMsgType.ERROR, aiohttp.WSMsgType.CLOSED, aiohttp.WSMsgType.CLOSING):
|
||||
await self.logger.error(f'WebSocket closed during auth: {msg.type}')
|
||||
return False
|
||||
await self.logger.error(f'Unexpected message type during auth: {msg.type}')
|
||||
return False
|
||||
except asyncio.TimeoutError:
|
||||
await self.logger.error('Auth response timeout')
|
||||
return False
|
||||
|
||||
async def _heartbeat_loop(self):
|
||||
"""Periodically send heartbeat pings."""
|
||||
try:
|
||||
while self._running and self._ws and not self._ws.closed:
|
||||
await asyncio.sleep(self.heartbeat_interval)
|
||||
if not self._running or not self._ws or self._ws.closed:
|
||||
break
|
||||
|
||||
if self._missed_pong_count >= self._max_missed_pong:
|
||||
await self.logger.warning(
|
||||
f'No heartbeat ack for {self._missed_pong_count} consecutive pings, connection considered dead'
|
||||
)
|
||||
await self._ws.close()
|
||||
break
|
||||
|
||||
self._missed_pong_count += 1
|
||||
frame = {
|
||||
'cmd': CMD_HEARTBEAT,
|
||||
'headers': {'req_id': _generate_req_id(CMD_HEARTBEAT)},
|
||||
}
|
||||
try:
|
||||
await self._send_frame(frame)
|
||||
except Exception:
|
||||
break
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
async def _listen_loop(self):
|
||||
"""Listen for incoming WebSocket frames and dispatch them."""
|
||||
async for msg in self._ws:
|
||||
if not self._running:
|
||||
break
|
||||
if msg.type == aiohttp.WSMsgType.TEXT:
|
||||
try:
|
||||
frame = json.loads(msg.data)
|
||||
await self._handle_frame(frame)
|
||||
except json.JSONDecodeError:
|
||||
await self.logger.error(f'Failed to parse WebSocket message: {str(msg.data)[:200]}')
|
||||
except Exception:
|
||||
await self.logger.error(f'Error handling frame: {traceback.format_exc()}')
|
||||
elif msg.type == aiohttp.WSMsgType.BINARY:
|
||||
try:
|
||||
frame = json.loads(msg.data)
|
||||
await self._handle_frame(frame)
|
||||
except Exception:
|
||||
await self.logger.error(f'Error handling binary frame: {traceback.format_exc()}')
|
||||
elif msg.type in (aiohttp.WSMsgType.ERROR, aiohttp.WSMsgType.CLOSED, aiohttp.WSMsgType.CLOSING):
|
||||
await self.logger.warning(f'WebSocket connection closed: {msg.type}')
|
||||
break
|
||||
|
||||
# ── Frame handling ──────────────────────────────────────────────
|
||||
|
||||
async def _handle_frame(self, frame: dict):
|
||||
"""Route an incoming frame to the appropriate handler."""
|
||||
cmd = frame.get('cmd', '')
|
||||
|
||||
# Message push
|
||||
if cmd == CMD_MSG_CALLBACK:
|
||||
asyncio.create_task(self._handle_message_callback(frame))
|
||||
return
|
||||
|
||||
# Event push
|
||||
if cmd == CMD_EVENT_CALLBACK:
|
||||
asyncio.create_task(self._handle_event_callback(frame))
|
||||
return
|
||||
|
||||
# No cmd → response/ACK frame, dispatch by req_id prefix
|
||||
req_id = frame.get('headers', {}).get('req_id', '')
|
||||
|
||||
# Check pending ACKs first
|
||||
if req_id in self._pending_acks:
|
||||
future = self._pending_acks.pop(req_id)
|
||||
if not future.done():
|
||||
future.set_result(frame)
|
||||
return
|
||||
|
||||
# Heartbeat response
|
||||
if req_id.startswith(CMD_HEARTBEAT):
|
||||
if frame.get('errcode') == 0:
|
||||
self._missed_pong_count = 0
|
||||
return
|
||||
|
||||
# Unknown frame
|
||||
await self.logger.warning(f'Unknown frame: {json.dumps(frame, ensure_ascii=False)[:200]}')
|
||||
|
||||
async def _handle_message_callback(self, frame: dict):
|
||||
"""Handle an incoming message callback frame."""
|
||||
try:
|
||||
body = frame.get('body', {})
|
||||
req_id = frame.get('headers', {}).get('req_id', '')
|
||||
|
||||
# Parse message using shared logic
|
||||
message_data = await parse_wecom_bot_message(body, self.encoding_aes_key, self.logger)
|
||||
if not message_data:
|
||||
return
|
||||
|
||||
# Generate stream_id for this message and store the mapping
|
||||
stream_id = _generate_req_id('stream')
|
||||
msg_id = message_data.get('msgid', '')
|
||||
if msg_id:
|
||||
self._stream_ids[msg_id] = f'{req_id}|{stream_id}'
|
||||
# Store session info for feedback tracking
|
||||
self._stream_sessions[msg_id] = {
|
||||
'req_id': req_id,
|
||||
'stream_id': stream_id,
|
||||
'msg_id': msg_id,
|
||||
'user_id': message_data.get('userid', ''),
|
||||
'chat_id': message_data.get('chatid', ''),
|
||||
'chat_type': message_data.get('type', 'single'),
|
||||
}
|
||||
message_data['stream_id'] = stream_id
|
||||
message_data['req_id'] = req_id
|
||||
|
||||
event = wecombotevent.WecomBotEvent(message_data)
|
||||
await self._dispatch_event(event)
|
||||
except Exception:
|
||||
await self.logger.error(f'Error in message callback: {traceback.format_exc()}')
|
||||
|
||||
async def _handle_event_callback(self, frame: dict):
|
||||
"""Handle an incoming event callback frame (enter_chat, template_card_event, feedback_event, disconnected_event)."""
|
||||
try:
|
||||
body = frame.get('body', {})
|
||||
req_id = frame.get('headers', {}).get('req_id', '')
|
||||
|
||||
event_info = body.get('event', {})
|
||||
event_type = event_info.get('eventtype', '')
|
||||
|
||||
message_data = {
|
||||
'msgtype': 'event',
|
||||
'type': body.get('chattype', 'single'),
|
||||
'event': event_info,
|
||||
'eventtype': event_type,
|
||||
'msgid': body.get('msgid', ''),
|
||||
'aibotid': body.get('aibotid', ''),
|
||||
'req_id': req_id,
|
||||
}
|
||||
|
||||
from_info = body.get('from', {})
|
||||
message_data['userid'] = from_info.get('userid', '')
|
||||
message_data['username'] = from_info.get('alias', '') or from_info.get('userid', '')
|
||||
|
||||
if body.get('chatid'):
|
||||
message_data['chatid'] = body.get('chatid', '')
|
||||
|
||||
if event_type == 'feedback_event':
|
||||
feedback_event = event_info.get('feedback_event', {})
|
||||
feedback_id = feedback_event.get('id', '')
|
||||
feedback_type = feedback_event.get('type', 0)
|
||||
feedback_content = feedback_event.get('content', '')
|
||||
inaccurate_reasons = feedback_event.get('inaccurate_reason_list', [])
|
||||
|
||||
await self.logger.info(
|
||||
f'收到用户反馈事件: feedback_id={feedback_id}, type={feedback_type}, '
|
||||
f'content={feedback_content}, reasons={inaccurate_reasons}'
|
||||
)
|
||||
|
||||
# Look up session by feedback_id
|
||||
session_info = self._feedback_sessions.get(feedback_id)
|
||||
session = None
|
||||
if session_info:
|
||||
session = StreamSession(
|
||||
stream_id=session_info.get('stream_id', ''),
|
||||
msg_id=session_info.get('msg_id', ''),
|
||||
chat_id=session_info.get('chat_id') or None,
|
||||
user_id=session_info.get('user_id') or None,
|
||||
feedback_id=feedback_id,
|
||||
)
|
||||
await self.logger.info(
|
||||
f'反馈关联到会话: stream_id={session.stream_id}, msg_id={session.msg_id}, user_id={session.user_id}'
|
||||
)
|
||||
else:
|
||||
await self.logger.warning(f'未找到 feedback_id={feedback_id} 对应的会话')
|
||||
|
||||
for handler in self._message_handlers.get('feedback', []):
|
||||
try:
|
||||
await handler(
|
||||
feedback_id=feedback_id,
|
||||
feedback_type=feedback_type,
|
||||
feedback_content=feedback_content,
|
||||
inaccurate_reasons=inaccurate_reasons,
|
||||
session=session,
|
||||
)
|
||||
except Exception:
|
||||
await self.logger.error(f'Error in feedback handler: {traceback.format_exc()}')
|
||||
return
|
||||
|
||||
event = wecombotevent.WecomBotEvent(message_data)
|
||||
|
||||
if event_type in self._message_handlers:
|
||||
for handler in self._message_handlers[event_type]:
|
||||
await handler(event)
|
||||
|
||||
if 'event' in self._message_handlers:
|
||||
for handler in self._message_handlers['event']:
|
||||
await handler(event)
|
||||
|
||||
except Exception:
|
||||
await self.logger.error(f'Error in event callback: {traceback.format_exc()}')
|
||||
|
||||
async def _dispatch_event(self, event: wecombotevent.WecomBotEvent):
|
||||
"""Dispatch a message event to registered handlers with deduplication."""
|
||||
try:
|
||||
message_id = event.message_id
|
||||
if message_id in self._msg_id_map:
|
||||
self._msg_id_map[message_id] += 1
|
||||
return
|
||||
self._msg_id_map[message_id] = 1
|
||||
|
||||
msg_type = event.type
|
||||
if msg_type in self._message_handlers:
|
||||
for handler in self._message_handlers[msg_type]:
|
||||
await handler(event)
|
||||
except Exception:
|
||||
await self.logger.error(f'Error dispatching event: {traceback.format_exc()}')
|
||||
|
||||
# ── Reply sending with serial queue ─────────────────────────────
|
||||
|
||||
async def _send_reply(
|
||||
self,
|
||||
req_id: str,
|
||||
body: dict,
|
||||
cmd: str = CMD_RESPOND_MSG,
|
||||
) -> Optional[dict]:
|
||||
"""Send a reply frame and wait for ACK.
|
||||
|
||||
Replies with the same req_id are serialized to maintain ordering.
|
||||
"""
|
||||
if not self._ws or self._ws.closed:
|
||||
return None
|
||||
|
||||
frame = {
|
||||
'cmd': cmd,
|
||||
'headers': {'req_id': req_id},
|
||||
'body': body,
|
||||
}
|
||||
|
||||
# Ensure serial delivery per req_id
|
||||
if req_id not in self._reply_queues:
|
||||
self._reply_queues[req_id] = asyncio.Queue()
|
||||
self._reply_workers[req_id] = asyncio.create_task(self._reply_queue_worker(req_id))
|
||||
|
||||
future: asyncio.Future = asyncio.get_event_loop().create_future()
|
||||
await self._reply_queues[req_id].put((frame, future))
|
||||
return await future
|
||||
|
||||
async def _reply_queue_worker(self, req_id: str):
|
||||
"""Process reply queue items serially for a given req_id."""
|
||||
queue = self._reply_queues[req_id]
|
||||
try:
|
||||
while self._running:
|
||||
try:
|
||||
frame, future = await asyncio.wait_for(queue.get(), timeout=60.0)
|
||||
except asyncio.TimeoutError:
|
||||
# Queue idle, clean up worker
|
||||
break
|
||||
|
||||
try:
|
||||
ack = await self._send_and_wait_ack(frame)
|
||||
if not future.done():
|
||||
future.set_result(ack)
|
||||
except Exception as e:
|
||||
if not future.done():
|
||||
future.set_exception(e)
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
finally:
|
||||
self._reply_queues.pop(req_id, None)
|
||||
self._reply_workers.pop(req_id, None)
|
||||
|
||||
async def _send_and_wait_ack(self, frame: dict) -> Optional[dict]:
|
||||
"""Send a frame and wait for the corresponding ACK."""
|
||||
req_id = frame['headers']['req_id']
|
||||
ack_future: asyncio.Future = asyncio.get_event_loop().create_future()
|
||||
self._pending_acks[req_id] = ack_future
|
||||
|
||||
try:
|
||||
await self._send_frame(frame)
|
||||
result = await asyncio.wait_for(ack_future, timeout=self._reply_ack_timeout)
|
||||
if result.get('errcode', 0) != 0:
|
||||
await self.logger.warning(
|
||||
f'Reply ACK error: errcode={result.get("errcode")}, errmsg={result.get("errmsg")}'
|
||||
)
|
||||
return result
|
||||
except asyncio.TimeoutError:
|
||||
self._pending_acks.pop(req_id, None)
|
||||
await self.logger.warning(f'Reply ACK timeout ({self._reply_ack_timeout}s) for req_id={req_id}')
|
||||
return None
|
||||
|
||||
async def _send_frame(self, frame: dict):
|
||||
"""Send a JSON frame over the WebSocket connection."""
|
||||
if self._ws and not self._ws.closed:
|
||||
await self._ws.send_str(json.dumps(frame, ensure_ascii=False))
|
||||
|
||||
def _clear_pending_acks(self, reason: str):
|
||||
"""Reject all pending ACK futures on disconnection."""
|
||||
for req_id, future in self._pending_acks.items():
|
||||
if not future.done():
|
||||
future.set_exception(ConnectionError(reason))
|
||||
self._pending_acks.clear()
|
||||
@@ -4,7 +4,6 @@ import base64
|
||||
import binascii
|
||||
import httpx
|
||||
import traceback
|
||||
from urllib.parse import quote
|
||||
from quart import Quart
|
||||
import xml.etree.ElementTree as ET
|
||||
from typing import Callable, Dict, Any
|
||||
@@ -68,31 +67,6 @@ class WecomClient:
|
||||
await self.logger.error(f'获取accesstoken失败:{response.json()}')
|
||||
raise Exception(f'未获取access token: {data}')
|
||||
|
||||
async def get_user_info(self, userid: str) -> dict:
|
||||
"""
|
||||
Get user information by user ID using the application secret.
|
||||
|
||||
Args:
|
||||
userid: The user ID to look up.
|
||||
|
||||
Returns:
|
||||
dict: User information including 'name' field.
|
||||
"""
|
||||
if not await self.check_access_token():
|
||||
self.access_token = await self.get_access_token(self.secret)
|
||||
|
||||
url = self.base_url + '/user/get?access_token=' + self.access_token + '&userid=' + quote(userid)
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.get(url)
|
||||
data = response.json()
|
||||
if data.get('errcode') == 40014 or data.get('errcode') == 42001:
|
||||
self.access_token = await self.get_access_token(self.secret)
|
||||
return await self.get_user_info(userid)
|
||||
if data.get('errcode', 0) != 0:
|
||||
await self.logger.error(f'获取用户信息失败:{data}')
|
||||
return {}
|
||||
return data
|
||||
|
||||
async def get_users(self):
|
||||
if not self.check_access_token_for_contacts():
|
||||
self.access_token_for_contacts = await self.get_access_token(self.secret_for_contacts)
|
||||
|
||||
@@ -10,7 +10,6 @@ from typing import Callable
|
||||
from .wecomcsevent import WecomCSEvent
|
||||
import langbot_plugin.api.entities.builtin.platform.message as platform_message
|
||||
import aiofiles
|
||||
import time
|
||||
|
||||
|
||||
class WecomCSClient:
|
||||
@@ -35,10 +34,6 @@ class WecomCSClient:
|
||||
self.unified_mode = unified_mode
|
||||
self.app = Quart(__name__)
|
||||
|
||||
# Customer info cache: {external_userid: (info_dict, timestamp)}
|
||||
self._customer_cache: dict[str, tuple[dict, float]] = {}
|
||||
self._cache_ttl = 60 # Cache TTL in seconds (1 minute)
|
||||
|
||||
# 只有在非统一模式下才注册独立路由
|
||||
if not self.unified_mode:
|
||||
self.app.add_url_rule(
|
||||
@@ -383,53 +378,3 @@ class WecomCSClient:
|
||||
async def get_media_id(self, image: platform_message.Image):
|
||||
media_id = await self.upload_to_work(image=image)
|
||||
return media_id
|
||||
|
||||
async def get_customer_info(self, external_userid: str) -> dict | None:
|
||||
"""
|
||||
Get customer information by external_userid with caching.
|
||||
|
||||
Uses a 1-minute cache to avoid repeated API calls for the same user.
|
||||
|
||||
Args:
|
||||
external_userid: The external user ID of the customer.
|
||||
|
||||
Returns:
|
||||
Customer info dict with 'nickname', 'avatar', etc., or None if not found.
|
||||
"""
|
||||
# Check cache first
|
||||
current_time = time.time()
|
||||
if external_userid in self._customer_cache:
|
||||
cached_info, cached_time = self._customer_cache[external_userid]
|
||||
if current_time - cached_time < self._cache_ttl:
|
||||
return cached_info
|
||||
|
||||
# Cache miss or expired, fetch from API
|
||||
if not await self.check_access_token():
|
||||
self.access_token = await self.get_access_token(self.secret)
|
||||
|
||||
url = f'{self.base_url}/kf/customer/batchget?access_token={self.access_token}'
|
||||
|
||||
payload = {
|
||||
'external_userid_list': [external_userid],
|
||||
}
|
||||
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.post(url, json=payload)
|
||||
data = response.json()
|
||||
|
||||
if data.get('errcode') in [40014, 42001]:
|
||||
self.access_token = await self.get_access_token(self.secret)
|
||||
return await self.get_customer_info(external_userid)
|
||||
|
||||
if data.get('errcode', 0) != 0:
|
||||
if self.logger:
|
||||
await self.logger.warning(f'Failed to get customer info: {data}')
|
||||
return None
|
||||
|
||||
customer_list = data.get('customer_list', [])
|
||||
if customer_list:
|
||||
customer_info = customer_list[0]
|
||||
# Store in cache
|
||||
self._customer_cache[external_userid] = (customer_info, current_time)
|
||||
return customer_info
|
||||
return None
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from .. import group
|
||||
|
||||
|
||||
@group.group_class('box', '/api/v1/box')
|
||||
class BoxRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('/status', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
status = await self.ap.box_service.get_status()
|
||||
return self.success(data=status)
|
||||
|
||||
@self.route('/sessions', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
sessions = await self.ap.box_service.get_sessions()
|
||||
return self.success(data=sessions)
|
||||
|
||||
@self.route('/errors', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
errors = self.ap.box_service.get_recent_errors()
|
||||
return self.success(data=errors)
|
||||
@@ -1,52 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import quart
|
||||
|
||||
from .. import group
|
||||
|
||||
|
||||
@group.group_class('extensions', '/api/v1/extensions')
|
||||
class ExtensionsRouterGroup(group.RouterGroup):
|
||||
"""Unified API for installed extensions (plugins, MCP servers, skills)."""
|
||||
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['GET'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _() -> quart.Response:
|
||||
plugins, mcp_servers, skills = await asyncio.gather(
|
||||
self.ap.plugin_connector.list_plugins(),
|
||||
self.ap.mcp_service.get_mcp_servers(contain_runtime_info=True),
|
||||
self.ap.skill_service.list_skills(),
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
def _sort_key(item: dict) -> str:
|
||||
if item['type'] == 'plugin':
|
||||
return (
|
||||
item['plugin']
|
||||
.get('manifest', {})
|
||||
.get('manifest', {})
|
||||
.get('metadata', {})
|
||||
.get('name', '')
|
||||
.lower()
|
||||
)
|
||||
if item['type'] == 'mcp':
|
||||
return (item['server'].get('name') or '').lower()
|
||||
if item['type'] == 'skill':
|
||||
return (item['skill'].get('display_name') or item['skill'].get('name') or '').lower()
|
||||
return ''
|
||||
|
||||
extensions: list[dict] = []
|
||||
if isinstance(plugins, list):
|
||||
for plugin in plugins:
|
||||
extensions.append({'type': 'plugin', 'plugin': plugin})
|
||||
if isinstance(mcp_servers, list):
|
||||
for server in mcp_servers:
|
||||
extensions.append({'type': 'mcp', 'server': server})
|
||||
if isinstance(skills, list):
|
||||
for skill in skills:
|
||||
extensions.append({'type': 'skill', 'skill': skill})
|
||||
|
||||
extensions.sort(key=_sort_key)
|
||||
|
||||
return self.success(data={'extensions': extensions})
|
||||
@@ -13,9 +13,9 @@ from .. import group
|
||||
@group.group_class('files', '/api/v1/files')
|
||||
class FilesRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('/image/<path:image_key>', methods=['GET'], auth_type=group.AuthType.NONE)
|
||||
@self.route('/image/<image_key>', methods=['GET'], auth_type=group.AuthType.NONE)
|
||||
async def _(image_key: str) -> quart.Response:
|
||||
if '..' in image_key or '\\' in image_key:
|
||||
if '/' in image_key or '\\' in image_key:
|
||||
return quart.Response(status=404)
|
||||
|
||||
if not await self.ap.storage_mgr.storage_provider.exists(image_key):
|
||||
|
||||
@@ -1,372 +0,0 @@
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import quart
|
||||
import sqlalchemy
|
||||
|
||||
from ... import group
|
||||
from ......core import taskmgr
|
||||
from ......entity.persistence import metadata as persistence_metadata
|
||||
from langbot_plugin.runtime.plugin.mgr import PluginInstallSource
|
||||
|
||||
LANGRAG_PLUGIN_AUTHOR = 'langbot-team'
|
||||
LANGRAG_PLUGIN_NAME = 'LangRAG'
|
||||
LANGRAG_PLUGIN_ID = f'{LANGRAG_PLUGIN_AUTHOR}/{LANGRAG_PLUGIN_NAME}'
|
||||
DEFAULT_SPACE_URL = 'https://space.langbot.app'
|
||||
|
||||
# Old Retriever plugin_name -> New Connector plugin_name
|
||||
EXTERNAL_PLUGIN_NAME_MAPPING = {
|
||||
'DifyDatasetsRetriever': 'DifyDatasetsConnector',
|
||||
'RAGFlowRetriever': 'RAGFlowConnector',
|
||||
'FastGPTRetriever': 'FastGPTConnector',
|
||||
}
|
||||
|
||||
# Per-plugin: which old retriever_config fields belong to creation_settings.
|
||||
# Remaining fields go to retrieval_settings.
|
||||
# None means ALL fields go to creation_settings (no retrieval_schema).
|
||||
EXTERNAL_PLUGIN_CREATION_FIELDS: dict[str, set[str] | None] = {
|
||||
'langbot-team/DifyDatasetsConnector': {'api_base_url', 'dify_apikey', 'dataset_id'},
|
||||
'langbot-team/RAGFlowConnector': {'api_base_url', 'api_key', 'dataset_ids'},
|
||||
'langbot-team/FastGPTConnector': None, # all fields -> creation_settings
|
||||
}
|
||||
|
||||
|
||||
@group.group_class('knowledge/migration', '/api/v1/knowledge/migration')
|
||||
class KnowledgeMigrationRouterGroup(group.RouterGroup):
|
||||
async def _get_migration_flag(self) -> bool:
|
||||
"""Check if rag_plugin_migration_needed flag is set."""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_metadata.Metadata).where(
|
||||
persistence_metadata.Metadata.key == 'rag_plugin_migration_needed'
|
||||
)
|
||||
)
|
||||
row = result.first()
|
||||
return row is not None and row.value == 'true'
|
||||
|
||||
async def _set_migration_flag(self, value: str):
|
||||
"""Set rag_plugin_migration_needed flag."""
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_metadata.Metadata)
|
||||
.where(persistence_metadata.Metadata.key == 'rag_plugin_migration_needed')
|
||||
.values(value=value)
|
||||
)
|
||||
|
||||
async def _table_exists(self, table_name: str) -> bool:
|
||||
"""Check if a table exists."""
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = :table_name);'
|
||||
).bindparams(table_name=table_name)
|
||||
)
|
||||
return result.scalar()
|
||||
else:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text("SELECT name FROM sqlite_master WHERE type='table' AND name=:table_name;").bindparams(
|
||||
table_name=table_name
|
||||
)
|
||||
)
|
||||
return result.first() is not None
|
||||
|
||||
async def _install_plugin_from_marketplace(
|
||||
self, plugin_id: str, task_context: taskmgr.TaskContext, space_url: str
|
||||
) -> None:
|
||||
"""Install a single plugin from the marketplace."""
|
||||
p_author, p_name = plugin_id.split('/', 1)
|
||||
self.ap.logger.info(f'RAG migration: installing plugin {plugin_id} from marketplace...')
|
||||
task_context.trace(f'Installing plugin {plugin_id} from marketplace...')
|
||||
|
||||
async with httpx.AsyncClient(trust_env=True, timeout=15) as client:
|
||||
resp = await client.get(f'{space_url}/api/v1/marketplace/plugins/{p_author}/{p_name}')
|
||||
resp.raise_for_status()
|
||||
p_data = resp.json().get('data', {}).get('plugin', {})
|
||||
p_version = p_data.get('latest_version')
|
||||
if not p_version:
|
||||
raise Exception(f'Could not determine latest version for {plugin_id}')
|
||||
|
||||
await self.ap.plugin_connector.install_plugin(
|
||||
PluginInstallSource.MARKETPLACE,
|
||||
{
|
||||
'plugin_author': p_author,
|
||||
'plugin_name': p_name,
|
||||
'plugin_version': p_version,
|
||||
},
|
||||
task_context=task_context,
|
||||
)
|
||||
self.ap.logger.info(f'RAG migration: plugin {plugin_id} install request sent.')
|
||||
|
||||
async def _execute_rag_migration(self, task_context: taskmgr.TaskContext, install_plugin: bool = True):
|
||||
"""Execute RAG migration: install required plugins and restore backup data."""
|
||||
warnings = []
|
||||
|
||||
# Collect all plugins we need: LangRAG (always) + connector plugins (from external KBs)
|
||||
needed_plugins: dict[str, str] = {
|
||||
LANGRAG_PLUGIN_ID: LANGRAG_PLUGIN_NAME,
|
||||
}
|
||||
|
||||
has_external = await self._table_exists('external_knowledge_bases')
|
||||
if has_external:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT DISTINCT plugin_author, plugin_name FROM external_knowledge_bases;')
|
||||
)
|
||||
for row in result.fetchall():
|
||||
plugin_author = row[0] or ''
|
||||
plugin_name = row[1] or ''
|
||||
mapped_name = EXTERNAL_PLUGIN_NAME_MAPPING.get(plugin_name, plugin_name)
|
||||
plugin_id = f'{plugin_author}/{mapped_name}'
|
||||
if plugin_id not in needed_plugins:
|
||||
needed_plugins[plugin_id] = mapped_name
|
||||
|
||||
self.ap.logger.info(f'RAG migration: plugins needed: {list(needed_plugins.keys())}')
|
||||
|
||||
if install_plugin:
|
||||
# Step 1: Install all required plugins from marketplace
|
||||
task_context.trace('Installing required plugins...', action='install-plugin')
|
||||
space_url = self.ap.instance_config.data.get('space', {}).get('url', DEFAULT_SPACE_URL).rstrip('/')
|
||||
|
||||
for plugin_id in needed_plugins:
|
||||
try:
|
||||
await self._install_plugin_from_marketplace(plugin_id, task_context, space_url)
|
||||
except Exception as e:
|
||||
self.ap.logger.warning(f'RAG migration: plugin {plugin_id} install returned: {e}')
|
||||
task_context.trace(f'Plugin install note ({plugin_id}): {e}')
|
||||
|
||||
# Step 2: Wait for all plugins to become available as knowledge engines
|
||||
task_context.trace(
|
||||
f'Waiting for plugins to become available: {list(needed_plugins.keys())}...',
|
||||
action='wait-plugin',
|
||||
)
|
||||
max_retries = 30
|
||||
engine_id_set: set[str] = set()
|
||||
for i in range(max_retries):
|
||||
try:
|
||||
engines = await self.ap.plugin_connector.list_knowledge_engines()
|
||||
engine_id_set = {e.get('plugin_id') for e in engines}
|
||||
except Exception:
|
||||
pass
|
||||
if all(pid in engine_id_set for pid in needed_plugins):
|
||||
self.ap.logger.info(f'RAG migration: all plugins ready: {engine_id_set}')
|
||||
task_context.trace('All required plugins are ready.')
|
||||
break
|
||||
if i == max_retries - 1:
|
||||
still_missing = [pid for pid in needed_plugins if pid not in engine_id_set]
|
||||
warning = f'Plugin(s) {still_missing} did not become available after {max_retries} retries'
|
||||
self.ap.logger.warning(f'RAG migration: {warning}')
|
||||
warnings.append(warning)
|
||||
task_context.trace(warning)
|
||||
await asyncio.sleep(2)
|
||||
else:
|
||||
try:
|
||||
engines = await self.ap.plugin_connector.list_knowledge_engines()
|
||||
engine_id_set = {e.get('plugin_id') for e in engines}
|
||||
except Exception:
|
||||
engine_id_set = set()
|
||||
|
||||
# Step 3: Restore internal knowledge bases from backup
|
||||
task_context.trace('Restoring internal knowledge bases...', action='restore-internal')
|
||||
if await self._table_exists('knowledge_bases_backup'):
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT * FROM knowledge_bases_backup;')
|
||||
)
|
||||
rows = result.fetchall()
|
||||
columns = result.keys()
|
||||
|
||||
for row in rows:
|
||||
row_dict = dict(zip(columns, row))
|
||||
kb_uuid = row_dict.get('uuid')
|
||||
name = row_dict.get('name', 'Untitled')
|
||||
description = row_dict.get('description', '')
|
||||
emoji = row_dict.get('emoji', '\U0001f4da')
|
||||
embedding_model_uuid = row_dict.get('embedding_model_uuid', '')
|
||||
top_k = row_dict.get('top_k', 5)
|
||||
created_at = row_dict.get('created_at')
|
||||
updated_at = row_dict.get('updated_at')
|
||||
|
||||
creation_settings = json.dumps({'embedding_model_uuid': embedding_model_uuid})
|
||||
retrieval_settings = json.dumps({'top_k': top_k})
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'INSERT INTO knowledge_bases '
|
||||
'(uuid, name, description, emoji, created_at, updated_at, '
|
||||
'knowledge_engine_plugin_id, collection_id, creation_settings, retrieval_settings) '
|
||||
'VALUES (:uuid, :name, :description, :emoji, :created_at, :updated_at, '
|
||||
':plugin_id, :collection_id, :creation_settings, :retrieval_settings);'
|
||||
).bindparams(
|
||||
uuid=kb_uuid,
|
||||
name=name,
|
||||
description=description,
|
||||
emoji=emoji,
|
||||
created_at=created_at,
|
||||
updated_at=updated_at,
|
||||
plugin_id=LANGRAG_PLUGIN_ID,
|
||||
collection_id=kb_uuid,
|
||||
creation_settings=creation_settings,
|
||||
retrieval_settings=retrieval_settings,
|
||||
)
|
||||
)
|
||||
|
||||
try:
|
||||
config = {'embedding_model_uuid': embedding_model_uuid}
|
||||
await self.ap.plugin_connector.rag_on_kb_create(LANGRAG_PLUGIN_ID, kb_uuid, config)
|
||||
task_context.trace(f'Restored internal KB: {name} ({kb_uuid})')
|
||||
except Exception as e:
|
||||
warning = f'Failed to notify plugin for KB {name} ({kb_uuid}): {e}'
|
||||
warnings.append(warning)
|
||||
task_context.trace(warning)
|
||||
|
||||
await self.ap.rag_mgr.load_knowledge_bases_from_db()
|
||||
|
||||
# Step 4: Restore external knowledge bases
|
||||
task_context.trace('Restoring external knowledge bases...', action='restore-external')
|
||||
if has_external:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT * FROM external_knowledge_bases;')
|
||||
)
|
||||
rows = result.fetchall()
|
||||
columns = result.keys()
|
||||
|
||||
self.ap.logger.info(
|
||||
f'RAG migration: {len(rows)} external KB(s) to restore. Available engines: {engine_id_set}'
|
||||
)
|
||||
task_context.trace(f'Found {len(rows)} external KB(s). Available engines: {engine_id_set}')
|
||||
|
||||
for row in rows:
|
||||
row_dict = dict(zip(columns, row))
|
||||
kb_uuid = row_dict.get('uuid')
|
||||
name = row_dict.get('name', 'Untitled')
|
||||
description = row_dict.get('description', '')
|
||||
emoji = row_dict.get('emoji', '\U0001f517')
|
||||
plugin_author = row_dict.get('plugin_author', '')
|
||||
plugin_name = row_dict.get('plugin_name', '')
|
||||
retriever_config = row_dict.get('retriever_config', {})
|
||||
created_at = row_dict.get('created_at')
|
||||
|
||||
mapped_plugin_name = EXTERNAL_PLUGIN_NAME_MAPPING.get(plugin_name, plugin_name)
|
||||
external_plugin_id = f'{plugin_author}/{mapped_plugin_name}'
|
||||
|
||||
self.ap.logger.info(
|
||||
f'RAG migration: processing external KB "{name}" ({kb_uuid}), '
|
||||
f'plugin: {plugin_author}/{plugin_name} -> {external_plugin_id}'
|
||||
)
|
||||
|
||||
if isinstance(retriever_config, str):
|
||||
try:
|
||||
retriever_config = json.loads(retriever_config)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
retriever_config = {}
|
||||
|
||||
creation_fields = EXTERNAL_PLUGIN_CREATION_FIELDS.get(external_plugin_id)
|
||||
if creation_fields is None:
|
||||
creation_settings_dict = retriever_config
|
||||
retrieval_settings_dict = {}
|
||||
else:
|
||||
creation_settings_dict = {k: v for k, v in retriever_config.items() if k in creation_fields}
|
||||
retrieval_settings_dict = {k: v for k, v in retriever_config.items() if k not in creation_fields}
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'INSERT INTO knowledge_bases '
|
||||
'(uuid, name, description, emoji, created_at, updated_at, '
|
||||
'knowledge_engine_plugin_id, collection_id, creation_settings, retrieval_settings) '
|
||||
'VALUES (:uuid, :name, :description, :emoji, :created_at, :updated_at, '
|
||||
':plugin_id, :collection_id, :creation_settings, :retrieval_settings);'
|
||||
).bindparams(
|
||||
uuid=kb_uuid,
|
||||
name=name,
|
||||
description=description,
|
||||
emoji=emoji,
|
||||
created_at=created_at,
|
||||
updated_at=created_at,
|
||||
plugin_id=external_plugin_id,
|
||||
collection_id=kb_uuid,
|
||||
creation_settings=json.dumps(creation_settings_dict),
|
||||
retrieval_settings=json.dumps(retrieval_settings_dict),
|
||||
)
|
||||
)
|
||||
|
||||
if external_plugin_id not in engine_id_set:
|
||||
warning = (
|
||||
f'External KB "{name}" ({kb_uuid}) record saved, but plugin {external_plugin_id} '
|
||||
f'is not installed yet. Install the connector plugin to use it.'
|
||||
)
|
||||
warnings.append(warning)
|
||||
task_context.trace(warning)
|
||||
else:
|
||||
try:
|
||||
await self.ap.plugin_connector.rag_on_kb_create(
|
||||
external_plugin_id, kb_uuid, creation_settings_dict
|
||||
)
|
||||
task_context.trace(f'Restored external KB: {name} ({kb_uuid})')
|
||||
except Exception as e:
|
||||
warning = f'Failed to notify plugin for external KB {name} ({kb_uuid}): {e}'
|
||||
warnings.append(warning)
|
||||
task_context.trace(warning)
|
||||
|
||||
await self.ap.rag_mgr.load_knowledge_bases_from_db()
|
||||
|
||||
# Step 5: Clear migration flag
|
||||
await self._set_migration_flag('false')
|
||||
task_context.trace('RAG migration completed.', action='done')
|
||||
|
||||
if warnings:
|
||||
task_context.trace(f'Completed with {len(warnings)} warning(s).')
|
||||
|
||||
async def initialize(self) -> None:
|
||||
@self.route('/status', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
needed = await self._get_migration_flag()
|
||||
|
||||
internal_kb_count = 0
|
||||
external_kb_count = 0
|
||||
|
||||
if needed:
|
||||
if await self._table_exists('knowledge_bases_backup'):
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT COUNT(*) FROM knowledge_bases_backup;')
|
||||
)
|
||||
internal_kb_count = result.scalar() or 0
|
||||
|
||||
if await self._table_exists('external_knowledge_bases'):
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT COUNT(*) FROM external_knowledge_bases;')
|
||||
)
|
||||
external_kb_count = result.scalar() or 0
|
||||
|
||||
return self.success(
|
||||
data={
|
||||
'needed': needed,
|
||||
'internal_kb_count': internal_kb_count,
|
||||
'external_kb_count': external_kb_count,
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/execute', methods=['POST'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
needed = await self._get_migration_flag()
|
||||
if not needed:
|
||||
return self.http_status(400, -1, 'RAG migration is not needed')
|
||||
|
||||
data = await quart.request.get_json(silent=True) or {}
|
||||
install_plugin = data.get('install_plugin', True)
|
||||
|
||||
ctx = taskmgr.TaskContext.new()
|
||||
wrapper = self.ap.task_mgr.create_user_task(
|
||||
self._execute_rag_migration(task_context=ctx, install_plugin=install_plugin),
|
||||
kind='rag-migration',
|
||||
name='rag-migration-execute',
|
||||
label='Migrating knowledge bases to plugin architecture',
|
||||
context=ctx,
|
||||
)
|
||||
|
||||
return self.success(data={'task_id': wrapper.id})
|
||||
|
||||
@self.route('/dismiss', methods=['POST'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
needed = await self._get_migration_flag()
|
||||
if not needed:
|
||||
return self.http_status(400, -1, 'RAG migration is not needed')
|
||||
|
||||
await self._set_migration_flag('false')
|
||||
return self.success()
|
||||
@@ -456,31 +456,6 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
'platform',
|
||||
'user_id',
|
||||
]
|
||||
elif export_type == 'feedback':
|
||||
data = await self.ap.monitoring_service.export_feedback(
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
limit=limit,
|
||||
)
|
||||
headers = [
|
||||
'id',
|
||||
'timestamp',
|
||||
'feedback_id',
|
||||
'feedback_type',
|
||||
'feedback_content',
|
||||
'inaccurate_reasons',
|
||||
'bot_id',
|
||||
'bot_name',
|
||||
'pipeline_id',
|
||||
'pipeline_name',
|
||||
'session_id',
|
||||
'message_id',
|
||||
'stream_id',
|
||||
'user_id',
|
||||
'platform',
|
||||
]
|
||||
else:
|
||||
return self.error(message=f'Invalid export type: {export_type}', code=400)
|
||||
|
||||
@@ -511,63 +486,3 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
)
|
||||
|
||||
return response, 200
|
||||
|
||||
@self.route('/feedback/stats', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def get_feedback_stats() -> str:
|
||||
"""Get feedback statistics"""
|
||||
# Parse query parameters
|
||||
bot_ids = quart.request.args.getlist('botId')
|
||||
pipeline_ids = quart.request.args.getlist('pipelineId')
|
||||
start_time_str = quart.request.args.get('startTime')
|
||||
end_time_str = quart.request.args.get('endTime')
|
||||
|
||||
# Parse datetime
|
||||
start_time = parse_iso_datetime(start_time_str)
|
||||
end_time = parse_iso_datetime(end_time_str)
|
||||
|
||||
stats = await self.ap.monitoring_service.get_feedback_stats(
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
)
|
||||
|
||||
return self.success(data=stats)
|
||||
|
||||
@self.route('/feedback', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def get_feedback() -> str:
|
||||
"""Get feedback list"""
|
||||
# Parse query parameters
|
||||
bot_ids = quart.request.args.getlist('botId')
|
||||
pipeline_ids = quart.request.args.getlist('pipelineId')
|
||||
feedback_type_str = quart.request.args.get('feedbackType')
|
||||
start_time_str = quart.request.args.get('startTime')
|
||||
end_time_str = quart.request.args.get('endTime')
|
||||
limit = int(quart.request.args.get('limit', 100))
|
||||
offset = int(quart.request.args.get('offset', 0))
|
||||
|
||||
# Parse datetime
|
||||
start_time = parse_iso_datetime(start_time_str)
|
||||
end_time = parse_iso_datetime(end_time_str)
|
||||
|
||||
# Parse feedback type
|
||||
feedback_type = int(feedback_type_str) if feedback_type_str else None
|
||||
|
||||
feedback_list, total = await self.ap.monitoring_service.get_feedback_list(
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
feedback_type=feedback_type,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
|
||||
return self.success(
|
||||
data={
|
||||
'feedback': feedback_list,
|
||||
'total': total,
|
||||
'limit': limit,
|
||||
'offset': offset,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -1,384 +0,0 @@
|
||||
"""Embed widget routes - serve embeddable chat widget for external websites.
|
||||
|
||||
All user-facing URLs are keyed by **bot_uuid** (not pipeline_uuid) so that
|
||||
internal pipeline identifiers are never exposed to end-users. Each handler
|
||||
resolves the bot_uuid to the owning ``web_page_bot`` RuntimeBot and extracts
|
||||
the bound pipeline_uuid for internal routing.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import datetime
|
||||
import json
|
||||
import logging
|
||||
import uuid
|
||||
import hmac
|
||||
import hashlib
|
||||
import time
|
||||
import re
|
||||
import httpx
|
||||
|
||||
import quart
|
||||
|
||||
from ... import group
|
||||
from ......utils import paths
|
||||
from ......platform.sources.websocket_manager import ws_connection_manager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Cache the widget template content
|
||||
_widget_template_cache: str | None = None
|
||||
_logo_bytes_cache: bytes | None = None
|
||||
|
||||
|
||||
def _is_valid_uuid(s: str) -> bool:
|
||||
return bool(re.match(r'^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$', s))
|
||||
|
||||
|
||||
def _get_widget_template() -> str:
|
||||
"""Load and cache the widget JS template."""
|
||||
global _widget_template_cache
|
||||
if _widget_template_cache is None:
|
||||
template_path = paths.get_resource_path('templates/embed/widget.js')
|
||||
with open(template_path, 'r', encoding='utf-8') as f:
|
||||
_widget_template_cache = f.read()
|
||||
return _widget_template_cache
|
||||
|
||||
|
||||
def _get_logo_bytes() -> bytes:
|
||||
"""Load and cache the logo image."""
|
||||
global _logo_bytes_cache
|
||||
if _logo_bytes_cache is None:
|
||||
logo_path = paths.get_resource_path('templates/embed/logo.webp')
|
||||
with open(logo_path, 'rb') as f:
|
||||
_logo_bytes_cache = f.read()
|
||||
return _logo_bytes_cache
|
||||
|
||||
|
||||
@group.group_class('embed', '/api/v1/embed')
|
||||
class EmbedRouterGroup(group.RouterGroup):
|
||||
# -- helpers -------------------------------------------------------------
|
||||
|
||||
def _resolve_bot(self, bot_uuid: str):
|
||||
"""Resolve *bot_uuid* to ``(runtime_bot, pipeline_uuid)``.
|
||||
|
||||
Returns ``(None, None)`` when the bot does not exist, is not a
|
||||
``web_page_bot``, is disabled, or has no pipeline bound.
|
||||
"""
|
||||
for bot in self.ap.platform_mgr.bots:
|
||||
if (
|
||||
bot.bot_entity.uuid == bot_uuid
|
||||
and bot.bot_entity.adapter == 'web_page_bot'
|
||||
and bot.bot_entity.enable
|
||||
and bot.bot_entity.use_pipeline_uuid
|
||||
):
|
||||
return bot, bot.bot_entity.use_pipeline_uuid
|
||||
return None, None
|
||||
|
||||
def _get_bot_config(self, bot_uuid: str) -> dict:
|
||||
for bot in self.ap.platform_mgr.bots:
|
||||
if bot.bot_entity.uuid == bot_uuid and bot.bot_entity.adapter == 'web_page_bot':
|
||||
return bot.bot_entity.adapter_config
|
||||
return {}
|
||||
|
||||
async def _verify_session_token(self, request, bot_uuid: str) -> bool:
|
||||
config = self._get_bot_config(bot_uuid)
|
||||
secret = config.get('turnstile_secret_key', '')
|
||||
if not secret:
|
||||
return True
|
||||
auth_header = request.headers.get('Authorization', '')
|
||||
if not auth_header.startswith('Bearer '):
|
||||
return False
|
||||
token = auth_header[7:]
|
||||
try:
|
||||
ts_str, mac = token.split('.', 1)
|
||||
ts = float(ts_str)
|
||||
if time.time() - ts > 86400:
|
||||
return False
|
||||
expected_mac = hmac.new(secret.encode(), f'{ts_str}'.encode(), hashlib.sha256).hexdigest()
|
||||
return hmac.compare_digest(mac, expected_mac)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
# -- routes --------------------------------------------------------------
|
||||
|
||||
async def initialize(self) -> None:
|
||||
@self.route('/<bot_uuid>/turnstile/verify', methods=['POST'], auth_type=group.AuthType.NONE)
|
||||
async def verify_turnstile(bot_uuid: str) -> str:
|
||||
if not _is_valid_uuid(bot_uuid):
|
||||
return self.http_status(400, -1, 'Invalid bot_uuid format')
|
||||
runtime_bot, pipeline_uuid = self._resolve_bot(bot_uuid)
|
||||
if runtime_bot is None:
|
||||
return self.http_status(404, -1, 'Bot not found or not available')
|
||||
try:
|
||||
data = await quart.request.get_json()
|
||||
token = data.get('token')
|
||||
if not token:
|
||||
return self.http_status(400, -1, 'Token is required')
|
||||
|
||||
config = self._get_bot_config(bot_uuid)
|
||||
secret = config.get('turnstile_secret_key', '')
|
||||
if not secret:
|
||||
ts = time.time()
|
||||
return self.success(data={'token': f'{ts}.dummy'})
|
||||
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.post(
|
||||
'https://challenges.cloudflare.com/turnstile/v0/siteverify',
|
||||
data={'secret': secret, 'response': token},
|
||||
)
|
||||
result = resp.json()
|
||||
|
||||
if not result.get('success'):
|
||||
return self.http_status(403, -1, 'Turnstile verification failed')
|
||||
|
||||
ts = time.time()
|
||||
mac = hmac.new(secret.encode(), f'{ts}'.encode(), hashlib.sha256).hexdigest()
|
||||
session_token = f'{ts}.{mac}'
|
||||
|
||||
return self.success(data={'token': session_token})
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f'Turnstile verify failed: {e}', exc_info=True)
|
||||
return self.http_status(500, -1, 'Internal server error')
|
||||
|
||||
@self.route('/<bot_uuid>/widget.js', methods=['GET'], auth_type=group.AuthType.NONE)
|
||||
async def serve_widget(bot_uuid: str) -> quart.Response:
|
||||
"""Serve the embed widget JavaScript with injected configuration."""
|
||||
if not _is_valid_uuid(bot_uuid):
|
||||
return self.http_status(400, -1, 'Invalid bot_uuid format')
|
||||
runtime_bot, pipeline_uuid = self._resolve_bot(bot_uuid)
|
||||
if runtime_bot is None:
|
||||
return quart.Response(
|
||||
'// Bot not found or not available', status=404, content_type='application/javascript'
|
||||
)
|
||||
try:
|
||||
template = _get_widget_template()
|
||||
except FileNotFoundError:
|
||||
return quart.Response('// Widget template not found', status=404, content_type='application/javascript')
|
||||
|
||||
base_url = quart.request.host_url.rstrip('/')
|
||||
webhook_prefix = self.ap.instance_config.data.get('api', {}).get('webhook_prefix', '')
|
||||
if webhook_prefix:
|
||||
base_url = webhook_prefix.rstrip('/')
|
||||
|
||||
if not re.match(r'^https?://[a-zA-Z0-9._:/-]+$', base_url):
|
||||
base_url = quart.request.host_url.rstrip('/')
|
||||
|
||||
config = self._get_bot_config(bot_uuid)
|
||||
site_key = config.get('turnstile_site_key', '')
|
||||
locale = config.get('language', 'en_US') or 'en_US'
|
||||
bubble_icon = config.get('bubble_icon', 'logo') or 'logo'
|
||||
widget_js = template.replace('__LANGBOT_TURNSTILE_SITE_KEY__', site_key)
|
||||
widget_js = widget_js.replace('__LANGBOT_BOT_UUID__', bot_uuid)
|
||||
widget_js = widget_js.replace('__LANGBOT_BASE_URL__', base_url)
|
||||
widget_js = widget_js.replace('__LANGBOT_LOCALE__', locale)
|
||||
widget_js = widget_js.replace('__LANGBOT_BUBBLE_ICON__', bubble_icon)
|
||||
|
||||
response = quart.Response(widget_js, content_type='application/javascript; charset=utf-8')
|
||||
response.headers['Cache-Control'] = 'public, max-age=300'
|
||||
return response
|
||||
|
||||
@self.route('/logo', methods=['GET'], auth_type=group.AuthType.NONE)
|
||||
async def serve_logo() -> quart.Response:
|
||||
"""Serve the LangBot logo for the embed widget."""
|
||||
try:
|
||||
logo_data = _get_logo_bytes()
|
||||
except FileNotFoundError:
|
||||
return quart.Response('', status=404)
|
||||
|
||||
response = quart.Response(logo_data, content_type='image/webp')
|
||||
response.headers['Cache-Control'] = 'public, max-age=86400'
|
||||
return response
|
||||
|
||||
@self.route('/<bot_uuid>/messages/<session_type>', methods=['GET'], auth_type=group.AuthType.NONE)
|
||||
async def get_embed_messages(bot_uuid: str, session_type: str) -> str:
|
||||
if not _is_valid_uuid(bot_uuid):
|
||||
return self.http_status(400, -1, 'Invalid bot_uuid format')
|
||||
runtime_bot, pipeline_uuid = self._resolve_bot(bot_uuid)
|
||||
if runtime_bot is None:
|
||||
return self.http_status(404, -1, 'Bot not found or not available')
|
||||
if not await self._verify_session_token(quart.request, bot_uuid):
|
||||
return self.http_status(403, -1, 'Unauthorized or session expired')
|
||||
try:
|
||||
if session_type not in ['person', 'group']:
|
||||
return self.http_status(400, -1, 'session_type must be person or group')
|
||||
|
||||
websocket_adapter = self.ap.platform_mgr.websocket_proxy_bot.adapter
|
||||
if not websocket_adapter:
|
||||
return self.http_status(404, -1, 'WebSocket adapter not found')
|
||||
|
||||
messages = websocket_adapter.get_websocket_messages(pipeline_uuid, session_type)
|
||||
return self.success(data={'messages': messages})
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to get embed messages: {e}', exc_info=True)
|
||||
return self.http_status(500, -1, 'Internal server error')
|
||||
|
||||
@self.route('/<bot_uuid>/reset/<session_type>', methods=['POST'], auth_type=group.AuthType.NONE)
|
||||
async def reset_embed_session(bot_uuid: str, session_type: str) -> str:
|
||||
if not _is_valid_uuid(bot_uuid):
|
||||
return self.http_status(400, -1, 'Invalid bot_uuid format')
|
||||
runtime_bot, pipeline_uuid = self._resolve_bot(bot_uuid)
|
||||
if runtime_bot is None:
|
||||
return self.http_status(404, -1, 'Bot not found or not available')
|
||||
if not await self._verify_session_token(quart.request, bot_uuid):
|
||||
return self.http_status(403, -1, 'Unauthorized or session expired')
|
||||
try:
|
||||
if session_type not in ['person', 'group']:
|
||||
return self.http_status(400, -1, 'session_type must be person or group')
|
||||
|
||||
websocket_adapter = self.ap.platform_mgr.websocket_proxy_bot.adapter
|
||||
if not websocket_adapter:
|
||||
return self.http_status(404, -1, 'WebSocket adapter not found')
|
||||
|
||||
websocket_adapter.reset_session(pipeline_uuid, session_type)
|
||||
return self.success(data={'message': 'Session reset successfully'})
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to reset embed session: {e}', exc_info=True)
|
||||
return self.http_status(500, -1, 'Internal server error')
|
||||
|
||||
@self.route('/<bot_uuid>/feedback', methods=['POST'], auth_type=group.AuthType.NONE)
|
||||
async def submit_feedback(bot_uuid: str) -> str:
|
||||
if not _is_valid_uuid(bot_uuid):
|
||||
return self.http_status(400, -1, 'Invalid bot_uuid format')
|
||||
runtime_bot, pipeline_uuid = self._resolve_bot(bot_uuid)
|
||||
if runtime_bot is None:
|
||||
return self.http_status(404, -1, 'Bot not found or not available')
|
||||
if not await self._verify_session_token(quart.request, bot_uuid):
|
||||
return self.http_status(403, -1, 'Unauthorized or session expired')
|
||||
try:
|
||||
data = await quart.request.get_json()
|
||||
message_id = data.get('message_id', '')
|
||||
feedback_type = data.get('feedback_type')
|
||||
|
||||
if feedback_type not in (1, 2, 3):
|
||||
return self.http_status(400, -1, 'feedback_type must be 1 (like), 2 (dislike), or 3 (cancel)')
|
||||
|
||||
feedback_id = f'embed_{uuid.uuid4().hex[:12]}'
|
||||
|
||||
await self.ap.monitoring_service.record_feedback(
|
||||
feedback_id=feedback_id,
|
||||
feedback_type=feedback_type,
|
||||
bot_id=runtime_bot.bot_entity.uuid,
|
||||
bot_name=runtime_bot.bot_entity.name or bot_uuid,
|
||||
pipeline_id=pipeline_uuid,
|
||||
message_id=str(message_id),
|
||||
platform='web_page_bot',
|
||||
)
|
||||
|
||||
return self.success(data={'feedback_id': feedback_id})
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to record feedback: {e}', exc_info=True)
|
||||
return self.http_status(500, -1, 'Internal server error')
|
||||
|
||||
# -- Embed WebSocket endpoint ----------------------------------------
|
||||
|
||||
@self.quart_app.websocket(self.path + '/<bot_uuid>/ws/connect')
|
||||
async def embed_websocket_connect(bot_uuid: str):
|
||||
"""WebSocket connection for embed widget, keyed by bot_uuid."""
|
||||
if not _is_valid_uuid(bot_uuid):
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Invalid bot_uuid format'}))
|
||||
return
|
||||
|
||||
runtime_bot, pipeline_uuid = self._resolve_bot(bot_uuid)
|
||||
if runtime_bot is None:
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Bot not found or not available'}))
|
||||
return
|
||||
|
||||
session_type = quart.websocket.args.get('session_type', 'person')
|
||||
if session_type not in ['person', 'group']:
|
||||
await quart.websocket.send(
|
||||
json.dumps({'type': 'error', 'message': 'session_type must be person or group'})
|
||||
)
|
||||
return
|
||||
|
||||
websocket_adapter = self.ap.platform_mgr.websocket_proxy_bot.adapter
|
||||
if not websocket_adapter:
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'WebSocket adapter not found'}))
|
||||
return
|
||||
|
||||
try:
|
||||
connection = await ws_connection_manager.add_connection(
|
||||
websocket=quart.websocket._get_current_object(),
|
||||
pipeline_uuid=pipeline_uuid,
|
||||
session_type=session_type,
|
||||
metadata={'user_agent': quart.websocket.headers.get('User-Agent', '')},
|
||||
)
|
||||
|
||||
await quart.websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
'type': 'connected',
|
||||
'connection_id': connection.connection_id,
|
||||
'bot_uuid': bot_uuid,
|
||||
'session_type': session_type,
|
||||
'timestamp': connection.created_at.isoformat(),
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
f'Embed WebSocket connected: {connection.connection_id} '
|
||||
f'(bot={bot_uuid}, pipeline={pipeline_uuid}, session_type={session_type})'
|
||||
)
|
||||
|
||||
receive_task = asyncio.create_task(self._handle_receive(connection, websocket_adapter, runtime_bot))
|
||||
send_task = asyncio.create_task(self._handle_send(connection))
|
||||
|
||||
try:
|
||||
await asyncio.gather(receive_task, send_task)
|
||||
except Exception as e:
|
||||
logger.error(f'Embed WebSocket task error: {e}')
|
||||
finally:
|
||||
await ws_connection_manager.remove_connection(connection.connection_id)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f'Embed WebSocket connection error: {e}', exc_info=True)
|
||||
try:
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Internal server error'}))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# -- WebSocket receive/send helpers --------------------------------------
|
||||
|
||||
async def _handle_receive(self, connection, websocket_adapter, owner_bot):
|
||||
try:
|
||||
while connection.is_active:
|
||||
message = await quart.websocket.receive()
|
||||
await ws_connection_manager.update_activity(connection.connection_id)
|
||||
|
||||
try:
|
||||
data = json.loads(message)
|
||||
message_type = data.get('type', 'message')
|
||||
|
||||
if message_type == 'ping':
|
||||
await connection.send_queue.put(
|
||||
{'type': 'pong', 'timestamp': datetime.datetime.now().isoformat()}
|
||||
)
|
||||
elif message_type == 'message':
|
||||
await websocket_adapter.handle_websocket_message(connection, data, owner_bot=owner_bot)
|
||||
elif message_type == 'disconnect':
|
||||
break
|
||||
|
||||
except json.JSONDecodeError:
|
||||
await connection.send_queue.put({'type': 'error', 'message': 'Invalid JSON format'})
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f'Embed receive error: {e}', exc_info=True)
|
||||
finally:
|
||||
connection.is_active = False
|
||||
|
||||
async def _handle_send(self, connection):
|
||||
try:
|
||||
while connection.is_active:
|
||||
try:
|
||||
message = await asyncio.wait_for(connection.send_queue.get(), timeout=1.0)
|
||||
await quart.websocket.send(json.dumps(message))
|
||||
except asyncio.TimeoutError:
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.error(f'Embed send error: {e}', exc_info=True)
|
||||
finally:
|
||||
connection.is_active = False
|
||||
@@ -73,21 +73,15 @@ class PipelinesRouterGroup(group.RouterGroup):
|
||||
plugins = await self.ap.plugin_connector.list_plugins(component_kinds=pipeline_component_kinds)
|
||||
mcp_servers = await self.ap.mcp_service.get_mcp_servers(contain_runtime_info=True)
|
||||
|
||||
# Get available skills
|
||||
available_skills = await self.ap.skill_service.list_skills()
|
||||
|
||||
extensions_prefs = pipeline.get('extensions_preferences', {})
|
||||
return self.success(
|
||||
data={
|
||||
'enable_all_plugins': extensions_prefs.get('enable_all_plugins', True),
|
||||
'enable_all_mcp_servers': extensions_prefs.get('enable_all_mcp_servers', True),
|
||||
'enable_all_skills': extensions_prefs.get('enable_all_skills', True),
|
||||
'bound_plugins': extensions_prefs.get('plugins', []),
|
||||
'available_plugins': plugins,
|
||||
'bound_mcp_servers': extensions_prefs.get('mcp_servers', []),
|
||||
'available_mcp_servers': mcp_servers,
|
||||
'bound_skills': extensions_prefs.get('skills', []),
|
||||
'available_skills': available_skills,
|
||||
}
|
||||
)
|
||||
elif quart.request.method == 'PUT':
|
||||
@@ -95,19 +89,11 @@ class PipelinesRouterGroup(group.RouterGroup):
|
||||
json_data = await quart.request.json
|
||||
enable_all_plugins = json_data.get('enable_all_plugins', True)
|
||||
enable_all_mcp_servers = json_data.get('enable_all_mcp_servers', True)
|
||||
enable_all_skills = json_data.get('enable_all_skills', True)
|
||||
bound_plugins = json_data.get('bound_plugins', [])
|
||||
bound_mcp_servers = json_data.get('bound_mcp_servers', [])
|
||||
bound_skills = json_data.get('bound_skills', [])
|
||||
|
||||
await self.ap.pipeline_service.update_pipeline_extensions(
|
||||
pipeline_uuid,
|
||||
bound_plugins,
|
||||
bound_mcp_servers,
|
||||
enable_all_plugins,
|
||||
enable_all_mcp_servers,
|
||||
bound_skills=bound_skills,
|
||||
enable_all_skills=enable_all_skills,
|
||||
pipeline_uuid, bound_plugins, bound_mcp_servers, enable_all_plugins, enable_all_mcp_servers
|
||||
)
|
||||
|
||||
return self.success()
|
||||
|
||||
@@ -43,9 +43,6 @@ class WebSocketChatRouterGroup(group.RouterGroup):
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'WebSocket adapter not found'}))
|
||||
return
|
||||
|
||||
# Find the owning bot for this pipeline (e.g. a web_page_bot)
|
||||
owner_bot = self._find_owner_bot(pipeline_uuid)
|
||||
|
||||
# 注册连接
|
||||
connection = await ws_connection_manager.add_connection(
|
||||
websocket=quart.websocket._get_current_object(),
|
||||
@@ -73,7 +70,7 @@ class WebSocketChatRouterGroup(group.RouterGroup):
|
||||
)
|
||||
|
||||
# 创建接收和发送任务
|
||||
receive_task = asyncio.create_task(self._handle_receive(connection, websocket_adapter, owner_bot))
|
||||
receive_task = asyncio.create_task(self._handle_receive(connection, websocket_adapter))
|
||||
send_task = asyncio.create_task(self._handle_send(connection))
|
||||
|
||||
# 等待任务完成
|
||||
@@ -181,14 +178,7 @@ class WebSocketChatRouterGroup(group.RouterGroup):
|
||||
except Exception as e:
|
||||
return self.http_status(500, -1, f'Internal server error: {str(e)}')
|
||||
|
||||
def _find_owner_bot(self, pipeline_uuid: str):
|
||||
"""Find a user-created bot (e.g. web_page_bot) that owns this pipeline."""
|
||||
for bot in self.ap.platform_mgr.bots:
|
||||
if bot.bot_entity.adapter == 'web_page_bot' and bot.bot_entity.use_pipeline_uuid == pipeline_uuid:
|
||||
return bot
|
||||
return None
|
||||
|
||||
async def _handle_receive(self, connection, websocket_adapter, owner_bot=None):
|
||||
async def _handle_receive(self, connection, websocket_adapter):
|
||||
"""处理接收消息的任务"""
|
||||
try:
|
||||
while connection.is_active:
|
||||
@@ -213,7 +203,7 @@ class WebSocketChatRouterGroup(group.RouterGroup):
|
||||
logger.debug(f'收到消息: {data} from {connection.connection_id}')
|
||||
|
||||
# 处理消息(不等待响应,响应会通过broadcast异步发送)
|
||||
await websocket_adapter.handle_websocket_message(connection, data, owner_bot=owner_bot)
|
||||
await websocket_adapter.handle_websocket_message(connection, data)
|
||||
|
||||
elif message_type == 'disconnect':
|
||||
# 客户端主动断开
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import quart
|
||||
import mimetypes
|
||||
import asyncio
|
||||
from ... import group
|
||||
from langbot.pkg.utils import importutil
|
||||
|
||||
@@ -36,640 +35,3 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
return quart.Response(
|
||||
importutil.read_resource_file_bytes(icon_path), mimetype=mimetypes.guess_type(icon_path)[0]
|
||||
)
|
||||
|
||||
# In-memory session store for active registrations
|
||||
_create_app_sessions: dict = {}
|
||||
_SESSION_TTL = 900 # 15 minutes
|
||||
|
||||
def _cleanup_expired_sessions():
|
||||
"""Remove sessions that have exceeded their TTL."""
|
||||
import time
|
||||
|
||||
now = time.time()
|
||||
expired = [sid for sid, s in _create_app_sessions.items() if now - s.get('created_at', 0) > _SESSION_TTL]
|
||||
for sid in expired:
|
||||
session = _create_app_sessions.pop(sid, None)
|
||||
if session and session.get('task') and not session['task'].done():
|
||||
session['task'].cancel()
|
||||
|
||||
@self.route('/lark/create-app', methods=['POST'])
|
||||
async def _() -> str:
|
||||
"""Start Feishu one-click app registration. Returns session_id + QR code URL."""
|
||||
import uuid
|
||||
import time
|
||||
import lark_oapi as lark
|
||||
from lark_oapi.scene.registration.errors import AppAccessDeniedError, AppExpiredError
|
||||
|
||||
_cleanup_expired_sessions()
|
||||
|
||||
session_id = str(uuid.uuid4())
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
session = {
|
||||
'status': 'pending',
|
||||
'qr_url': None,
|
||||
'expire_at': None,
|
||||
'app_id': None,
|
||||
'app_secret': None,
|
||||
'error': None,
|
||||
'created_at': time.time(),
|
||||
}
|
||||
_create_app_sessions[session_id] = session
|
||||
|
||||
def on_qr_code(info):
|
||||
# May be called from a background thread by the SDK;
|
||||
# use call_soon_threadsafe to safely update session state.
|
||||
def _update():
|
||||
session['qr_url'] = info['url']
|
||||
session['expire_at'] = time.time() + 600 # 10 minutes
|
||||
session['status'] = 'waiting'
|
||||
|
||||
loop.call_soon_threadsafe(_update)
|
||||
|
||||
async def run_registration():
|
||||
try:
|
||||
result = await lark.aregister_app(
|
||||
on_qr_code=on_qr_code,
|
||||
source='langbot',
|
||||
)
|
||||
session['status'] = 'success'
|
||||
session['app_id'] = result['client_id']
|
||||
session['app_secret'] = result['client_secret']
|
||||
except AppAccessDeniedError:
|
||||
session['status'] = 'error'
|
||||
session['error'] = 'User denied authorization'
|
||||
except AppExpiredError:
|
||||
session['status'] = 'error'
|
||||
session['error'] = 'QR code expired'
|
||||
except Exception as e:
|
||||
session['status'] = 'error'
|
||||
session['error'] = str(e)
|
||||
|
||||
task = asyncio.create_task(run_registration())
|
||||
session['task'] = task
|
||||
|
||||
# Wait for QR code to be ready (max 10 seconds)
|
||||
for _ in range(20):
|
||||
if session['qr_url']:
|
||||
break
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
if not session['qr_url']:
|
||||
task.cancel()
|
||||
session['status'] = 'error'
|
||||
session['error'] = 'Timeout waiting for QR code'
|
||||
return self.http_status(504, -1, 'Timeout waiting for QR code')
|
||||
|
||||
return self.success(
|
||||
data={
|
||||
'session_id': session_id,
|
||||
'qr_url': session['qr_url'],
|
||||
'expire_at': session['expire_at'],
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/lark/create-app/status/<session_id>', methods=['GET'])
|
||||
async def _(session_id: str) -> str:
|
||||
"""Poll registration status."""
|
||||
session = _create_app_sessions.get(session_id)
|
||||
if not session:
|
||||
return self.http_status(404, -1, 'Session not found')
|
||||
|
||||
data = {'status': session['status']}
|
||||
|
||||
if session['status'] == 'success':
|
||||
data['app_id'] = session['app_id']
|
||||
data['app_secret'] = session['app_secret']
|
||||
_create_app_sessions.pop(session_id, None)
|
||||
elif session['status'] == 'error':
|
||||
data['error'] = session['error']
|
||||
_create_app_sessions.pop(session_id, None)
|
||||
|
||||
return self.success(data=data)
|
||||
|
||||
@self.route('/lark/create-app/<session_id>', methods=['DELETE'])
|
||||
async def _(session_id: str) -> str:
|
||||
"""Cancel and clean up a registration session."""
|
||||
session = _create_app_sessions.pop(session_id, None)
|
||||
if session and session.get('task') and not session['task'].done():
|
||||
session['task'].cancel()
|
||||
return self.success(data={})
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# WeChat QR Code Login
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
_weixin_login_sessions: dict = {}
|
||||
_WEIXIN_SESSION_TTL = 600 # 10 minutes (3 retries × 3 min QR validity)
|
||||
|
||||
def _cleanup_expired_weixin_sessions():
|
||||
import time
|
||||
|
||||
now = time.time()
|
||||
expired = [
|
||||
sid for sid, s in _weixin_login_sessions.items() if now - s.get('created_at', 0) > _WEIXIN_SESSION_TTL
|
||||
]
|
||||
for sid in expired:
|
||||
session = _weixin_login_sessions.pop(sid, None)
|
||||
if session and session.get('task') and not session['task'].done():
|
||||
session['task'].cancel()
|
||||
|
||||
@self.route('/weixin/login', methods=['POST'])
|
||||
async def _() -> str:
|
||||
"""Start WeChat QR code login. Returns session_id + QR code data URL."""
|
||||
import uuid
|
||||
import time
|
||||
import io
|
||||
import base64
|
||||
|
||||
from langbot.libs.openclaw_weixin_api.client import OpenClawWeixinClient, DEFAULT_BASE_URL
|
||||
|
||||
_cleanup_expired_weixin_sessions()
|
||||
|
||||
session_id = str(uuid.uuid4())
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
session = {
|
||||
'status': 'pending',
|
||||
'qr_data_url': None,
|
||||
'expire_at': None,
|
||||
'token': None,
|
||||
'base_url': None,
|
||||
'account_id': None,
|
||||
'error': None,
|
||||
'created_at': time.time(),
|
||||
}
|
||||
_weixin_login_sessions[session_id] = session
|
||||
|
||||
client = OpenClawWeixinClient(
|
||||
base_url=DEFAULT_BASE_URL,
|
||||
token='',
|
||||
)
|
||||
|
||||
async def run_login():
|
||||
try:
|
||||
import qrcode as qr_lib
|
||||
|
||||
for _attempt in range(3):
|
||||
qr_resp = await client.fetch_qrcode()
|
||||
if not qr_resp.qrcode or not qr_resp.qrcode_img_content:
|
||||
raise Exception('Failed to get QR code from server')
|
||||
|
||||
# Generate QR code image locally
|
||||
qr = qr_lib.QRCode(error_correction=qr_lib.constants.ERROR_CORRECT_L)
|
||||
qr.add_data(qr_resp.qrcode_img_content)
|
||||
qr.make(fit=True)
|
||||
img = qr.make_image(fill_color='black', back_color='white')
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, format='PNG')
|
||||
b64 = base64.b64encode(buf.getvalue()).decode('utf-8')
|
||||
data_url = f'data:image/png;base64,{b64}'
|
||||
|
||||
def _update_qr():
|
||||
session['qr_data_url'] = data_url
|
||||
session['expire_at'] = time.time() + 480 # 8 minutes
|
||||
session['status'] = 'waiting'
|
||||
|
||||
loop.call_soon_threadsafe(_update_qr)
|
||||
|
||||
# Poll for scan status
|
||||
deadline = loop.time() + 180
|
||||
while loop.time() < deadline:
|
||||
try:
|
||||
status_resp = await client.poll_qrcode_status(qr_resp.qrcode)
|
||||
except Exception:
|
||||
await asyncio.sleep(2)
|
||||
continue
|
||||
|
||||
if status_resp.status == 'confirmed' and status_resp.bot_token:
|
||||
session['status'] = 'success'
|
||||
session['token'] = status_resp.bot_token
|
||||
session['base_url'] = status_resp.baseurl or client.base_url
|
||||
session['account_id'] = status_resp.ilink_bot_id or ''
|
||||
return
|
||||
|
||||
if status_resp.status == 'expired':
|
||||
break # retry with new QR code
|
||||
|
||||
await asyncio.sleep(1)
|
||||
else:
|
||||
pass # timeout, retry
|
||||
|
||||
# All retries exhausted
|
||||
session['status'] = 'error'
|
||||
session['error'] = 'QR code login failed: max retries exceeded'
|
||||
|
||||
except Exception as e:
|
||||
session['status'] = 'error'
|
||||
session['error'] = str(e)
|
||||
finally:
|
||||
await client.close()
|
||||
|
||||
task = asyncio.create_task(run_login())
|
||||
session['task'] = task
|
||||
|
||||
# Wait for QR code to be ready (max 10 seconds)
|
||||
for _ in range(20):
|
||||
if session['qr_data_url']:
|
||||
break
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
if not session['qr_data_url']:
|
||||
task.cancel()
|
||||
session['status'] = 'error'
|
||||
session['error'] = 'Timeout waiting for QR code'
|
||||
return self.http_status(504, -1, 'Timeout waiting for QR code')
|
||||
|
||||
return self.success(
|
||||
data={
|
||||
'session_id': session_id,
|
||||
'qr_data_url': session['qr_data_url'],
|
||||
'expire_at': session['expire_at'],
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/weixin/login/status/<session_id>', methods=['GET'])
|
||||
async def _(session_id: str) -> str:
|
||||
"""Poll WeChat login status."""
|
||||
session = _weixin_login_sessions.get(session_id)
|
||||
if not session:
|
||||
return self.http_status(404, -1, 'Session not found')
|
||||
|
||||
data = {'status': session['status']}
|
||||
|
||||
if session['status'] == 'success':
|
||||
data['token'] = session['token']
|
||||
data['base_url'] = session['base_url']
|
||||
data['account_id'] = session['account_id']
|
||||
_weixin_login_sessions.pop(session_id, None)
|
||||
elif session['status'] == 'error':
|
||||
data['error'] = session['error']
|
||||
_weixin_login_sessions.pop(session_id, None)
|
||||
|
||||
return self.success(data=data)
|
||||
|
||||
@self.route('/weixin/login/<session_id>', methods=['DELETE'])
|
||||
async def _(session_id: str) -> str:
|
||||
"""Cancel and clean up a WeChat login session."""
|
||||
session = _weixin_login_sessions.pop(session_id, None)
|
||||
if session and session.get('task') and not session['task'].done():
|
||||
session['task'].cancel()
|
||||
return self.success(data={})
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# DingTalk Device Flow QR Code Login
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
_dingtalk_sessions: dict = {}
|
||||
_DINGTALK_SESSION_TTL = 600 # 10 minutes (QR code validity window)
|
||||
|
||||
def _cleanup_expired_dingtalk_sessions():
|
||||
import time
|
||||
|
||||
now = time.time()
|
||||
expired = [
|
||||
sid for sid, s in _dingtalk_sessions.items() if now - s.get('created_at', 0) > _DINGTALK_SESSION_TTL
|
||||
]
|
||||
for sid in expired:
|
||||
session = _dingtalk_sessions.pop(sid, None)
|
||||
if session and session.get('task') and not session['task'].done():
|
||||
session['task'].cancel()
|
||||
|
||||
@self.route('/dingtalk/create-app', methods=['POST'])
|
||||
async def _() -> str:
|
||||
"""Start DingTalk one-click app creation via Device Flow. Returns session_id + QR code URL."""
|
||||
import uuid
|
||||
import time
|
||||
import aiohttp
|
||||
|
||||
DINGTALK_BASE_URL = 'https://oapi.dingtalk.com'
|
||||
|
||||
_cleanup_expired_dingtalk_sessions()
|
||||
|
||||
session_id = str(uuid.uuid4())
|
||||
|
||||
session = {
|
||||
'status': 'pending',
|
||||
'qr_url': None,
|
||||
'expire_at': None,
|
||||
'client_id': None,
|
||||
'client_secret': None,
|
||||
'error': None,
|
||||
'created_at': time.time(),
|
||||
'device_code': None,
|
||||
'interval': 5,
|
||||
}
|
||||
_dingtalk_sessions[session_id] = session
|
||||
|
||||
async def run_device_flow():
|
||||
try:
|
||||
timeout = aiohttp.ClientTimeout(total=10)
|
||||
async with aiohttp.ClientSession(timeout=timeout) as http:
|
||||
# Step 1: Init — get nonce
|
||||
async with http.post(
|
||||
f'{DINGTALK_BASE_URL}/app/registration/init',
|
||||
json={'source': 'langbot'},
|
||||
) as resp:
|
||||
try:
|
||||
data = await resp.json()
|
||||
except (aiohttp.ContentTypeError, ValueError):
|
||||
session['status'] = 'error'
|
||||
session['error'] = 'Invalid response from DingTalk service'
|
||||
return
|
||||
if data.get('errcode', -1) != 0:
|
||||
session['status'] = 'error'
|
||||
session['error'] = data.get('errmsg', 'Failed to init')
|
||||
return
|
||||
nonce = data['nonce']
|
||||
|
||||
# Step 2: Begin — get device_code + QR URL
|
||||
async with http.post(
|
||||
f'{DINGTALK_BASE_URL}/app/registration/begin',
|
||||
json={'nonce': nonce},
|
||||
) as resp:
|
||||
try:
|
||||
data = await resp.json()
|
||||
except (aiohttp.ContentTypeError, ValueError):
|
||||
session['status'] = 'error'
|
||||
session['error'] = 'Invalid response from DingTalk service'
|
||||
return
|
||||
if data.get('errcode', -1) != 0:
|
||||
session['status'] = 'error'
|
||||
session['error'] = data.get('errmsg', 'Failed to begin authorization')
|
||||
return
|
||||
|
||||
device_code = data['device_code']
|
||||
verification_uri_complete = data.get('verification_uri_complete', '')
|
||||
expires_in = data.get('expires_in', 7200)
|
||||
interval = data.get('interval', 5)
|
||||
|
||||
session['device_code'] = device_code
|
||||
session['interval'] = interval
|
||||
session['qr_url'] = verification_uri_complete
|
||||
session['expire_at'] = time.time() + 600 # QR code valid for ~10 min
|
||||
session['status'] = 'waiting'
|
||||
|
||||
# Step 3: Poll for authorization result
|
||||
deadline = time.time() + expires_in
|
||||
while time.time() < deadline:
|
||||
await asyncio.sleep(interval)
|
||||
|
||||
async with http.post(
|
||||
f'{DINGTALK_BASE_URL}/app/registration/poll',
|
||||
json={'device_code': device_code},
|
||||
) as poll_resp:
|
||||
try:
|
||||
poll_data = await poll_resp.json()
|
||||
except (aiohttp.ContentTypeError, ValueError):
|
||||
continue
|
||||
|
||||
if poll_data.get('errcode', -1) != 0:
|
||||
session['status'] = 'error'
|
||||
session['error'] = poll_data.get('errmsg', 'Poll failed')
|
||||
return
|
||||
|
||||
status = poll_data.get('status', '')
|
||||
|
||||
if status == 'SUCCESS':
|
||||
session['status'] = 'success'
|
||||
session['client_id'] = poll_data.get('client_id', '')
|
||||
session['client_secret'] = poll_data.get('client_secret', '')
|
||||
return
|
||||
elif status == 'FAIL':
|
||||
session['status'] = 'error'
|
||||
session['error'] = poll_data.get('fail_reason', 'Authorization failed')
|
||||
return
|
||||
elif status == 'EXPIRED':
|
||||
session['status'] = 'error'
|
||||
session['error'] = 'QR code expired'
|
||||
return
|
||||
# status == 'WAITING': continue polling
|
||||
|
||||
# Timeout
|
||||
session['status'] = 'error'
|
||||
session['error'] = 'QR code expired'
|
||||
|
||||
except asyncio.CancelledError:
|
||||
return
|
||||
except Exception as e:
|
||||
session['status'] = 'error'
|
||||
session['error'] = str(e)
|
||||
|
||||
task = asyncio.create_task(run_device_flow())
|
||||
session['task'] = task
|
||||
|
||||
# Wait for QR code to be ready (max 10 seconds)
|
||||
for _ in range(20):
|
||||
if session['qr_url'] or session['error']:
|
||||
break
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
if session['error']:
|
||||
task.cancel()
|
||||
return self.http_status(502, -1, session['error'])
|
||||
|
||||
if not session['qr_url']:
|
||||
task.cancel()
|
||||
session['status'] = 'error'
|
||||
session['error'] = 'Timeout waiting for QR code'
|
||||
return self.http_status(504, -1, 'Timeout waiting for QR code')
|
||||
|
||||
return self.success(
|
||||
data={
|
||||
'session_id': session_id,
|
||||
'qr_url': session['qr_url'],
|
||||
'expire_at': session['expire_at'],
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/dingtalk/create-app/status/<session_id>', methods=['GET'])
|
||||
async def _(session_id: str) -> str:
|
||||
"""Poll DingTalk Device Flow status."""
|
||||
_cleanup_expired_dingtalk_sessions()
|
||||
session = _dingtalk_sessions.get(session_id)
|
||||
if not session:
|
||||
return self.http_status(404, -1, 'Session not found')
|
||||
|
||||
data = {'status': session['status']}
|
||||
|
||||
if session['status'] == 'success':
|
||||
data['client_id'] = session['client_id']
|
||||
data['client_secret'] = session['client_secret']
|
||||
_dingtalk_sessions.pop(session_id, None)
|
||||
elif session['status'] == 'error':
|
||||
data['error'] = session['error']
|
||||
_dingtalk_sessions.pop(session_id, None)
|
||||
|
||||
return self.success(data=data)
|
||||
|
||||
@self.route('/dingtalk/create-app/<session_id>', methods=['DELETE'])
|
||||
async def _(session_id: str) -> str:
|
||||
"""Cancel and clean up a DingTalk Device Flow session."""
|
||||
session = _dingtalk_sessions.pop(session_id, None)
|
||||
if session and session.get('task') and not session['task'].done():
|
||||
session['task'].cancel()
|
||||
return self.success(data={})
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# WeComBot QR Code One-Click Create
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
_wecombot_sessions: dict = {}
|
||||
_WECOMBOT_SESSION_TTL = 300 # 5 minutes (WeCom QR validity window)
|
||||
|
||||
def _cleanup_expired_wecombot_sessions():
|
||||
import time
|
||||
|
||||
now = time.time()
|
||||
expired = [
|
||||
sid for sid, s in _wecombot_sessions.items() if now - s.get('created_at', 0) > _WECOMBOT_SESSION_TTL
|
||||
]
|
||||
for sid in expired:
|
||||
session = _wecombot_sessions.pop(sid, None)
|
||||
if session and session.get('task') and not session['task'].done():
|
||||
session['task'].cancel()
|
||||
|
||||
@self.route('/wecombot/create-bot', methods=['POST'])
|
||||
async def _() -> str:
|
||||
"""Start WeComBot one-click creation via QR code. Returns session_id + QR code URL."""
|
||||
import uuid
|
||||
import time
|
||||
import aiohttp
|
||||
|
||||
WECOM_QC_GENERATE_URL = 'https://work.weixin.qq.com/ai/qc/generate'
|
||||
WECOM_QC_QUERY_URL = 'https://work.weixin.qq.com/ai/qc/query_result'
|
||||
|
||||
_cleanup_expired_wecombot_sessions()
|
||||
|
||||
session_id = str(uuid.uuid4())
|
||||
|
||||
session = {
|
||||
'status': 'pending',
|
||||
'qr_url': None,
|
||||
'expire_at': None,
|
||||
'botid': None,
|
||||
'secret': None,
|
||||
'error': None,
|
||||
'created_at': time.time(),
|
||||
'scode': None,
|
||||
'task': None,
|
||||
}
|
||||
_wecombot_sessions[session_id] = session
|
||||
|
||||
async def run_qr_flow():
|
||||
try:
|
||||
timeout = aiohttp.ClientTimeout(total=10)
|
||||
async with aiohttp.ClientSession(timeout=timeout) as http:
|
||||
# Step 1: Generate QR code
|
||||
async with http.get(
|
||||
f'{WECOM_QC_GENERATE_URL}?source=langbot&plat=0',
|
||||
) as resp:
|
||||
try:
|
||||
data = await resp.json()
|
||||
except (aiohttp.ContentTypeError, ValueError):
|
||||
session['status'] = 'error'
|
||||
session['error'] = 'Invalid response from WeCom service'
|
||||
return
|
||||
if not data.get('data', {}).get('scode') or not data.get('data', {}).get('auth_url'):
|
||||
session['status'] = 'error'
|
||||
session['error'] = data.get('errmsg', 'Failed to generate QR code')
|
||||
return
|
||||
|
||||
scode = data['data']['scode']
|
||||
auth_url = data['data']['auth_url']
|
||||
|
||||
session['scode'] = scode
|
||||
session['qr_url'] = auth_url
|
||||
session['expire_at'] = time.time() + _WECOMBOT_SESSION_TTL
|
||||
session['status'] = 'waiting'
|
||||
|
||||
# Step 2: Poll for scan result
|
||||
deadline = time.time() + _WECOMBOT_SESSION_TTL
|
||||
while time.time() < deadline:
|
||||
await asyncio.sleep(3)
|
||||
|
||||
async with http.get(
|
||||
f'{WECOM_QC_QUERY_URL}?scode={scode}',
|
||||
) as poll_resp:
|
||||
try:
|
||||
poll_data = await poll_resp.json()
|
||||
except (aiohttp.ContentTypeError, ValueError):
|
||||
continue
|
||||
|
||||
status = poll_data.get('data', {}).get('status', '')
|
||||
if status == 'success':
|
||||
bot_info = poll_data.get('data', {}).get('bot_info', {})
|
||||
if bot_info.get('botid') and bot_info.get('secret'):
|
||||
session['status'] = 'success'
|
||||
session['botid'] = bot_info['botid']
|
||||
session['secret'] = bot_info['secret']
|
||||
return
|
||||
else:
|
||||
session['status'] = 'error'
|
||||
session['error'] = 'Scan succeeded but bot info is incomplete'
|
||||
return
|
||||
|
||||
# Timeout
|
||||
session['status'] = 'error'
|
||||
session['error'] = 'QR code expired'
|
||||
|
||||
except asyncio.CancelledError:
|
||||
return
|
||||
except Exception as e:
|
||||
session['status'] = 'error'
|
||||
session['error'] = str(e)
|
||||
|
||||
task = asyncio.create_task(run_qr_flow())
|
||||
session['task'] = task
|
||||
|
||||
# Wait for QR code to be ready (max 10 seconds)
|
||||
for _ in range(20):
|
||||
if session['qr_url'] or session['error']:
|
||||
break
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
if session['error']:
|
||||
task.cancel()
|
||||
return self.http_status(502, -1, session['error'])
|
||||
|
||||
if not session['qr_url']:
|
||||
task.cancel()
|
||||
session['status'] = 'error'
|
||||
session['error'] = 'Timeout waiting for QR code'
|
||||
return self.http_status(504, -1, 'Timeout waiting for QR code')
|
||||
|
||||
return self.success(
|
||||
data={
|
||||
'session_id': session_id,
|
||||
'qr_url': session['qr_url'],
|
||||
'expire_at': session['expire_at'],
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/wecombot/create-bot/status/<session_id>', methods=['GET'])
|
||||
async def _(session_id: str) -> str:
|
||||
"""Poll WeComBot creation status."""
|
||||
_cleanup_expired_wecombot_sessions()
|
||||
session = _wecombot_sessions.get(session_id)
|
||||
if not session:
|
||||
return self.http_status(404, -1, 'Session not found')
|
||||
|
||||
data = {'status': session['status']}
|
||||
|
||||
if session['status'] == 'success':
|
||||
data['botid'] = session['botid']
|
||||
data['secret'] = session['secret']
|
||||
_wecombot_sessions.pop(session_id, None)
|
||||
elif session['status'] == 'error':
|
||||
data['error'] = session['error']
|
||||
_wecombot_sessions.pop(session_id, None)
|
||||
|
||||
return self.success(data=data)
|
||||
|
||||
@self.route('/wecombot/create-bot/<session_id>', methods=['DELETE'])
|
||||
async def _(session_id: str) -> str:
|
||||
"""Cancel and clean up a WeComBot creation session."""
|
||||
session = _wecombot_sessions.pop(session_id, None)
|
||||
if session and session.get('task') and not session['task'].done():
|
||||
session['task'].cancel()
|
||||
return self.success(data={})
|
||||
|
||||
@@ -1,153 +1,19 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import io
|
||||
import quart
|
||||
import re
|
||||
import httpx
|
||||
import uuid
|
||||
import os
|
||||
import zipfile
|
||||
import yaml
|
||||
from urllib.parse import urlparse
|
||||
import posixpath
|
||||
import sqlalchemy
|
||||
|
||||
from .....core import taskmgr
|
||||
from .....entity.persistence import plugin as persistence_plugin
|
||||
from .. import group
|
||||
from langbot_plugin.runtime.plugin.mgr import PluginInstallSource
|
||||
|
||||
# Resolve the built-in page SDK JS from the langbot_plugin package
|
||||
_PAGE_SDK_PATH = None
|
||||
try:
|
||||
import langbot_plugin.assets as _assets_pkg
|
||||
|
||||
_candidate = os.path.join(os.path.dirname(_assets_pkg.__file__), 'langbot-page-sdk.js')
|
||||
if os.path.exists(_candidate):
|
||||
_PAGE_SDK_PATH = _candidate
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _normalize_plugin_asset_path(filepath: str) -> str | None:
|
||||
filepath = filepath.replace('\\', '/')
|
||||
if filepath.startswith('/'):
|
||||
return None
|
||||
|
||||
normalized = posixpath.normpath(filepath)
|
||||
if normalized == '.' or normalized.startswith('../') or normalized == '..':
|
||||
return None
|
||||
|
||||
if normalized.startswith('components/pages/'):
|
||||
return normalized
|
||||
|
||||
return f'assets/{normalized}'
|
||||
|
||||
|
||||
def _get_request_origin() -> str:
|
||||
"""Return the public request origin, respecting reverse-proxy headers."""
|
||||
forwarded_proto = quart.request.headers.get('X-Forwarded-Proto', '').split(',')[0].strip()
|
||||
forwarded_host = quart.request.headers.get('X-Forwarded-Host', '').split(',')[0].strip()
|
||||
|
||||
scheme = forwarded_proto or quart.request.scheme
|
||||
host = forwarded_host or quart.request.host
|
||||
return f'{scheme}://{host}'
|
||||
|
||||
|
||||
@group.group_class('plugins', '/api/v1/plugins')
|
||||
class PluginsRouterGroup(group.RouterGroup):
|
||||
@staticmethod
|
||||
def _normalize_archive_path(path: str) -> str:
|
||||
normalized = str(path or '').replace('\\', '/').strip('/')
|
||||
return posixpath.normpath(normalized) if normalized else ''
|
||||
|
||||
@classmethod
|
||||
def _component_source_path(cls, entry) -> str:
|
||||
if isinstance(entry, dict):
|
||||
return cls._normalize_archive_path(entry.get('path') or '')
|
||||
return cls._normalize_archive_path(str(entry or ''))
|
||||
|
||||
@classmethod
|
||||
def _count_component_configs(cls, component_config, archive_names: list[str]) -> int:
|
||||
normalized_names = [cls._normalize_archive_path(name) for name in archive_names]
|
||||
component_files: set[str] = set()
|
||||
|
||||
if isinstance(component_config, list):
|
||||
return len(component_config)
|
||||
if not isinstance(component_config, dict):
|
||||
return 1 if component_config else 0
|
||||
|
||||
for entry in component_config.get('fromFiles') or []:
|
||||
source_path = cls._component_source_path(entry)
|
||||
if source_path and source_path in normalized_names:
|
||||
component_files.add(source_path)
|
||||
|
||||
for entry in component_config.get('fromDirs') or []:
|
||||
source_dir = cls._component_source_path(entry).rstrip('/')
|
||||
if not source_dir:
|
||||
continue
|
||||
prefix = f'{source_dir}/'
|
||||
for archive_name in normalized_names:
|
||||
if not archive_name.startswith(prefix):
|
||||
continue
|
||||
if archive_name.lower().endswith(('.yaml', '.yml')):
|
||||
component_files.add(archive_name)
|
||||
|
||||
if component_files:
|
||||
return len(component_files)
|
||||
|
||||
return 1 if any(key in component_config for key in ('path', 'name', 'kind')) else 0
|
||||
|
||||
@classmethod
|
||||
def _count_plugin_components(cls, components, archive_names: list[str]) -> dict[str, int]:
|
||||
if not isinstance(components, dict):
|
||||
return {}
|
||||
|
||||
component_counts: dict[str, int] = {}
|
||||
for kind, component_config in components.items():
|
||||
count = cls._count_component_configs(component_config, archive_names)
|
||||
if count > 0:
|
||||
component_counts[str(kind)] = count
|
||||
return component_counts
|
||||
|
||||
@staticmethod
|
||||
def _parse_github_repo_url(repo_url: str) -> dict | None:
|
||||
raw_url = str(repo_url or '').strip()
|
||||
if not raw_url:
|
||||
return None
|
||||
|
||||
if not re.match(r'^[a-zA-Z][a-zA-Z0-9+.-]*://', raw_url):
|
||||
raw_url = f'https://{raw_url}'
|
||||
|
||||
parsed = urlparse(raw_url)
|
||||
if parsed.netloc.lower() not in ('github.com', 'www.github.com'):
|
||||
return None
|
||||
|
||||
parts = [part for part in parsed.path.strip('/').split('/') if part]
|
||||
if len(parts) < 2:
|
||||
return None
|
||||
|
||||
owner = parts[0]
|
||||
repo = parts[1]
|
||||
if repo.endswith('.git'):
|
||||
repo = repo[:-4]
|
||||
if not owner or not repo:
|
||||
return None
|
||||
|
||||
ref = ''
|
||||
subdir = ''
|
||||
if len(parts) >= 4 and parts[2] in ('tree', 'blob'):
|
||||
ref = parts[3]
|
||||
subdir = '/'.join(parts[4:]).strip('/')
|
||||
|
||||
return {
|
||||
'owner': owner,
|
||||
'repo': repo,
|
||||
'ref': ref,
|
||||
'subdir': subdir,
|
||||
}
|
||||
|
||||
async def _check_extensions_limit(self) -> str | None:
|
||||
"""Check if extensions limit is reached. Returns error response if limit exceeded, None otherwise."""
|
||||
limitation = self.ap.instance_config.data.get('system', {}).get('limitation', {})
|
||||
@@ -161,15 +27,6 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
return None
|
||||
|
||||
async def initialize(self) -> None:
|
||||
@self.route('/_sdk/page-sdk.js', methods=['GET'], auth_type=group.AuthType.NONE)
|
||||
async def _() -> quart.Response:
|
||||
"""Serve the built-in LangBot page SDK JavaScript."""
|
||||
if _PAGE_SDK_PATH and os.path.exists(_PAGE_SDK_PATH):
|
||||
with open(_PAGE_SDK_PATH, 'r') as f:
|
||||
content = f.read()
|
||||
return quart.Response(content, mimetype='application/javascript')
|
||||
return quart.Response('// SDK not found', status=404, mimetype='application/javascript')
|
||||
|
||||
@self.route('', methods=['GET'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _() -> str:
|
||||
plugins = await self.ap.plugin_connector.list_plugins()
|
||||
@@ -245,15 +102,7 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
return self.http_status(404, -1, 'plugin not found')
|
||||
|
||||
if quart.request.method == 'GET':
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_plugin.PluginSetting.config)
|
||||
.where(persistence_plugin.PluginSetting.plugin_author == author)
|
||||
.where(persistence_plugin.PluginSetting.plugin_name == plugin_name)
|
||||
)
|
||||
persisted_config = result.scalar_one_or_none()
|
||||
|
||||
config = persisted_config if persisted_config is not None else plugin['plugin_config']
|
||||
return self.success(data={'config': config})
|
||||
return self.success(data={'config': plugin['plugin_config']})
|
||||
elif quart.request.method == 'PUT':
|
||||
data = await quart.request.json
|
||||
|
||||
@@ -286,62 +135,15 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
return quart.Response(icon_data, mimetype=mime_type)
|
||||
|
||||
@self.route(
|
||||
'/<author>/<plugin_name>/assets/<path:filepath>',
|
||||
'/<author>/<plugin_name>/assets/<filepath>',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.NONE,
|
||||
)
|
||||
async def _(author: str, plugin_name: str, filepath: str) -> quart.Response:
|
||||
asset_path = _normalize_plugin_asset_path(filepath)
|
||||
if asset_path is None:
|
||||
return quart.Response('Asset not found', status=404)
|
||||
|
||||
asset_data = await self.ap.plugin_connector.get_plugin_assets(author, plugin_name, asset_path)
|
||||
if not asset_data.get('asset_base64'):
|
||||
return quart.Response('Asset not found', status=404)
|
||||
asset_data = await self.ap.plugin_connector.get_plugin_assets(author, plugin_name, filepath)
|
||||
asset_bytes = base64.b64decode(asset_data['asset_base64'])
|
||||
mime_type = asset_data['mime_type']
|
||||
resp = quart.Response(asset_bytes, mimetype=mime_type)
|
||||
# CSP for HTML pages served to sandboxed iframes (opaque origin).
|
||||
# 'self' doesn't work in sandboxed iframes — use actual server origin.
|
||||
if mime_type and mime_type.startswith('text/html'):
|
||||
origin = _get_request_origin()
|
||||
resp.headers['Content-Security-Policy'] = (
|
||||
f'default-src {origin}; '
|
||||
f"script-src {origin} 'unsafe-inline'; "
|
||||
f"style-src {origin} 'unsafe-inline'; "
|
||||
f'img-src {origin} data:; '
|
||||
f'connect-src {origin}; '
|
||||
"frame-src 'none'; "
|
||||
"object-src 'none'"
|
||||
)
|
||||
return resp
|
||||
|
||||
@self.route(
|
||||
'/<author>/<plugin_name>/page-api',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
)
|
||||
async def _(author: str, plugin_name: str) -> str:
|
||||
"""Forward a page API request to the plugin."""
|
||||
data = await quart.request.json
|
||||
if not isinstance(data, dict):
|
||||
return self.http_status(400, -1, 'invalid request body')
|
||||
|
||||
page_id = data.get('page_id', '')
|
||||
endpoint = data.get('endpoint', '')
|
||||
method = data.get('method', 'POST')
|
||||
body = data.get('body')
|
||||
if not isinstance(page_id, str) or not isinstance(endpoint, str) or not isinstance(method, str):
|
||||
return self.http_status(400, -1, 'invalid page api request')
|
||||
if not endpoint.startswith('/') or '..' in endpoint:
|
||||
return self.http_status(400, -1, 'invalid endpoint')
|
||||
|
||||
result = await self.ap.plugin_connector.handle_page_api(
|
||||
author, plugin_name, page_id, endpoint, method.upper(), body
|
||||
)
|
||||
if result.get('error'):
|
||||
return self.http_status(400, -1, result['error'])
|
||||
return self.success(data=result.get('data'))
|
||||
return quart.Response(asset_bytes, mimetype=mime_type)
|
||||
|
||||
@self.route('/github/releases', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _() -> str:
|
||||
@@ -349,37 +151,17 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
data = await quart.request.json
|
||||
repo_url = data.get('repo_url', '')
|
||||
|
||||
parsed_repo = self._parse_github_repo_url(repo_url)
|
||||
if not parsed_repo:
|
||||
# Parse GitHub repository URL to extract owner and repo
|
||||
# Supports: https://github.com/owner/repo or github.com/owner/repo
|
||||
pattern = r'github\.com/([^/]+)/([^/]+?)(?:\.git)?(?:/.*)?$'
|
||||
match = re.search(pattern, repo_url)
|
||||
|
||||
if not match:
|
||||
return self.http_status(400, -1, 'Invalid GitHub repository URL')
|
||||
|
||||
owner = parsed_repo['owner']
|
||||
repo = parsed_repo['repo']
|
||||
requested_ref = parsed_repo['ref']
|
||||
requested_subdir = parsed_repo['subdir']
|
||||
owner, repo = match.groups()
|
||||
|
||||
try:
|
||||
if requested_ref:
|
||||
return self.success(
|
||||
data={
|
||||
'releases': [
|
||||
{
|
||||
'id': 0,
|
||||
'tag_name': requested_ref,
|
||||
'name': requested_ref,
|
||||
'published_at': '',
|
||||
'prerelease': False,
|
||||
'draft': False,
|
||||
'source_type': 'branch',
|
||||
'archive_url': f'https://api.github.com/repos/{owner}/{repo}/zipball/{requested_ref}',
|
||||
}
|
||||
],
|
||||
'owner': owner,
|
||||
'repo': repo,
|
||||
'source_subdir': requested_subdir,
|
||||
}
|
||||
)
|
||||
|
||||
# Fetch releases from GitHub API
|
||||
url = f'https://api.github.com/repos/{owner}/{repo}/releases'
|
||||
async with httpx.AsyncClient(
|
||||
@@ -405,14 +187,7 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
}
|
||||
)
|
||||
|
||||
return self.success(
|
||||
data={
|
||||
'releases': formatted_releases,
|
||||
'owner': owner,
|
||||
'repo': repo,
|
||||
'source_subdir': requested_subdir,
|
||||
}
|
||||
)
|
||||
return self.success(data={'releases': formatted_releases, 'owner': owner, 'repo': repo})
|
||||
except httpx.RequestError as e:
|
||||
return self.http_status(500, -1, f'Failed to fetch releases: {str(e)}')
|
||||
|
||||
@@ -490,8 +265,6 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
return self.http_status(400, -1, 'Missing asset_url parameter')
|
||||
|
||||
ctx = taskmgr.TaskContext.new()
|
||||
ctx.metadata['plugin_name'] = f'{owner}/{repo}'
|
||||
ctx.metadata['install_source'] = 'github'
|
||||
install_info = {
|
||||
'asset_url': asset_url,
|
||||
'owner': owner,
|
||||
@@ -522,17 +295,12 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
|
||||
data = await quart.request.json
|
||||
|
||||
plugin_author = data.get('plugin_author', '')
|
||||
plugin_name = data.get('plugin_name', '')
|
||||
|
||||
ctx = taskmgr.TaskContext.new()
|
||||
ctx.metadata['plugin_name'] = f'{plugin_author}/{plugin_name}'
|
||||
ctx.metadata['install_source'] = 'marketplace'
|
||||
wrapper = self.ap.task_mgr.create_user_task(
|
||||
self.ap.plugin_connector.install_plugin(PluginInstallSource.MARKETPLACE, data, task_context=ctx),
|
||||
kind='plugin-operation',
|
||||
name='plugin-install-marketplace',
|
||||
label=f'Installing plugin from marketplace {plugin_author}/{plugin_name}',
|
||||
label=f'Installing plugin from marketplace ...{data}',
|
||||
context=ctx,
|
||||
)
|
||||
|
||||
@@ -555,74 +323,16 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
}
|
||||
|
||||
ctx = taskmgr.TaskContext.new()
|
||||
ctx.metadata['plugin_name'] = file.filename or 'local plugin'
|
||||
ctx.metadata['install_source'] = 'local'
|
||||
wrapper = self.ap.task_mgr.create_user_task(
|
||||
self.ap.plugin_connector.install_plugin(PluginInstallSource.LOCAL, data, task_context=ctx),
|
||||
kind='plugin-operation',
|
||||
name='plugin-install-local',
|
||||
label=f'Installing plugin from local {file.filename}',
|
||||
label=f'Installing plugin from local ...{file.filename}',
|
||||
context=ctx,
|
||||
)
|
||||
|
||||
return self.success(data={'task_id': wrapper.id})
|
||||
|
||||
@self.route('/install/local/preview', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _() -> str:
|
||||
file = (await quart.request.files).get('file')
|
||||
if file is None:
|
||||
return self.http_status(400, -1, 'file is required')
|
||||
|
||||
file_bytes = file.read()
|
||||
try:
|
||||
with zipfile.ZipFile(io.BytesIO(file_bytes)) as zf:
|
||||
names = [name for name in zf.namelist() if not name.endswith('/')]
|
||||
manifest_name = next(
|
||||
(
|
||||
name
|
||||
for name in names
|
||||
if name.replace('\\', '/').strip('/').lower() in ('manifest.yaml', 'manifest.yml')
|
||||
),
|
||||
None,
|
||||
)
|
||||
if manifest_name is None:
|
||||
return self.http_status(400, -1, 'manifest.yaml is required')
|
||||
|
||||
manifest = yaml.safe_load(zf.read(manifest_name).decode('utf-8')) or {}
|
||||
requirements: list[str] = []
|
||||
requirements_name = next(
|
||||
(name for name in names if name.replace('\\', '/').strip('/').lower() == 'requirements.txt'),
|
||||
None,
|
||||
)
|
||||
if requirements_name is not None:
|
||||
requirements = [
|
||||
line.strip()
|
||||
for line in zf.read(requirements_name).decode('utf-8', errors='ignore').splitlines()
|
||||
if line.strip() and not line.strip().startswith('#')
|
||||
]
|
||||
|
||||
spec = manifest.get('spec') or {}
|
||||
components = spec.get('components') or {}
|
||||
component_counts = self._count_plugin_components(components, names)
|
||||
component_types = list(component_counts.keys())
|
||||
|
||||
return self.success(
|
||||
data={
|
||||
'filename': file.filename or 'local plugin',
|
||||
'size': len(file_bytes),
|
||||
'manifest': manifest,
|
||||
'metadata': manifest.get('metadata') or {},
|
||||
'component_types': component_types,
|
||||
'component_counts': component_counts,
|
||||
'requirements': requirements,
|
||||
'file_count': len(names),
|
||||
}
|
||||
)
|
||||
except zipfile.BadZipFile:
|
||||
return self.http_status(400, -1, 'invalid .lbpkg file')
|
||||
except Exception as exc:
|
||||
return self.http_status(500, -1, f'Failed to preview plugin package: {exc}')
|
||||
|
||||
@self.route('/config-files', methods=['POST'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
"""Upload a file for plugin configuration"""
|
||||
|
||||
@@ -97,51 +97,3 @@ class EmbeddingModelsRouterGroup(group.RouterGroup):
|
||||
await self.ap.embedding_models_service.test_embedding_model(model_uuid, json_data)
|
||||
|
||||
return self.success()
|
||||
|
||||
|
||||
@group.group_class('models/rerank', '/api/v1/provider/models/rerank')
|
||||
class RerankModelsRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['GET', 'POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _() -> str:
|
||||
if quart.request.method == 'GET':
|
||||
provider_uuid = quart.request.args.get('provider_uuid')
|
||||
if provider_uuid:
|
||||
return self.success(
|
||||
data={
|
||||
'models': await self.ap.rerank_models_service.get_rerank_models_by_provider(provider_uuid)
|
||||
}
|
||||
)
|
||||
return self.success(data={'models': await self.ap.rerank_models_service.get_rerank_models()})
|
||||
elif quart.request.method == 'POST':
|
||||
json_data = await quart.request.json
|
||||
model_uuid = await self.ap.rerank_models_service.create_rerank_model(json_data)
|
||||
return self.success(data={'uuid': model_uuid})
|
||||
|
||||
@self.route('/<model_uuid>', methods=['GET', 'PUT', 'DELETE'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _(model_uuid: str) -> str:
|
||||
if quart.request.method == 'GET':
|
||||
model = await self.ap.rerank_models_service.get_rerank_model(model_uuid)
|
||||
|
||||
if model is None:
|
||||
return self.http_status(404, -1, 'model not found')
|
||||
|
||||
return self.success(data={'model': model})
|
||||
elif quart.request.method == 'PUT':
|
||||
json_data = await quart.request.json
|
||||
|
||||
await self.ap.rerank_models_service.update_rerank_model(model_uuid, json_data)
|
||||
|
||||
return self.success()
|
||||
elif quart.request.method == 'DELETE':
|
||||
await self.ap.rerank_models_service.delete_rerank_model(model_uuid)
|
||||
|
||||
return self.success()
|
||||
|
||||
@self.route('/<model_uuid>/test', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _(model_uuid: str) -> str:
|
||||
json_data = await quart.request.json
|
||||
|
||||
await self.ap.rerank_models_service.test_rerank_model(model_uuid, json_data)
|
||||
|
||||
return self.success()
|
||||
|
||||
@@ -15,7 +15,6 @@ class ModelProvidersRouterGroup(group.RouterGroup):
|
||||
counts = await self.ap.provider_service.get_provider_model_counts(provider['uuid'])
|
||||
provider['llm_count'] = counts['llm_count']
|
||||
provider['embedding_count'] = counts['embedding_count']
|
||||
provider['rerank_count'] = counts['rerank_count']
|
||||
return self.success(data={'providers': providers})
|
||||
elif quart.request.method == 'POST':
|
||||
json_data = await quart.request.json
|
||||
@@ -33,7 +32,6 @@ class ModelProvidersRouterGroup(group.RouterGroup):
|
||||
counts = await self.ap.provider_service.get_provider_model_counts(provider_uuid)
|
||||
provider['llm_count'] = counts['llm_count']
|
||||
provider['embedding_count'] = counts['embedding_count']
|
||||
provider['rerank_count'] = counts['rerank_count']
|
||||
return self.success(data={'provider': provider})
|
||||
elif quart.request.method == 'PUT':
|
||||
json_data = await quart.request.json
|
||||
@@ -45,12 +43,3 @@ class ModelProvidersRouterGroup(group.RouterGroup):
|
||||
return self.success()
|
||||
except ValueError as e:
|
||||
return self.http_status(400, -1, str(e))
|
||||
|
||||
@self.route('/<provider_uuid>/scan-models', methods=['GET'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _(provider_uuid: str) -> str:
|
||||
try:
|
||||
model_type = quart.request.args.get('type')
|
||||
result = await self.ap.provider_service.scan_provider_models(provider_uuid, model_type)
|
||||
return self.success(data=result)
|
||||
except ValueError as e:
|
||||
return self.http_status(400, -1, str(e))
|
||||
|
||||
@@ -31,9 +31,6 @@ class MCPRouterGroup(group.RouterGroup):
|
||||
@self.route('/servers/<server_name>', methods=['GET', 'PUT', 'DELETE'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _(server_name: str) -> str:
|
||||
"""获取、更新或删除MCP服务器配置"""
|
||||
from urllib.parse import unquote
|
||||
|
||||
server_name = unquote(server_name)
|
||||
|
||||
server_data = await self.ap.mcp_service.get_mcp_server_by_name(server_name)
|
||||
if server_data is None:
|
||||
@@ -60,9 +57,6 @@ class MCPRouterGroup(group.RouterGroup):
|
||||
@self.route('/servers/<server_name>/test', methods=['POST'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _(server_name: str) -> str:
|
||||
"""测试MCP服务器连接"""
|
||||
from urllib.parse import unquote
|
||||
|
||||
server_name = unquote(server_name)
|
||||
server_data = await quart.request.json
|
||||
task_id = await self.ap.mcp_service.test_mcp_server(server_name=server_name, server_data=server_data)
|
||||
return self.success(data={'task_id': task_id})
|
||||
|
||||
@@ -1,45 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from ... import group
|
||||
|
||||
|
||||
@group.group_class('tools', '/api/v1/tools')
|
||||
class ToolsRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
"""获取所有可用工具列表"""
|
||||
tools = await self.ap.tool_mgr.get_all_tools()
|
||||
|
||||
tool_list = []
|
||||
for tool in tools:
|
||||
tool_list.append(
|
||||
{
|
||||
'name': tool.name,
|
||||
'description': tool.description,
|
||||
'human_desc': tool.human_desc,
|
||||
'parameters': tool.parameters,
|
||||
}
|
||||
)
|
||||
|
||||
return self.success(data={'tools': tool_list})
|
||||
|
||||
@self.route('/<tool_name>', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _(tool_name: str) -> str:
|
||||
"""获取特定工具详情"""
|
||||
tools = await self.ap.tool_mgr.get_all_tools()
|
||||
|
||||
for tool in tools:
|
||||
if tool.name == tool_name:
|
||||
return self.success(
|
||||
data={
|
||||
'tool': {
|
||||
'name': tool.name,
|
||||
'description': tool.description,
|
||||
'human_desc': tool.human_desc,
|
||||
'parameters': tool.parameters,
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
return self.http_status(404, -1, f'Tool not found: {tool_name}')
|
||||
@@ -1,190 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import quart
|
||||
|
||||
from langbot_plugin.box.errors import BoxError
|
||||
|
||||
from .. import group
|
||||
|
||||
|
||||
@group.group_class('skills', '/api/v1/skills')
|
||||
class SkillsRouterGroup(group.RouterGroup):
|
||||
"""Skills management API endpoints."""
|
||||
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['GET', 'POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def list_or_create_skills() -> quart.Response:
|
||||
if quart.request.method == 'GET':
|
||||
try:
|
||||
skills = await self.ap.skill_service.list_skills()
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
return self.success(data={'skills': skills})
|
||||
|
||||
data = await quart.request.json
|
||||
if 'name' not in data or not data['name']:
|
||||
return self.http_status(400, -1, 'Missing required field: name')
|
||||
|
||||
try:
|
||||
skill = await self.ap.skill_service.create_skill(data)
|
||||
return self.success(data={'skill': skill})
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
|
||||
@self.route('/<skill_name>', methods=['GET', 'PUT', 'DELETE'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def get_update_delete_skill(skill_name: str) -> quart.Response:
|
||||
if quart.request.method == 'GET':
|
||||
try:
|
||||
skill = await self.ap.skill_service.get_skill(skill_name)
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
if not skill:
|
||||
return self.http_status(404, -1, 'Skill not found')
|
||||
return self.success(data={'skill': skill})
|
||||
|
||||
if quart.request.method == 'PUT':
|
||||
data = await quart.request.json
|
||||
try:
|
||||
skill = await self.ap.skill_service.update_skill(skill_name, data)
|
||||
return self.success(data={'skill': skill})
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
|
||||
try:
|
||||
await self.ap.skill_service.delete_skill(skill_name)
|
||||
return self.success()
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
|
||||
@self.route('/<skill_name>/files', methods=['GET'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def list_skill_files(skill_name: str) -> quart.Response:
|
||||
"""List files in skill package directory."""
|
||||
path = quart.request.args.get('path', '.').strip()
|
||||
include_hidden = quart.request.args.get('include_hidden', 'false').lower() == 'true'
|
||||
|
||||
try:
|
||||
result = await self.ap.skill_service.list_skill_files(
|
||||
skill_name,
|
||||
path=path,
|
||||
include_hidden=include_hidden,
|
||||
)
|
||||
return self.success(data=result)
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
|
||||
@self.route(
|
||||
'/<skill_name>/files/<path:path>', methods=['GET', 'PUT'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY
|
||||
)
|
||||
async def read_or_write_skill_file(skill_name: str, path: str) -> quart.Response:
|
||||
"""Read or write a file in skill package."""
|
||||
if quart.request.method == 'GET':
|
||||
try:
|
||||
result = await self.ap.skill_service.read_skill_file(skill_name, path)
|
||||
return self.success(data=result)
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
|
||||
# PUT - write file
|
||||
data = await quart.request.json
|
||||
content = data.get('content', '')
|
||||
if content is None:
|
||||
return self.http_status(400, -1, 'Missing required field: content')
|
||||
|
||||
try:
|
||||
result = await self.ap.skill_service.write_skill_file(skill_name, path, content)
|
||||
return self.success(data=result)
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
|
||||
@self.route('/<skill_name>/preview', methods=['GET'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def preview_skill(skill_name: str) -> quart.Response:
|
||||
skill = self.ap.skill_mgr.get_skill_by_name(skill_name)
|
||||
if not skill:
|
||||
return self.http_status(404, -1, 'Skill not found')
|
||||
return self.success(data={'instructions': skill.get('instructions', '')})
|
||||
|
||||
@self.route('/install/github', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def install_skill_from_github() -> quart.Response:
|
||||
data = await quart.request.json
|
||||
required_fields = ['asset_url', 'owner', 'repo']
|
||||
for field in required_fields:
|
||||
if field not in data or not data[field]:
|
||||
return self.http_status(400, -1, f'Missing required field: {field}')
|
||||
asset_url = str(data['asset_url']).strip().lower().split('?', 1)[0].split('#', 1)[0]
|
||||
if not asset_url.endswith('skill.md') and not data.get('release_tag'):
|
||||
return self.http_status(400, -1, 'Missing required field: release_tag')
|
||||
|
||||
try:
|
||||
skill = await self.ap.skill_service.install_from_github(data)
|
||||
return self.success(data={'skills': skill})
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
except Exception as exc:
|
||||
return self.http_status(500, -1, f'Failed to install skill: {exc}')
|
||||
|
||||
@self.route('/install/github/preview', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def preview_skill_from_github() -> quart.Response:
|
||||
data = await quart.request.json
|
||||
required_fields = ['asset_url', 'owner', 'repo']
|
||||
for field in required_fields:
|
||||
if field not in data or not data[field]:
|
||||
return self.http_status(400, -1, f'Missing required field: {field}')
|
||||
asset_url = str(data['asset_url']).strip().lower().split('?', 1)[0].split('#', 1)[0]
|
||||
if not asset_url.endswith('skill.md') and not data.get('release_tag'):
|
||||
return self.http_status(400, -1, 'Missing required field: release_tag')
|
||||
|
||||
try:
|
||||
preview = await self.ap.skill_service.preview_install_from_github(data)
|
||||
return self.success(data={'skills': preview})
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
except Exception as exc:
|
||||
return self.http_status(500, -1, f'Failed to preview skill: {exc}')
|
||||
|
||||
@self.route('/install/upload', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def install_skill_from_upload() -> quart.Response:
|
||||
file = (await quart.request.files).get('file')
|
||||
if file is None:
|
||||
return self.http_status(400, -1, 'file is required')
|
||||
form = await quart.request.form
|
||||
|
||||
try:
|
||||
skill = await self.ap.skill_service.install_from_zip_upload(
|
||||
file_bytes=file.read(),
|
||||
filename=file.filename or '',
|
||||
source_paths=form.getlist('source_paths'),
|
||||
)
|
||||
return self.success(data={'skills': skill})
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
except Exception as exc:
|
||||
return self.http_status(500, -1, f'Failed to install skill: {exc}')
|
||||
|
||||
@self.route('/install/upload/preview', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def preview_skill_from_upload() -> quart.Response:
|
||||
file = (await quart.request.files).get('file')
|
||||
if file is None:
|
||||
return self.http_status(400, -1, 'file is required')
|
||||
|
||||
try:
|
||||
preview = await self.ap.skill_service.preview_install_from_zip_upload(
|
||||
file_bytes=file.read(),
|
||||
filename=file.filename or '',
|
||||
)
|
||||
return self.success(data={'skills': preview})
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
except Exception as exc:
|
||||
return self.http_status(500, -1, f'Failed to preview skill: {exc}')
|
||||
|
||||
@self.route('/scan', methods=['GET'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def scan_skill_directory() -> quart.Response:
|
||||
path = quart.request.args.get('path', '').strip()
|
||||
if not path:
|
||||
return self.http_status(400, -1, 'Missing required parameter: path')
|
||||
|
||||
try:
|
||||
result = await self.ap.skill_service.scan_directory_async(path)
|
||||
return self.success(data=result)
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
@@ -1,11 +1,7 @@
|
||||
import json
|
||||
|
||||
import quart
|
||||
import sqlalchemy
|
||||
|
||||
from .. import group
|
||||
from .....utils import constants
|
||||
from .....entity.persistence.metadata import Metadata
|
||||
|
||||
|
||||
@group.group_class('system', '/api/v1/system')
|
||||
@@ -13,24 +9,6 @@ class SystemRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('/info', methods=['GET'], auth_type=group.AuthType.NONE)
|
||||
async def _() -> str:
|
||||
# Read wizard_status and wizard_progress from metadata table
|
||||
wizard_status = 'none'
|
||||
wizard_progress = None
|
||||
try:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(Metadata).where(Metadata.key.in_(['wizard_status', 'wizard_progress']))
|
||||
)
|
||||
for row in result:
|
||||
if row.key == 'wizard_status':
|
||||
wizard_status = row.value
|
||||
elif row.key == 'wizard_progress':
|
||||
try:
|
||||
wizard_progress = json.loads(row.value)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
wizard_progress = None
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return self.success(
|
||||
data={
|
||||
'version': constants.semantic_version,
|
||||
@@ -49,83 +27,17 @@ class SystemRouterGroup(group.RouterGroup):
|
||||
'disable_models_service', False
|
||||
),
|
||||
'limitation': self.ap.instance_config.data.get('system', {}).get('limitation', {}),
|
||||
'wizard_status': wizard_status,
|
||||
'wizard_progress': wizard_progress,
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/wizard/completed', methods=['POST'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
"""Mark wizard status in metadata table and clear progress.
|
||||
|
||||
Accepts JSON body: { "status": "skipped" | "completed" }
|
||||
"""
|
||||
data = await quart.request.get_json(silent=True) or {}
|
||||
status = data.get('status', 'completed')
|
||||
if status not in ('skipped', 'completed'):
|
||||
return self.http_status(400, 400, f'Invalid wizard status: {status}')
|
||||
|
||||
try:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(Metadata).where(Metadata.key == 'wizard_status')
|
||||
)
|
||||
if result.first():
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(Metadata).where(Metadata.key == 'wizard_status').values(value=status)
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(Metadata).values(key='wizard_status', value=status)
|
||||
)
|
||||
|
||||
# Clear wizard progress when wizard is completed/skipped
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(Metadata).where(Metadata.key == 'wizard_progress')
|
||||
)
|
||||
except Exception as e:
|
||||
return self.http_status(500, 500, f'Failed to update wizard status: {e}')
|
||||
|
||||
return self.success(data={})
|
||||
|
||||
@self.route('/wizard/progress', methods=['PUT'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
"""Save wizard progress to metadata table.
|
||||
|
||||
Accepts JSON body with wizard state fields:
|
||||
{ "step": int, "selected_adapter": str|null, "created_bot_uuid": str|null,
|
||||
"bot_saved": bool, "selected_runner": str|null }
|
||||
"""
|
||||
data = await quart.request.get_json(silent=True) or {}
|
||||
progress_json = json.dumps(data, ensure_ascii=False)
|
||||
|
||||
try:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(Metadata).where(Metadata.key == 'wizard_progress')
|
||||
)
|
||||
if result.first():
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(Metadata).where(Metadata.key == 'wizard_progress').values(value=progress_json)
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(Metadata).values(key='wizard_progress', value=progress_json)
|
||||
)
|
||||
except Exception as e:
|
||||
return self.http_status(500, 500, f'Failed to save wizard progress: {e}')
|
||||
|
||||
return self.success(data={})
|
||||
|
||||
@self.route('/tasks', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
task_type = quart.request.args.get('type')
|
||||
task_kind = quart.request.args.get('kind')
|
||||
|
||||
if task_type == '':
|
||||
task_type = None
|
||||
if task_kind == '':
|
||||
task_kind = None
|
||||
|
||||
return self.success(data=self.ap.task_mgr.get_tasks_dict(task_type, task_kind))
|
||||
return self.success(data=self.ap.task_mgr.get_tasks_dict(task_type))
|
||||
|
||||
@self.route('/tasks/<task_id>', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _(task_id: str) -> str:
|
||||
@@ -136,9 +48,16 @@ class SystemRouterGroup(group.RouterGroup):
|
||||
|
||||
return self.success(data=task.to_dict())
|
||||
|
||||
@self.route('/storage-analysis', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
@self.route('/debug/exec', methods=['POST'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
return self.success(data=await self.ap.maintenance_service.get_storage_analysis())
|
||||
if not constants.debug_mode:
|
||||
return self.http_status(403, 403, 'Forbidden')
|
||||
|
||||
py_code = await quart.request.data
|
||||
|
||||
ap = self.ap
|
||||
|
||||
return self.success(data=exec(py_code, {'ap': ap}))
|
||||
|
||||
@self.route(
|
||||
'/debug/plugin/action',
|
||||
|
||||
@@ -146,7 +146,6 @@ class UserRouterGroup(group.RouterGroup):
|
||||
return self.fail(3, str(e))
|
||||
except ValueError as e:
|
||||
traceback.print_exc()
|
||||
self.ap.logger.warning(f'Space OAuth callback failed: {e}')
|
||||
return self.fail(1, str(e))
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
|
||||
@@ -105,29 +105,6 @@ class HTTPController:
|
||||
):
|
||||
if os.path.exists(os.path.join(frontend_path, path + '.html')):
|
||||
path += '.html'
|
||||
elif not path.startswith('api/'):
|
||||
# SPA fallback: serve index.html for all non-API, non-static routes
|
||||
# so that React Router can handle client-side routing (Vite SPA).
|
||||
# For /home/* sub-routes, first try parent .html files (pre-rendered pages).
|
||||
if path.startswith('home/'):
|
||||
segments = path.rstrip('/').split('/')
|
||||
for i in range(len(segments) - 1, 0, -1):
|
||||
parent_path = '/'.join(segments[:i]) + '.html'
|
||||
if os.path.exists(os.path.join(frontend_path, parent_path)):
|
||||
response = await quart.send_from_directory(
|
||||
frontend_path, parent_path, mimetype='text/html'
|
||||
)
|
||||
response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
|
||||
response.headers['Pragma'] = 'no-cache'
|
||||
response.headers['Expires'] = '0'
|
||||
return response
|
||||
|
||||
# Fallback to index.html for SPA client-side routing
|
||||
response = await quart.send_from_directory(frontend_path, 'index.html', mimetype='text/html')
|
||||
response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
|
||||
response.headers['Pragma'] = 'no-cache'
|
||||
response.headers['Expires'] = '0'
|
||||
return response
|
||||
else:
|
||||
return await quart.send_from_directory(frontend_path, '404.html')
|
||||
|
||||
|
||||
@@ -52,9 +52,6 @@ class ApiKeyService:
|
||||
|
||||
async def verify_api_key(self, key: str) -> bool:
|
||||
"""Verify if an API key is valid"""
|
||||
if not isinstance(key, str) or not key.startswith('lbk_'):
|
||||
return False
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(apikey.ApiKey).where(apikey.ApiKey.key == key)
|
||||
)
|
||||
|
||||
@@ -70,17 +70,12 @@ class BotService:
|
||||
'lark',
|
||||
]:
|
||||
webhook_prefix = self.ap.instance_config.data['api'].get('webhook_prefix', 'http://127.0.0.1:5300')
|
||||
extra_webhook_prefix = self.ap.instance_config.data['api'].get('extra_webhook_prefix', '')
|
||||
webhook_url = f'/bots/{bot_uuid}'
|
||||
adapter_runtime_values['webhook_url'] = webhook_url
|
||||
adapter_runtime_values['webhook_full_url'] = f'{webhook_prefix}{webhook_url}'
|
||||
adapter_runtime_values['extra_webhook_full_url'] = (
|
||||
f'{extra_webhook_prefix}{webhook_url}' if extra_webhook_prefix else ''
|
||||
)
|
||||
else:
|
||||
adapter_runtime_values['webhook_url'] = None
|
||||
adapter_runtime_values['webhook_full_url'] = None
|
||||
adapter_runtime_values['extra_webhook_full_url'] = None
|
||||
|
||||
persistence_bot['adapter_runtime_values'] = adapter_runtime_values
|
||||
|
||||
@@ -99,11 +94,11 @@ class BotService:
|
||||
# TODO: 检查配置信息格式
|
||||
bot_data['uuid'] = str(uuid.uuid4())
|
||||
|
||||
# bind the most recently updated pipeline if any exist
|
||||
# checkout the default pipeline
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline)
|
||||
.order_by(persistence_pipeline.LegacyPipeline.updated_at.desc())
|
||||
.limit(1)
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.is_default == True
|
||||
)
|
||||
)
|
||||
pipeline = result.first()
|
||||
if pipeline is not None:
|
||||
@@ -120,26 +115,24 @@ class BotService:
|
||||
|
||||
async def update_bot(self, bot_uuid: str, bot_data: dict) -> None:
|
||||
"""Update bot"""
|
||||
update_data = bot_data.copy()
|
||||
|
||||
if 'uuid' in update_data:
|
||||
del update_data['uuid']
|
||||
if 'uuid' in bot_data:
|
||||
del bot_data['uuid']
|
||||
|
||||
# set use_pipeline_name
|
||||
if 'use_pipeline_uuid' in update_data:
|
||||
if 'use_pipeline_uuid' in bot_data:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.uuid == update_data['use_pipeline_uuid']
|
||||
persistence_pipeline.LegacyPipeline.uuid == bot_data['use_pipeline_uuid']
|
||||
)
|
||||
)
|
||||
pipeline = result.first()
|
||||
if pipeline is not None:
|
||||
update_data['use_pipeline_name'] = pipeline.name
|
||||
bot_data['use_pipeline_name'] = pipeline.name
|
||||
else:
|
||||
raise Exception('Pipeline not found')
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_bot.Bot).values(update_data).where(persistence_bot.Bot.uuid == bot_uuid)
|
||||
sqlalchemy.update(persistence_bot.Bot).values(bot_data).where(persistence_bot.Bot.uuid == bot_uuid)
|
||||
)
|
||||
await self.ap.platform_mgr.remove_bot(bot_uuid)
|
||||
|
||||
|
||||
@@ -31,126 +31,15 @@ class KnowledgeService:
|
||||
if not knowledge_engine_plugin_id:
|
||||
raise ValueError('knowledge_engine_plugin_id is required')
|
||||
|
||||
creation_settings = kb_data.get('creation_settings', {})
|
||||
retrieval_settings = kb_data.get('retrieval_settings', {})
|
||||
|
||||
# Validate required fields based on plugin's creation_schema and retrieval_schema
|
||||
await self._validate_schema_required_fields(
|
||||
knowledge_engine_plugin_id,
|
||||
creation_settings,
|
||||
retrieval_settings,
|
||||
)
|
||||
|
||||
kb = await self.ap.rag_mgr.create_knowledge_base(
|
||||
name=kb_data.get('name', 'Untitled'),
|
||||
knowledge_engine_plugin_id=knowledge_engine_plugin_id,
|
||||
creation_settings=creation_settings,
|
||||
retrieval_settings=retrieval_settings,
|
||||
creation_settings=kb_data.get('creation_settings', {}),
|
||||
retrieval_settings=kb_data.get('retrieval_settings', {}),
|
||||
description=kb_data.get('description', ''),
|
||||
)
|
||||
return kb.uuid
|
||||
|
||||
async def _validate_schema_required_fields(
|
||||
self,
|
||||
plugin_id: str,
|
||||
creation_settings: dict,
|
||||
retrieval_settings: dict,
|
||||
) -> None:
|
||||
"""Validate required fields based on plugin's creation_schema and retrieval_schema.
|
||||
|
||||
This is a business-agnostic validation that checks all fields marked as
|
||||
required in the plugin's schema, regardless of field type.
|
||||
|
||||
Args:
|
||||
plugin_id: Knowledge Engine plugin ID.
|
||||
creation_settings: User-provided creation settings.
|
||||
retrieval_settings: User-provided retrieval settings.
|
||||
|
||||
Raises:
|
||||
ValueError: If any required field is missing or empty.
|
||||
"""
|
||||
# Validate creation_schema
|
||||
try:
|
||||
creation_schema = await self.ap.plugin_connector.get_rag_creation_schema(plugin_id)
|
||||
self._check_required_fields(creation_schema, creation_settings, 'creation_settings')
|
||||
except ValueError:
|
||||
raise
|
||||
except Exception as e:
|
||||
self.ap.logger.warning(f'Failed to get creation_schema for validation: {e}')
|
||||
|
||||
# Validate retrieval_schema
|
||||
try:
|
||||
retrieval_schema = await self.ap.plugin_connector.get_rag_retrieval_schema(plugin_id)
|
||||
self._check_required_fields(retrieval_schema, retrieval_settings, 'retrieval_settings')
|
||||
except ValueError:
|
||||
raise
|
||||
except Exception as e:
|
||||
self.ap.logger.warning(f'Failed to get retrieval_schema for validation: {e}')
|
||||
|
||||
def _check_required_fields(
|
||||
self,
|
||||
schema: dict | list,
|
||||
settings: dict,
|
||||
context: str,
|
||||
) -> None:
|
||||
"""Check required fields in schema against provided settings.
|
||||
|
||||
Args:
|
||||
schema: Plugin-defined schema (can be list or dict with 'schema' key).
|
||||
settings: User-provided settings values.
|
||||
context: Context name for error messages (e.g., 'creation_settings').
|
||||
|
||||
Raises:
|
||||
ValueError: If a required field is missing or empty.
|
||||
"""
|
||||
if not schema:
|
||||
return
|
||||
|
||||
# schema can be a list directly, or a dict with 'schema' key
|
||||
items = schema if isinstance(schema, list) else schema.get('schema', [])
|
||||
if not items:
|
||||
return
|
||||
|
||||
for item in items:
|
||||
field_name = item.get('name')
|
||||
if not field_name:
|
||||
continue
|
||||
|
||||
is_required = item.get('required', False)
|
||||
if not is_required:
|
||||
continue
|
||||
|
||||
# Check show_if condition - if field is conditionally shown, only validate when condition is met
|
||||
show_if = item.get('show_if')
|
||||
if show_if:
|
||||
depend_field = show_if.get('field')
|
||||
operator = show_if.get('operator')
|
||||
expected_value = show_if.get('value')
|
||||
|
||||
if depend_field and operator:
|
||||
depend_value = settings.get(depend_field)
|
||||
# If show_if condition is not met, skip validation for this field
|
||||
if operator == 'eq' and depend_value != expected_value:
|
||||
continue
|
||||
if operator == 'neq' and depend_value == expected_value:
|
||||
continue
|
||||
if operator == 'in' and isinstance(expected_value, list) and depend_value not in expected_value:
|
||||
continue
|
||||
|
||||
value = settings.get(field_name)
|
||||
|
||||
# Validate required field has a non-empty value
|
||||
if value is None or (isinstance(value, str) and value.strip() == ''):
|
||||
# Get field label for friendly error message
|
||||
label = item.get('label', {})
|
||||
field_label = (
|
||||
label.get('en_US', field_name)
|
||||
or label.get('zh_Hans', field_name)
|
||||
or label.get('zh_Hant', field_name)
|
||||
or field_name
|
||||
)
|
||||
raise ValueError(f'{field_label} is required ({context}.{field_name})')
|
||||
|
||||
async def update_knowledge_base(self, kb_uuid: str, kb_data: dict) -> None:
|
||||
"""更新知识库"""
|
||||
# Filter to only mutable fields
|
||||
|
||||
@@ -1,309 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import sqlalchemy
|
||||
|
||||
from ....core import app
|
||||
from ....entity.persistence import bstorage as persistence_bstorage
|
||||
from ....entity.persistence import monitoring as persistence_monitoring
|
||||
|
||||
|
||||
LOG_FILE_PATTERN = re.compile(r'^langbot-(\d{4}-\d{2}-\d{2})\.log(?:\.\d+)?$')
|
||||
DEFAULT_UPLOAD_FILE_RETENTION_DAYS = 7
|
||||
DEFAULT_LOG_RETENTION_DAYS = 3
|
||||
|
||||
|
||||
class MaintenanceService:
|
||||
"""Storage maintenance and diagnostics."""
|
||||
|
||||
ap: app.Application
|
||||
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
async def cleanup_expired_files(self) -> dict[str, int]:
|
||||
cleanup_cfg = self.ap.instance_config.data.get('storage', {}).get('cleanup', {})
|
||||
upload_retention_days = self._positive_int(
|
||||
cleanup_cfg.get('uploaded_file_retention_days'),
|
||||
DEFAULT_UPLOAD_FILE_RETENTION_DAYS,
|
||||
'storage.cleanup.uploaded_file_retention_days',
|
||||
)
|
||||
log_retention_days = self._positive_int(
|
||||
cleanup_cfg.get('log_retention_days'),
|
||||
DEFAULT_LOG_RETENTION_DAYS,
|
||||
'storage.cleanup.log_retention_days',
|
||||
)
|
||||
|
||||
return {
|
||||
'uploaded_files': await self._cleanup_expired_uploaded_files(upload_retention_days),
|
||||
'log_files': self._cleanup_expired_log_files(log_retention_days),
|
||||
}
|
||||
|
||||
async def get_storage_analysis(self) -> dict[str, Any]:
|
||||
cleanup_cfg = self.ap.instance_config.data.get('storage', {}).get('cleanup', {})
|
||||
upload_retention_days = self._positive_int(
|
||||
cleanup_cfg.get('uploaded_file_retention_days'),
|
||||
DEFAULT_UPLOAD_FILE_RETENTION_DAYS,
|
||||
'storage.cleanup.uploaded_file_retention_days',
|
||||
)
|
||||
log_retention_days = self._positive_int(
|
||||
cleanup_cfg.get('log_retention_days'),
|
||||
DEFAULT_LOG_RETENTION_DAYS,
|
||||
'storage.cleanup.log_retention_days',
|
||||
)
|
||||
|
||||
database_cfg = self.ap.instance_config.data.get('database', {})
|
||||
database_type = database_cfg.get('use', 'sqlite')
|
||||
database_path = (
|
||||
Path(database_cfg.get('sqlite', {}).get('path', 'data/langbot.db')) if database_type == 'sqlite' else None
|
||||
)
|
||||
roots: list[tuple[str, Path | None]] = [
|
||||
('database', database_path),
|
||||
('logs', Path('data/logs')),
|
||||
('storage', Path('data/storage')),
|
||||
('vector_store', Path('data/chroma')),
|
||||
('plugins', Path('data/plugins')),
|
||||
('mcp', Path('data/mcp')),
|
||||
('temp', Path('data/temp')),
|
||||
]
|
||||
|
||||
sections = []
|
||||
for key, path in roots:
|
||||
sections.append(
|
||||
{
|
||||
'key': key,
|
||||
'path': str(path) if path else '',
|
||||
'exists': path.exists() if path else False,
|
||||
'size_bytes': self._path_size(path) if path else 0,
|
||||
'file_count': self._file_count(path) if path else 0,
|
||||
}
|
||||
)
|
||||
|
||||
monitoring_counts = await self._monitoring_counts()
|
||||
binary_storage = await self._binary_storage_stats()
|
||||
upload_candidates = await self._expired_uploaded_candidates(upload_retention_days)
|
||||
log_candidates = self._expired_log_candidates(log_retention_days)
|
||||
|
||||
return {
|
||||
'generated_at': datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
'cleanup_policy': {
|
||||
'uploaded_file_retention_days': upload_retention_days,
|
||||
'log_retention_days': log_retention_days,
|
||||
},
|
||||
'sections': sections,
|
||||
'database': {
|
||||
'type': database_type,
|
||||
'monitoring_counts': monitoring_counts,
|
||||
'binary_storage': binary_storage,
|
||||
},
|
||||
'cleanup_candidates': {
|
||||
'uploaded_files': upload_candidates,
|
||||
'log_files': log_candidates,
|
||||
},
|
||||
'tasks': self.ap.task_mgr.get_stats() if self.ap.task_mgr else {},
|
||||
}
|
||||
|
||||
async def _cleanup_expired_uploaded_files(self, retention_days: int) -> int:
|
||||
provider = self.ap.storage_mgr.storage_provider
|
||||
provider_name = provider.__class__.__name__
|
||||
if provider_name == 'LocalStorageProvider':
|
||||
candidates = self._expired_local_upload_candidates(retention_days, include_paths=True)
|
||||
deleted = 0
|
||||
for item in candidates:
|
||||
try:
|
||||
os.remove(item['path'])
|
||||
deleted += 1
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except Exception as e:
|
||||
self.ap.logger.warning(f'Failed to delete expired uploaded file {item["key"]}: {e}')
|
||||
return deleted
|
||||
|
||||
if provider_name == 'S3StorageProvider':
|
||||
return await self._cleanup_expired_s3_uploaded_files(retention_days)
|
||||
|
||||
return 0
|
||||
|
||||
async def _expired_uploaded_candidates(self, retention_days: int) -> list[dict[str, Any]]:
|
||||
provider_name = self.ap.storage_mgr.storage_provider.__class__.__name__
|
||||
if provider_name == 'LocalStorageProvider':
|
||||
return self._expired_local_upload_candidates(retention_days)
|
||||
if provider_name == 'S3StorageProvider':
|
||||
return await self._expired_s3_upload_candidates(retention_days)
|
||||
return []
|
||||
|
||||
async def _cleanup_expired_s3_uploaded_files(self, retention_days: int) -> int:
|
||||
provider = self.ap.storage_mgr.storage_provider
|
||||
candidates = await self._expired_s3_upload_candidates(retention_days)
|
||||
deleted = 0
|
||||
for item in candidates:
|
||||
await provider.delete(item['key'])
|
||||
deleted += 1
|
||||
return deleted
|
||||
|
||||
async def _expired_s3_upload_candidates(self, retention_days: int) -> list[dict[str, Any]]:
|
||||
provider = self.ap.storage_mgr.storage_provider
|
||||
cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=retention_days)
|
||||
candidates = []
|
||||
paginator = provider.s3_client.get_paginator('list_objects_v2')
|
||||
|
||||
for page in paginator.paginate(Bucket=provider.bucket_name):
|
||||
for obj in page.get('Contents', []):
|
||||
key = obj.get('Key', '')
|
||||
last_modified = obj.get('LastModified')
|
||||
if not self._is_uploaded_file_key(key):
|
||||
continue
|
||||
if last_modified and last_modified < cutoff:
|
||||
candidates.append(
|
||||
{
|
||||
'key': key,
|
||||
'size_bytes': obj.get('Size', 0),
|
||||
'modified_at': last_modified.isoformat(),
|
||||
}
|
||||
)
|
||||
|
||||
return candidates
|
||||
|
||||
def _cleanup_expired_log_files(self, retention_days: int) -> int:
|
||||
deleted = 0
|
||||
for item in self._expired_log_candidates(retention_days, include_paths=True):
|
||||
try:
|
||||
os.remove(item['path'])
|
||||
deleted += 1
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except Exception as e:
|
||||
self.ap.logger.warning(f'Failed to delete expired log file {item["name"]}: {e}')
|
||||
return deleted
|
||||
|
||||
def _expired_local_upload_candidates(
|
||||
self, retention_days: int, include_paths: bool = False
|
||||
) -> list[dict[str, Any]]:
|
||||
storage_root = Path('data/storage')
|
||||
if not storage_root.exists():
|
||||
return []
|
||||
|
||||
cutoff = datetime.datetime.now().timestamp() - retention_days * 86400
|
||||
candidates = []
|
||||
for entry in storage_root.iterdir():
|
||||
if not entry.is_file() or not self._is_uploaded_file_key(entry.name):
|
||||
continue
|
||||
stat = entry.stat()
|
||||
if stat.st_mtime >= cutoff:
|
||||
continue
|
||||
item = {
|
||||
'key': entry.name,
|
||||
'size_bytes': stat.st_size,
|
||||
'modified_at': datetime.datetime.fromtimestamp(stat.st_mtime, datetime.timezone.utc).isoformat(),
|
||||
}
|
||||
if include_paths:
|
||||
item['path'] = str(entry)
|
||||
candidates.append(item)
|
||||
return candidates
|
||||
|
||||
def _expired_log_candidates(self, retention_days: int, include_paths: bool = False) -> list[dict[str, Any]]:
|
||||
log_root = Path('data/logs')
|
||||
if not log_root.exists():
|
||||
return []
|
||||
|
||||
cutoff_date = datetime.date.today() - datetime.timedelta(days=retention_days - 1)
|
||||
candidates = []
|
||||
for entry in log_root.iterdir():
|
||||
if not entry.is_file():
|
||||
continue
|
||||
match = LOG_FILE_PATTERN.match(entry.name)
|
||||
if not match:
|
||||
continue
|
||||
try:
|
||||
file_date = datetime.date.fromisoformat(match.group(1))
|
||||
except ValueError:
|
||||
continue
|
||||
if file_date >= cutoff_date:
|
||||
continue
|
||||
stat = entry.stat()
|
||||
item = {
|
||||
'name': entry.name,
|
||||
'date': file_date.isoformat(),
|
||||
'size_bytes': stat.st_size,
|
||||
}
|
||||
if include_paths:
|
||||
item['path'] = str(entry)
|
||||
candidates.append(item)
|
||||
return candidates
|
||||
|
||||
def _is_uploaded_file_key(self, key: str) -> bool:
|
||||
return '/' not in key and not key.startswith('plugin_config_')
|
||||
|
||||
async def _monitoring_counts(self) -> dict[str, int]:
|
||||
tables = {
|
||||
'messages': persistence_monitoring.MonitoringMessage.id,
|
||||
'llm_calls': persistence_monitoring.MonitoringLLMCall.id,
|
||||
'embedding_calls': persistence_monitoring.MonitoringEmbeddingCall.id,
|
||||
'errors': persistence_monitoring.MonitoringError.id,
|
||||
'sessions': persistence_monitoring.MonitoringSession.session_id,
|
||||
'feedback': persistence_monitoring.MonitoringFeedback.id,
|
||||
}
|
||||
counts: dict[str, int] = {}
|
||||
for key, column in tables.items():
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(sqlalchemy.func.count(column)))
|
||||
counts[key] = result.scalar() or 0
|
||||
return counts
|
||||
|
||||
async def _binary_storage_stats(self) -> dict[str, Any]:
|
||||
count_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.count(persistence_bstorage.BinaryStorage.unique_key))
|
||||
)
|
||||
size_bytes = None
|
||||
try:
|
||||
size_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.sum(sqlalchemy.func.length(persistence_bstorage.BinaryStorage.value)))
|
||||
)
|
||||
size_bytes = size_result.scalar() or 0
|
||||
except Exception as e:
|
||||
self.ap.logger.warning(f'Failed to estimate binary storage size: {e}')
|
||||
|
||||
return {
|
||||
'count': count_result.scalar() or 0,
|
||||
'size_bytes': size_bytes,
|
||||
}
|
||||
|
||||
def _path_size(self, path: Path) -> int:
|
||||
if not path.exists():
|
||||
return 0
|
||||
if path.is_file():
|
||||
return path.stat().st_size
|
||||
total = 0
|
||||
for root, _, files in os.walk(path):
|
||||
for file_name in files:
|
||||
file_path = Path(root) / file_name
|
||||
try:
|
||||
total += file_path.stat().st_size
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
return total
|
||||
|
||||
def _file_count(self, path: Path) -> int:
|
||||
if not path.exists():
|
||||
return 0
|
||||
if path.is_file():
|
||||
return 1
|
||||
count = 0
|
||||
for _, _, files in os.walk(path):
|
||||
count += len(files)
|
||||
return count
|
||||
|
||||
def _positive_int(self, value: Any, default: int, name: str) -> int:
|
||||
try:
|
||||
parsed = int(value)
|
||||
except (TypeError, ValueError):
|
||||
self.ap.logger.warning(f'Invalid {name}: {value!r}, using {default}')
|
||||
return default
|
||||
if parsed < 1:
|
||||
self.ap.logger.warning(f'Invalid {name}: {value!r}, using {default}')
|
||||
return default
|
||||
return parsed
|
||||
@@ -23,17 +23,6 @@ def _parse_provider_api_keys(provider_dict: dict) -> dict:
|
||||
return provider_dict
|
||||
|
||||
|
||||
def _runtime_model_data(model_uuid: str, model_data: dict) -> dict:
|
||||
"""Return model data for rebuilding runtime models after an update.
|
||||
|
||||
Update payloads intentionally omit uuid before writing to the database.
|
||||
Runtime model entities still need the stable uuid so pipeline configs can
|
||||
resolve the in-memory model immediately after an edit, without requiring a
|
||||
process restart.
|
||||
"""
|
||||
return {**model_data, 'uuid': model_uuid}
|
||||
|
||||
|
||||
class LLMModelsService:
|
||||
ap: app.Application
|
||||
|
||||
@@ -116,16 +105,11 @@ class LLMModelsService:
|
||||
)
|
||||
)
|
||||
pipeline = result.first()
|
||||
if pipeline is not None:
|
||||
model_config = pipeline.config.get('ai', {}).get('local-agent', {}).get('model', {})
|
||||
if not model_config.get('primary', ''):
|
||||
pipeline_config = pipeline.config
|
||||
pipeline_config['ai']['local-agent']['model'] = {
|
||||
'primary': model_data['uuid'],
|
||||
'fallbacks': [],
|
||||
}
|
||||
pipeline_data = {'config': pipeline_config}
|
||||
await self.ap.pipeline_service.update_pipeline(pipeline.uuid, pipeline_data)
|
||||
if pipeline is not None and pipeline.config['ai']['local-agent']['model'] == '':
|
||||
pipeline_config = pipeline.config
|
||||
pipeline_config['ai']['local-agent']['model'] = model_data['uuid']
|
||||
pipeline_data = {'config': pipeline_config}
|
||||
await self.ap.pipeline_service.update_pipeline(pipeline.uuid, pipeline_data)
|
||||
|
||||
return model_data['uuid']
|
||||
|
||||
@@ -184,7 +168,7 @@ class LLMModelsService:
|
||||
raise Exception('provider not found')
|
||||
|
||||
runtime_llm_model = await self.ap.model_mgr.load_llm_model_with_provider(
|
||||
persistence_model.LLMModel(**_runtime_model_data(model_uuid, model_data)),
|
||||
persistence_model.LLMModel(**model_data),
|
||||
runtime_provider,
|
||||
)
|
||||
self.ap.model_mgr.llm_models.append(runtime_llm_model)
|
||||
@@ -345,7 +329,7 @@ class EmbeddingModelsService:
|
||||
raise Exception('provider not found')
|
||||
|
||||
runtime_embedding_model = await self.ap.model_mgr.load_embedding_model_with_provider(
|
||||
persistence_model.EmbeddingModel(**_runtime_model_data(model_uuid, model_data)),
|
||||
persistence_model.EmbeddingModel(**model_data),
|
||||
runtime_provider,
|
||||
)
|
||||
self.ap.model_mgr.embedding_models.append(runtime_embedding_model)
|
||||
@@ -378,162 +362,3 @@ class EmbeddingModelsService:
|
||||
input_text=['Hello, world!'],
|
||||
extra_args={},
|
||||
)
|
||||
|
||||
|
||||
class RerankModelsService:
|
||||
ap: app.Application
|
||||
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
async def get_rerank_models(self) -> list[dict]:
|
||||
"""Get all rerank models with provider info"""
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_model.RerankModel))
|
||||
models = result.all()
|
||||
|
||||
providers_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.ModelProvider)
|
||||
)
|
||||
providers = {p.uuid: p for p in providers_result.all()}
|
||||
|
||||
models_list = []
|
||||
for model in models:
|
||||
model_dict = self.ap.persistence_mgr.serialize_model(persistence_model.RerankModel, model)
|
||||
provider = providers.get(model.provider_uuid)
|
||||
if provider:
|
||||
provider_dict = self.ap.persistence_mgr.serialize_model(persistence_model.ModelProvider, provider)
|
||||
model_dict['provider'] = _parse_provider_api_keys(provider_dict)
|
||||
models_list.append(model_dict)
|
||||
|
||||
return models_list
|
||||
|
||||
async def get_rerank_models_by_provider(self, provider_uuid: str) -> list[dict]:
|
||||
"""Get rerank models by provider UUID"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.RerankModel).where(
|
||||
persistence_model.RerankModel.provider_uuid == provider_uuid
|
||||
)
|
||||
)
|
||||
models = result.all()
|
||||
return [self.ap.persistence_mgr.serialize_model(persistence_model.RerankModel, m) for m in models]
|
||||
|
||||
async def create_rerank_model(self, model_data: dict, preserve_uuid: bool = False) -> str:
|
||||
"""Create a new rerank model"""
|
||||
if not preserve_uuid:
|
||||
model_data['uuid'] = str(uuid.uuid4())
|
||||
|
||||
if 'provider' in model_data:
|
||||
provider_data = model_data.pop('provider')
|
||||
if provider_data.get('uuid'):
|
||||
model_data['provider_uuid'] = provider_data['uuid']
|
||||
else:
|
||||
provider_uuid = await self.ap.provider_service.find_or_create_provider(
|
||||
requester=provider_data.get('requester', ''),
|
||||
base_url=provider_data.get('base_url', ''),
|
||||
api_keys=provider_data.get('api_keys', []),
|
||||
)
|
||||
model_data['provider_uuid'] = provider_uuid
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(persistence_model.RerankModel).values(**model_data)
|
||||
)
|
||||
|
||||
runtime_provider = self.ap.model_mgr.provider_dict.get(model_data['provider_uuid'])
|
||||
if runtime_provider is None:
|
||||
raise Exception('provider not found')
|
||||
|
||||
runtime_rerank_model = await self.ap.model_mgr.load_rerank_model_with_provider(
|
||||
persistence_model.RerankModel(**model_data),
|
||||
runtime_provider,
|
||||
)
|
||||
self.ap.model_mgr.rerank_models.append(runtime_rerank_model)
|
||||
|
||||
return model_data['uuid']
|
||||
|
||||
async def get_rerank_model(self, model_uuid: str) -> dict | None:
|
||||
"""Get a single rerank model with provider info"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.RerankModel).where(persistence_model.RerankModel.uuid == model_uuid)
|
||||
)
|
||||
model = result.first()
|
||||
if model is None:
|
||||
return None
|
||||
|
||||
model_dict = self.ap.persistence_mgr.serialize_model(persistence_model.RerankModel, model)
|
||||
|
||||
provider_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.uuid == model.provider_uuid
|
||||
)
|
||||
)
|
||||
provider = provider_result.first()
|
||||
if provider:
|
||||
provider_dict = self.ap.persistence_mgr.serialize_model(persistence_model.ModelProvider, provider)
|
||||
model_dict['provider'] = _parse_provider_api_keys(provider_dict)
|
||||
|
||||
return model_dict
|
||||
|
||||
async def update_rerank_model(self, model_uuid: str, model_data: dict) -> None:
|
||||
"""Update an existing rerank model"""
|
||||
if 'uuid' in model_data:
|
||||
del model_data['uuid']
|
||||
|
||||
if 'provider' in model_data:
|
||||
provider_data = model_data.pop('provider')
|
||||
if provider_data.get('uuid'):
|
||||
model_data['provider_uuid'] = provider_data['uuid']
|
||||
else:
|
||||
provider_uuid = await self.ap.provider_service.find_or_create_provider(
|
||||
requester=provider_data.get('requester', ''),
|
||||
base_url=provider_data.get('base_url', ''),
|
||||
api_keys=provider_data.get('api_keys', []),
|
||||
)
|
||||
model_data['provider_uuid'] = provider_uuid
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_model.RerankModel)
|
||||
.where(persistence_model.RerankModel.uuid == model_uuid)
|
||||
.values(**model_data)
|
||||
)
|
||||
|
||||
await self.ap.model_mgr.remove_rerank_model(model_uuid)
|
||||
|
||||
runtime_provider = self.ap.model_mgr.provider_dict.get(model_data['provider_uuid'])
|
||||
if runtime_provider is None:
|
||||
raise Exception('provider not found')
|
||||
|
||||
runtime_rerank_model = await self.ap.model_mgr.load_rerank_model_with_provider(
|
||||
persistence_model.RerankModel(**_runtime_model_data(model_uuid, model_data)),
|
||||
runtime_provider,
|
||||
)
|
||||
self.ap.model_mgr.rerank_models.append(runtime_rerank_model)
|
||||
|
||||
async def delete_rerank_model(self, model_uuid: str) -> None:
|
||||
"""Delete a rerank model"""
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_model.RerankModel).where(persistence_model.RerankModel.uuid == model_uuid)
|
||||
)
|
||||
await self.ap.model_mgr.remove_rerank_model(model_uuid)
|
||||
|
||||
async def test_rerank_model(self, model_uuid: str, model_data: dict) -> None:
|
||||
"""Test a rerank model"""
|
||||
runtime_rerank_model: model_requester.RuntimeRerankModel | None = None
|
||||
|
||||
if model_uuid != '_':
|
||||
for model in self.ap.model_mgr.rerank_models:
|
||||
if model.model_entity.uuid == model_uuid:
|
||||
runtime_rerank_model = model
|
||||
break
|
||||
if runtime_rerank_model is None:
|
||||
raise Exception('model not found')
|
||||
else:
|
||||
runtime_rerank_model = await self.ap.model_mgr.init_temporary_runtime_rerank_model(model_data)
|
||||
|
||||
await runtime_rerank_model.provider.invoke_rerank(
|
||||
model=runtime_rerank_model,
|
||||
query='What is artificial intelligence?',
|
||||
documents=[
|
||||
'Artificial intelligence is a branch of computer science.',
|
||||
'The weather is nice today.',
|
||||
],
|
||||
)
|
||||
|
||||
@@ -16,121 +16,6 @@ class MonitoringService:
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
# ========== Cleanup Methods ==========
|
||||
|
||||
async def cleanup_expired_records(self, retention_days: int, batch_size: int = 1000) -> dict[str, int]:
|
||||
"""Delete monitoring records older than the specified retention period.
|
||||
|
||||
Args:
|
||||
retention_days: Number of days to retain records.
|
||||
batch_size: Maximum rows to delete per table batch.
|
||||
|
||||
Returns:
|
||||
A dict mapping table name to the number of deleted rows.
|
||||
"""
|
||||
if retention_days < 1:
|
||||
raise ValueError('retention_days must be >= 1')
|
||||
if batch_size < 1:
|
||||
raise ValueError('batch_size must be >= 1')
|
||||
|
||||
cutoff = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) - datetime.timedelta(
|
||||
days=retention_days
|
||||
)
|
||||
|
||||
tables_and_columns: list[tuple[str, type, sqlalchemy.Column, sqlalchemy.Column]] = [
|
||||
(
|
||||
'monitoring_messages',
|
||||
persistence_monitoring.MonitoringMessage,
|
||||
persistence_monitoring.MonitoringMessage.timestamp,
|
||||
persistence_monitoring.MonitoringMessage.id,
|
||||
),
|
||||
(
|
||||
'monitoring_llm_calls',
|
||||
persistence_monitoring.MonitoringLLMCall,
|
||||
persistence_monitoring.MonitoringLLMCall.timestamp,
|
||||
persistence_monitoring.MonitoringLLMCall.id,
|
||||
),
|
||||
(
|
||||
'monitoring_embedding_calls',
|
||||
persistence_monitoring.MonitoringEmbeddingCall,
|
||||
persistence_monitoring.MonitoringEmbeddingCall.timestamp,
|
||||
persistence_monitoring.MonitoringEmbeddingCall.id,
|
||||
),
|
||||
(
|
||||
'monitoring_errors',
|
||||
persistence_monitoring.MonitoringError,
|
||||
persistence_monitoring.MonitoringError.timestamp,
|
||||
persistence_monitoring.MonitoringError.id,
|
||||
),
|
||||
(
|
||||
'monitoring_sessions',
|
||||
persistence_monitoring.MonitoringSession,
|
||||
persistence_monitoring.MonitoringSession.last_activity,
|
||||
persistence_monitoring.MonitoringSession.session_id,
|
||||
),
|
||||
(
|
||||
'monitoring_feedback',
|
||||
persistence_monitoring.MonitoringFeedback,
|
||||
persistence_monitoring.MonitoringFeedback.timestamp,
|
||||
persistence_monitoring.MonitoringFeedback.id,
|
||||
),
|
||||
]
|
||||
|
||||
deleted_counts: dict[str, int] = {}
|
||||
|
||||
for table_name, model_cls, ts_column, pk_column in tables_and_columns:
|
||||
deleted_counts[table_name] = await self._delete_expired_in_batches(
|
||||
model_cls=model_cls,
|
||||
ts_column=ts_column,
|
||||
pk_column=pk_column,
|
||||
cutoff=cutoff,
|
||||
batch_size=batch_size,
|
||||
)
|
||||
|
||||
if sum(deleted_counts.values()) > 0:
|
||||
await self._release_sqlite_space()
|
||||
|
||||
return deleted_counts
|
||||
|
||||
async def _delete_expired_in_batches(
|
||||
self,
|
||||
model_cls: type,
|
||||
ts_column: sqlalchemy.Column,
|
||||
pk_column: sqlalchemy.Column,
|
||||
cutoff: datetime.datetime,
|
||||
batch_size: int,
|
||||
) -> int:
|
||||
deleted_total = 0
|
||||
|
||||
while True:
|
||||
select_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(pk_column).where(ts_column < cutoff).limit(batch_size)
|
||||
)
|
||||
pk_values = list(select_result.scalars().all())
|
||||
if not pk_values:
|
||||
break
|
||||
|
||||
delete_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(model_cls).where(pk_column.in_(pk_values))
|
||||
)
|
||||
deleted = delete_result.rowcount or 0
|
||||
deleted_total += deleted
|
||||
|
||||
if len(pk_values) < batch_size:
|
||||
break
|
||||
|
||||
return deleted_total
|
||||
|
||||
async def _release_sqlite_space(self) -> None:
|
||||
database_type = self.ap.instance_config.data.get('database', {}).get('use', 'sqlite')
|
||||
if database_type != 'sqlite':
|
||||
return
|
||||
|
||||
async with self.ap.persistence_mgr.get_db_engine().connect() as conn:
|
||||
autocommit_conn = await conn.execution_options(isolation_level='AUTOCOMMIT')
|
||||
await autocommit_conn.execute(sqlalchemy.text('PRAGMA wal_checkpoint(TRUNCATE)'))
|
||||
await autocommit_conn.execute(sqlalchemy.text('VACUUM'))
|
||||
|
||||
# ========== Recording Methods ==========
|
||||
|
||||
async def record_message(
|
||||
@@ -145,7 +30,6 @@ class MonitoringService:
|
||||
level: str = 'info',
|
||||
platform: str | None = None,
|
||||
user_id: str | None = None,
|
||||
user_name: str | None = None,
|
||||
runner_name: str | None = None,
|
||||
variables: str | None = None,
|
||||
role: str = 'user',
|
||||
@@ -165,7 +49,6 @@ class MonitoringService:
|
||||
'level': level,
|
||||
'platform': platform,
|
||||
'user_id': user_id,
|
||||
'user_name': user_name,
|
||||
'runner_name': runner_name,
|
||||
'variables': variables,
|
||||
'role': role,
|
||||
@@ -269,7 +152,6 @@ class MonitoringService:
|
||||
pipeline_name: str,
|
||||
platform: str | None = None,
|
||||
user_id: str | None = None,
|
||||
user_name: str | None = None,
|
||||
) -> None:
|
||||
"""Record a new session"""
|
||||
session_data = {
|
||||
@@ -284,7 +166,6 @@ class MonitoringService:
|
||||
'is_active': True,
|
||||
'platform': platform,
|
||||
'user_id': user_id,
|
||||
'user_name': user_name,
|
||||
}
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
@@ -1247,314 +1128,3 @@ class MonitoringService:
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
|
||||
# ========== Feedback Methods ==========
|
||||
|
||||
async def record_feedback(
|
||||
self,
|
||||
feedback_id: str,
|
||||
feedback_type: int,
|
||||
feedback_content: str | None = None,
|
||||
inaccurate_reasons: list[str] | None = None,
|
||||
bot_id: str | None = None,
|
||||
bot_name: str | None = None,
|
||||
pipeline_id: str | None = None,
|
||||
pipeline_name: str | None = None,
|
||||
session_id: str | None = None,
|
||||
message_id: str | None = None,
|
||||
stream_id: str | None = None,
|
||||
user_id: str | None = None,
|
||||
platform: str | None = None,
|
||||
) -> str:
|
||||
"""Record user feedback (like/dislike) from AI Bot conversation.
|
||||
|
||||
Args:
|
||||
feedback_id: Unique feedback identifier from platform (e.g., WeChat Work)
|
||||
feedback_type: 1 = like (thumbs up), 2 = dislike (thumbs down)
|
||||
feedback_content: Optional user feedback text
|
||||
inaccurate_reasons: List of reasons for inaccurate response (for dislike)
|
||||
bot_id: Bot ID
|
||||
bot_name: Bot name
|
||||
pipeline_id: Pipeline ID
|
||||
pipeline_name: Pipeline name
|
||||
session_id: Session ID
|
||||
message_id: Message ID
|
||||
stream_id: Stream ID (for WeChat Work streaming messages)
|
||||
user_id: User ID
|
||||
platform: Platform name (e.g., 'wecom')
|
||||
|
||||
Returns:
|
||||
The record ID
|
||||
"""
|
||||
import json
|
||||
|
||||
now = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
|
||||
reasons_json = json.dumps(inaccurate_reasons, ensure_ascii=False) if inaccurate_reasons else None
|
||||
|
||||
MonitoringFeedback = persistence_monitoring.MonitoringFeedback
|
||||
|
||||
# Handle cancel feedback (type=3): delete existing record
|
||||
if feedback_type == 3:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(MonitoringFeedback).where(MonitoringFeedback.feedback_id == feedback_id)
|
||||
)
|
||||
return None
|
||||
|
||||
# Check if record with this feedback_id already exists
|
||||
existing_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(MonitoringFeedback).where(MonitoringFeedback.feedback_id == feedback_id)
|
||||
)
|
||||
existing_row = existing_result.first()
|
||||
|
||||
if existing_row:
|
||||
# UPDATE existing record
|
||||
existing = existing_row[0] if isinstance(existing_row, tuple) else existing_row
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(MonitoringFeedback)
|
||||
.where(MonitoringFeedback.feedback_id == feedback_id)
|
||||
.values(
|
||||
timestamp=now,
|
||||
feedback_type=feedback_type,
|
||||
feedback_content=feedback_content,
|
||||
inaccurate_reasons=reasons_json,
|
||||
bot_id=bot_id or existing.bot_id,
|
||||
bot_name=bot_name or existing.bot_name,
|
||||
pipeline_id=pipeline_id or existing.pipeline_id,
|
||||
pipeline_name=pipeline_name or existing.pipeline_name,
|
||||
session_id=session_id or existing.session_id,
|
||||
message_id=message_id or existing.message_id,
|
||||
stream_id=stream_id or existing.stream_id,
|
||||
user_id=user_id or existing.user_id,
|
||||
platform=platform or existing.platform,
|
||||
)
|
||||
)
|
||||
return existing.id
|
||||
else:
|
||||
# INSERT new record with IntegrityError defense
|
||||
record_id = str(uuid.uuid4())
|
||||
record_data = {
|
||||
'id': record_id,
|
||||
'timestamp': now,
|
||||
'feedback_id': feedback_id,
|
||||
'feedback_type': feedback_type,
|
||||
'feedback_content': feedback_content,
|
||||
'inaccurate_reasons': reasons_json,
|
||||
'bot_id': bot_id,
|
||||
'bot_name': bot_name,
|
||||
'pipeline_id': pipeline_id,
|
||||
'pipeline_name': pipeline_name,
|
||||
'session_id': session_id,
|
||||
'message_id': message_id,
|
||||
'stream_id': stream_id,
|
||||
'user_id': user_id,
|
||||
'platform': platform,
|
||||
}
|
||||
try:
|
||||
await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(MonitoringFeedback).values(record_data))
|
||||
return record_id
|
||||
except Exception:
|
||||
# UNIQUE constraint conflict (concurrent feedback for same feedback_id)
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(MonitoringFeedback)
|
||||
.where(MonitoringFeedback.feedback_id == feedback_id)
|
||||
.values(
|
||||
timestamp=now,
|
||||
feedback_type=feedback_type,
|
||||
feedback_content=feedback_content,
|
||||
inaccurate_reasons=reasons_json,
|
||||
)
|
||||
)
|
||||
return feedback_id
|
||||
|
||||
async def get_feedback_stats(
|
||||
self,
|
||||
bot_ids: list[str] | None = None,
|
||||
pipeline_ids: list[str] | None = None,
|
||||
start_time: datetime.datetime | None = None,
|
||||
end_time: datetime.datetime | None = None,
|
||||
) -> dict:
|
||||
"""Get feedback statistics.
|
||||
|
||||
Returns:
|
||||
Dictionary with total likes, dislikes, and breakdown by bot/pipeline
|
||||
"""
|
||||
conditions = []
|
||||
|
||||
if bot_ids:
|
||||
conditions.append(persistence_monitoring.MonitoringFeedback.bot_id.in_(bot_ids))
|
||||
if pipeline_ids:
|
||||
conditions.append(persistence_monitoring.MonitoringFeedback.pipeline_id.in_(pipeline_ids))
|
||||
if start_time:
|
||||
conditions.append(persistence_monitoring.MonitoringFeedback.timestamp >= start_time)
|
||||
if end_time:
|
||||
conditions.append(persistence_monitoring.MonitoringFeedback.timestamp <= end_time)
|
||||
|
||||
# Get total likes (feedback_type = 1)
|
||||
likes_query = sqlalchemy.select(sqlalchemy.func.count(persistence_monitoring.MonitoringFeedback.id)).where(
|
||||
persistence_monitoring.MonitoringFeedback.feedback_type == 1
|
||||
)
|
||||
if conditions:
|
||||
likes_query = likes_query.where(sqlalchemy.and_(*conditions))
|
||||
likes_result = await self.ap.persistence_mgr.execute_async(likes_query)
|
||||
total_likes = likes_result.scalar() or 0
|
||||
|
||||
# Get total dislikes (feedback_type = 2)
|
||||
dislikes_query = sqlalchemy.select(sqlalchemy.func.count(persistence_monitoring.MonitoringFeedback.id)).where(
|
||||
persistence_monitoring.MonitoringFeedback.feedback_type == 2
|
||||
)
|
||||
if conditions:
|
||||
dislikes_query = dislikes_query.where(sqlalchemy.and_(*conditions))
|
||||
dislikes_result = await self.ap.persistence_mgr.execute_async(dislikes_query)
|
||||
total_dislikes = dislikes_result.scalar() or 0
|
||||
|
||||
# Get total feedback count
|
||||
total_query = sqlalchemy.select(sqlalchemy.func.count(persistence_monitoring.MonitoringFeedback.id))
|
||||
if conditions:
|
||||
total_query = total_query.where(sqlalchemy.and_(*conditions))
|
||||
total_result = await self.ap.persistence_mgr.execute_async(total_query)
|
||||
total_feedback = total_result.scalar() or 0
|
||||
|
||||
# Calculate satisfaction rate
|
||||
satisfaction_rate = (total_likes / total_feedback * 100) if total_feedback > 0 else 0
|
||||
|
||||
# Get feedback by bot
|
||||
bot_stats_query = sqlalchemy.select(
|
||||
persistence_monitoring.MonitoringFeedback.bot_id,
|
||||
persistence_monitoring.MonitoringFeedback.bot_name,
|
||||
sqlalchemy.func.count(persistence_monitoring.MonitoringFeedback.id).label('total'),
|
||||
sqlalchemy.func.sum(
|
||||
sqlalchemy.case((persistence_monitoring.MonitoringFeedback.feedback_type == 1, 1), else_=0)
|
||||
).label('likes'),
|
||||
sqlalchemy.func.sum(
|
||||
sqlalchemy.case((persistence_monitoring.MonitoringFeedback.feedback_type == 2, 1), else_=0)
|
||||
).label('dislikes'),
|
||||
).group_by(
|
||||
persistence_monitoring.MonitoringFeedback.bot_id,
|
||||
persistence_monitoring.MonitoringFeedback.bot_name,
|
||||
)
|
||||
if conditions:
|
||||
bot_stats_query = bot_stats_query.where(sqlalchemy.and_(*conditions))
|
||||
bot_stats_result = await self.ap.persistence_mgr.execute_async(bot_stats_query)
|
||||
bot_stats = [
|
||||
{
|
||||
'bot_id': row.bot_id,
|
||||
'bot_name': row.bot_name,
|
||||
'total': row.total,
|
||||
'likes': row.likes or 0,
|
||||
'dislikes': row.dislikes or 0,
|
||||
}
|
||||
for row in bot_stats_result.all()
|
||||
]
|
||||
|
||||
return {
|
||||
'total_feedback': total_feedback,
|
||||
'total_likes': total_likes,
|
||||
'total_dislikes': total_dislikes,
|
||||
'satisfaction_rate': round(satisfaction_rate, 2),
|
||||
'by_bot': bot_stats,
|
||||
}
|
||||
|
||||
async def get_feedback_list(
|
||||
self,
|
||||
bot_ids: list[str] | None = None,
|
||||
pipeline_ids: list[str] | None = None,
|
||||
feedback_type: int | None = None,
|
||||
start_time: datetime.datetime | None = None,
|
||||
end_time: datetime.datetime | None = None,
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
) -> tuple[list[dict], int]:
|
||||
"""Get feedback list with filters."""
|
||||
conditions = []
|
||||
|
||||
if bot_ids:
|
||||
conditions.append(persistence_monitoring.MonitoringFeedback.bot_id.in_(bot_ids))
|
||||
if pipeline_ids:
|
||||
conditions.append(persistence_monitoring.MonitoringFeedback.pipeline_id.in_(pipeline_ids))
|
||||
if feedback_type is not None:
|
||||
conditions.append(persistence_monitoring.MonitoringFeedback.feedback_type == feedback_type)
|
||||
if start_time:
|
||||
conditions.append(persistence_monitoring.MonitoringFeedback.timestamp >= start_time)
|
||||
if end_time:
|
||||
conditions.append(persistence_monitoring.MonitoringFeedback.timestamp <= end_time)
|
||||
|
||||
# Get total count
|
||||
count_query = sqlalchemy.select(sqlalchemy.func.count(persistence_monitoring.MonitoringFeedback.id))
|
||||
if conditions:
|
||||
count_query = count_query.where(sqlalchemy.and_(*conditions))
|
||||
count_result = await self.ap.persistence_mgr.execute_async(count_query)
|
||||
total = count_result.scalar() or 0
|
||||
|
||||
# Get feedback list
|
||||
query = sqlalchemy.select(persistence_monitoring.MonitoringFeedback).order_by(
|
||||
persistence_monitoring.MonitoringFeedback.timestamp.desc()
|
||||
)
|
||||
if conditions:
|
||||
query = query.where(sqlalchemy.and_(*conditions))
|
||||
query = query.limit(limit).offset(offset)
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(query)
|
||||
rows = result.all()
|
||||
|
||||
return (
|
||||
[
|
||||
self.ap.persistence_mgr.serialize_model(
|
||||
persistence_monitoring.MonitoringFeedback, row[0] if isinstance(row, tuple) else row
|
||||
)
|
||||
for row in rows
|
||||
],
|
||||
total,
|
||||
)
|
||||
|
||||
async def export_feedback(
|
||||
self,
|
||||
bot_ids: list[str] | None = None,
|
||||
pipeline_ids: list[str] | None = None,
|
||||
start_time: datetime.datetime | None = None,
|
||||
end_time: datetime.datetime | None = None,
|
||||
limit: int = 100000,
|
||||
) -> list[dict]:
|
||||
"""Export feedback as list of dictionaries for CSV conversion."""
|
||||
conditions = []
|
||||
|
||||
if bot_ids:
|
||||
conditions.append(persistence_monitoring.MonitoringFeedback.bot_id.in_(bot_ids))
|
||||
if pipeline_ids:
|
||||
conditions.append(persistence_monitoring.MonitoringFeedback.pipeline_id.in_(pipeline_ids))
|
||||
if start_time:
|
||||
conditions.append(persistence_monitoring.MonitoringFeedback.timestamp >= start_time)
|
||||
if end_time:
|
||||
conditions.append(persistence_monitoring.MonitoringFeedback.timestamp <= end_time)
|
||||
|
||||
query = sqlalchemy.select(persistence_monitoring.MonitoringFeedback).order_by(
|
||||
persistence_monitoring.MonitoringFeedback.timestamp.desc()
|
||||
)
|
||||
if conditions:
|
||||
query = query.where(sqlalchemy.and_(*conditions))
|
||||
query = query.limit(limit)
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(query)
|
||||
rows = result.all()
|
||||
|
||||
return [
|
||||
{
|
||||
'id': row[0].id if isinstance(row, tuple) else row.id,
|
||||
'timestamp': self._format_timestamp(row[0].timestamp if isinstance(row, tuple) else row.timestamp),
|
||||
'feedback_id': row[0].feedback_id if isinstance(row, tuple) else row.feedback_id,
|
||||
'feedback_type': 'like'
|
||||
if (row[0].feedback_type if isinstance(row, tuple) else row.feedback_type) == 1
|
||||
else 'dislike',
|
||||
'feedback_content': row[0].feedback_content if isinstance(row, tuple) else row.feedback_content,
|
||||
'inaccurate_reasons': row[0].inaccurate_reasons if isinstance(row, tuple) else row.inaccurate_reasons,
|
||||
'bot_id': row[0].bot_id if isinstance(row, tuple) else row.bot_id,
|
||||
'bot_name': row[0].bot_name if isinstance(row, tuple) else row.bot_name,
|
||||
'pipeline_id': row[0].pipeline_id if isinstance(row, tuple) else row.pipeline_id,
|
||||
'pipeline_name': row[0].pipeline_name if isinstance(row, tuple) else row.pipeline_name,
|
||||
'session_id': row[0].session_id if isinstance(row, tuple) else row.session_id,
|
||||
'message_id': row[0].message_id if isinstance(row, tuple) else row.message_id,
|
||||
'stream_id': row[0].stream_id if isinstance(row, tuple) else row.stream_id,
|
||||
'user_id': row[0].user_id if isinstance(row, tuple) else row.user_id,
|
||||
'platform': row[0].platform if isinstance(row, tuple) else row.platform,
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
|
||||
@@ -113,9 +113,14 @@ class PipelineService:
|
||||
return pipeline_data['uuid']
|
||||
|
||||
async def update_pipeline(self, pipeline_uuid: str, pipeline_data: dict) -> None:
|
||||
pipeline_data = pipeline_data.copy()
|
||||
for protected_field in ('uuid', 'for_version', 'stages', 'is_default'):
|
||||
pipeline_data.pop(protected_field, None)
|
||||
if 'uuid' in pipeline_data:
|
||||
del pipeline_data['uuid']
|
||||
if 'for_version' in pipeline_data:
|
||||
del pipeline_data['for_version']
|
||||
if 'stages' in pipeline_data:
|
||||
del pipeline_data['stages']
|
||||
if 'is_default' in pipeline_data:
|
||||
del pipeline_data['is_default']
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_pipeline.LegacyPipeline)
|
||||
@@ -215,8 +220,6 @@ class PipelineService:
|
||||
bound_mcp_servers: list[str] = None,
|
||||
enable_all_plugins: bool = True,
|
||||
enable_all_mcp_servers: bool = True,
|
||||
bound_skills: list[str] = None,
|
||||
enable_all_skills: bool = True,
|
||||
) -> None:
|
||||
"""Update the bound plugins and MCP servers for a pipeline"""
|
||||
# Get current pipeline
|
||||
@@ -234,12 +237,9 @@ class PipelineService:
|
||||
extensions_preferences = pipeline.extensions_preferences or {}
|
||||
extensions_preferences['enable_all_plugins'] = enable_all_plugins
|
||||
extensions_preferences['enable_all_mcp_servers'] = enable_all_mcp_servers
|
||||
extensions_preferences['enable_all_skills'] = enable_all_skills
|
||||
extensions_preferences['plugins'] = bound_plugins
|
||||
if bound_mcp_servers is not None:
|
||||
extensions_preferences['mcp_servers'] = bound_mcp_servers
|
||||
if bound_skills is not None:
|
||||
extensions_preferences['skills'] = bound_skills
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_pipeline.LegacyPipeline)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
import traceback
|
||||
|
||||
import sqlalchemy
|
||||
|
||||
@@ -17,24 +16,6 @@ class ModelProviderService:
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
@staticmethod
|
||||
def _normalize_api_keys(api_keys: str | list[str] | tuple[str, ...] | None) -> list[str]:
|
||||
if api_keys is None:
|
||||
return []
|
||||
|
||||
raw_keys = [api_keys] if isinstance(api_keys, str) else list(api_keys)
|
||||
normalized_keys = []
|
||||
seen_keys = set()
|
||||
|
||||
for raw_key in raw_keys:
|
||||
normalized_key = raw_key.strip() if isinstance(raw_key, str) else ''
|
||||
if not normalized_key or normalized_key in seen_keys:
|
||||
continue
|
||||
normalized_keys.append(normalized_key)
|
||||
seen_keys.add(normalized_key)
|
||||
|
||||
return normalized_keys
|
||||
|
||||
async def get_providers(self) -> list[dict]:
|
||||
"""Get all providers"""
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_model.ModelProvider))
|
||||
@@ -77,7 +58,6 @@ class ModelProviderService:
|
||||
async def create_provider(self, provider_data: dict) -> str:
|
||||
"""Create a new provider"""
|
||||
provider_data['uuid'] = str(uuid.uuid4())
|
||||
provider_data['api_keys'] = self._normalize_api_keys(provider_data.get('api_keys'))
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(persistence_model.ModelProvider).values(**provider_data)
|
||||
)
|
||||
@@ -91,8 +71,6 @@ class ModelProviderService:
|
||||
"""Update an existing provider"""
|
||||
if 'uuid' in provider_data:
|
||||
del provider_data['uuid']
|
||||
if 'api_keys' in provider_data:
|
||||
provider_data['api_keys'] = self._normalize_api_keys(provider_data.get('api_keys'))
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_model.ModelProvider)
|
||||
.where(persistence_model.ModelProvider.uuid == provider_uuid)
|
||||
@@ -119,14 +97,6 @@ class ModelProviderService:
|
||||
if embedding_result.first() is not None:
|
||||
raise ValueError('Cannot delete provider: Embedding models still reference it')
|
||||
|
||||
rerank_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.RerankModel).where(
|
||||
persistence_model.RerankModel.provider_uuid == provider_uuid
|
||||
)
|
||||
)
|
||||
if rerank_result.first() is not None:
|
||||
raise ValueError('Cannot delete provider: Rerank models still reference it')
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.uuid == provider_uuid
|
||||
@@ -151,19 +121,10 @@ class ModelProviderService:
|
||||
)
|
||||
embedding_count = embedding_result.scalar() or 0
|
||||
|
||||
rerank_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.count())
|
||||
.select_from(persistence_model.RerankModel)
|
||||
.where(persistence_model.RerankModel.provider_uuid == provider_uuid)
|
||||
)
|
||||
rerank_count = rerank_result.scalar() or 0
|
||||
|
||||
return {'llm_count': llm_count, 'embedding_count': embedding_count, 'rerank_count': rerank_count}
|
||||
return {'llm_count': llm_count, 'embedding_count': embedding_count}
|
||||
|
||||
async def find_or_create_provider(self, requester: str, base_url: str, api_keys: list) -> str:
|
||||
"""Find existing provider or create new one"""
|
||||
api_keys = self._normalize_api_keys(api_keys)
|
||||
|
||||
# Try to find existing provider with same config
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.ModelProvider).where(
|
||||
@@ -191,7 +152,7 @@ class ModelProviderService:
|
||||
'name': provider_name,
|
||||
'requester': requester,
|
||||
'base_url': base_url,
|
||||
'api_keys': api_keys,
|
||||
'api_keys': api_keys or [],
|
||||
}
|
||||
)
|
||||
|
||||
@@ -200,69 +161,6 @@ class ModelProviderService:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_model.ModelProvider)
|
||||
.where(persistence_model.ModelProvider.uuid == '00000000-0000-0000-0000-000000000000')
|
||||
.values(api_keys=self._normalize_api_keys(api_key))
|
||||
.values(api_keys=[api_key])
|
||||
)
|
||||
await self.ap.model_mgr.reload_provider('00000000-0000-0000-0000-000000000000')
|
||||
|
||||
async def scan_provider_models(self, provider_uuid: str, model_type: str | None = None) -> dict:
|
||||
provider = await self.get_provider(provider_uuid)
|
||||
if provider is None:
|
||||
raise ValueError('provider not found')
|
||||
|
||||
runtime_provider = await self.ap.model_mgr.load_provider(provider)
|
||||
|
||||
try:
|
||||
scan_result = await runtime_provider.requester.scan_models(
|
||||
runtime_provider.token_mgr.get_token() if runtime_provider.token_mgr.tokens else None
|
||||
)
|
||||
except NotImplementedError:
|
||||
raise ValueError('current provider does not support model scanning')
|
||||
except Exception as exc:
|
||||
self.ap.logger.warning(
|
||||
f'Failed to scan models for provider {provider_uuid}: {exc}\n{traceback.format_exc()}'
|
||||
)
|
||||
raise ValueError(str(exc)) from exc
|
||||
|
||||
if isinstance(scan_result, dict):
|
||||
scanned_models = scan_result.get('models', [])
|
||||
debug_info = scan_result.get('debug')
|
||||
else:
|
||||
scanned_models = scan_result
|
||||
debug_info = None
|
||||
|
||||
llm_models = await self.ap.llm_model_service.get_llm_models_by_provider(provider_uuid)
|
||||
embedding_models = await self.ap.embedding_models_service.get_embedding_models_by_provider(provider_uuid)
|
||||
existing_llm_names = {model['name'] for model in llm_models}
|
||||
existing_embedding_names = {model['name'] for model in embedding_models}
|
||||
|
||||
filtered_models = []
|
||||
for model in scanned_models:
|
||||
scanned_type = model.get('type', 'llm')
|
||||
if model_type and scanned_type != model_type:
|
||||
continue
|
||||
|
||||
model_name = model.get('name') or model.get('id')
|
||||
if not model_name:
|
||||
continue
|
||||
|
||||
filtered_models.append(
|
||||
{
|
||||
'id': model.get('id', model_name),
|
||||
'name': model_name,
|
||||
'type': scanned_type,
|
||||
'abilities': model.get('abilities', []),
|
||||
'display_name': model.get('display_name'),
|
||||
'description': model.get('description'),
|
||||
'context_length': model.get('context_length'),
|
||||
'owned_by': model.get('owned_by'),
|
||||
'input_modalities': model.get('input_modalities', []),
|
||||
'output_modalities': model.get('output_modalities', []),
|
||||
'already_added': (
|
||||
model_name in existing_embedding_names
|
||||
if scanned_type == 'embedding'
|
||||
else model_name in existing_llm_names
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
return {'models': filtered_models, 'debug': debug_info}
|
||||
|
||||
@@ -1,428 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import inspect
|
||||
import os
|
||||
import posixpath
|
||||
import zipfile
|
||||
from typing import Optional
|
||||
from urllib.parse import quote, unquote, urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from ....core import app
|
||||
from ....skill.utils import parse_frontmatter
|
||||
|
||||
|
||||
_PUBLIC_SKILL_FIELDS = (
|
||||
'name',
|
||||
'display_name',
|
||||
'description',
|
||||
'instructions',
|
||||
'package_root',
|
||||
'created_at',
|
||||
'updated_at',
|
||||
)
|
||||
|
||||
_GITHUB_ASSET_HOSTS = {
|
||||
'github.com',
|
||||
'api.github.com',
|
||||
'objects.githubusercontent.com',
|
||||
'githubusercontent.com',
|
||||
'raw.githubusercontent.com',
|
||||
'codeload.github.com',
|
||||
}
|
||||
|
||||
|
||||
class SkillService:
|
||||
"""Filesystem-backed skill management service."""
|
||||
|
||||
ap: app.Application
|
||||
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
def _box_service(self):
|
||||
box_service = getattr(self.ap, 'box_service', None)
|
||||
if box_service is not None and getattr(box_service, 'available', False):
|
||||
return box_service
|
||||
return None
|
||||
|
||||
def _require_box(self, action: str):
|
||||
"""Return the Box service or raise if it is not available.
|
||||
|
||||
Box is the only source of truth for skills. Every read and write
|
||||
operation goes through it — there is no local-filesystem fallback.
|
||||
"""
|
||||
box_service = self._box_service()
|
||||
if box_service is not None:
|
||||
return box_service
|
||||
ap_box = getattr(self.ap, 'box_service', None)
|
||||
if ap_box is None:
|
||||
reason = 'not initialised'
|
||||
elif not getattr(ap_box, 'enabled', True):
|
||||
reason = 'disabled in config (box.enabled = false)'
|
||||
else:
|
||||
connector_error = getattr(ap_box, '_connector_error', '') or 'currently unavailable'
|
||||
reason = f'unavailable: {connector_error}'
|
||||
raise ValueError(
|
||||
f'{action} requires the Box runtime, which is {reason}. '
|
||||
f'Enable Box in config.yaml (box.enabled = true) and ensure the '
|
||||
f'runtime is reachable before retrying.'
|
||||
)
|
||||
|
||||
def _require_box_for_write(self, action: str) -> None:
|
||||
"""Backwards-compatible alias preserved for clarity at call sites."""
|
||||
self._require_box(action)
|
||||
|
||||
@staticmethod
|
||||
def _serialize_skill(skill: dict) -> dict:
|
||||
return {field: skill.get(field) for field in _PUBLIC_SKILL_FIELDS if field in skill}
|
||||
|
||||
async def list_skills(self) -> list[dict]:
|
||||
# When Box is unavailable, surface an empty list rather than raising —
|
||||
# the skills page should render cleanly, and the UI separately renders
|
||||
# a "Box disabled / unavailable" banner via useBoxStatus.
|
||||
box_service = self._box_service()
|
||||
if box_service is None:
|
||||
return []
|
||||
return [self._serialize_skill(skill) for skill in await box_service.list_skills()]
|
||||
|
||||
async def get_skill(self, skill_name: str) -> Optional[dict]:
|
||||
box_service = self._box_service()
|
||||
if box_service is None:
|
||||
return None
|
||||
skill = await box_service.get_skill(skill_name)
|
||||
return self._serialize_skill(skill) if skill else None
|
||||
|
||||
async def get_skill_by_name(self, name: str) -> Optional[dict]:
|
||||
return await self.get_skill(name)
|
||||
|
||||
async def create_skill(self, data: dict) -> dict:
|
||||
box_service = self._require_box('Creating a skill')
|
||||
created = await box_service.create_skill(data)
|
||||
await self._reload_skills()
|
||||
return self._serialize_skill(created)
|
||||
|
||||
async def update_skill(self, skill_name: str, data: dict) -> dict:
|
||||
box_service = self._require_box('Editing a skill')
|
||||
updated = await box_service.update_skill(skill_name, data)
|
||||
await self._reload_skills()
|
||||
return self._serialize_skill(updated)
|
||||
|
||||
async def delete_skill(self, skill_name: str) -> bool:
|
||||
box_service = self._require_box('Deleting a skill')
|
||||
await box_service.delete_skill(skill_name)
|
||||
await self._reload_skills()
|
||||
return True
|
||||
|
||||
async def list_skill_files(
|
||||
self,
|
||||
skill_name: str,
|
||||
path: str = '.',
|
||||
include_hidden: bool = False,
|
||||
max_entries: int = 200,
|
||||
) -> dict:
|
||||
box_service = self._require_box('Browsing skill files')
|
||||
return await box_service.list_skill_files(skill_name, path, include_hidden, max_entries)
|
||||
|
||||
async def read_skill_file(self, skill_name: str, path: str) -> dict:
|
||||
box_service = self._require_box('Reading a skill file')
|
||||
return await box_service.read_skill_file(skill_name, path)
|
||||
|
||||
async def write_skill_file(self, skill_name: str, path: str, content: str) -> dict:
|
||||
box_service = self._require_box('Editing skill files')
|
||||
result = await box_service.write_skill_file(skill_name, path, content)
|
||||
await self._reload_skills()
|
||||
return result
|
||||
|
||||
async def install_from_github(self, data: dict) -> list[dict]:
|
||||
box_service = self._require_box('Installing a skill from GitHub')
|
||||
owner = str(data['owner']).strip()
|
||||
repo = str(data['repo']).strip()
|
||||
release_tag = str(data.get('release_tag', '')).strip()
|
||||
raw_asset_url = str(data['asset_url']).strip()
|
||||
if self._is_github_skill_md_url(raw_asset_url):
|
||||
return await self._install_github_skill_md(raw_asset_url, owner=owner, repo=repo, data=data)
|
||||
|
||||
asset_url = self._validate_github_asset_url(raw_asset_url, owner=owner, repo=repo, release_tag=release_tag)
|
||||
source_subdir = str(data.get('source_subdir', '') or '').strip()
|
||||
|
||||
zip_bytes = await self._download_github_asset(asset_url)
|
||||
filename = f'{repo}-{release_tag.lstrip("v").replace("/", "-") or "source"}.zip'
|
||||
installed = await box_service.install_skill_zip(
|
||||
zip_bytes,
|
||||
filename,
|
||||
source_paths=data.get('source_paths') or [],
|
||||
source_path=str(data.get('source_path', '') or ''),
|
||||
source_subdir=source_subdir,
|
||||
)
|
||||
await self._reload_skills()
|
||||
return [self._serialize_skill(skill) for skill in installed]
|
||||
|
||||
async def preview_install_from_github(self, data: dict) -> list[dict]:
|
||||
box_service = self._require_box('Previewing a skill from GitHub')
|
||||
owner = str(data['owner']).strip()
|
||||
repo = str(data['repo']).strip()
|
||||
release_tag = str(data.get('release_tag', '')).strip()
|
||||
raw_asset_url = str(data['asset_url']).strip()
|
||||
if self._is_github_skill_md_url(raw_asset_url):
|
||||
return await self._preview_github_skill_md(raw_asset_url, owner=owner, repo=repo)
|
||||
|
||||
asset_url = self._validate_github_asset_url(raw_asset_url, owner=owner, repo=repo, release_tag=release_tag)
|
||||
source_subdir = str(data.get('source_subdir', '') or '').strip()
|
||||
|
||||
zip_bytes = await self._download_github_asset(asset_url)
|
||||
return await box_service.preview_skill_zip(
|
||||
zip_bytes,
|
||||
f'{repo}-{release_tag.lstrip("v").replace("/", "-") or "source"}.zip',
|
||||
source_subdir=source_subdir,
|
||||
)
|
||||
|
||||
async def install_from_zip_upload(
|
||||
self,
|
||||
*,
|
||||
file_bytes: bytes,
|
||||
filename: str,
|
||||
source_paths: list[str] | None = None,
|
||||
source_path: str = '',
|
||||
) -> list[dict]:
|
||||
box_service = self._require_box('Installing a skill from upload')
|
||||
installed = await box_service.install_skill_zip(
|
||||
file_bytes,
|
||||
filename,
|
||||
source_paths=source_paths or [],
|
||||
source_path=source_path,
|
||||
)
|
||||
await self._reload_skills()
|
||||
return [self._serialize_skill(skill) for skill in installed]
|
||||
|
||||
async def preview_install_from_zip_upload(self, *, file_bytes: bytes, filename: str) -> list[dict]:
|
||||
box_service = self._require_box('Previewing a skill upload')
|
||||
return await box_service.preview_skill_zip(file_bytes, filename)
|
||||
|
||||
async def _install_github_skill_md(self, asset_url: str, *, owner: str, repo: str, data: dict) -> list[dict]:
|
||||
box_service = self._require_box('Installing a skill from GitHub')
|
||||
zip_bytes, filename, _package_name = await self._download_github_skill_directory_as_zip(
|
||||
asset_url,
|
||||
owner=owner,
|
||||
repo=repo,
|
||||
)
|
||||
|
||||
installed = await box_service.install_skill_zip(
|
||||
zip_bytes,
|
||||
filename,
|
||||
source_paths=data.get('source_paths') or [],
|
||||
source_path=str(data.get('source_path', '') or ''),
|
||||
target_suffix='',
|
||||
)
|
||||
await self._reload_skills()
|
||||
return [self._serialize_skill(skill) for skill in installed]
|
||||
|
||||
async def _preview_github_skill_md(self, asset_url: str, *, owner: str, repo: str) -> list[dict]:
|
||||
box_service = self._require_box('Previewing a skill from GitHub')
|
||||
zip_bytes, _filename, package_name = await self._download_github_skill_directory_as_zip(
|
||||
asset_url,
|
||||
owner=owner,
|
||||
repo=repo,
|
||||
)
|
||||
return await box_service.preview_skill_zip(zip_bytes, f'{package_name}.zip', target_suffix='')
|
||||
|
||||
async def reload_skills(self) -> list[dict]:
|
||||
await self._reload_skills()
|
||||
return await self.list_skills()
|
||||
|
||||
async def scan_directory_async(self, path: str) -> dict:
|
||||
box_service = self._require_box('Scanning a skill directory')
|
||||
return await box_service.scan_skill_directory(path)
|
||||
|
||||
async def _reload_skills(self) -> None:
|
||||
skill_mgr = getattr(self.ap, 'skill_mgr', None)
|
||||
reload_skills = getattr(skill_mgr, 'reload_skills', None)
|
||||
if not callable(reload_skills):
|
||||
return
|
||||
result = reload_skills()
|
||||
if inspect.isawaitable(result):
|
||||
await result
|
||||
|
||||
async def _download_github_asset(self, asset_url: str) -> bytes:
|
||||
async with httpx.AsyncClient(follow_redirects=True, timeout=120) as client:
|
||||
resp = await client.get(asset_url)
|
||||
resp.raise_for_status()
|
||||
return resp.content
|
||||
|
||||
async def _download_github_skill_directory_as_zip(
|
||||
self, asset_url: str, *, owner: str, repo: str
|
||||
) -> tuple[bytes, str, str]:
|
||||
info = self._parse_github_skill_md_url(asset_url, owner=owner, repo=repo)
|
||||
archive_url = f'https://codeload.github.com/{owner}/{repo}/zip/{quote(info["ref"], safe="/")}'
|
||||
archive_bytes = await self._download_github_asset(archive_url)
|
||||
|
||||
try:
|
||||
source_archive = zipfile.ZipFile(io.BytesIO(archive_bytes), 'r')
|
||||
except zipfile.BadZipFile as exc:
|
||||
raise ValueError('GitHub repository archive must be a valid .zip archive') from exc
|
||||
|
||||
with source_archive as source_zip:
|
||||
skill_entry = self._find_github_skill_archive_entry(source_zip, info['file_path'])
|
||||
try:
|
||||
skill_md_content = source_zip.read(skill_entry).decode('utf-8')
|
||||
except UnicodeDecodeError as exc:
|
||||
raise ValueError('GitHub SKILL.md must be valid UTF-8 text') from exc
|
||||
|
||||
package_name = self._resolve_github_skill_md_package_name(skill_md_content, info['package_name'])
|
||||
source_skill_dir = posixpath.dirname(posixpath.normpath(skill_entry.filename))
|
||||
|
||||
buffer = io.BytesIO()
|
||||
with zipfile.ZipFile(buffer, 'w', zipfile.ZIP_DEFLATED) as target_zip:
|
||||
self._copy_github_skill_directory_to_zip(source_zip, target_zip, source_skill_dir, package_name)
|
||||
return buffer.getvalue(), f'{package_name}.zip', package_name
|
||||
|
||||
def _find_github_skill_archive_entry(self, archive: zipfile.ZipFile, file_path: str) -> zipfile.ZipInfo:
|
||||
normalized_file_path = posixpath.normpath(file_path).lower()
|
||||
for member in archive.infolist():
|
||||
if member.is_dir():
|
||||
continue
|
||||
normalized_member = posixpath.normpath(member.filename)
|
||||
path_parts = normalized_member.split('/', 1)
|
||||
if len(path_parts) != 2:
|
||||
continue
|
||||
archive_relative_path = path_parts[1].lower()
|
||||
if archive_relative_path == normalized_file_path:
|
||||
return member
|
||||
raise ValueError(f'GitHub archive does not contain requested SKILL.md: {file_path}')
|
||||
|
||||
def _copy_github_skill_directory_to_zip(
|
||||
self,
|
||||
source_zip: zipfile.ZipFile,
|
||||
target_zip: zipfile.ZipFile,
|
||||
source_skill_dir: str,
|
||||
package_name: str,
|
||||
) -> None:
|
||||
normalized_source_dir = posixpath.normpath(source_skill_dir)
|
||||
source_prefix = f'{normalized_source_dir}/'
|
||||
copied_files = 0
|
||||
|
||||
for member in source_zip.infolist():
|
||||
normalized_member = posixpath.normpath(member.filename)
|
||||
if normalized_member != normalized_source_dir and not normalized_member.startswith(source_prefix):
|
||||
continue
|
||||
|
||||
relative_path = posixpath.relpath(normalized_member, normalized_source_dir)
|
||||
if relative_path in ('', '.'):
|
||||
continue
|
||||
if relative_path.startswith('../') or relative_path == '..' or posixpath.isabs(relative_path):
|
||||
raise ValueError(f'GitHub archive contains an unsafe skill path: {member.filename}')
|
||||
|
||||
target_name = f'{package_name}/{relative_path}'
|
||||
if member.is_dir() and not target_name.endswith('/'):
|
||||
target_name = f'{target_name}/'
|
||||
target_info = zipfile.ZipInfo(target_name, date_time=member.date_time)
|
||||
target_info.external_attr = member.external_attr
|
||||
target_info.compress_type = zipfile.ZIP_DEFLATED
|
||||
|
||||
if member.is_dir():
|
||||
target_zip.writestr(target_info, b'')
|
||||
continue
|
||||
|
||||
target_zip.writestr(target_info, source_zip.read(member))
|
||||
copied_files += 1
|
||||
|
||||
if copied_files == 0:
|
||||
raise ValueError('GitHub skill directory is empty')
|
||||
|
||||
def _uploaded_skill_target_stem(self, filename: str) -> str:
|
||||
stem = os.path.splitext(os.path.basename(str(filename or '').strip()))[0]
|
||||
safe_stem = ''.join(ch if ch.isalnum() or ch in ('-', '_') else '-' for ch in stem).strip('-_')
|
||||
if not safe_stem:
|
||||
safe_stem = 'uploaded-skill'
|
||||
return safe_stem
|
||||
|
||||
@staticmethod
|
||||
def _is_github_skill_md_url(asset_url: str) -> bool:
|
||||
parsed = urlparse(str(asset_url or '').strip())
|
||||
normalized_path = posixpath.normpath(parsed.path or '/')
|
||||
return normalized_path.lower().endswith('/skill.md')
|
||||
|
||||
def _parse_github_skill_md_url(self, asset_url: str, *, owner: str, repo: str) -> dict:
|
||||
parsed = urlparse(str(asset_url or '').strip())
|
||||
if parsed.scheme != 'https' or not parsed.netloc:
|
||||
raise ValueError('asset_url must be a valid HTTPS GitHub SKILL.md URL')
|
||||
|
||||
host = parsed.netloc.lower()
|
||||
path_parts = [unquote(part) for part in (parsed.path or '').split('/') if part]
|
||||
if host == 'github.com':
|
||||
if (
|
||||
len(path_parts) < 5
|
||||
or path_parts[0] != owner
|
||||
or path_parts[1] != repo
|
||||
or path_parts[2]
|
||||
not in (
|
||||
'blob',
|
||||
'raw',
|
||||
)
|
||||
):
|
||||
raise ValueError('GitHub SKILL.md URL must point to the requested owner/repo blob path')
|
||||
ref = path_parts[3]
|
||||
file_path = '/'.join(path_parts[4:])
|
||||
elif host == 'raw.githubusercontent.com':
|
||||
if len(path_parts) < 4 or path_parts[0] != owner or path_parts[1] != repo:
|
||||
raise ValueError('GitHub SKILL.md URL must point to the requested owner/repo raw path')
|
||||
ref = path_parts[2]
|
||||
file_path = '/'.join(path_parts[3:])
|
||||
else:
|
||||
raise ValueError('asset_url must point to a GitHub SKILL.md file')
|
||||
|
||||
normalized_file_path = posixpath.normpath(file_path)
|
||||
normalized_file_path_lower = normalized_file_path.lower()
|
||||
if normalized_file_path_lower != 'skill.md' and not normalized_file_path_lower.endswith('/skill.md'):
|
||||
raise ValueError('GitHub skill import requires a URL ending with SKILL.md')
|
||||
|
||||
parent_dir = posixpath.basename(posixpath.dirname(normalized_file_path)) or repo
|
||||
return {
|
||||
'ref': ref,
|
||||
'file_path': normalized_file_path,
|
||||
'package_name': self._uploaded_skill_target_stem(parent_dir),
|
||||
}
|
||||
|
||||
def _resolve_github_skill_md_package_name(self, content: str, fallback: str) -> str:
|
||||
metadata, _instructions = parse_frontmatter(content)
|
||||
candidate = str(metadata.get('name') or fallback or '').strip()
|
||||
try:
|
||||
return self._validate_skill_name(candidate)
|
||||
except ValueError:
|
||||
return self._validate_skill_name(fallback)
|
||||
|
||||
@staticmethod
|
||||
def _validate_github_asset_url(asset_url: str, *, owner: str, repo: str, release_tag: str) -> str:
|
||||
parsed = urlparse(str(asset_url).strip())
|
||||
if parsed.scheme != 'https' or not parsed.netloc:
|
||||
raise ValueError('asset_url must be a valid HTTPS GitHub asset URL')
|
||||
|
||||
host = parsed.netloc.lower()
|
||||
if host not in _GITHUB_ASSET_HOSTS:
|
||||
raise ValueError('asset_url must point to a GitHub-hosted release asset or archive')
|
||||
|
||||
normalized_path = posixpath.normpath(parsed.path or '/')
|
||||
allowed_prefixes = [
|
||||
f'/repos/{owner}/{repo}/',
|
||||
f'/{owner}/{repo}/',
|
||||
]
|
||||
if not any(normalized_path.startswith(prefix) for prefix in allowed_prefixes):
|
||||
raise ValueError('asset_url does not match the requested owner/repo')
|
||||
|
||||
if release_tag and release_tag not in parsed.path and release_tag not in parsed.query:
|
||||
raise ValueError('asset_url does not match the requested release_tag')
|
||||
|
||||
return parsed.geturl()
|
||||
|
||||
@staticmethod
|
||||
def _validate_skill_name(name: str) -> str:
|
||||
name = str(name or '').strip()
|
||||
if not name:
|
||||
raise ValueError('Skill name is required')
|
||||
if not name.replace('-', '').replace('_', '').isalnum():
|
||||
raise ValueError('Skill name can only contain letters, numbers, hyphens and underscores')
|
||||
if len(name) > 64:
|
||||
raise ValueError('Skill name cannot exceed 64 characters')
|
||||
return name
|
||||
@@ -179,7 +179,7 @@ class SpaceService:
|
||||
space_url = space_config['url']
|
||||
|
||||
session = httpclient.get_session()
|
||||
async with session.get(f'{space_url}/api/v1/models', params={'page_size': 100}) as response:
|
||||
async with session.get(f'{space_url}/api/v1/models') as response:
|
||||
if response.status != 200:
|
||||
raise ValueError(f'Failed to get models: {await response.text()}')
|
||||
data = await response.json()
|
||||
|
||||
@@ -65,8 +65,8 @@ class UserService:
|
||||
|
||||
user_obj = result_list[0]
|
||||
|
||||
# Check if this user has a local password set
|
||||
if not user_obj.password:
|
||||
# Check if this is a Space account
|
||||
if user_obj.account_type == 'space':
|
||||
raise ValueError('请使用 Space 账户登录')
|
||||
|
||||
ph = argon2.PasswordHasher()
|
||||
@@ -108,8 +108,9 @@ class UserService:
|
||||
if user_obj is None:
|
||||
raise ValueError('User not found')
|
||||
|
||||
if not user_obj.password:
|
||||
raise ValueError('No local password set, please set a password first')
|
||||
# Space accounts cannot change password locally
|
||||
if user_obj.account_type == 'space':
|
||||
raise ValueError('Space account cannot change password locally')
|
||||
|
||||
ph.verify(user_obj.password, current_password)
|
||||
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
"""LangBot Box runtime package."""
|
||||
|
||||
from .workspace import BoxWorkspaceSession
|
||||
|
||||
__all__ = ['BoxWorkspaceSession']
|
||||
@@ -1,354 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import typing
|
||||
from typing import TYPE_CHECKING
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from langbot_plugin.entities.io.actions.enums import CommonAction
|
||||
from langbot_plugin.runtime.io.handler import Handler
|
||||
from langbot_plugin.runtime.io.connection import Connection
|
||||
|
||||
from langbot_plugin.box.client import ActionRPCBoxClient
|
||||
from langbot_plugin.box.errors import BoxRuntimeUnavailableError
|
||||
from langbot_plugin.box.actions import LangBotToBoxAction
|
||||
|
||||
from ..utils import platform
|
||||
from ..utils.managed_runtime import ManagedRuntimeConnector
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..core import app as core_app
|
||||
|
||||
|
||||
# Default Docker Compose service name for the standalone Box container.
|
||||
_DOCKER_BOX_HOST = 'langbot_box'
|
||||
_DEFAULT_PORT = 5410
|
||||
|
||||
_HEARTBEAT_INTERVAL_SEC = 20
|
||||
|
||||
# Top-level keys under ``box`` that are LangBot-internal and should not be
|
||||
# forwarded to the Box runtime.
|
||||
_INTERNAL_BOX_CONFIG_KEYS = frozenset({'runtime'})
|
||||
|
||||
|
||||
def _get_box_config(ap) -> dict:
|
||||
"""Return the 'box' section from instance config.
|
||||
|
||||
Environment-variable overrides are handled uniformly by
|
||||
``LoadConfigStage._apply_env_overrides_to_config`` using the
|
||||
``SECTION__SUBSECTION__KEY`` convention (e.g. ``BOX__LOCAL__HOST_ROOT``,
|
||||
``BOX__LOCAL__ALLOWED_MOUNT_ROOTS="/a,/b"``) before this is read, so no
|
||||
box-specific env parsing is needed here.
|
||||
"""
|
||||
instance_config = getattr(ap, 'instance_config', None)
|
||||
config_data = getattr(instance_config, 'data', {}) if instance_config is not None else {}
|
||||
return dict(config_data.get('box', {}) or {})
|
||||
|
||||
|
||||
def _get_runtime_endpoint(box_cfg: dict) -> str:
|
||||
runtime_cfg = box_cfg.get('runtime') or {}
|
||||
return str(runtime_cfg.get('endpoint', '')).strip()
|
||||
|
||||
|
||||
def _filter_config_for_runtime(box_cfg: dict) -> dict:
|
||||
return {k: v for k, v in box_cfg.items() if k not in _INTERNAL_BOX_CONFIG_KEYS}
|
||||
|
||||
|
||||
def resolve_box_ws_relay_url(ap: core_app.Application) -> str:
|
||||
"""Derive the WS relay base URL used for managed-process attach.
|
||||
|
||||
The WS relay serves the ``/v1/sessions/{id}/managed-process/ws`` endpoint
|
||||
on the *relay* port (default 5410).
|
||||
"""
|
||||
box_cfg = _get_box_config(ap)
|
||||
|
||||
# Explicit runtime endpoint takes precedence. The config value is a base
|
||||
# URL; endpoint-specific paths are appended by the SDK client.
|
||||
endpoint = _get_runtime_endpoint(box_cfg)
|
||||
if endpoint:
|
||||
parsed = urlparse(endpoint)
|
||||
scheme = parsed.scheme or 'ws'
|
||||
if scheme == 'ws':
|
||||
scheme = 'http'
|
||||
elif scheme == 'wss':
|
||||
scheme = 'https'
|
||||
host = parsed.hostname or '127.0.0.1'
|
||||
port = parsed.port or _DEFAULT_PORT
|
||||
return f'{scheme}://{host}:{port}'
|
||||
|
||||
# In Docker, relay lives on the box runtime container.
|
||||
if platform.get_platform() == 'docker':
|
||||
return f'http://{_DOCKER_BOX_HOST}:{_DEFAULT_PORT}'
|
||||
|
||||
return f'http://127.0.0.1:{_DEFAULT_PORT}'
|
||||
|
||||
|
||||
class BoxRuntimeConnector(ManagedRuntimeConnector):
|
||||
"""Connect to the Box runtime via action RPC.
|
||||
|
||||
Transport decision (mirrors Plugin runtime logic):
|
||||
1. Docker / --standalone-box / explicit runtime.endpoint -> WebSocket to external Box process
|
||||
2. Windows (non-Docker) -> subprocess + WebSocket (Windows lacks async stdio pipe)
|
||||
3. Unix / macOS -> subprocess + stdio pipe
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
ap: core_app.Application,
|
||||
runtime_disconnect_callback: typing.Callable[
|
||||
['BoxRuntimeConnector'], typing.Coroutine[typing.Any, typing.Any, None]
|
||||
]
|
||||
| None = None,
|
||||
):
|
||||
super().__init__(ap)
|
||||
self.runtime_disconnect_callback = runtime_disconnect_callback
|
||||
self.configured_runtime_endpoint = self._load_configured_runtime_endpoint()
|
||||
self.ws_relay_base_url = resolve_box_ws_relay_url(ap)
|
||||
self.client = ActionRPCBoxClient(logger=ap.logger)
|
||||
|
||||
self._handler: Handler | None = None
|
||||
self._handler_task: asyncio.Task | None = None
|
||||
self._ctrl_task: asyncio.Task | None = None
|
||||
self._heartbeat_task: asyncio.Task | None = None
|
||||
|
||||
# Parse the relay URL once for reuse.
|
||||
parsed = urlparse(self.ws_relay_base_url)
|
||||
self._relay_host = parsed.hostname or '127.0.0.1'
|
||||
self._relay_port = parsed.port or _DEFAULT_PORT
|
||||
self._filtered_box_config = _filter_config_for_runtime(_get_box_config(ap))
|
||||
|
||||
def _uses_websocket(self) -> bool:
|
||||
"""Whether the connector should use WebSocket to reach the Box runtime.
|
||||
|
||||
True when:
|
||||
- Running inside Docker (Box runtime is a separate container)
|
||||
- The ``--standalone-box`` CLI flag was passed
|
||||
- An explicit ``runtime.endpoint`` was configured
|
||||
"""
|
||||
return bool(
|
||||
self.configured_runtime_endpoint
|
||||
or platform.get_platform() == 'docker'
|
||||
or platform.use_websocket_to_connect_box_runtime()
|
||||
)
|
||||
|
||||
async def initialize(self) -> None:
|
||||
if self._uses_websocket():
|
||||
if platform.get_platform() == 'win32' and not self.configured_runtime_endpoint:
|
||||
await self._start_subprocess_then_ws()
|
||||
else:
|
||||
await self._connect_remote_ws()
|
||||
else:
|
||||
await self._start_local_stdio()
|
||||
|
||||
# Start heartbeat after successful connection
|
||||
if self._heartbeat_task is None:
|
||||
self._heartbeat_task = asyncio.create_task(self._heartbeat_loop())
|
||||
|
||||
# -- heartbeat -----------------------------------------------------------
|
||||
|
||||
async def _heartbeat_loop(self) -> None:
|
||||
"""Periodically ping the Box runtime to detect silent disconnections."""
|
||||
while True:
|
||||
await asyncio.sleep(_HEARTBEAT_INTERVAL_SEC)
|
||||
try:
|
||||
await self.ping()
|
||||
self.ap.logger.debug('Heartbeat to Box runtime success.')
|
||||
except Exception as e:
|
||||
self.ap.logger.debug(f'Failed to heartbeat to Box runtime: {e}')
|
||||
|
||||
async def ping(self) -> None:
|
||||
if self._handler is None:
|
||||
raise BoxRuntimeUnavailableError('Box runtime is not connected')
|
||||
await self._handler.call_action(CommonAction.PING, {})
|
||||
|
||||
# -- transport paths -----------------------------------------------------
|
||||
|
||||
async def _start_local_stdio(self) -> None:
|
||||
"""Launch box server as subprocess and connect via stdio (Unix/macOS)."""
|
||||
from langbot_plugin.runtime.io.controllers.stdio.client import StdioClientController
|
||||
|
||||
self.ap.logger.info('Use stdio to connect to box runtime')
|
||||
python_path = sys.executable
|
||||
env = os.environ.copy()
|
||||
if self._filtered_box_config:
|
||||
env['LANGBOT_BOX_CONFIG'] = json.dumps(self._filtered_box_config)
|
||||
|
||||
connected = asyncio.Event()
|
||||
connect_error: list[Exception] = []
|
||||
|
||||
ctrl = StdioClientController(
|
||||
command=python_path,
|
||||
# Launched through the same CLI entry point as the plugin runtime
|
||||
# (cli.__init__ <subcommand>); `-s` selects the stdio transport,
|
||||
# mirroring `rt -s`.
|
||||
args=['-m', 'langbot_plugin.cli.__init__', 'box', '-s', '--ws-control-port', str(self._relay_port)],
|
||||
env=env,
|
||||
)
|
||||
self._ctrl_task = asyncio.create_task(
|
||||
ctrl.run(self._make_connection_callback('stdio', connected, connect_error))
|
||||
)
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(connected.wait(), timeout=30.0)
|
||||
except asyncio.TimeoutError:
|
||||
raise BoxRuntimeUnavailableError('box runtime subprocess did not connect in time')
|
||||
|
||||
if connect_error:
|
||||
raise BoxRuntimeUnavailableError(f'box runtime connection failed: {connect_error[0]}')
|
||||
|
||||
self._subprocess = ctrl.process
|
||||
|
||||
async def _start_subprocess_then_ws(self) -> None:
|
||||
"""Launch box server as detached subprocess, then connect via WS (Windows)."""
|
||||
self.ap.logger.info('(windows) Use cmd to launch box runtime and communicate via ws')
|
||||
|
||||
env = os.environ.copy()
|
||||
if self._filtered_box_config:
|
||||
env['LANGBOT_BOX_CONFIG'] = json.dumps(self._filtered_box_config)
|
||||
|
||||
python_path = sys.executable
|
||||
# Launched through the same CLI entry point as the plugin runtime
|
||||
# (cli.__init__ <subcommand>); no flag => WebSocket transport.
|
||||
self.runtime_subprocess = await asyncio.create_subprocess_exec(
|
||||
python_path,
|
||||
'-m',
|
||||
'langbot_plugin.cli.__init__',
|
||||
'box',
|
||||
'--ws-control-port',
|
||||
str(self._relay_port),
|
||||
env=env,
|
||||
)
|
||||
self.runtime_subprocess_task = asyncio.create_task(self.runtime_subprocess.wait())
|
||||
|
||||
ws_url = f'ws://localhost:{self._relay_port}/rpc/ws'
|
||||
await self._connect_ws(ws_url, '(windows) WebSocket')
|
||||
|
||||
async def _connect_remote_ws(self) -> None:
|
||||
"""Connect to a remote (or Docker) box server via WebSocket."""
|
||||
ws_url = self._resolve_rpc_ws_url()
|
||||
self.ap.logger.info(f'Use WebSocket to connect to box runtime ({ws_url})')
|
||||
await self._connect_ws(ws_url, 'WebSocket')
|
||||
|
||||
# -- helpers -------------------------------------------------------------
|
||||
|
||||
def _resolve_rpc_ws_url(self) -> str:
|
||||
"""Determine the action-RPC WebSocket URL.
|
||||
|
||||
All endpoints share a single port; action RPC is at ``/rpc/ws``.
|
||||
"""
|
||||
if self.configured_runtime_endpoint:
|
||||
base = self.configured_runtime_endpoint.rstrip('/')
|
||||
parsed = urlparse(base)
|
||||
scheme = parsed.scheme or 'ws'
|
||||
if scheme in ('http', 'https'):
|
||||
scheme = 'wss' if scheme == 'https' else 'ws'
|
||||
host = parsed.hostname or '127.0.0.1'
|
||||
port = parsed.port or _DEFAULT_PORT
|
||||
return f'{scheme}://{host}:{port}/rpc/ws'
|
||||
|
||||
if platform.get_platform() == 'docker':
|
||||
return f'ws://{_DOCKER_BOX_HOST}:{_DEFAULT_PORT}/rpc/ws'
|
||||
|
||||
return f'ws://localhost:{self._relay_port}/rpc/ws'
|
||||
|
||||
async def _connect_ws(self, ws_url: str, transport_name: str) -> None:
|
||||
"""Shared WebSocket connection procedure."""
|
||||
from langbot_plugin.runtime.io.controllers.ws.client import WebSocketClientController
|
||||
|
||||
connected = asyncio.Event()
|
||||
connect_error: list[Exception] = []
|
||||
|
||||
async def on_connect_failed(ctrl, exc):
|
||||
if exc is not None:
|
||||
self.ap.logger.error(f'Failed to connect to Box runtime ({ws_url}): {exc}')
|
||||
else:
|
||||
self.ap.logger.error(f'Failed to connect to Box runtime ({ws_url}), trying to reconnect...')
|
||||
connect_error.append(exc or BoxRuntimeUnavailableError('ws connection failed'))
|
||||
connected.set()
|
||||
if self.runtime_disconnect_callback is not None:
|
||||
await self.runtime_disconnect_callback(self)
|
||||
|
||||
ctrl = WebSocketClientController(ws_url=ws_url, make_connection_failed_callback=on_connect_failed)
|
||||
self._ctrl_task = asyncio.create_task(
|
||||
ctrl.run(self._make_connection_callback(transport_name, connected, connect_error))
|
||||
)
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(connected.wait(), timeout=30.0)
|
||||
except asyncio.TimeoutError:
|
||||
raise BoxRuntimeUnavailableError(f'box runtime ws connection timed out ({ws_url})')
|
||||
|
||||
if connect_error:
|
||||
raise BoxRuntimeUnavailableError(f'box runtime connection failed: {connect_error[0]}')
|
||||
|
||||
def _make_connection_callback(
|
||||
self,
|
||||
transport_name: str,
|
||||
connected: asyncio.Event,
|
||||
connect_error: list[Exception],
|
||||
):
|
||||
async def new_connection_callback(connection: Connection) -> None:
|
||||
handler = Handler(connection)
|
||||
self._handler = handler
|
||||
self.client.set_handler(handler)
|
||||
self._handler_task = asyncio.create_task(handler.run())
|
||||
try:
|
||||
await handler.call_action(CommonAction.PING, {})
|
||||
if self._filtered_box_config:
|
||||
await handler.call_action(LangBotToBoxAction.INIT, self._filtered_box_config)
|
||||
self.ap.logger.debug('Sent box configuration to Box runtime via INIT.')
|
||||
self.ap.logger.info(f'Connected to Box runtime via {transport_name}.')
|
||||
connected.set()
|
||||
await self._handler_task
|
||||
except Exception as exc:
|
||||
if not connected.is_set():
|
||||
connect_error.append(exc)
|
||||
connected.set()
|
||||
return
|
||||
|
||||
# If we reach here, handler.run() returned normally (connection
|
||||
# closed) or raised after the initial handshake succeeded.
|
||||
# Either way, treat it as a disconnect.
|
||||
if connected.is_set():
|
||||
if self._uses_websocket():
|
||||
self.ap.logger.error('Disconnected from Box runtime, trying to reconnect...')
|
||||
if self.runtime_disconnect_callback is not None:
|
||||
await self.runtime_disconnect_callback(self)
|
||||
else:
|
||||
self.ap.logger.error(
|
||||
'Disconnected from Box runtime via stdio. '
|
||||
'Cannot automatically reconnect — please restart LangBot.'
|
||||
)
|
||||
|
||||
return new_connection_callback
|
||||
|
||||
# -- lifecycle -----------------------------------------------------------
|
||||
|
||||
def dispose(self) -> None:
|
||||
if self._heartbeat_task is not None:
|
||||
self._heartbeat_task.cancel()
|
||||
self._heartbeat_task = None
|
||||
|
||||
if self._handler_task is not None:
|
||||
self._handler_task.cancel()
|
||||
self._handler_task = None
|
||||
|
||||
if self._ctrl_task is not None:
|
||||
self._ctrl_task.cancel()
|
||||
self._ctrl_task = None
|
||||
|
||||
# stdio-managed subprocess (stored as self._subprocess by _start_local_stdio)
|
||||
if hasattr(self, '_subprocess') and self._subprocess is not None and self._subprocess.returncode is None:
|
||||
self.ap.logger.info('Terminating managed box runtime process...')
|
||||
self._subprocess.terminate()
|
||||
|
||||
# Subprocess launched by ManagedRuntimeConnector._start_runtime_subprocess (Windows path)
|
||||
self._dispose_subprocess()
|
||||
|
||||
# -- config helpers ------------------------------------------------------
|
||||
|
||||
def _load_configured_runtime_endpoint(self) -> str:
|
||||
return _get_runtime_endpoint(_get_box_config(self.ap))
|
||||
@@ -1,98 +0,0 @@
|
||||
"""Three-layer security policy for LangBot Box.
|
||||
|
||||
The design separates concerns into three independent layers, aligned with
|
||||
OpenCode / OpenClaw patterns:
|
||||
|
||||
1. **SandboxPolicy** – *where* tools run (host vs sandbox).
|
||||
2. **ToolPolicy** – *which* tools are allowed (allow/deny lists).
|
||||
3. **ElevatedPolicy** – *whether* a single exec call may temporarily
|
||||
escape the default sandbox boundary.
|
||||
|
||||
These three layers are orthogonal:
|
||||
- ToolPolicy is a hard boundary; ``elevated`` cannot bypass a denied tool.
|
||||
- SandboxPolicy decides the default execution location.
|
||||
- ElevatedPolicy only affects ``exec`` and only when the framework allows it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import enum
|
||||
from typing import Sequence
|
||||
|
||||
|
||||
# ── Layer 1: Sandbox Policy ──────────────────────────────────────────
|
||||
|
||||
|
||||
class SandboxMode(str, enum.Enum):
|
||||
"""Determines when agent execution is routed through the sandbox."""
|
||||
|
||||
OFF = 'off'
|
||||
"""Sandbox disabled; all exec runs on the host."""
|
||||
|
||||
NON_DEFAULT = 'non_default'
|
||||
"""Only non-default sessions are sandboxed (e.g. sub-agents, MCP)."""
|
||||
|
||||
ALL = 'all'
|
||||
"""Every agent exec call is routed through the sandbox."""
|
||||
|
||||
|
||||
class SandboxPolicy:
|
||||
"""Decides whether a given execution context should use the sandbox."""
|
||||
|
||||
def __init__(self, mode: SandboxMode = SandboxMode.ALL):
|
||||
self.mode = mode
|
||||
|
||||
def should_sandbox(self, *, is_default_session: bool = True) -> bool:
|
||||
if self.mode == SandboxMode.OFF:
|
||||
return False
|
||||
if self.mode == SandboxMode.ALL:
|
||||
return True
|
||||
# NON_DEFAULT: sandbox everything except the default session
|
||||
return not is_default_session
|
||||
|
||||
|
||||
# ── Layer 2: Tool Policy ─────────────────────────────────────────────
|
||||
|
||||
|
||||
class ToolPolicy:
|
||||
"""Controls which tools are available to the current agent/session.
|
||||
|
||||
Rules:
|
||||
- ``deny`` always takes precedence over ``allow``.
|
||||
- An empty ``allow`` list means "all tools allowed" (no allowlist filter).
|
||||
- ``elevated`` cannot bypass a denied tool.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
allow: Sequence[str] = (),
|
||||
deny: Sequence[str] = (),
|
||||
):
|
||||
self._allow: frozenset[str] = frozenset(allow)
|
||||
self._deny: frozenset[str] = frozenset(deny)
|
||||
|
||||
def is_tool_allowed(self, tool_name: str) -> bool:
|
||||
if tool_name in self._deny:
|
||||
return False
|
||||
if self._allow and tool_name not in self._allow:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
# ── Layer 3: Elevated Policy ─────────────────────────────────────────
|
||||
|
||||
|
||||
class ElevatedPolicy:
|
||||
"""Controls whether ``exec`` may request temporary privilege escalation.
|
||||
|
||||
``elevated`` only applies to the ``exec`` tool. It means "run this
|
||||
command outside the default sandbox boundary" (e.g. with network, or
|
||||
on the host). The framework decides whether to honor the request.
|
||||
"""
|
||||
|
||||
def __init__(self, *, allow_elevated: bool = False, require_approval: bool = True):
|
||||
self.allow_elevated = allow_elevated
|
||||
self.require_approval = require_approval
|
||||
|
||||
def is_elevation_permitted(self) -> bool:
|
||||
return self.allow_elevated
|
||||
@@ -1,794 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import collections
|
||||
import datetime as _dt
|
||||
import enum
|
||||
import json
|
||||
import os
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pydantic
|
||||
|
||||
from langbot_plugin.box.client import BoxRuntimeClient
|
||||
from .connector import BoxRuntimeConnector, _get_box_config
|
||||
from langbot_plugin.box.errors import BoxError, BoxValidationError
|
||||
from langbot_plugin.box.models import (
|
||||
BUILTIN_PROFILES,
|
||||
BoxExecutionResult,
|
||||
BoxManagedProcessInfo,
|
||||
BoxManagedProcessSpec,
|
||||
BoxProfile,
|
||||
BoxSpec,
|
||||
)
|
||||
|
||||
_INT_ADAPTER = pydantic.TypeAdapter(int)
|
||||
_UTC = _dt.timezone.utc
|
||||
_MAX_RECENT_ERRORS = 50
|
||||
_MIB = 1024 * 1024
|
||||
|
||||
|
||||
def _is_path_under(path: str, root: str) -> bool:
|
||||
"""Check whether *path* equals *root* or is a child of *root*."""
|
||||
return path == root or path.startswith(f'{root}{os.sep}')
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..core import app as core_app
|
||||
import langbot_plugin.api.entities.builtin.pipeline.query as pipeline_query
|
||||
|
||||
|
||||
class BoxService:
|
||||
def __init__(
|
||||
self,
|
||||
ap: core_app.Application,
|
||||
client: BoxRuntimeClient | None = None,
|
||||
output_limit_chars: int = 4000,
|
||||
):
|
||||
self.ap = ap
|
||||
self._enabled = self._load_enabled()
|
||||
self._runtime_connector: BoxRuntimeConnector | None = None
|
||||
if client is None:
|
||||
# Always construct a connector — its __init__ is side-effect free
|
||||
# (no I/O, no subprocess). When ``box.enabled = false`` we simply
|
||||
# skip ``connector.initialize()`` so no connection is attempted.
|
||||
self._runtime_connector = BoxRuntimeConnector(ap, runtime_disconnect_callback=self._on_runtime_disconnect)
|
||||
client = self._runtime_connector.client
|
||||
self.client = client
|
||||
self.output_limit_chars = output_limit_chars
|
||||
self.host_root = self._load_host_root()
|
||||
self.allowed_mount_roots = self._load_allowed_mount_roots()
|
||||
self.default_workspace = self._load_default_workspace()
|
||||
self.profile = self._load_profile()
|
||||
self.custom_image = self._load_custom_image()
|
||||
self.workspace_quota_mb = self._load_workspace_quota_mb()
|
||||
self._recent_errors: collections.deque[dict] = collections.deque(maxlen=_MAX_RECENT_ERRORS)
|
||||
self._shutdown_task = None
|
||||
self._available = False
|
||||
self._connector_error: str = ''
|
||||
self._reconnecting = False
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
"""Whether Box is enabled in config. False means the operator has
|
||||
deliberately turned the sandbox off via ``box.enabled = false``.
|
||||
Disabled and "enabled but unavailable" are reported as the same
|
||||
``available = False`` to consumers, but distinguished in get_status."""
|
||||
return self._enabled
|
||||
|
||||
async def initialize(self):
|
||||
self._ensure_default_workspace()
|
||||
if not self._enabled:
|
||||
# Disabled by config: do NOT connect to a remote runtime, do NOT
|
||||
# fork a stdio subprocess. Every consumer of box_service should
|
||||
# gate on ``available`` and degrade gracefully.
|
||||
self._available = False
|
||||
self._connector_error = 'Box runtime is disabled in config (box.enabled = false)'
|
||||
self.ap.logger.info(
|
||||
'Box runtime disabled by config; sandbox features (exec/read/write/edit, '
|
||||
'skill add/edit, stdio MCP) will be unavailable.'
|
||||
)
|
||||
return
|
||||
try:
|
||||
if self._runtime_connector is not None:
|
||||
await self._runtime_connector.initialize()
|
||||
else:
|
||||
await self.client.initialize()
|
||||
self._available = True
|
||||
self._connector_error = ''
|
||||
self.ap.logger.info(
|
||||
f'LangBot Box runtime initialized: profile={self.profile.name} '
|
||||
f'default_workspace={self.default_workspace or "(none)"}'
|
||||
)
|
||||
except Exception as exc:
|
||||
self.ap.logger.warning(f'LangBot Box runtime unavailable, sandbox features disabled: {exc}')
|
||||
self._available = False
|
||||
self._connector_error = str(exc)
|
||||
|
||||
async def _on_runtime_disconnect(self, connector: BoxRuntimeConnector) -> None:
|
||||
"""Called by the connector when the Box runtime connection drops.
|
||||
|
||||
Spawns a background reconnection loop so the caller is not blocked.
|
||||
Skipped entirely when Box is disabled by config — that path should
|
||||
never have connected in the first place.
|
||||
"""
|
||||
if not self._enabled:
|
||||
return
|
||||
if self._reconnecting:
|
||||
return # Another reconnect loop is already running
|
||||
self._reconnecting = True
|
||||
self._available = False
|
||||
self._connector_error = 'Disconnected from Box runtime'
|
||||
self.ap.logger.warning('Box runtime disconnected, sandbox features temporarily disabled.')
|
||||
asyncio.create_task(self._reconnect_loop(connector))
|
||||
|
||||
async def _reconnect_loop(self, connector: BoxRuntimeConnector) -> None:
|
||||
"""Retry reconnection with exponential backoff (3s → 60s max)."""
|
||||
delay = 3
|
||||
max_delay = 60
|
||||
try:
|
||||
while True:
|
||||
self.ap.logger.info(f'Attempting to reconnect to Box runtime in {delay}s...')
|
||||
await asyncio.sleep(delay)
|
||||
try:
|
||||
connector.dispose()
|
||||
await connector.initialize()
|
||||
self._available = True
|
||||
self._connector_error = ''
|
||||
self.ap.logger.info('Box runtime reconnected, sandbox features restored.')
|
||||
return
|
||||
except Exception as exc:
|
||||
self._connector_error = str(exc)
|
||||
self.ap.logger.warning(f'Box runtime reconnection failed: {exc}')
|
||||
delay = min(delay * 2, max_delay)
|
||||
finally:
|
||||
self._reconnecting = False
|
||||
|
||||
@property
|
||||
def available(self) -> bool:
|
||||
return self._available
|
||||
|
||||
async def execute_spec_payload(
|
||||
self,
|
||||
spec_payload: dict,
|
||||
query: pipeline_query.Query,
|
||||
*,
|
||||
skip_host_mount_validation: bool = False,
|
||||
) -> dict:
|
||||
if not self._available:
|
||||
raise BoxError('Box runtime is not available. Install and start Docker to use sandbox features.')
|
||||
try:
|
||||
spec = self.build_spec(spec_payload, skip_host_mount_validation=skip_host_mount_validation)
|
||||
except BoxError as exc:
|
||||
self._record_error(exc, query)
|
||||
raise
|
||||
self.ap.logger.info(
|
||||
'LangBot Box request: '
|
||||
f'query_id={query.query_id} '
|
||||
f'spec={json.dumps(self._summarize_spec(spec), ensure_ascii=False)}'
|
||||
)
|
||||
try:
|
||||
self._enforce_workspace_quota(spec, phase='before execution')
|
||||
except BoxError as exc:
|
||||
self._record_error(exc, query)
|
||||
raise
|
||||
try:
|
||||
result = await self.client.execute(spec)
|
||||
except BoxError as exc:
|
||||
self._record_error(exc, query)
|
||||
raise
|
||||
try:
|
||||
self._enforce_workspace_quota(spec, phase='after execution')
|
||||
except BoxError as exc:
|
||||
await self._cleanup_exceeded_session(spec)
|
||||
self._record_error(exc, query)
|
||||
raise
|
||||
self.ap.logger.info(
|
||||
'LangBot Box result: '
|
||||
f'query_id={query.query_id} '
|
||||
f'summary={json.dumps(self._summarize_result(result), ensure_ascii=False)}'
|
||||
)
|
||||
return self._serialize_result(result)
|
||||
|
||||
def resolve_box_session_id(self, query: pipeline_query.Query) -> str:
|
||||
"""Resolve the Box session_id from the pipeline's template and query variables."""
|
||||
template = (
|
||||
(query.pipeline_config or {})
|
||||
.get('ai', {})
|
||||
.get('local-agent', {})
|
||||
.get('box-session-id-template', '{launcher_type}_{launcher_id}')
|
||||
)
|
||||
variables = dict(query.variables or {})
|
||||
launcher_type = getattr(query, 'launcher_type', None)
|
||||
if hasattr(launcher_type, 'value'):
|
||||
launcher_type = launcher_type.value
|
||||
launcher_id = getattr(query, 'launcher_id', None)
|
||||
sender_id = getattr(query, 'sender_id', None)
|
||||
query_id = getattr(query, 'query_id', None)
|
||||
|
||||
variables.setdefault('query_id', str(query_id or 'unknown'))
|
||||
variables.setdefault('launcher_type', str(launcher_type or 'query'))
|
||||
variables.setdefault('launcher_id', str(launcher_id or query_id or 'unknown'))
|
||||
variables.setdefault('sender_id', str(sender_id or launcher_id or query_id or 'unknown'))
|
||||
variables.setdefault('global', 'global')
|
||||
return template.format_map(collections.defaultdict(lambda: 'unknown', variables))
|
||||
|
||||
def build_skill_extra_mounts(self, query: pipeline_query.Query) -> list[dict]:
|
||||
"""Build extra_mounts entries for all pipeline-bound skills.
|
||||
|
||||
This ensures that when a container is first created it already has
|
||||
all skill packages mounted, regardless of which skill is currently
|
||||
activated.
|
||||
|
||||
Skills whose ``package_root`` is missing or no longer a directory on
|
||||
the LangBot-visible filesystem are skipped with a warning instead of
|
||||
being passed through to the backend. Without this guard the three
|
||||
backends behave inconsistently on a stale mount: nsjail refuses to
|
||||
start the sandbox (failing every exec in the session), Docker
|
||||
silently auto-creates a root-owned empty directory on the host, and
|
||||
E2B silently skips the upload — none of which surfaces an
|
||||
actionable error to the agent or operator.
|
||||
"""
|
||||
skill_mgr = getattr(self.ap, 'skill_mgr', None)
|
||||
if skill_mgr is None:
|
||||
return []
|
||||
|
||||
from ..provider.tools.loaders import skill as skill_loader
|
||||
|
||||
visible_skills = skill_loader.get_visible_skills(self.ap, query)
|
||||
mounts: list[dict] = []
|
||||
for skill_name, skill_data in visible_skills.items():
|
||||
package_root = str(skill_data.get('package_root', '') or '').strip()
|
||||
if not package_root:
|
||||
continue
|
||||
if not os.path.isdir(package_root):
|
||||
self.ap.logger.warning(
|
||||
f'Skill "{skill_name}" package_root missing on filesystem '
|
||||
f'({package_root}); skipping mount to prevent sandbox failures. '
|
||||
f'The skill cache may be stale — consider reloading skills.'
|
||||
)
|
||||
continue
|
||||
mounts.append(
|
||||
{
|
||||
'host_path': package_root,
|
||||
'mount_path': f'/workspace/.skills/{skill_name}',
|
||||
'mode': 'rw',
|
||||
}
|
||||
)
|
||||
return mounts
|
||||
|
||||
async def execute_tool(self, parameters: dict, query: pipeline_query.Query) -> dict:
|
||||
"""Execute an agent-facing ``exec`` tool call.
|
||||
|
||||
Translates the agent-facing ``command`` field to the internal
|
||||
``BoxSpec.cmd`` field and injects the session id from the query.
|
||||
"""
|
||||
spec_payload: dict = {'cmd': parameters['command']}
|
||||
|
||||
# Pass through allowed agent-facing fields
|
||||
for key in ('workdir', 'timeout_sec', 'env'):
|
||||
if key in parameters:
|
||||
spec_payload[key] = parameters[key]
|
||||
|
||||
# Inject context the agent must not control
|
||||
spec_payload.setdefault('session_id', self.resolve_box_session_id(query))
|
||||
|
||||
# Mount all pipeline-bound skills so they are available in the container
|
||||
if 'extra_mounts' not in spec_payload:
|
||||
spec_payload['extra_mounts'] = self.build_skill_extra_mounts(query)
|
||||
|
||||
return await self.execute_spec_payload(spec_payload, query)
|
||||
|
||||
async def shutdown(self):
|
||||
await self.client.shutdown()
|
||||
|
||||
def dispose(self):
|
||||
if self._runtime_connector is not None:
|
||||
self._runtime_connector.dispose()
|
||||
loop = getattr(self.ap, 'event_loop', None)
|
||||
if loop is not None and not loop.is_closed() and (self._shutdown_task is None or self._shutdown_task.done()):
|
||||
self._shutdown_task = loop.create_task(self.shutdown())
|
||||
|
||||
async def get_sessions(self) -> list[dict]:
|
||||
if not self._available:
|
||||
return []
|
||||
try:
|
||||
return await self.client.get_sessions()
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
def build_spec(self, spec_payload: dict, skip_host_mount_validation: bool = False) -> BoxSpec:
|
||||
spec_payload = dict(spec_payload)
|
||||
spec_payload.setdefault('env', {})
|
||||
if spec_payload.get('host_path') in (None, '') and self.default_workspace is not None:
|
||||
spec_payload['host_path'] = self.default_workspace
|
||||
if spec_payload.get('workspace_quota_mb') in (None, '') and self.workspace_quota_mb is not None:
|
||||
spec_payload['workspace_quota_mb'] = self.workspace_quota_mb
|
||||
|
||||
# Global custom image overrides profile default (but not caller-specified image)
|
||||
if self.custom_image and 'image' not in spec_payload:
|
||||
spec_payload['image'] = self.custom_image
|
||||
|
||||
self._apply_profile(spec_payload)
|
||||
|
||||
try:
|
||||
spec = BoxSpec.model_validate(spec_payload)
|
||||
except pydantic.ValidationError as exc:
|
||||
first_error = exc.errors()[0]
|
||||
raise BoxValidationError(first_error.get('msg', 'invalid box arguments')) from exc
|
||||
|
||||
if not skip_host_mount_validation:
|
||||
self._validate_host_mount(spec)
|
||||
return spec
|
||||
|
||||
async def create_session(self, spec_payload: dict, *, skip_host_mount_validation: bool = False) -> dict:
|
||||
spec = self.build_spec(spec_payload, skip_host_mount_validation=skip_host_mount_validation)
|
||||
return await self.client.create_session(spec)
|
||||
|
||||
async def start_managed_process(self, session_id: str, process_payload: dict) -> BoxManagedProcessInfo:
|
||||
process_spec = BoxManagedProcessSpec.model_validate(process_payload)
|
||||
return await self.client.start_managed_process(session_id, process_spec)
|
||||
|
||||
async def get_managed_process(self, session_id: str, process_id: str = 'default') -> BoxManagedProcessInfo:
|
||||
return await self.client.get_managed_process(session_id, process_id)
|
||||
|
||||
async def stop_managed_process(self, session_id: str, process_id: str = 'default') -> None:
|
||||
return await self.client.stop_managed_process(session_id, process_id)
|
||||
|
||||
def get_managed_process_websocket_url(self, session_id: str, process_id: str = 'default') -> str:
|
||||
getter = getattr(self.client, 'get_managed_process_websocket_url', None)
|
||||
if getter is None:
|
||||
raise BoxValidationError('box runtime client does not support managed process websocket attach')
|
||||
ws_relay_base_url = (
|
||||
self._runtime_connector.ws_relay_base_url
|
||||
if self._runtime_connector is not None
|
||||
else 'http://127.0.0.1:5410'
|
||||
)
|
||||
return getter(session_id, ws_relay_base_url, process_id)
|
||||
|
||||
async def list_skills(self) -> list[dict]:
|
||||
return await self.client.list_skills()
|
||||
|
||||
async def get_skill(self, name: str) -> dict | None:
|
||||
return await self.client.get_skill(name)
|
||||
|
||||
async def create_skill(self, skill: dict) -> dict:
|
||||
return await self.client.create_skill(skill)
|
||||
|
||||
async def update_skill(self, name: str, skill: dict) -> dict:
|
||||
return await self.client.update_skill(name, skill)
|
||||
|
||||
async def delete_skill(self, name: str) -> None:
|
||||
await self.client.delete_skill(name)
|
||||
|
||||
async def scan_skill_directory(self, path: str) -> dict:
|
||||
return await self.client.scan_skill_directory(path)
|
||||
|
||||
async def list_skill_files(
|
||||
self,
|
||||
name: str,
|
||||
path: str = '.',
|
||||
include_hidden: bool = False,
|
||||
max_entries: int = 200,
|
||||
) -> dict:
|
||||
return await self.client.list_skill_files(name, path, include_hidden, max_entries)
|
||||
|
||||
async def read_skill_file(self, name: str, path: str) -> dict:
|
||||
return await self.client.read_skill_file(name, path)
|
||||
|
||||
async def write_skill_file(self, name: str, path: str, content: str) -> dict:
|
||||
return await self.client.write_skill_file(name, path, content)
|
||||
|
||||
async def preview_skill_zip(
|
||||
self,
|
||||
file_bytes: bytes,
|
||||
filename: str,
|
||||
source_subdir: str = '',
|
||||
target_suffix: str = 'upload',
|
||||
) -> list[dict]:
|
||||
return await self.client.preview_skill_zip(file_bytes, filename, source_subdir, target_suffix)
|
||||
|
||||
async def install_skill_zip(
|
||||
self,
|
||||
file_bytes: bytes,
|
||||
filename: str,
|
||||
source_paths: list[str] | None = None,
|
||||
source_path: str = '',
|
||||
source_subdir: str = '',
|
||||
target_suffix: str = 'upload',
|
||||
) -> list[dict]:
|
||||
return await self.client.install_skill_zip(
|
||||
file_bytes,
|
||||
filename,
|
||||
source_paths,
|
||||
source_path,
|
||||
source_subdir,
|
||||
target_suffix,
|
||||
)
|
||||
|
||||
def _serialize_result(self, result: BoxExecutionResult) -> dict:
|
||||
stdout, stdout_truncated = self._truncate(result.stdout)
|
||||
stderr, stderr_truncated = self._truncate(result.stderr)
|
||||
|
||||
return {
|
||||
'session_id': result.session_id,
|
||||
'backend': result.backend_name,
|
||||
'status': result.status.value,
|
||||
'ok': result.ok,
|
||||
'exit_code': result.exit_code,
|
||||
'stdout': stdout,
|
||||
'stderr': stderr,
|
||||
'stdout_truncated': stdout_truncated,
|
||||
'stderr_truncated': stderr_truncated,
|
||||
'duration_ms': result.duration_ms,
|
||||
}
|
||||
|
||||
def _truncate(self, text: str) -> tuple[str, bool]:
|
||||
if len(text) <= self.output_limit_chars:
|
||||
return text, False
|
||||
if self.output_limit_chars <= 0:
|
||||
return '', True
|
||||
|
||||
head_size = 0
|
||||
tail_size = 0
|
||||
notice = ''
|
||||
# Recompute once the omitted count is known so the final payload
|
||||
# stays within output_limit_chars even after adding the notice.
|
||||
for _ in range(4):
|
||||
omitted = max(len(text) - head_size - tail_size, 0)
|
||||
notice = f'\n\n... [{omitted} characters truncated] ...\n\n'
|
||||
available = self.output_limit_chars - len(notice)
|
||||
if available <= 0:
|
||||
return notice[: self.output_limit_chars], True
|
||||
|
||||
new_head_size = int(available * 0.6)
|
||||
new_tail_size = available - new_head_size
|
||||
if new_head_size == head_size and new_tail_size == tail_size:
|
||||
break
|
||||
head_size = new_head_size
|
||||
tail_size = new_tail_size
|
||||
|
||||
head = text[:head_size]
|
||||
tail = text[-tail_size:] if tail_size else ''
|
||||
truncated = f'{head}{notice}{tail}'
|
||||
return truncated[: self.output_limit_chars], True
|
||||
|
||||
def _summarize_spec(self, spec: BoxSpec) -> dict:
|
||||
cmd = spec.cmd.strip()
|
||||
if len(cmd) > 400:
|
||||
cmd = f'{cmd[:397]}...'
|
||||
|
||||
return {
|
||||
'session_id': spec.session_id,
|
||||
'workdir': spec.workdir,
|
||||
'mount_path': spec.mount_path,
|
||||
'timeout_sec': spec.timeout_sec,
|
||||
'network': spec.network.value,
|
||||
'image': spec.image,
|
||||
'host_path': spec.host_path,
|
||||
'host_path_mode': spec.host_path_mode.value,
|
||||
'cpus': spec.cpus,
|
||||
'memory_mb': spec.memory_mb,
|
||||
'pids_limit': spec.pids_limit,
|
||||
'read_only_rootfs': spec.read_only_rootfs,
|
||||
'workspace_quota_mb': spec.workspace_quota_mb,
|
||||
'env_keys': sorted(spec.env.keys()),
|
||||
'cmd': cmd,
|
||||
}
|
||||
|
||||
def _summarize_result(self, result: BoxExecutionResult) -> dict:
|
||||
stdout_preview = result.stdout[:200]
|
||||
stderr_preview = result.stderr[:200]
|
||||
if len(result.stdout) > 200:
|
||||
stdout_preview = f'{stdout_preview}...'
|
||||
if len(result.stderr) > 200:
|
||||
stderr_preview = f'{stderr_preview}...'
|
||||
|
||||
return {
|
||||
'session_id': result.session_id,
|
||||
'backend': result.backend_name,
|
||||
'status': result.status.value,
|
||||
'exit_code': result.exit_code,
|
||||
'duration_ms': result.duration_ms,
|
||||
'stdout_preview': stdout_preview,
|
||||
'stderr_preview': stderr_preview,
|
||||
}
|
||||
|
||||
def _local_config(self) -> dict:
|
||||
"""Return ``box.local`` from instance config.
|
||||
|
||||
Environment overrides are applied uniformly by
|
||||
``LoadConfigStage._apply_env_overrides_to_config`` (e.g.
|
||||
``BOX__LOCAL__HOST_ROOT``) before this is read, so no box-specific
|
||||
env parsing happens here.
|
||||
"""
|
||||
return dict(_get_box_config(self.ap).get('local') or {})
|
||||
|
||||
def _load_allowed_mount_roots(self) -> list[str]:
|
||||
configured_roots = self._local_config().get('allowed_mount_roots', [])
|
||||
# The unified env-override mechanism stores a brand-new key as a raw
|
||||
# string when the key is absent from config.yaml. Accept a
|
||||
# comma-separated string as well as a list so that
|
||||
# ``BOX__LOCAL__ALLOWED_MOUNT_ROOTS="/a,/b"`` keeps working even when
|
||||
# the config file has no ``box.local.allowed_mount_roots`` entry.
|
||||
if isinstance(configured_roots, str):
|
||||
configured_roots = [item.strip() for item in configured_roots.split(',') if item.strip()]
|
||||
|
||||
normalized_roots: list[str] = []
|
||||
for root in configured_roots:
|
||||
root_value = str(root).strip()
|
||||
if not root_value:
|
||||
continue
|
||||
normalized_roots.append(os.path.realpath(os.path.abspath(root_value)))
|
||||
|
||||
if not normalized_roots and self.host_root is not None:
|
||||
normalized_roots.append(self.host_root)
|
||||
|
||||
return normalized_roots
|
||||
|
||||
def _load_host_root(self) -> str | None:
|
||||
host_root = str(self._local_config().get('host_root', '')).strip()
|
||||
if not host_root:
|
||||
return None
|
||||
return os.path.realpath(os.path.abspath(host_root))
|
||||
|
||||
def _load_default_workspace(self) -> str | None:
|
||||
default_workspace = str(self._local_config().get('default_workspace', '')).strip()
|
||||
if not default_workspace:
|
||||
if self.host_root is None:
|
||||
return None
|
||||
default_workspace = os.path.join(self.host_root, 'default')
|
||||
elif not os.path.isabs(default_workspace) and self.host_root is not None:
|
||||
default_workspace = os.path.join(self.host_root, default_workspace)
|
||||
return os.path.realpath(os.path.abspath(default_workspace))
|
||||
|
||||
def get_skills_root(self) -> str | None:
|
||||
skills_root = str(self._local_config().get('skills_root', '') or 'skills').strip()
|
||||
if not skills_root:
|
||||
skills_root = 'skills'
|
||||
if not os.path.isabs(skills_root) and self.host_root is not None:
|
||||
skills_root = os.path.join(self.host_root, skills_root)
|
||||
return os.path.realpath(os.path.abspath(skills_root))
|
||||
|
||||
def _load_enabled(self) -> bool:
|
||||
"""Read ``box.enabled`` (top-level, not ``box.local.*``). Default True
|
||||
— disabling is opt-in. Accepts bool, ``'true'``/``'false'`` strings,
|
||||
and the standard env-overridden truthy values that
|
||||
``LoadConfigStage._apply_env_overrides_to_config`` produces."""
|
||||
raw = _get_box_config(self.ap).get('enabled', True)
|
||||
if isinstance(raw, bool):
|
||||
return raw
|
||||
return str(raw).strip().lower() not in ('false', '0', 'no', 'off', '')
|
||||
|
||||
def _load_custom_image(self) -> str | None:
|
||||
raw = str(self._local_config().get('image', '') or '').strip()
|
||||
return raw or None
|
||||
|
||||
def _load_workspace_quota_mb(self) -> int | None:
|
||||
raw_value = self._local_config().get('workspace_quota_mb')
|
||||
if raw_value in (None, ''):
|
||||
return None
|
||||
try:
|
||||
value = _INT_ADAPTER.validate_python(raw_value)
|
||||
except pydantic.ValidationError as exc:
|
||||
raise BoxValidationError('workspace_quota_mb must be an integer greater than or equal to 0') from exc
|
||||
if value < 0:
|
||||
raise BoxValidationError('workspace_quota_mb must be greater than or equal to 0')
|
||||
return value
|
||||
|
||||
def _ensure_default_workspace(self):
|
||||
if self.default_workspace is None:
|
||||
return
|
||||
|
||||
if os.path.isdir(self.default_workspace):
|
||||
return
|
||||
|
||||
if os.path.exists(self.default_workspace):
|
||||
raise BoxValidationError('box.local.default_workspace must point to a directory on the host')
|
||||
|
||||
if not self.allowed_mount_roots:
|
||||
raise BoxValidationError(
|
||||
'box.local.default_workspace cannot be created because no allowed_mount_roots are configured'
|
||||
)
|
||||
|
||||
for allowed_root in self.allowed_mount_roots:
|
||||
if _is_path_under(self.default_workspace, allowed_root):
|
||||
os.makedirs(self.default_workspace, exist_ok=True)
|
||||
return
|
||||
|
||||
allowed_roots = ', '.join(self.allowed_mount_roots)
|
||||
raise BoxValidationError(f'box.local.default_workspace is outside allowed_mount_roots: {allowed_roots}')
|
||||
|
||||
def _validate_host_mount(self, spec: BoxSpec):
|
||||
if spec.host_path is None:
|
||||
return
|
||||
|
||||
host_path = os.path.realpath(spec.host_path)
|
||||
if not os.path.isdir(host_path):
|
||||
raise BoxValidationError('host_path must point to an existing directory on the host')
|
||||
|
||||
if not self.allowed_mount_roots:
|
||||
raise BoxValidationError('host_path mounting is disabled because no allowed_mount_roots are configured')
|
||||
|
||||
for allowed_root in self.allowed_mount_roots:
|
||||
if _is_path_under(host_path, allowed_root):
|
||||
return
|
||||
|
||||
allowed_roots = ', '.join(self.allowed_mount_roots)
|
||||
raise BoxValidationError(f'host_path is outside allowed_mount_roots: {allowed_roots}')
|
||||
|
||||
def _load_profile(self) -> BoxProfile:
|
||||
profile_name = str(self._local_config().get('profile', 'default')).strip() or 'default'
|
||||
|
||||
profile = BUILTIN_PROFILES.get(profile_name)
|
||||
if profile is None:
|
||||
available = ', '.join(sorted(BUILTIN_PROFILES))
|
||||
raise BoxValidationError(f"unknown box profile '{profile_name}', available profiles: {available}")
|
||||
return profile
|
||||
|
||||
def _apply_profile(self, params: dict):
|
||||
"""Merge profile defaults into *params* in-place, enforce locked fields and clamp timeout."""
|
||||
profile = self.profile
|
||||
_PROFILE_FIELDS = (
|
||||
'image',
|
||||
'network',
|
||||
'timeout_sec',
|
||||
'host_path_mode',
|
||||
'cpus',
|
||||
'memory_mb',
|
||||
'pids_limit',
|
||||
'read_only_rootfs',
|
||||
'workspace_quota_mb',
|
||||
)
|
||||
|
||||
for field in _PROFILE_FIELDS:
|
||||
profile_value = getattr(profile, field)
|
||||
raw_value = profile_value.value if isinstance(profile_value, enum.Enum) else profile_value
|
||||
|
||||
if field in profile.locked:
|
||||
params[field] = raw_value
|
||||
elif field not in params:
|
||||
params[field] = raw_value
|
||||
|
||||
timeout = params.get('timeout_sec')
|
||||
try:
|
||||
normalized_timeout = _INT_ADAPTER.validate_python(timeout)
|
||||
except pydantic.ValidationError:
|
||||
return
|
||||
|
||||
if normalized_timeout > profile.max_timeout_sec:
|
||||
params['timeout_sec'] = profile.max_timeout_sec
|
||||
|
||||
def _get_workspace_size_bytes(self, root: str) -> int:
|
||||
total = 0
|
||||
|
||||
def _walk(path: str):
|
||||
nonlocal total
|
||||
try:
|
||||
with os.scandir(path) as entries:
|
||||
for entry in entries:
|
||||
try:
|
||||
if entry.is_symlink():
|
||||
total += entry.stat(follow_symlinks=False).st_size
|
||||
continue
|
||||
if entry.is_dir(follow_symlinks=False):
|
||||
_walk(entry.path)
|
||||
continue
|
||||
total += entry.stat(follow_symlinks=False).st_size
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
except FileNotFoundError:
|
||||
return
|
||||
|
||||
_walk(root)
|
||||
return total
|
||||
|
||||
def _enforce_workspace_quota(self, spec: BoxSpec, *, phase: str) -> None:
|
||||
if spec.host_path is None or spec.workspace_quota_mb <= 0:
|
||||
return
|
||||
|
||||
host_path = os.path.realpath(spec.host_path)
|
||||
if not os.path.isdir(host_path):
|
||||
return
|
||||
|
||||
used_bytes = self._get_workspace_size_bytes(host_path)
|
||||
limit_bytes = spec.workspace_quota_mb * _MIB
|
||||
if used_bytes <= limit_bytes:
|
||||
return
|
||||
|
||||
raise BoxValidationError(
|
||||
f'workspace quota exceeded {phase}: '
|
||||
f'used={used_bytes} bytes limit={limit_bytes} bytes '
|
||||
f'host_path={host_path} session_id={spec.session_id}'
|
||||
)
|
||||
|
||||
async def _cleanup_exceeded_session(self, spec: BoxSpec) -> None:
|
||||
try:
|
||||
await self.client.delete_session(spec.session_id)
|
||||
except Exception as exc:
|
||||
self.ap.logger.warning(
|
||||
'Failed to clean up Box session after workspace quota was exceeded: '
|
||||
f'session_id={spec.session_id} error={exc}'
|
||||
)
|
||||
|
||||
# ── Observability ─────────────────────────────────────────────────
|
||||
|
||||
def _record_error(self, exc: Exception, query: pipeline_query.Query):
|
||||
self._recent_errors.append(
|
||||
{
|
||||
'timestamp': _dt.datetime.now(_UTC).isoformat(),
|
||||
'type': type(exc).__name__,
|
||||
'message': str(exc),
|
||||
'query_id': str(query.query_id),
|
||||
}
|
||||
)
|
||||
|
||||
def get_recent_errors(self) -> list[dict]:
|
||||
return list(self._recent_errors)
|
||||
|
||||
def get_system_guidance(self) -> str:
|
||||
"""Return LLM system-prompt guidance for the exec tool.
|
||||
|
||||
All execution-specific prompt text is kept here so that callers
|
||||
(e.g. LocalAgentRunner) stay free of box domain knowledge.
|
||||
"""
|
||||
guidance = (
|
||||
'When the exec tool is available, use it for exact calculations, statistics, structured data parsing, '
|
||||
'and code execution instead of estimating mentally. If the user provides numbers, tables, CSV-like text, '
|
||||
'JSON, or other data and asks for a computed answer, prefer running a short Python script via exec '
|
||||
'and then answer from the tool result. Unless the user explicitly asks for the script, code, or implementation '
|
||||
'details, do not include the generated script in the final answer; return the result and a brief explanation only.'
|
||||
)
|
||||
if self.default_workspace:
|
||||
guidance += (
|
||||
' A default workspace is mounted at /workspace for file tasks. When the user asks to read, create, or '
|
||||
'modify local files in the working directory, use exec with /workspace paths directly; do not ask the '
|
||||
'user for directory parameters unless they explicitly need a different directory.'
|
||||
)
|
||||
return guidance
|
||||
|
||||
async def get_status(self) -> dict:
|
||||
if not self._available:
|
||||
return {
|
||||
'available': False,
|
||||
'enabled': self._enabled,
|
||||
'profile': self.profile.name,
|
||||
'recent_error_count': len(self._recent_errors),
|
||||
'connector_error': self._connector_error,
|
||||
}
|
||||
try:
|
||||
runtime_status = await self.client.get_status()
|
||||
except Exception as exc:
|
||||
# RPC failed — the runtime likely just disconnected and the
|
||||
# heartbeat hasn't flipped _available yet.
|
||||
return {
|
||||
'available': False,
|
||||
'enabled': self._enabled,
|
||||
'profile': self.profile.name,
|
||||
'recent_error_count': len(self._recent_errors),
|
||||
'connector_error': str(exc),
|
||||
}
|
||||
# Backend state can be unavailable even when the connector is healthy
|
||||
# (operator selected nsjail but the binary is missing, Docker daemon
|
||||
# went down after the runtime started, E2B credentials wrong, ...).
|
||||
# Report the combined state in the top-level ``available`` so the
|
||||
# frontend banner / ``useBoxStatus`` hook / native-tool gate all
|
||||
# agree on "actually usable" rather than "connector alive". The
|
||||
# detailed ``backend`` object stays in the payload so the dialog
|
||||
# can still show which backend was tried.
|
||||
backend_info = runtime_status.get('backend') if isinstance(runtime_status, dict) else None
|
||||
backend_ok = bool(backend_info and backend_info.get('available', False))
|
||||
payload = {
|
||||
**runtime_status,
|
||||
'available': backend_ok,
|
||||
'enabled': self._enabled,
|
||||
'profile': self.profile.name,
|
||||
'recent_error_count': len(self._recent_errors),
|
||||
}
|
||||
if not backend_ok and 'connector_error' not in payload:
|
||||
backend_name = backend_info.get('name') if backend_info else None
|
||||
if backend_name:
|
||||
payload['connector_error'] = f'Configured sandbox backend "{backend_name}" is unavailable'
|
||||
else:
|
||||
payload['connector_error'] = 'No supported sandbox backend (Docker / nsjail / E2B) is available'
|
||||
return payload
|
||||
@@ -1,413 +0,0 @@
|
||||
"""Reusable workspace/session helpers built on top of Box.
|
||||
|
||||
This module is the middle layer between the raw Box runtime primitives and
|
||||
application-specific flows such as skills or MCP stdio.
|
||||
|
||||
It intentionally stays generic:
|
||||
- path and virtualenv rewriting are workspace concerns
|
||||
- Python project detection/bootstrap are workspace concerns
|
||||
- session exec / managed-process helpers are workspace concerns
|
||||
|
||||
Higher layers add their own semantics on top, for example:
|
||||
- skills choose a stable per-skill session id and use repeated exec
|
||||
- MCP stdio chooses how to prepare dependencies and attaches to a managed process
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import textwrap
|
||||
from typing import Any
|
||||
|
||||
PYTHON_MANIFEST_FILES = (
|
||||
'requirements.txt',
|
||||
'pyproject.toml',
|
||||
'setup.py',
|
||||
'setup.cfg',
|
||||
)
|
||||
_VENV_DIRS = frozenset({'.venv', 'venv', 'env', '.env'})
|
||||
_VENV_BIN_DIRS = frozenset({'bin', 'Scripts'})
|
||||
|
||||
|
||||
def normalize_host_path(path: str | None) -> str:
|
||||
if path is None:
|
||||
return ''
|
||||
stripped = str(path).strip()
|
||||
if not stripped:
|
||||
return ''
|
||||
return os.path.realpath(os.path.abspath(stripped))
|
||||
|
||||
|
||||
def rewrite_mounted_path(path: str, host_path: str | None, *, mount_path: str = '/workspace') -> str:
|
||||
"""Translate a host path into the path visible inside the sandbox mount."""
|
||||
if not host_path or not path:
|
||||
return path
|
||||
normalized_host = os.path.realpath(host_path)
|
||||
normalized_path = os.path.realpath(path)
|
||||
if normalized_path.startswith(normalized_host + '/'):
|
||||
return mount_path + normalized_path[len(normalized_host) :]
|
||||
if normalized_path == normalized_host:
|
||||
return mount_path
|
||||
return path
|
||||
|
||||
|
||||
def unwrap_venv_path(directory: str) -> str:
|
||||
"""Collapse ``.../.venv/bin`` style paths back to the project root."""
|
||||
parts = directory.replace('\\', '/').split('/')
|
||||
for i in range(len(parts) - 1, 0, -1):
|
||||
if parts[i] in _VENV_BIN_DIRS and i >= 1:
|
||||
venv_dir = parts[i - 1]
|
||||
if venv_dir in _VENV_DIRS:
|
||||
project_root = '/'.join(parts[: i - 1])
|
||||
return project_root if project_root else '/'
|
||||
return directory
|
||||
|
||||
|
||||
def infer_workspace_host_path(command: str, args: list[str] | None = None) -> str | None:
|
||||
"""Infer the project/workspace root from absolute command/arg paths."""
|
||||
candidates: list[str] = []
|
||||
for part in [command, *(args or [])]:
|
||||
if not os.path.isabs(part):
|
||||
continue
|
||||
if os.path.exists(part):
|
||||
directory = os.path.dirname(part)
|
||||
candidates.append(os.path.realpath(unwrap_venv_path(directory)))
|
||||
if not candidates:
|
||||
return None
|
||||
common = os.path.commonpath(candidates)
|
||||
return common if common != '/' else None
|
||||
|
||||
|
||||
def rewrite_venv_command(command: str, host_path: str | None, *, mount_path: str = '/workspace') -> str:
|
||||
"""Rewrite host venv interpreters to plain ``python`` inside the sandbox.
|
||||
|
||||
Once a project is mounted into the sandbox, host virtualenv paths are no
|
||||
longer valid. For those paths we intentionally drop down to ``python`` and
|
||||
let the sandbox-side environment/bootstrap decide what interpreter to use.
|
||||
"""
|
||||
if not host_path or not command:
|
||||
return command
|
||||
normalized_host = os.path.realpath(host_path)
|
||||
normalized_command = os.path.realpath(command)
|
||||
if not normalized_command.startswith(normalized_host + '/'):
|
||||
return command
|
||||
rel = normalized_command[len(normalized_host) + 1 :]
|
||||
parts = rel.replace('\\', '/').split('/')
|
||||
if len(parts) >= 3 and parts[0] in _VENV_DIRS and parts[1] in _VENV_BIN_DIRS and parts[2].startswith('python'):
|
||||
return 'python'
|
||||
return rewrite_mounted_path(normalized_command, host_path, mount_path=mount_path)
|
||||
|
||||
|
||||
def list_python_manifest_files(host_path: str | None) -> list[str]:
|
||||
normalized_root = normalize_host_path(host_path)
|
||||
if not normalized_root:
|
||||
return []
|
||||
return [filename for filename in PYTHON_MANIFEST_FILES if os.path.isfile(os.path.join(normalized_root, filename))]
|
||||
|
||||
|
||||
def classify_python_workspace(host_path: str | None) -> str | None:
|
||||
"""Return the generic Python workspace shape, without app-specific policy."""
|
||||
manifest_files = set(list_python_manifest_files(host_path))
|
||||
if not manifest_files:
|
||||
return None
|
||||
if {'pyproject.toml', 'setup.py', 'setup.cfg'} & manifest_files:
|
||||
return 'package'
|
||||
if 'requirements.txt' in manifest_files:
|
||||
return 'requirements'
|
||||
return None
|
||||
|
||||
|
||||
def should_prepare_python_env(host_path: str | None) -> bool:
|
||||
normalized_root = normalize_host_path(host_path)
|
||||
if not normalized_root:
|
||||
return False
|
||||
if os.path.isdir(os.path.join(normalized_root, '.venv')):
|
||||
return True
|
||||
return bool(list_python_manifest_files(normalized_root))
|
||||
|
||||
|
||||
def wrap_python_command_with_env(command: str, *, mount_path: str = '/workspace') -> str:
|
||||
"""Wrap a command with a reusable sandbox-local Python env bootstrap.
|
||||
|
||||
This is the generic "workspace is a Python project" path used by mutable
|
||||
workspaces such as skills. Read-only installation strategies stay in the
|
||||
higher-level caller because they are application policy, not workspace
|
||||
semantics.
|
||||
"""
|
||||
bootstrap = textwrap.dedent(
|
||||
f"""
|
||||
set -e
|
||||
|
||||
_LB_VENV_DIR="{mount_path}/.venv"
|
||||
_LB_META_DIR="{mount_path}/.langbot"
|
||||
_LB_META_FILE="$_LB_META_DIR/python-env.json"
|
||||
_LB_LOCK_DIR="$_LB_META_DIR/python-env.lock"
|
||||
_LB_TMP_DIR="{mount_path}/.tmp"
|
||||
_LB_PIP_CACHE_DIR="{mount_path}/.cache/pip"
|
||||
|
||||
mkdir -p "$_LB_META_DIR" "$_LB_TMP_DIR" "$_LB_PIP_CACHE_DIR"
|
||||
export TMPDIR="$_LB_TMP_DIR"
|
||||
export TEMP="$_LB_TMP_DIR"
|
||||
export TMP="$_LB_TMP_DIR"
|
||||
export PIP_CACHE_DIR="$_LB_PIP_CACHE_DIR"
|
||||
|
||||
_lb_python_meta() {{
|
||||
python - <<'PY'
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
root = "{mount_path}"
|
||||
digest = hashlib.sha256()
|
||||
manifest_files = []
|
||||
for rel in ("requirements.txt", "pyproject.toml", "setup.py", "setup.cfg"):
|
||||
path = os.path.join(root, rel)
|
||||
if not os.path.isfile(path):
|
||||
continue
|
||||
manifest_files.append(rel)
|
||||
with open(path, "rb") as handle:
|
||||
digest.update(rel.encode("utf-8"))
|
||||
digest.update(b"\\0")
|
||||
digest.update(handle.read())
|
||||
digest.update(b"\\0")
|
||||
|
||||
print(
|
||||
json.dumps(
|
||||
{{
|
||||
"python_executable": sys.executable,
|
||||
"python_version": list(sys.version_info[:3]),
|
||||
"manifest_files": manifest_files,
|
||||
"manifest_sha256": digest.hexdigest(),
|
||||
}},
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
PY
|
||||
}}
|
||||
|
||||
_LB_CURRENT_META="$(_lb_python_meta)"
|
||||
_LB_NEEDS_BOOTSTRAP=0
|
||||
|
||||
if [ ! -x "$_LB_VENV_DIR/bin/python" ]; then
|
||||
_LB_NEEDS_BOOTSTRAP=1
|
||||
elif [ ! -f "$_LB_META_FILE" ]; then
|
||||
_LB_NEEDS_BOOTSTRAP=1
|
||||
elif [ "$(cat "$_LB_META_FILE")" != "$_LB_CURRENT_META" ]; then
|
||||
_LB_NEEDS_BOOTSTRAP=1
|
||||
fi
|
||||
|
||||
if [ "$_LB_NEEDS_BOOTSTRAP" -eq 1 ]; then
|
||||
_LB_LOCK_WAIT=0
|
||||
while ! mkdir "$_LB_LOCK_DIR" 2>/dev/null; do
|
||||
if [ "$_LB_LOCK_WAIT" -ge 120 ]; then
|
||||
echo "Timed out waiting for Python environment lock: $_LB_LOCK_DIR" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
_LB_LOCK_WAIT=$((_LB_LOCK_WAIT + 1))
|
||||
done
|
||||
|
||||
_lb_cleanup_lock() {{
|
||||
rmdir "$_LB_LOCK_DIR" >/dev/null 2>&1 || true
|
||||
}}
|
||||
trap _lb_cleanup_lock EXIT INT TERM
|
||||
|
||||
_LB_CURRENT_META="$(_lb_python_meta)"
|
||||
_LB_NEEDS_BOOTSTRAP=0
|
||||
if [ ! -x "$_LB_VENV_DIR/bin/python" ]; then
|
||||
_LB_NEEDS_BOOTSTRAP=1
|
||||
elif [ ! -f "$_LB_META_FILE" ]; then
|
||||
_LB_NEEDS_BOOTSTRAP=1
|
||||
elif [ "$(cat "$_LB_META_FILE")" != "$_LB_CURRENT_META" ]; then
|
||||
_LB_NEEDS_BOOTSTRAP=1
|
||||
fi
|
||||
|
||||
if [ "$_LB_NEEDS_BOOTSTRAP" -eq 1 ]; then
|
||||
rm -rf "$_LB_VENV_DIR"
|
||||
python -m venv "$_LB_VENV_DIR"
|
||||
. "$_LB_VENV_DIR/bin/activate"
|
||||
python -m pip install --upgrade pip setuptools wheel
|
||||
if [ -f "{mount_path}/requirements.txt" ]; then
|
||||
python -m pip install -r "{mount_path}/requirements.txt"
|
||||
elif [ -f "{mount_path}/pyproject.toml" ] || [ -f "{mount_path}/setup.py" ] || [ -f "{mount_path}/setup.cfg" ]; then
|
||||
python -m pip install "{mount_path}"
|
||||
fi
|
||||
printf '%s' "$_LB_CURRENT_META" > "$_LB_META_FILE"
|
||||
fi
|
||||
fi
|
||||
|
||||
export VIRTUAL_ENV="$_LB_VENV_DIR"
|
||||
export PATH="$_LB_VENV_DIR/bin:$PATH"
|
||||
{command}
|
||||
"""
|
||||
).strip()
|
||||
return bootstrap + '\n'
|
||||
|
||||
|
||||
class BoxWorkspaceSession:
|
||||
"""High-level handle for one reusable workspace-backed Box session.
|
||||
|
||||
The Box runtime already understands sessions and managed processes. This
|
||||
wrapper adds LangBot's workspace-centric view on top: a mounted host path,
|
||||
a stable ``session_id``, optional environment defaults, and convenience
|
||||
helpers for exec or long-running processes inside that workspace.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
box_service,
|
||||
session_id: str,
|
||||
*,
|
||||
host_path: str | None = None,
|
||||
host_path_mode: str = 'rw',
|
||||
workdir: str = '/workspace',
|
||||
env: dict[str, str] | None = None,
|
||||
mount_path: str = '/workspace',
|
||||
network: str | None = None,
|
||||
read_only_rootfs: bool | None = None,
|
||||
image: str | None = None,
|
||||
cpus: float | None = None,
|
||||
memory_mb: int | None = None,
|
||||
pids_limit: int | None = None,
|
||||
persistent: bool = False,
|
||||
):
|
||||
self.box_service = box_service
|
||||
self.session_id = session_id
|
||||
self.host_path = host_path
|
||||
self.host_path_mode = host_path_mode
|
||||
self.workdir = workdir
|
||||
self.env = dict(env or {})
|
||||
self.mount_path = mount_path
|
||||
self.network = network
|
||||
self.read_only_rootfs = read_only_rootfs
|
||||
self.image = image
|
||||
self.cpus = cpus
|
||||
self.memory_mb = memory_mb
|
||||
self.pids_limit = pids_limit
|
||||
self.persistent = persistent
|
||||
|
||||
def rewrite_path(self, path: str) -> str:
|
||||
return rewrite_mounted_path(path, self.host_path, mount_path=self.mount_path)
|
||||
|
||||
def rewrite_venv_command(self, command: str) -> str:
|
||||
return rewrite_venv_command(command, self.host_path, mount_path=self.mount_path)
|
||||
|
||||
def build_session_payload(self) -> dict[str, Any]:
|
||||
# Keep this payload generic so callers can reuse the same workspace
|
||||
# handle for plain exec, file-producing tasks, or managed processes.
|
||||
payload: dict[str, Any] = {
|
||||
'session_id': self.session_id,
|
||||
'workdir': self.workdir,
|
||||
'env': self.env,
|
||||
'persistent': self.persistent,
|
||||
}
|
||||
if self.network is not None:
|
||||
payload['network'] = self.network
|
||||
if self.read_only_rootfs is not None:
|
||||
payload['read_only_rootfs'] = self.read_only_rootfs
|
||||
if self.host_path:
|
||||
payload['host_path'] = self.host_path
|
||||
payload['host_path_mode'] = self.host_path_mode
|
||||
for key in ('image', 'cpus', 'memory_mb', 'pids_limit'):
|
||||
value = getattr(self, key)
|
||||
if value is not None:
|
||||
payload[key] = value
|
||||
return payload
|
||||
|
||||
def build_exec_payload(
|
||||
self,
|
||||
cmd: str,
|
||||
*,
|
||||
workdir: str | None = None,
|
||||
env: dict[str, str] | None = None,
|
||||
timeout_sec: int | None = None,
|
||||
) -> dict[str, Any]:
|
||||
# Exec payloads inherit the session-level workspace config, then layer
|
||||
# per-call command/workdir/env overrides on top.
|
||||
payload = self.build_session_payload()
|
||||
payload['cmd'] = cmd
|
||||
payload['workdir'] = workdir or self.workdir
|
||||
if timeout_sec is not None:
|
||||
payload['timeout_sec'] = timeout_sec
|
||||
resolved_env = self.env if env is None else env
|
||||
if resolved_env:
|
||||
payload['env'] = resolved_env
|
||||
elif 'env' in payload and not payload['env']:
|
||||
payload.pop('env')
|
||||
return payload
|
||||
|
||||
async def execute_raw(
|
||||
self,
|
||||
cmd: str,
|
||||
*,
|
||||
workdir: str | None = None,
|
||||
env: dict[str, str] | None = None,
|
||||
timeout_sec: int | None = None,
|
||||
):
|
||||
payload = self.build_exec_payload(cmd, workdir=workdir, env=env, timeout_sec=timeout_sec)
|
||||
return await self.box_service.client.execute(self.box_service.build_spec(payload))
|
||||
|
||||
async def execute_for_query(
|
||||
self,
|
||||
query,
|
||||
cmd: str,
|
||||
*,
|
||||
workdir: str | None = None,
|
||||
env: dict[str, str] | None = None,
|
||||
timeout_sec: int | None = None,
|
||||
) -> dict:
|
||||
payload = self.build_exec_payload(cmd, workdir=workdir, env=env, timeout_sec=timeout_sec)
|
||||
return await self.box_service.execute_spec_payload(payload, query)
|
||||
|
||||
async def create_session(self):
|
||||
return await self.box_service.create_session(self.build_session_payload())
|
||||
|
||||
def build_process_payload(
|
||||
self,
|
||||
command: str,
|
||||
args: list[str] | None = None,
|
||||
*,
|
||||
env: dict[str, str] | None = None,
|
||||
cwd: str = '/workspace',
|
||||
) -> dict[str, Any]:
|
||||
# Managed processes run inside the same workspace model as one-shot
|
||||
# execs, so path/venv rewriting is shared here.
|
||||
normalized_command = command
|
||||
normalized_args = list(args or [])
|
||||
normalized_cwd = cwd
|
||||
if self.host_path:
|
||||
normalized_command = self.rewrite_venv_command(command)
|
||||
normalized_args = [self.rewrite_path(arg) for arg in normalized_args]
|
||||
normalized_cwd = self.rewrite_path(cwd)
|
||||
return {
|
||||
'command': normalized_command,
|
||||
'args': normalized_args,
|
||||
'env': dict(env or {}),
|
||||
'cwd': normalized_cwd,
|
||||
}
|
||||
|
||||
async def start_managed_process(
|
||||
self,
|
||||
command: str,
|
||||
args: list[str] | None = None,
|
||||
*,
|
||||
process_id: str = 'default',
|
||||
env: dict[str, str] | None = None,
|
||||
cwd: str = '/workspace',
|
||||
):
|
||||
payload = self.build_process_payload(command, args, env=env, cwd=cwd)
|
||||
payload['process_id'] = process_id
|
||||
return await self.box_service.start_managed_process(self.session_id, payload)
|
||||
|
||||
async def get_managed_process(self, process_id: str = 'default'):
|
||||
return await self.box_service.get_managed_process(self.session_id, process_id)
|
||||
|
||||
async def stop_managed_process(self, process_id: str = 'default') -> None:
|
||||
await self.box_service.stop_managed_process(self.session_id, process_id)
|
||||
|
||||
def get_managed_process_websocket_url(self, process_id: str = 'default') -> str:
|
||||
return self.box_service.get_managed_process_websocket_url(self.session_id, process_id)
|
||||
|
||||
async def cleanup(self) -> None:
|
||||
await self.box_service.client.delete_session(self.session_id)
|
||||
@@ -9,8 +9,6 @@ from ..platform import botmgr as im_mgr
|
||||
from ..platform.webhook_pusher import WebhookPusher
|
||||
from ..provider.session import sessionmgr as llm_session_mgr
|
||||
from ..provider.modelmgr import modelmgr as llm_model_mgr
|
||||
from ..box import service as box_service_module
|
||||
|
||||
from langbot.pkg.provider.tools import toolmgr as llm_tool_mgr
|
||||
from ..config import manager as config_mgr
|
||||
from ..command import cmdmgr
|
||||
@@ -32,8 +30,6 @@ from ..api.http.service import mcp as mcp_service
|
||||
from ..api.http.service import apikey as apikey_service
|
||||
from ..api.http.service import webhook as webhook_service
|
||||
from ..api.http.service import monitoring as monitoring_service
|
||||
from ..api.http.service import skill as skill_service
|
||||
from ..api.http.service import maintenance as maintenance_service
|
||||
from ..discover import engine as discover_engine
|
||||
from ..storage import mgr as storagemgr
|
||||
from ..utils import logcache
|
||||
@@ -44,7 +40,6 @@ from ..rag.service import RAGRuntimeService
|
||||
from ..vector import mgr as vectordb_mgr
|
||||
from ..telemetry import telemetry as telemetry_module
|
||||
from ..survey import manager as survey_module
|
||||
from ..skill import manager as skill_mgr
|
||||
|
||||
|
||||
class Application:
|
||||
@@ -72,7 +67,6 @@ class Application:
|
||||
|
||||
# TODO move to pipeline
|
||||
tool_mgr: llm_tool_mgr.ToolManager = None
|
||||
box_service: box_service_module.BoxService = None
|
||||
|
||||
# ======= Config manager =======
|
||||
|
||||
@@ -137,8 +131,6 @@ class Application:
|
||||
|
||||
embedding_models_service: model_service.EmbeddingModelsService = None
|
||||
|
||||
rerank_models_service: model_service.RerankModelsService = None
|
||||
|
||||
provider_service: provider_service.ModelProviderService = None
|
||||
|
||||
pipeline_service: pipeline_service.PipelineService = None
|
||||
@@ -159,12 +151,6 @@ class Application:
|
||||
|
||||
monitoring_service: monitoring_service.MonitoringService = None
|
||||
|
||||
skill_service: skill_service.SkillService = None
|
||||
|
||||
skill_mgr: skill_mgr.SkillManager = None
|
||||
|
||||
maintenance_service: maintenance_service.MaintenanceService = None
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
@@ -200,77 +186,6 @@ class Application:
|
||||
scopes=[core_entities.LifecycleControlScope.APPLICATION],
|
||||
)
|
||||
|
||||
# Start monitoring data cleanup task if enabled
|
||||
monitoring_cfg = self.instance_config.data.get('monitoring', {})
|
||||
auto_cleanup_cfg = monitoring_cfg.get('auto_cleanup', {})
|
||||
if auto_cleanup_cfg.get('enabled', True):
|
||||
retention_days = self._get_positive_int_config(
|
||||
auto_cleanup_cfg.get('retention_days', 30),
|
||||
default=30,
|
||||
name='monitoring.auto_cleanup.retention_days',
|
||||
)
|
||||
delete_batch_size = self._get_positive_int_config(
|
||||
auto_cleanup_cfg.get('delete_batch_size', 1000),
|
||||
default=1000,
|
||||
name='monitoring.auto_cleanup.delete_batch_size',
|
||||
)
|
||||
check_interval_hours = self._get_positive_float_config(
|
||||
auto_cleanup_cfg.get('check_interval_hours', 1),
|
||||
default=1,
|
||||
name='monitoring.auto_cleanup.check_interval_hours',
|
||||
)
|
||||
|
||||
async def monitoring_cleanup_loop():
|
||||
check_interval_seconds = check_interval_hours * 3600
|
||||
while True:
|
||||
try:
|
||||
deleted = await self.monitoring_service.cleanup_expired_records(
|
||||
retention_days,
|
||||
batch_size=delete_batch_size,
|
||||
)
|
||||
total_deleted = sum(deleted.values())
|
||||
if total_deleted > 0:
|
||||
self.logger.info(
|
||||
f'Monitoring auto-cleanup: deleted {total_deleted} expired records '
|
||||
f'(retention={retention_days}d): {deleted}'
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.warning(f'Monitoring auto-cleanup error: {e}')
|
||||
await asyncio.sleep(check_interval_seconds)
|
||||
|
||||
self.task_mgr.create_task(
|
||||
monitoring_cleanup_loop(),
|
||||
name='monitoring-cleanup',
|
||||
scopes=[core_entities.LifecycleControlScope.APPLICATION],
|
||||
)
|
||||
|
||||
# Start storage/log maintenance task if enabled
|
||||
storage_cleanup_cfg = self.instance_config.data.get('storage', {}).get('cleanup', {})
|
||||
if storage_cleanup_cfg.get('enabled', True) and self.maintenance_service is not None:
|
||||
check_interval_hours = self._get_positive_float_config(
|
||||
storage_cleanup_cfg.get('check_interval_hours', 1),
|
||||
default=1,
|
||||
name='storage.cleanup.check_interval_hours',
|
||||
)
|
||||
|
||||
async def storage_cleanup_loop():
|
||||
check_interval_seconds = check_interval_hours * 3600
|
||||
while True:
|
||||
try:
|
||||
deleted = await self.maintenance_service.cleanup_expired_files()
|
||||
total_deleted = sum(deleted.values())
|
||||
if total_deleted > 0:
|
||||
self.logger.info(f'Storage maintenance: deleted expired files: {deleted}')
|
||||
except Exception as e:
|
||||
self.logger.warning(f'Storage maintenance error: {e}')
|
||||
await asyncio.sleep(check_interval_seconds)
|
||||
|
||||
self.task_mgr.create_task(
|
||||
storage_cleanup_loop(),
|
||||
name='storage-maintenance',
|
||||
scopes=[core_entities.LifecycleControlScope.APPLICATION],
|
||||
)
|
||||
|
||||
self.task_mgr.create_task(
|
||||
never_ending(),
|
||||
name='never-ending-task',
|
||||
@@ -285,33 +200,8 @@ class Application:
|
||||
self.logger.error(f'Application runtime fatal exception: {e}')
|
||||
self.logger.debug(f'Traceback: {traceback.format_exc()}')
|
||||
|
||||
def _get_positive_int_config(self, value, default: int, name: str) -> int:
|
||||
try:
|
||||
parsed = int(value)
|
||||
except (TypeError, ValueError):
|
||||
self.logger.warning(f'Invalid {name}: {value!r}, using {default}')
|
||||
return default
|
||||
if parsed < 1:
|
||||
self.logger.warning(f'Invalid {name}: {value!r}, using {default}')
|
||||
return default
|
||||
return parsed
|
||||
|
||||
def _get_positive_float_config(self, value, default: float, name: str) -> float:
|
||||
try:
|
||||
parsed = float(value)
|
||||
except (TypeError, ValueError):
|
||||
self.logger.warning(f'Invalid {name}: {value!r}, using {default}')
|
||||
return default
|
||||
if parsed <= 0:
|
||||
self.logger.warning(f'Invalid {name}: {value!r}, using {default}')
|
||||
return default
|
||||
return parsed
|
||||
|
||||
def dispose(self):
|
||||
if self.plugin_connector is not None:
|
||||
self.plugin_connector.dispose()
|
||||
if self.box_service is not None:
|
||||
self.box_service.dispose()
|
||||
self.plugin_connector.dispose()
|
||||
|
||||
async def print_web_access_info(self):
|
||||
"""Print access webui tips"""
|
||||
|
||||
@@ -46,14 +46,12 @@ async def make_app(loop: asyncio.AbstractEventLoop) -> app.Application:
|
||||
|
||||
|
||||
async def main(loop: asyncio.AbstractEventLoop):
|
||||
app_inst: app.Application | None = None
|
||||
try:
|
||||
# Hang system signal processing
|
||||
import signal
|
||||
|
||||
def signal_handler(sig, frame):
|
||||
if app_inst is not None:
|
||||
app_inst.dispose()
|
||||
app_inst.dispose()
|
||||
print('[Signal] Program exit.')
|
||||
os._exit(0)
|
||||
|
||||
@@ -62,6 +60,4 @@ async def main(loop: asyncio.AbstractEventLoop):
|
||||
app_inst = await make_app(loop)
|
||||
await app_inst.run()
|
||||
except Exception:
|
||||
if app_inst is not None:
|
||||
app_inst.dispose()
|
||||
traceback.print_exc()
|
||||
|
||||
@@ -6,7 +6,6 @@ from .. import stage, app
|
||||
from ...utils import version, proxy
|
||||
from ...pipeline import pool, controller, pipelinemgr
|
||||
from ...pipeline import aggregator as message_aggregator
|
||||
from ...box import service as box_service
|
||||
from ...plugin import connector as plugin_connector
|
||||
from ...command import cmdmgr
|
||||
from ...provider.session import sessionmgr as llm_session_mgr
|
||||
@@ -29,9 +28,6 @@ from ...api.http.service import mcp as mcp_service
|
||||
from ...api.http.service import apikey as apikey_service
|
||||
from ...api.http.service import webhook as webhook_service
|
||||
from ...api.http.service import monitoring as monitoring_service
|
||||
from ...api.http.service import skill as skill_service
|
||||
from ...skill import manager as skill_mgr
|
||||
from ...api.http.service import maintenance as maintenance_service
|
||||
from ...discover import engine as discover_engine
|
||||
from ...storage import mgr as storagemgr
|
||||
from ...utils import logcache
|
||||
@@ -65,9 +61,6 @@ class BuildAppStage(stage.BootingStage):
|
||||
embedding_models_service_inst = model_service.EmbeddingModelsService(ap)
|
||||
ap.embedding_models_service = embedding_models_service_inst
|
||||
|
||||
rerank_models_service_inst = model_service.RerankModelsService(ap)
|
||||
ap.rerank_models_service = rerank_models_service_inst
|
||||
|
||||
provider_service_inst = provider_service.ModelProviderService(ap)
|
||||
ap.provider_service = provider_service_inst
|
||||
|
||||
@@ -89,9 +82,6 @@ class BuildAppStage(stage.BootingStage):
|
||||
webhook_service_inst = webhook_service.WebhookService(ap)
|
||||
ap.webhook_service = webhook_service_inst
|
||||
|
||||
skill_service_inst = skill_service.SkillService(ap)
|
||||
ap.skill_service = skill_service_inst
|
||||
|
||||
proxy_mgr = proxy.ProxyManager(ap)
|
||||
await proxy_mgr.initialize()
|
||||
ap.proxy_mgr = proxy_mgr
|
||||
@@ -135,10 +125,6 @@ class BuildAppStage(stage.BootingStage):
|
||||
await llm_session_mgr_inst.initialize()
|
||||
ap.sess_mgr = llm_session_mgr_inst
|
||||
|
||||
box_service_inst = box_service.BoxService(ap)
|
||||
await box_service_inst.initialize()
|
||||
ap.box_service = box_service_inst
|
||||
|
||||
llm_tool_mgr_inst = llm_tool_mgr.ToolManager(ap)
|
||||
await llm_tool_mgr_inst.initialize()
|
||||
ap.tool_mgr = llm_tool_mgr_inst
|
||||
@@ -159,11 +145,6 @@ class BuildAppStage(stage.BootingStage):
|
||||
msg_aggregator_inst = message_aggregator.MessageAggregator(ap)
|
||||
ap.msg_aggregator = msg_aggregator_inst
|
||||
|
||||
# Initialize skill manager
|
||||
skill_mgr_inst = skill_mgr.SkillManager(ap)
|
||||
await skill_mgr_inst.initialize()
|
||||
ap.skill_mgr = skill_mgr_inst
|
||||
|
||||
rag_mgr_inst = rag_mgr.RAGManager(ap)
|
||||
await rag_mgr_inst.initialize()
|
||||
ap.rag_mgr = rag_mgr_inst
|
||||
@@ -183,9 +164,6 @@ class BuildAppStage(stage.BootingStage):
|
||||
monitoring_service_inst = monitoring_service.MonitoringService(ap)
|
||||
ap.monitoring_service = monitoring_service_inst
|
||||
|
||||
maintenance_service_inst = maintenance_service.MaintenanceService(ap)
|
||||
ap.maintenance_service = maintenance_service_inst
|
||||
|
||||
async def runtime_disconnect_callback(connector: plugin_connector.PluginRuntimeConnector) -> None:
|
||||
await asyncio.sleep(3)
|
||||
await plugin_connector_inst.initialize()
|
||||
|
||||
@@ -74,30 +74,20 @@ def _apply_env_overrides_to_config(cfg: dict) -> dict:
|
||||
current = cfg
|
||||
|
||||
for i, key in enumerate(keys):
|
||||
if not isinstance(current, dict):
|
||||
if not isinstance(current, dict) or key not in current:
|
||||
break
|
||||
|
||||
if i == len(keys) - 1:
|
||||
# At the final key
|
||||
if key in current:
|
||||
if isinstance(current[key], list):
|
||||
# Convert comma-separated string to list
|
||||
# e.g., SYSTEM__DISABLED_ADAPTERS="aiocqhttp,dingtalk"
|
||||
current[key] = [item.strip() for item in env_value.split(',') if item.strip()]
|
||||
elif isinstance(current[key], dict):
|
||||
# Skip dict types
|
||||
pass
|
||||
else:
|
||||
# Valid scalar value - convert and set it
|
||||
converted_value = convert_value(env_value, current[key])
|
||||
current[key] = converted_value
|
||||
# At the final key - check if it's a scalar value
|
||||
if isinstance(current[key], (dict, list)):
|
||||
# Skip dict and list types
|
||||
pass
|
||||
else:
|
||||
# Key doesn't exist yet - create it as string
|
||||
current[key] = env_value
|
||||
# Valid scalar value - convert and set it
|
||||
converted_value = convert_value(env_value, current[key])
|
||||
current[key] = converted_value
|
||||
else:
|
||||
# Navigate deeper - create intermediate dict if needed
|
||||
if key not in current:
|
||||
current[key] = {}
|
||||
# Navigate deeper
|
||||
current = current[key]
|
||||
|
||||
return cfg
|
||||
@@ -156,50 +146,16 @@ class LoadConfigStage(stage.BootingStage):
|
||||
await ap.instance_config.dump_config()
|
||||
|
||||
# load or generate instance id
|
||||
# Priority:
|
||||
# 1. system.instance_id from config.yaml (can be set via SYSTEM__INSTANCE_ID env var)
|
||||
# 2. data/labels/instance_id.json (if file exists)
|
||||
# 3. Generate new and save to file
|
||||
config_instance_id = ap.instance_config.data.get('system', {}).get('instance_id', '')
|
||||
ap.instance_id = await config.load_json_config(
|
||||
'data/labels/instance_id.json',
|
||||
template_data={
|
||||
'instance_id': f'instance_{str(uuid.uuid4())}',
|
||||
'instance_create_ts': int(time.time()),
|
||||
},
|
||||
completion=False,
|
||||
)
|
||||
|
||||
if config_instance_id:
|
||||
# Use the instance_id from config.yaml
|
||||
constants.instance_id = config_instance_id
|
||||
# Still load/create the file for backward compat, but don't use its value
|
||||
ap.instance_id = await config.load_json_config(
|
||||
'data/labels/instance_id.json',
|
||||
template_data={
|
||||
'instance_id': f'instance_{str(uuid.uuid4())}',
|
||||
'instance_create_ts': int(time.time()),
|
||||
},
|
||||
completion=False,
|
||||
)
|
||||
else:
|
||||
# Try loading file-based instance id
|
||||
instance_id_path = os.path.join('data', 'labels', 'instance_id.json')
|
||||
if os.path.exists(instance_id_path):
|
||||
# File exists, read it
|
||||
ap.instance_id = await config.load_json_config(
|
||||
'data/labels/instance_id.json',
|
||||
template_data={
|
||||
'instance_id': '',
|
||||
'instance_create_ts': 0,
|
||||
},
|
||||
completion=False,
|
||||
)
|
||||
constants.instance_id = ap.instance_id.data['instance_id']
|
||||
else:
|
||||
# Neither config nor file, generate new and save to file
|
||||
new_id = f'instance_{str(uuid.uuid4())}'
|
||||
ap.instance_id = await config.load_json_config(
|
||||
'data/labels/instance_id.json',
|
||||
template_data={
|
||||
'instance_id': new_id,
|
||||
'instance_create_ts': int(time.time()),
|
||||
},
|
||||
completion=False,
|
||||
)
|
||||
constants.instance_id = new_id
|
||||
constants.instance_id = ap.instance_id.data['instance_id']
|
||||
constants.edition = ap.instance_config.data.get('system', {}).get('edition', 'community')
|
||||
|
||||
print(f'LangBot instance id: {constants.instance_id}')
|
||||
|
||||
@@ -3,7 +3,6 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import typing
|
||||
import datetime
|
||||
import time
|
||||
|
||||
from . import app
|
||||
from . import entities as core_entities
|
||||
@@ -18,13 +17,9 @@ class TaskContext:
|
||||
log: str
|
||||
"""Log"""
|
||||
|
||||
metadata: dict
|
||||
"""Structured metadata for progress reporting"""
|
||||
|
||||
def __init__(self):
|
||||
self.current_action = 'default'
|
||||
self.log = ''
|
||||
self.metadata = {}
|
||||
|
||||
def _log(self, msg: str):
|
||||
self.log += msg + '\n'
|
||||
@@ -43,7 +38,7 @@ class TaskContext:
|
||||
self._log(f'{datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")} | {self.current_action} | {msg}')
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {'current_action': self.current_action, 'log': self.log, 'metadata': self.metadata}
|
||||
return {'current_action': self.current_action, 'log': self.log}
|
||||
|
||||
@staticmethod
|
||||
def new() -> TaskContext:
|
||||
@@ -120,7 +115,6 @@ class TaskWrapper:
|
||||
self.label = label if label != '' else name
|
||||
self.task.set_name(name)
|
||||
self.scopes = scopes
|
||||
self.created_at = time.time()
|
||||
|
||||
def assume_exception(self):
|
||||
try:
|
||||
@@ -156,7 +150,6 @@ class TaskWrapper:
|
||||
'name': self.name,
|
||||
'label': self.label,
|
||||
'scopes': [scope.value for scope in self.scopes],
|
||||
'created_at': self.created_at,
|
||||
'task_context': self.task_context.to_dict(),
|
||||
'runtime': {
|
||||
'done': self.task.done(),
|
||||
@@ -196,8 +189,6 @@ class AsyncTaskManager:
|
||||
) -> TaskWrapper:
|
||||
wrapper = TaskWrapper(self.ap, coro, task_type, kind, name, label, context, scopes)
|
||||
self.tasks.append(wrapper)
|
||||
wrapper.task.add_done_callback(lambda _: self._prune_completed_tasks())
|
||||
self._prune_completed_tasks()
|
||||
return wrapper
|
||||
|
||||
def create_user_task(
|
||||
@@ -220,23 +211,9 @@ class AsyncTaskManager:
|
||||
def get_tasks_dict(
|
||||
self,
|
||||
type: str = None,
|
||||
kind: str = None,
|
||||
) -> dict:
|
||||
return {
|
||||
'tasks': [
|
||||
t.to_dict()
|
||||
for t in self.tasks
|
||||
if (type is None or t.task_type == type) and (kind is None or t.kind == kind)
|
||||
],
|
||||
'id_index': TaskWrapper._id_index,
|
||||
}
|
||||
|
||||
def get_stats(self) -> dict:
|
||||
completed = sum(1 for t in self.tasks if t.task.done())
|
||||
return {
|
||||
'total': len(self.tasks),
|
||||
'running': len(self.tasks) - completed,
|
||||
'completed': completed,
|
||||
'tasks': [t.to_dict() for t in self.tasks if type is None or t.task_type == type],
|
||||
'id_index': TaskWrapper._id_index,
|
||||
}
|
||||
|
||||
@@ -257,27 +234,3 @@ class AsyncTaskManager:
|
||||
if not wrapper.task.done():
|
||||
wrapper.task.cancel()
|
||||
return
|
||||
|
||||
def _prune_completed_tasks(self):
|
||||
completed_limit = (
|
||||
self.ap.instance_config.data.get('system', {})
|
||||
.get('task_retention', {})
|
||||
.get(
|
||||
'completed_limit',
|
||||
200,
|
||||
)
|
||||
)
|
||||
try:
|
||||
completed_limit = int(completed_limit)
|
||||
except (TypeError, ValueError):
|
||||
completed_limit = 200
|
||||
if completed_limit < 1:
|
||||
completed_limit = 1
|
||||
|
||||
completed_tasks = [wrapper for wrapper in self.tasks if wrapper.task.done()]
|
||||
overflow = len(completed_tasks) - completed_limit
|
||||
if overflow <= 0:
|
||||
return
|
||||
|
||||
remove_ids = {wrapper.id for wrapper in completed_tasks[:overflow]}
|
||||
self.tasks = [wrapper for wrapper in self.tasks if wrapper.id not in remove_ids]
|
||||
|
||||
@@ -17,23 +17,11 @@ class I18nString(pydantic.BaseModel):
|
||||
"""英文"""
|
||||
|
||||
zh_Hans: typing.Optional[str] = None
|
||||
"""简体中文"""
|
||||
|
||||
zh_Hant: typing.Optional[str] = None
|
||||
"""繁体中文"""
|
||||
"""中文"""
|
||||
|
||||
ja_JP: typing.Optional[str] = None
|
||||
"""日文"""
|
||||
|
||||
th_TH: typing.Optional[str] = None
|
||||
"""泰文"""
|
||||
|
||||
vi_VN: typing.Optional[str] = None
|
||||
"""越南文"""
|
||||
|
||||
es_ES: typing.Optional[str] = None
|
||||
"""西班牙文"""
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""转换为字典"""
|
||||
dic = {}
|
||||
@@ -41,16 +29,8 @@ class I18nString(pydantic.BaseModel):
|
||||
dic['en_US'] = self.en_US
|
||||
if self.zh_Hans is not None:
|
||||
dic['zh_Hans'] = self.zh_Hans
|
||||
if self.zh_Hant is not None:
|
||||
dic['zh_Hant'] = self.zh_Hant
|
||||
if self.ja_JP is not None:
|
||||
dic['ja_JP'] = self.ja_JP
|
||||
if self.th_TH is not None:
|
||||
dic['th_TH'] = self.th_TH
|
||||
if self.vi_VN is not None:
|
||||
dic['vi_VN'] = self.vi_VN
|
||||
if self.es_ES is not None:
|
||||
dic['es_ES'] = self.es_ES
|
||||
return dic
|
||||
|
||||
|
||||
|
||||
@@ -16,7 +16,6 @@ class Bot(Base):
|
||||
enable = sqlalchemy.Column(sqlalchemy.Boolean, nullable=False, default=False)
|
||||
use_pipeline_name = sqlalchemy.Column(sqlalchemy.String(255), nullable=True)
|
||||
use_pipeline_uuid = sqlalchemy.Column(sqlalchemy.String(255), nullable=True)
|
||||
pipeline_routing_rules = sqlalchemy.Column(sqlalchemy.JSON, nullable=False, server_default='[]')
|
||||
created_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False, server_default=sqlalchemy.func.now())
|
||||
updated_at = sqlalchemy.Column(
|
||||
sqlalchemy.DateTime,
|
||||
|
||||
@@ -59,22 +59,3 @@ class EmbeddingModel(Base):
|
||||
server_default=sqlalchemy.func.now(),
|
||||
onupdate=sqlalchemy.func.now(),
|
||||
)
|
||||
|
||||
|
||||
class RerankModel(Base):
|
||||
"""Rerank model"""
|
||||
|
||||
__tablename__ = 'rerank_models'
|
||||
|
||||
uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True, unique=True)
|
||||
name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
provider_uuid = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
extra_args = sqlalchemy.Column(sqlalchemy.JSON, nullable=False, default={})
|
||||
prefered_ranking = sqlalchemy.Column(sqlalchemy.Integer, nullable=False, default=0)
|
||||
created_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False, server_default=sqlalchemy.func.now())
|
||||
updated_at = sqlalchemy.Column(
|
||||
sqlalchemy.DateTime,
|
||||
nullable=False,
|
||||
server_default=sqlalchemy.func.now(),
|
||||
onupdate=sqlalchemy.func.now(),
|
||||
)
|
||||
|
||||
@@ -20,7 +20,6 @@ class MonitoringMessage(Base):
|
||||
level = sqlalchemy.Column(sqlalchemy.String(50), nullable=False) # info, warning, error, debug
|
||||
platform = sqlalchemy.Column(sqlalchemy.String(255), nullable=True)
|
||||
user_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=True)
|
||||
user_name = sqlalchemy.Column(sqlalchemy.String(255), nullable=True) # User display name
|
||||
runner_name = sqlalchemy.Column(sqlalchemy.String(255), nullable=True) # Runner name for this query
|
||||
variables = sqlalchemy.Column(sqlalchemy.Text, nullable=True) # Query variables as JSON string
|
||||
role = sqlalchemy.Column(sqlalchemy.String(50), nullable=True, default='user') # user, assistant
|
||||
@@ -65,7 +64,6 @@ class MonitoringSession(Base):
|
||||
is_active = sqlalchemy.Column(sqlalchemy.Boolean, nullable=False, default=True, index=True)
|
||||
platform = sqlalchemy.Column(sqlalchemy.String(255), nullable=True)
|
||||
user_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=True)
|
||||
user_name = sqlalchemy.Column(sqlalchemy.String(255), nullable=True) # User display name
|
||||
|
||||
|
||||
class MonitoringError(Base):
|
||||
@@ -106,26 +104,3 @@ class MonitoringEmbeddingCall(Base):
|
||||
session_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=True, index=True)
|
||||
message_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=True, index=True)
|
||||
call_type = sqlalchemy.Column(sqlalchemy.String(50), nullable=True) # embedding, retrieve
|
||||
|
||||
|
||||
class MonitoringFeedback(Base):
|
||||
"""User feedback records (like/dislike) from AI Bot conversations"""
|
||||
|
||||
__tablename__ = 'monitoring_feedback'
|
||||
|
||||
id = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True)
|
||||
timestamp = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False, index=True)
|
||||
feedback_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=False, unique=True, index=True)
|
||||
feedback_type = sqlalchemy.Column(sqlalchemy.Integer, nullable=False) # 1=like, 2=dislike
|
||||
feedback_content = sqlalchemy.Column(sqlalchemy.Text, nullable=True) # User feedback text
|
||||
inaccurate_reasons = sqlalchemy.Column(sqlalchemy.Text, nullable=True) # JSON list of inaccurate reasons
|
||||
# Context fields
|
||||
bot_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=True, index=True)
|
||||
bot_name = sqlalchemy.Column(sqlalchemy.String(255), nullable=True)
|
||||
pipeline_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=True, index=True)
|
||||
pipeline_name = sqlalchemy.Column(sqlalchemy.String(255), nullable=True)
|
||||
session_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=True, index=True)
|
||||
message_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=True, index=True)
|
||||
stream_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=True, index=True)
|
||||
user_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=True)
|
||||
platform = sqlalchemy.Column(sqlalchemy.String(255), nullable=True) # e.g., wecom
|
||||
|
||||
@@ -1,51 +0,0 @@
|
||||
"""Alembic environment for LangBot.
|
||||
|
||||
This env.py is designed to be called programmatically (not via CLI).
|
||||
It supports both SQLite and PostgreSQL.
|
||||
|
||||
The sync connection is passed via config attributes by the runner.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from alembic import context
|
||||
from sqlalchemy.engine import Connection
|
||||
|
||||
from langbot.pkg.entity.persistence.base import Base
|
||||
|
||||
target_metadata = Base.metadata
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
"""Run migrations in 'offline' mode — emit SQL without a live connection."""
|
||||
url = context.config.get_main_option('sqlalchemy.url')
|
||||
context.configure(
|
||||
url=url,
|
||||
target_metadata=target_metadata,
|
||||
literal_binds=True,
|
||||
dialect_opts={'paramstyle': 'named'},
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
"""Run migrations with a live sync connection passed via config attributes."""
|
||||
connection: Connection = context.config.attributes.get('connection')
|
||||
if connection is None:
|
||||
raise RuntimeError('connection not provided in alembic config attributes')
|
||||
|
||||
context.configure(
|
||||
connection=connection,
|
||||
target_metadata=target_metadata,
|
||||
# render_as_batch=True is critical for SQLite ALTER TABLE support
|
||||
render_as_batch=True,
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
||||
@@ -1,24 +0,0 @@
|
||||
# Alembic script.py.mako — template for auto-generated revisions
|
||||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
# revision identifiers
|
||||
revision = ${repr(up_revision)}
|
||||
down_revision = ${repr(down_revision)}
|
||||
branch_labels = ${repr(branch_labels)}
|
||||
depends_on = ${repr(depends_on)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
${downgrades if downgrades else "pass"}
|
||||
@@ -1,24 +0,0 @@
|
||||
"""baseline: stamp existing schema (db version 25)
|
||||
|
||||
This is a no-op migration that marks the starting point for Alembic.
|
||||
All tables already exist via create_all() + legacy DBMigration system.
|
||||
|
||||
Revision ID: 0001_baseline
|
||||
Revises: None
|
||||
Create Date: 2026-04-08
|
||||
"""
|
||||
|
||||
revision = '0001_baseline'
|
||||
down_revision = None
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# No-op: existing schema is already at database_version=25
|
||||
# This revision serves as the Alembic baseline.
|
||||
pass
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
pass
|
||||
@@ -1,62 +0,0 @@
|
||||
"""example: sample migration demonstrating Alembic patterns
|
||||
|
||||
This is a SAMPLE showing how to write migrations that work
|
||||
seamlessly across SQLite and PostgreSQL. Delete or adapt as needed.
|
||||
|
||||
Revision ID: 0002_sample
|
||||
Revises: 0001_baseline
|
||||
Create Date: 2026-04-08
|
||||
|
||||
Patterns demonstrated:
|
||||
1. Schema change (add column) — works on both DBs via render_as_batch
|
||||
2. Data migration (read + modify JSON) — pure SQLAlchemy, no dialect branching
|
||||
"""
|
||||
|
||||
revision = '0002_sample'
|
||||
down_revision = '0001_baseline'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""
|
||||
EXAMPLE: Uncomment to use. This shows the patterns.
|
||||
|
||||
# --- Pattern 1: Schema change (add/drop column) ---
|
||||
# render_as_batch=True in env.py makes this work on SQLite too.
|
||||
#
|
||||
# op.add_column('pipelines', sa.Column('description', sa.String(512), server_default=''))
|
||||
|
||||
# --- Pattern 2: Data migration (read + modify JSON field) ---
|
||||
# No if/else for sqlite vs postgres needed!
|
||||
#
|
||||
# conn = op.get_bind()
|
||||
# rows = conn.execute(sa.text("SELECT uuid, config FROM pipelines")).fetchall()
|
||||
# for row in rows:
|
||||
# config = json.loads(row[1]) if isinstance(row[1], str) else row[1]
|
||||
# # Modify the config
|
||||
# config.setdefault('ai', {}).setdefault('some_new_key', 'default_value')
|
||||
# conn.execute(
|
||||
# sa.text("UPDATE pipelines SET config = :cfg WHERE uuid = :uuid"),
|
||||
# {"cfg": json.dumps(config), "uuid": row[0]}
|
||||
# )
|
||||
|
||||
# --- Pattern 3: Create a new table ---
|
||||
#
|
||||
# op.create_table(
|
||||
# 'audit_log',
|
||||
# sa.Column('id', sa.Integer, primary_key=True, autoincrement=True),
|
||||
# sa.Column('action', sa.String(255), nullable=False),
|
||||
# sa.Column('detail', sa.Text),
|
||||
# sa.Column('created_at', sa.DateTime, server_default=sa.func.now()),
|
||||
# )
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""
|
||||
# op.drop_column('pipelines', 'description')
|
||||
# op.drop_table('audit_log')
|
||||
"""
|
||||
pass
|
||||
@@ -1,35 +0,0 @@
|
||||
"""add rerank_models table
|
||||
|
||||
Revision ID: 0003_add_rerank_models
|
||||
Revises: 0002_sample
|
||||
Create Date: 2026-04-19
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = '0003_add_rerank_models'
|
||||
down_revision = '0002_sample'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# Check if table already exists (may have been created by create_all())
|
||||
conn = op.get_bind()
|
||||
inspector = sa.inspect(conn)
|
||||
if 'rerank_models' not in inspector.get_table_names():
|
||||
op.create_table(
|
||||
'rerank_models',
|
||||
sa.Column('uuid', sa.String(255), primary_key=True, unique=True),
|
||||
sa.Column('name', sa.String(255), nullable=False),
|
||||
sa.Column('provider_uuid', sa.String(255), nullable=False),
|
||||
sa.Column('extra_args', sa.JSON, nullable=False, server_default='{}'),
|
||||
sa.Column('prefered_ranking', sa.Integer, nullable=False, server_default='0'),
|
||||
sa.Column('created_at', sa.DateTime, nullable=False, server_default=sa.func.now()),
|
||||
sa.Column('updated_at', sa.DateTime, nullable=False, server_default=sa.func.now()),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table('rerank_models')
|
||||
@@ -1,150 +0,0 @@
|
||||
"""Programmatic Alembic runner for LangBot.
|
||||
|
||||
Usage from async code:
|
||||
from langbot.pkg.persistence.alembic_runner import run_alembic_upgrade
|
||||
await run_alembic_upgrade(async_engine)
|
||||
|
||||
CLI usage (autogenerate):
|
||||
python -m langbot.pkg.persistence.alembic_runner autogenerate "add description column"
|
||||
python -m langbot.pkg.persistence.alembic_runner upgrade
|
||||
python -m langbot.pkg.persistence.alembic_runner current
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from alembic.config import Config
|
||||
from alembic import command
|
||||
from alembic.runtime.migration import MigrationContext
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine
|
||||
from sqlalchemy.engine import Connection
|
||||
|
||||
|
||||
_ALEMBIC_DIR = os.path.join(os.path.dirname(__file__), 'alembic')
|
||||
|
||||
|
||||
def _build_config(connection: Connection) -> Config:
|
||||
"""Build an Alembic Config with sync connection attached."""
|
||||
cfg = Config()
|
||||
cfg.set_main_option('script_location', _ALEMBIC_DIR)
|
||||
cfg.attributes['connection'] = connection
|
||||
return cfg
|
||||
|
||||
|
||||
def _do_upgrade(connection: Connection, revision: str = 'head') -> None:
|
||||
"""Synchronous upgrade — runs inside run_sync."""
|
||||
cfg = _build_config(connection)
|
||||
command.upgrade(cfg, revision)
|
||||
|
||||
|
||||
def _do_stamp(connection: Connection, revision: str = 'head') -> None:
|
||||
"""Synchronous stamp — runs inside run_sync."""
|
||||
cfg = _build_config(connection)
|
||||
command.stamp(cfg, revision)
|
||||
|
||||
|
||||
def _do_get_current(connection: Connection) -> str | None:
|
||||
"""Get current alembic revision synchronously."""
|
||||
ctx = MigrationContext.configure(connection)
|
||||
return ctx.get_current_revision()
|
||||
|
||||
|
||||
def _do_autogenerate(connection: Connection, message: str = 'auto migration') -> None:
|
||||
"""Synchronous autogenerate — runs inside run_sync."""
|
||||
cfg = _build_config(connection)
|
||||
command.revision(cfg, message=message, autogenerate=True)
|
||||
|
||||
|
||||
async def run_alembic_upgrade(async_engine: AsyncEngine, revision: str = 'head') -> None:
|
||||
"""Run Alembic upgrade to the given revision."""
|
||||
async with async_engine.connect() as conn:
|
||||
await conn.run_sync(_do_upgrade, revision)
|
||||
await conn.commit()
|
||||
|
||||
|
||||
async def run_alembic_stamp(async_engine: AsyncEngine, revision: str = 'head') -> None:
|
||||
"""Stamp the database with a revision without running migrations."""
|
||||
async with async_engine.connect() as conn:
|
||||
await conn.run_sync(_do_stamp, revision)
|
||||
await conn.commit()
|
||||
|
||||
|
||||
async def get_alembic_current(async_engine: AsyncEngine) -> str | None:
|
||||
"""Get current alembic revision, or None if not stamped."""
|
||||
async with async_engine.connect() as conn:
|
||||
return await conn.run_sync(_do_get_current)
|
||||
|
||||
|
||||
async def run_alembic_autogenerate(async_engine: AsyncEngine, message: str = 'auto migration') -> None:
|
||||
"""Compare ORM models against DB schema and generate a migration script."""
|
||||
async with async_engine.connect() as conn:
|
||||
await conn.run_sync(_do_autogenerate, message)
|
||||
|
||||
|
||||
# CLI entrypoint: python -m langbot.pkg.persistence.alembic_runner <command> [args]
|
||||
if __name__ == '__main__':
|
||||
import sys
|
||||
import asyncio
|
||||
|
||||
def _get_engine():
|
||||
"""Create engine from data/config.yaml or default SQLite."""
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
|
||||
try:
|
||||
import yaml
|
||||
|
||||
with open('data/config.yaml') as f:
|
||||
config = yaml.safe_load(f)
|
||||
db_cfg = config.get('database', {})
|
||||
db_type = db_cfg.get('use', 'sqlite')
|
||||
if db_type == 'postgresql':
|
||||
pg = db_cfg.get('postgresql', {})
|
||||
url = (
|
||||
f'postgresql+asyncpg://{pg.get("user", "postgres")}:{pg.get("password", "postgres")}'
|
||||
f'@{pg.get("host", "127.0.0.1")}:{pg.get("port", 5432)}/{pg.get("database", "postgres")}'
|
||||
)
|
||||
else:
|
||||
path = db_cfg.get('sqlite', {}).get('path', 'data/langbot.db')
|
||||
url = f'sqlite+aiosqlite:///{path}'
|
||||
except Exception:
|
||||
url = 'sqlite+aiosqlite:///data/langbot.db'
|
||||
|
||||
return create_async_engine(url)
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
print('Usage: python -m langbot.pkg.persistence.alembic_runner <command> [args]')
|
||||
print('Commands:')
|
||||
print(' autogenerate "message" — Generate migration from ORM model diff')
|
||||
print(' upgrade [revision] — Upgrade database (default: head)')
|
||||
print(' stamp [revision] — Stamp revision without running (default: head)')
|
||||
print(' current — Show current revision')
|
||||
sys.exit(1)
|
||||
|
||||
cmd = sys.argv[1]
|
||||
engine = _get_engine()
|
||||
|
||||
if cmd == 'autogenerate':
|
||||
msg = sys.argv[2] if len(sys.argv) > 2 else 'auto migration'
|
||||
asyncio.run(run_alembic_autogenerate(engine, msg))
|
||||
print(f'Migration generated: {msg}')
|
||||
elif cmd == 'upgrade':
|
||||
rev = sys.argv[2] if len(sys.argv) > 2 else 'head'
|
||||
asyncio.run(run_alembic_upgrade(engine, rev))
|
||||
print(f'Upgraded to: {rev}')
|
||||
elif cmd == 'stamp':
|
||||
rev = sys.argv[2] if len(sys.argv) > 2 else 'head'
|
||||
asyncio.run(run_alembic_stamp(engine, rev))
|
||||
print(f'Stamped: {rev}')
|
||||
elif cmd == 'current':
|
||||
rev = asyncio.run(get_alembic_current(engine))
|
||||
print(f'Current revision: {rev}')
|
||||
else:
|
||||
print(f'Unknown command: {cmd}')
|
||||
sys.exit(1)
|
||||
|
||||
main()
|
||||
@@ -2,16 +2,18 @@ from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
import typing
|
||||
|
||||
import json
|
||||
import uuid
|
||||
|
||||
import sqlalchemy.ext.asyncio as sqlalchemy_asyncio
|
||||
import sqlalchemy
|
||||
|
||||
from . import database, migration
|
||||
from ..entity.persistence import base, metadata, model as persistence_model
|
||||
from ..entity.persistence import base, pipeline, metadata, model as persistence_model
|
||||
from ..entity import persistence
|
||||
from ..core import app
|
||||
from ..utils import constants, importutil
|
||||
from ..api.http.service import pipeline as pipeline_service
|
||||
from . import databases, migrations
|
||||
|
||||
importutil.import_modules_in_pkg(databases)
|
||||
@@ -76,9 +78,7 @@ class PersistenceManager:
|
||||
|
||||
self.ap.logger.info(f'Successfully upgraded database to version {last_migration_number}.')
|
||||
|
||||
# Run Alembic migrations (new migration system)
|
||||
await self._run_alembic_migrations()
|
||||
|
||||
await self.write_default_pipeline()
|
||||
await self.write_space_model_providers()
|
||||
|
||||
async def create_tables(self):
|
||||
@@ -101,6 +101,29 @@ class PersistenceManager:
|
||||
if row is None:
|
||||
await self.execute_async(sqlalchemy.insert(metadata.Metadata).values(item))
|
||||
|
||||
async def write_default_pipeline(self):
|
||||
# write default pipeline
|
||||
result = await self.execute_async(sqlalchemy.select(pipeline.LegacyPipeline))
|
||||
default_pipeline_uuid = None
|
||||
if result.first() is None:
|
||||
self.ap.logger.info('Creating default pipeline...')
|
||||
|
||||
pipeline_config = json.loads(importutil.read_resource_file('templates/default-pipeline-config.json'))
|
||||
|
||||
default_pipeline_uuid = str(uuid.uuid4())
|
||||
pipeline_data = {
|
||||
'uuid': default_pipeline_uuid,
|
||||
'for_version': self.ap.ver_mgr.get_current_version(),
|
||||
'stages': pipeline_service.default_stage_order,
|
||||
'is_default': True,
|
||||
'name': 'ChatPipeline',
|
||||
'description': 'Default pipeline, new bots will be bound to this pipeline | 默认提供的流水线,您配置的机器人将自动绑定到此流水线',
|
||||
'config': pipeline_config,
|
||||
'extensions_preferences': {},
|
||||
}
|
||||
|
||||
await self.execute_async(sqlalchemy.insert(pipeline.LegacyPipeline).values(pipeline_data))
|
||||
|
||||
async def write_space_model_providers(self):
|
||||
space_models_gateway_api_url = self.ap.instance_config.data.get('space', {}).get(
|
||||
'models_gateway_api_url', 'https://api.langbot.cloud/v1'
|
||||
@@ -138,28 +161,6 @@ class PersistenceManager:
|
||||
|
||||
# =================================
|
||||
|
||||
async def _run_alembic_migrations(self):
|
||||
"""Run Alembic-based migrations after legacy migrations complete."""
|
||||
from . import alembic_runner
|
||||
|
||||
engine = self.get_db_engine()
|
||||
|
||||
try:
|
||||
current_rev = await alembic_runner.get_alembic_current(engine)
|
||||
|
||||
if current_rev is None:
|
||||
# First time: stamp baseline so Alembic knows existing schema is up-to-date
|
||||
self.ap.logger.info('Alembic: no revision found, stamping baseline...')
|
||||
await alembic_runner.run_alembic_stamp(engine, '0001_baseline')
|
||||
current_rev = '0001_baseline'
|
||||
|
||||
# Upgrade to head
|
||||
await alembic_runner.run_alembic_upgrade(engine, 'head')
|
||||
self.ap.logger.info('Alembic migrations completed.')
|
||||
except Exception as e:
|
||||
self.ap.logger.error(f'Alembic migration failed: {e}', exc_info=True)
|
||||
raise
|
||||
|
||||
async def execute_async(self, *args, **kwargs) -> sqlalchemy.engine.cursor.CursorResult:
|
||||
async with self.get_db_engine().connect() as conn:
|
||||
result = await conn.execute(*args, **kwargs)
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import json
|
||||
|
||||
import sqlalchemy
|
||||
from .. import migration
|
||||
|
||||
@@ -7,22 +9,20 @@ class DBMigrateKnowledgeEnginePluginArchitecture(migration.DBMigration):
|
||||
"""Migrate to unified Knowledge Engine plugin architecture.
|
||||
|
||||
Changes:
|
||||
- Backup existing knowledge_bases data to knowledge_bases_backup
|
||||
- Clear knowledge_bases table and add new plugin architecture columns
|
||||
- Drop old columns (PostgreSQL only; SQLite leaves them unmapped)
|
||||
- Preserve external_knowledge_bases table as-is for future migration
|
||||
- Set rag_plugin_migration_needed flag in metadata if old data exists
|
||||
- Add knowledge_engine_plugin_id, collection_id, creation_settings, retrieval_settings columns to knowledge_bases
|
||||
- Migrate existing top_k values into retrieval_settings JSON
|
||||
- Migrate existing embedding_model_uuid into creation_settings JSON
|
||||
- Drop embedding_model_uuid and top_k columns (PostgreSQL only; SQLite leaves them unmapped)
|
||||
- Drop external_knowledge_bases table (no longer needed; external KB data is not migrated)
|
||||
"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""Upgrade"""
|
||||
has_internal_data = await self._backup_knowledge_bases()
|
||||
has_external_data = await self._check_external_knowledge_bases()
|
||||
await self._clear_knowledge_bases()
|
||||
await self._add_columns_to_knowledge_bases()
|
||||
await self._migrate_top_k_to_retrieval_settings()
|
||||
await self._migrate_embedding_model_uuid_to_creation_settings()
|
||||
await self._drop_old_columns()
|
||||
if has_internal_data or has_external_data:
|
||||
await self._set_migration_flag()
|
||||
await self._drop_external_knowledge_bases_table()
|
||||
|
||||
async def _get_table_columns(self, table_name: str) -> list[str]:
|
||||
"""Get column names from a table (works for both SQLite and PostgreSQL)."""
|
||||
@@ -57,50 +57,6 @@ class DBMigrateKnowledgeEnginePluginArchitecture(migration.DBMigration):
|
||||
)
|
||||
return result.first() is not None
|
||||
|
||||
async def _backup_knowledge_bases(self) -> bool:
|
||||
"""Backup knowledge_bases data. Returns True if data was backed up."""
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.text('SELECT COUNT(*) FROM knowledge_bases;'))
|
||||
count = result.scalar()
|
||||
if count == 0:
|
||||
return False
|
||||
|
||||
# Drop backup table if it already exists (from a previous failed migration)
|
||||
if await self._table_exists('knowledge_bases_backup'):
|
||||
await self.ap.persistence_mgr.execute_async(sqlalchemy.text('DROP TABLE knowledge_bases_backup;'))
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('CREATE TABLE knowledge_bases_backup AS SELECT * FROM knowledge_bases;')
|
||||
)
|
||||
self.ap.logger.info(
|
||||
'Backed up %d knowledge base(s) to knowledge_bases_backup table.',
|
||||
count,
|
||||
)
|
||||
return True
|
||||
|
||||
async def _check_external_knowledge_bases(self) -> bool:
|
||||
"""Check if external_knowledge_bases table exists and has data.
|
||||
|
||||
The table is preserved as-is (not dropped) for future migration.
|
||||
"""
|
||||
if not await self._table_exists('external_knowledge_bases'):
|
||||
return False
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT COUNT(*) FROM external_knowledge_bases;')
|
||||
)
|
||||
count = result.scalar()
|
||||
if count > 0:
|
||||
self.ap.logger.info(
|
||||
'Found %d external knowledge base(s) in external_knowledge_bases table. '
|
||||
'Table preserved for future migration.',
|
||||
count,
|
||||
)
|
||||
return count > 0
|
||||
|
||||
async def _clear_knowledge_bases(self):
|
||||
"""Clear all rows from knowledge_bases table (preserve table structure)."""
|
||||
await self.ap.persistence_mgr.execute_async(sqlalchemy.text('DELETE FROM knowledge_bases;'))
|
||||
|
||||
async def _add_columns_to_knowledge_bases(self):
|
||||
"""Add new RAG plugin architecture columns to knowledge_bases table."""
|
||||
columns = await self._get_table_columns('knowledge_bases')
|
||||
@@ -118,6 +74,73 @@ class DBMigrateKnowledgeEnginePluginArchitecture(migration.DBMigration):
|
||||
sqlalchemy.text(f'ALTER TABLE knowledge_bases ADD COLUMN {col_name} {col_type};')
|
||||
)
|
||||
|
||||
# For existing knowledge bases without knowledge_engine_plugin_id,
|
||||
# set collection_id = uuid (same default as new KBs)
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('UPDATE knowledge_bases SET collection_id = uuid WHERE collection_id IS NULL;')
|
||||
)
|
||||
|
||||
async def _migrate_top_k_to_retrieval_settings(self):
|
||||
"""Migrate existing top_k values into retrieval_settings JSON."""
|
||||
columns = await self._get_table_columns('knowledge_bases')
|
||||
if 'top_k' not in columns:
|
||||
return
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'SELECT uuid, top_k FROM knowledge_bases WHERE top_k IS NOT NULL AND retrieval_settings IS NULL;'
|
||||
)
|
||||
)
|
||||
rows = result.fetchall()
|
||||
|
||||
for row in rows:
|
||||
kb_uuid = row[0]
|
||||
top_k = row[1]
|
||||
retrieval_settings = json.dumps({'top_k': top_k})
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('UPDATE knowledge_bases SET retrieval_settings = :rs WHERE uuid = :uuid;').bindparams(
|
||||
rs=retrieval_settings, uuid=kb_uuid
|
||||
)
|
||||
)
|
||||
|
||||
async def _migrate_embedding_model_uuid_to_creation_settings(self):
|
||||
"""Migrate existing embedding_model_uuid into creation_settings JSON."""
|
||||
columns = await self._get_table_columns('knowledge_bases')
|
||||
if 'embedding_model_uuid' not in columns:
|
||||
return
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'SELECT uuid, embedding_model_uuid, creation_settings FROM knowledge_bases '
|
||||
"WHERE embedding_model_uuid IS NOT NULL AND embedding_model_uuid != '';"
|
||||
)
|
||||
)
|
||||
rows = result.fetchall()
|
||||
|
||||
for row in rows:
|
||||
kb_uuid = row[0]
|
||||
emb_uuid = row[1]
|
||||
existing_settings = row[2]
|
||||
|
||||
if existing_settings and isinstance(existing_settings, str):
|
||||
try:
|
||||
settings = json.loads(existing_settings)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
settings = {}
|
||||
elif isinstance(existing_settings, dict):
|
||||
settings = existing_settings
|
||||
else:
|
||||
settings = {}
|
||||
|
||||
if 'embedding_model_uuid' not in settings:
|
||||
settings['embedding_model_uuid'] = emb_uuid
|
||||
new_settings = json.dumps(settings)
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'UPDATE knowledge_bases SET creation_settings = :cs WHERE uuid = :uuid;'
|
||||
).bindparams(cs=new_settings, uuid=kb_uuid)
|
||||
)
|
||||
|
||||
async def _drop_old_columns(self):
|
||||
"""Drop embedding_model_uuid and top_k columns (PostgreSQL only).
|
||||
|
||||
@@ -139,22 +162,22 @@ class DBMigrateKnowledgeEnginePluginArchitecture(migration.DBMigration):
|
||||
sqlalchemy.text('ALTER TABLE knowledge_bases DROP COLUMN top_k;')
|
||||
)
|
||||
|
||||
async def _set_migration_flag(self):
|
||||
"""Set rag_plugin_migration_needed flag in metadata table."""
|
||||
# Check if the key already exists
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text("SELECT value FROM metadata WHERE key = 'rag_plugin_migration_needed';")
|
||||
)
|
||||
row = result.first()
|
||||
if row is not None:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text("UPDATE metadata SET value = 'true' WHERE key = 'rag_plugin_migration_needed';")
|
||||
async def _drop_external_knowledge_bases_table(self):
|
||||
"""Drop the external_knowledge_bases table if it exists."""
|
||||
if await self._table_exists('external_knowledge_bases'):
|
||||
# Log existing external KBs before dropping, so users are aware of data loss
|
||||
rows = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT * FROM external_knowledge_bases;')
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text("INSERT INTO metadata (key, value) VALUES ('rag_plugin_migration_needed', 'true');")
|
||||
)
|
||||
self.ap.logger.info('Set rag_plugin_migration_needed=true in metadata.')
|
||||
existing = rows.fetchall()
|
||||
if existing:
|
||||
self.ap.logger.warning(
|
||||
'Dropping external_knowledge_bases table with %d existing record(s). '
|
||||
'These external KB configurations will be removed: %s',
|
||||
len(existing),
|
||||
[dict(row._mapping) for row in existing],
|
||||
)
|
||||
await self.ap.persistence_mgr.execute_async(sqlalchemy.text('DROP TABLE external_knowledge_bases;'))
|
||||
|
||||
async def downgrade(self):
|
||||
"""Downgrade"""
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user