feat: external knowledge bases (#1783)

* Initial plan

* Add backend support for external knowledge bases

Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com>

* Add frontend support for external knowledge bases with tabs UI

Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com>

* Add i18n translations for all languages (Traditional Chinese and Japanese)

Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com>

* Update knowledge base tab list styling to match plugins page

Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com>

* perf: margin-top for kb page

* refactor: switch RetrievalResultEntry to langbot_plugin pkg ones

* feat: knowledge retriever listing and creating

* stash

* refactor: unify sync mechanism for polymorphic components

* feat: use unified retireval result struct in retrieval test page

* chore: remove unused methods

* feat: retriever icon displaying

* feat: localagent retrieval with external kbs

* chore: bump version of langbot-plugin to 0.2.0b1

* fix: i18n

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com>
Co-authored-by: Junyan Qin <rockchinq@gmail.com>

src/langbot/pkg/api/http/controller/groups/knowledge/external.py

@@ -0,0 +1,61 @@
+import quart
+from ... import group
+
+
+@group.group_class('external_knowledge_base', '/api/v1/knowledge/external-bases')
+class ExternalKnowledgeBaseRouterGroup(group.RouterGroup):
+    async def initialize(self) -> None:
+        @self.route('/retrievers', methods=['GET'])
+        async def list_knowledge_retrievers() -> quart.Response:
+            """List all available knowledge retrievers from plugins."""
+            retrievers = await self.ap.plugin_connector.list_knowledge_retrievers()
+            return self.success(data={'retrievers': retrievers})
+
+        @self.route('', methods=['POST', 'GET'])
+        async def handle_external_knowledge_bases() -> quart.Response:
+            if quart.request.method == 'GET':
+                external_kbs = await self.ap.external_kb_service.get_external_knowledge_bases()
+                return self.success(data={'bases': external_kbs})
+
+            elif quart.request.method == 'POST':
+                json_data = await quart.request.json
+                kb_uuid = await self.ap.external_kb_service.create_external_knowledge_base(json_data)
+                return self.success(data={'uuid': kb_uuid})
+
+            return self.http_status(405, -1, 'Method not allowed')
+
+        @self.route(
+            '/<kb_uuid>',
+            methods=['GET', 'DELETE', 'PUT'],
+        )
+        async def handle_specific_external_knowledge_base(kb_uuid: str) -> quart.Response:
+            if quart.request.method == 'GET':
+                external_kb = await self.ap.external_kb_service.get_external_knowledge_base(kb_uuid)
+
+                if external_kb is None:
+                    return self.http_status(404, -1, 'external knowledge base not found')
+
+                return self.success(
+                    data={
+                        'base': external_kb,
+                    }
+                )
+
+            elif quart.request.method == 'PUT':
+                json_data = await quart.request.json
+                await self.ap.external_kb_service.update_external_knowledge_base(kb_uuid, json_data)
+                return self.success({})
+
+            elif quart.request.method == 'DELETE':
+                await self.ap.external_kb_service.delete_external_knowledge_base(kb_uuid)
+                return self.success({})
+
+        @self.route(
+            '/<kb_uuid>/retrieve',
+            methods=['POST'],
+        )
+        async def retrieve_external_knowledge_base(kb_uuid: str) -> str:
+            json_data = await quart.request.json
+            query = json_data.get('query')
+            results = await self.ap.external_kb_service.retrieve_external_knowledge_base(kb_uuid, query)
+            return self.success(data={'results': results})

src/langbot/pkg/api/http/service/external_kb.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+from ....core import app
+import sqlalchemy
+from langbot.pkg.entity.persistence import rag as persistence_rag
+import uuid
+
+
+class ExternalKBService:
+    """External KB service"""
+
+    ap: app.Application
+
+    def __init__(self, ap: app.Application) -> None:
+        self.ap = ap
+
+    # External Knowledge Base methods
+    async def get_external_knowledge_bases(self) -> list[dict]:
+        result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_rag.ExternalKnowledgeBase))
+        external_kbs = result.all()
+        return [
+            self.ap.persistence_mgr.serialize_model(persistence_rag.ExternalKnowledgeBase, external_kb)
+            for external_kb in external_kbs
+        ]
+
+    async def get_external_knowledge_base(self, kb_uuid: str) -> dict | None:
+        result = await self.ap.persistence_mgr.execute_async(
+            sqlalchemy.select(persistence_rag.ExternalKnowledgeBase).where(
+                persistence_rag.ExternalKnowledgeBase.uuid == kb_uuid
+            )
+        )
+        external_kb = result.first()
+        if external_kb is None:
+            return None
+        return self.ap.persistence_mgr.serialize_model(persistence_rag.ExternalKnowledgeBase, external_kb)
+
+    async def create_external_knowledge_base(self, kb_data: dict) -> str:
+        kb_data['uuid'] = str(uuid.uuid4())
+        await self.ap.persistence_mgr.execute_async(
+            sqlalchemy.insert(persistence_rag.ExternalKnowledgeBase).values(kb_data)
+        )
+
+        kb = await self.get_external_knowledge_base(kb_data['uuid'])
+
+        await self.ap.rag_mgr.load_external_knowledge_base(kb)
+
+        return kb_data['uuid']
+
+    async def retrieve_external_knowledge_base(self, kb_uuid: str, query: str) -> list[dict]:
+        """Retrieve external knowledge base"""
+        runtime_kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(kb_uuid)
+        if runtime_kb is None:
+            raise Exception('Knowledge base not found')
+        return [
+            result.model_dump() for result in await runtime_kb.retrieve(query, 5)
+        ]  # top_k is just a placeholder for external knowledge base
+
+    async def update_external_knowledge_base(self, kb_uuid: str, kb_data: dict) -> None:
+        if 'uuid' in kb_data:
+            del kb_data['uuid']
+
+        await self.ap.persistence_mgr.execute_async(
+            sqlalchemy.update(persistence_rag.ExternalKnowledgeBase)
+            .values(kb_data)
+            .where(persistence_rag.ExternalKnowledgeBase.uuid == kb_uuid)
+        )
+        await self.ap.rag_mgr.remove_knowledge_base_from_runtime(kb_uuid)
+
+        kb = await self.get_external_knowledge_base(kb_uuid)
+
+        await self.ap.rag_mgr.load_external_knowledge_base(kb)
+
+    async def delete_external_knowledge_base(self, kb_uuid: str) -> None:
+        await self.ap.rag_mgr.delete_knowledge_base(kb_uuid)
+
+        await self.ap.persistence_mgr.execute_async(
+            sqlalchemy.delete(persistence_rag.ExternalKnowledgeBase).where(
+                persistence_rag.ExternalKnowledgeBase.uuid == kb_uuid
+            )
+        )

src/langbot/pkg/api/http/service/knowledge.py

@@ -71,6 +71,9 @@ class KnowledgeService:
         runtime_kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(kb_uuid)
         if runtime_kb is None:
             raise Exception('Knowledge base not found')
+        # Only internal KBs support file storage
+        if runtime_kb.get_type() != 'internal':
+            raise Exception('Only internal knowledge bases support file storage')
         return await runtime_kb.store_file(file_id)
 
     async def retrieve_knowledge_base(self, kb_uuid: str, query: str) -> list[dict]:
@@ -78,9 +81,16 @@ class KnowledgeService:
         runtime_kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(kb_uuid)
         if runtime_kb is None:
             raise Exception('Knowledge base not found')
-        return [
-            result.model_dump() for result in await runtime_kb.retrieve(query, runtime_kb.knowledge_base_entity.top_k)
-        ]
+
+        # Get top_k based on KB type
+        if runtime_kb.get_type() == 'internal':
+            top_k = runtime_kb.knowledge_base_entity.top_k
+        elif runtime_kb.get_type() == 'external':
+            top_k = runtime_kb.external_kb_entity.top_k
+        else:
+            top_k = 5  # default fallback
+
+        return [result.model_dump() for result in await runtime_kb.retrieve(query, top_k)]
 
     async def get_files_by_knowledge_base(self, kb_uuid: str) -> list[dict]:
         """获取知识库文件"""
@@ -95,6 +105,9 @@ class KnowledgeService:
         runtime_kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(kb_uuid)
         if runtime_kb is None:
             raise Exception('Knowledge base not found')
+        # Only internal KBs support file deletion
+        if runtime_kb.get_type() != 'internal':
+            raise Exception('Only internal knowledge bases support file deletion')
         await runtime_kb.delete_file(file_id)
 
     async def delete_knowledge_base(self, kb_uuid: str) -> None: