feat: external knowledge bases (#1783)

* Initial plan

* Add backend support for external knowledge bases

Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com>

* Add frontend support for external knowledge bases with tabs UI

Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com>

* Add i18n translations for all languages (Traditional Chinese and Japanese)

Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com>

* Update knowledge base tab list styling to match plugins page

Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com>

* perf: margin-top for kb page

* refactor: switch RetrievalResultEntry to langbot_plugin pkg ones

* feat: knowledge retriever listing and creating

* stash

* refactor: unify sync mechanism for polymorphic components

* feat: use unified retireval result struct in retrieval test page

* chore: remove unused methods

* feat: retriever icon displaying

* feat: localagent retrieval with external kbs

* chore: bump version of langbot-plugin to 0.2.0b1

* fix: i18n

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com>
Co-authored-by: Junyan Qin <rockchinq@gmail.com>

src/langbot/pkg/plugin/connector.py

@@ -7,6 +7,7 @@ import typing
 import os
 import sys
 import httpx
+import traceback
 import sqlalchemy
 from async_lru import alru_cache
 from langbot_plugin.api.entities.builtin.pipeline.query import provider_session
@@ -101,6 +102,12 @@ class PluginRuntimeConnector:
             self.handler_task = asyncio.create_task(self.handler.run())
             _ = await self.handler.ping()
             self.ap.logger.info('Connected to plugin runtime.')
+            # Sync polymorphic component instances after connection
+            try:
+                await self.sync_polymorphic_component_instances()
+            except Exception as e:
+                traceback.print_exc()
+                self.ap.logger.error(f'Failed to sync polymorphic component instances: {e}')
             await self.handler_task
 
         task: asyncio.Task | None = None
@@ -427,6 +434,31 @@ class PluginRuntimeConnector:
 
             yield cmd_ret
 
+    # KnowledgeRetriever methods
+    async def list_knowledge_retrievers(self, bound_plugins: list[str] | None = None) -> list[dict[str, Any]]:
+        """List all available KnowledgeRetriever components."""
+        if not self.is_enable_plugin:
+            return []
+
+        retrievers_data = await self.handler.list_knowledge_retrievers(include_plugins=bound_plugins)
+        return retrievers_data
+
+    async def retrieve_knowledge(
+        self,
+        plugin_author: str,
+        plugin_name: str,
+        retriever_name: str,
+        instance_id: str,
+        retrieval_context: dict[str, Any],
+    ) -> list[dict[str, Any]]:
+        """Retrieve knowledge using a KnowledgeRetriever instance."""
+        if not self.is_enable_plugin:
+            return []
+
+        return await self.handler.retrieve_knowledge(
+            plugin_author, plugin_name, retriever_name, instance_id, retrieval_context
+        )
+
     def dispose(self):
         # No need to consider the shutdown on Windows
         # for Windows can kill processes and subprocesses chainly
@@ -438,3 +470,42 @@ class PluginRuntimeConnector:
         if self.heartbeat_task is not None:
             self.heartbeat_task.cancel()
             self.heartbeat_task = None
+
+    async def sync_polymorphic_component_instances(self) -> dict[str, Any]:
+        """Sync polymorphic component instances with runtime.
+
+        This collects all external knowledge bases from database and sends to runtime
+        to ensure instance integrity across restarts.
+        """
+        if not self.is_enable_plugin:
+            return {}
+
+        # ===== external knowledge bases =====
+
+        external_kbs = await self.ap.external_kb_service.get_external_knowledge_bases()
+
+        # Build required_instances list
+        required_instances = []
+        for kb in external_kbs:
+            required_instances.append(
+                {
+                    'instance_id': kb['uuid'],
+                    'plugin_author': kb['plugin_author'],
+                    'plugin_name': kb['plugin_name'],
+                    'component_kind': 'KnowledgeRetriever',
+                    'component_name': kb['retriever_name'],
+                    'config': kb['retriever_config'],
+                }
+            )
+
+        self.ap.logger.info(f'Syncing {len(required_instances)} polymorphic component instances to runtime')
+
+        # Send to runtime
+        sync_result = await self.handler.sync_polymorphic_component_instances(required_instances)
+
+        self.ap.logger.info(
+            f'Sync complete: {len(sync_result.get("success_instances", []))} succeeded, '
+            f'{len(sync_result.get("failed_instances", []))} failed'
+        )
+
+        return sync_result

src/langbot/pkg/plugin/handler.py

@@ -713,3 +713,48 @@ class RuntimeConnectionHandler(handler.Handler):
 
         async for ret in gen:
             yield ret
+
+    # KnowledgeRetriever methods
+    async def list_knowledge_retrievers(self, include_plugins: list[str] | None = None) -> list[dict[str, Any]]:
+        """List knowledge retrievers"""
+        result = await self.call_action(
+            LangBotToRuntimeAction.LIST_KNOWLEDGE_RETRIEVERS,
+            {
+                'include_plugins': include_plugins,
+            },
+            timeout=10,
+        )
+        return result['retrievers']
+
+    async def retrieve_knowledge(
+        self,
+        plugin_author: str,
+        plugin_name: str,
+        retriever_name: str,
+        instance_id: str,
+        retrieval_context: dict[str, Any],
+    ) -> list[dict[str, Any]]:
+        """Retrieve knowledge"""
+        result = await self.call_action(
+            LangBotToRuntimeAction.RETRIEVE_KNOWLEDGE,
+            {
+                'plugin_author': plugin_author,
+                'plugin_name': plugin_name,
+                'retriever_name': retriever_name,
+                'instance_id': instance_id,
+                'retrieval_context': retrieval_context,
+            },
+            timeout=30,
+        )
+        return result['retrieval_results']
+
+    async def sync_polymorphic_component_instances(self, required_instances: list[dict[str, Any]]) -> dict[str, Any]:
+        """Sync polymorphic component instances with runtime"""
+        result = await self.call_action(
+            LangBotToRuntimeAction.SYNC_POLYMORPHIC_COMPONENT_INSTANCES,
+            {
+                'required_instances': required_instances,
+            },
+            timeout=30,
+        )
+        return result