feat: external knowledge bases (#1783)

* Initial plan

* Add backend support for external knowledge bases

Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com>

* Add frontend support for external knowledge bases with tabs UI

Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com>

* Add i18n translations for all languages (Traditional Chinese and Japanese)

Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com>

* Update knowledge base tab list styling to match plugins page

Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com>

* perf: margin-top for kb page

* refactor: switch RetrievalResultEntry to langbot_plugin pkg ones

* feat: knowledge retriever listing and creating

* stash

* refactor: unify sync mechanism for polymorphic components

* feat: use unified retireval result struct in retrieval test page

* chore: remove unused methods

* feat: retriever icon displaying

* feat: localagent retrieval with external kbs

* chore: bump version of langbot-plugin to 0.2.0b1

* fix: i18n

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com>
Co-authored-by: Junyan Qin <rockchinq@gmail.com>
This commit is contained in:
Copilot
2025-11-27 23:19:43 +08:00
committed by GitHub
parent 3c04eeaff9
commit a8481e43f0
33 changed files with 1924 additions and 161 deletions

View File

@@ -63,7 +63,7 @@ dependencies = [
"langchain-text-splitters>=0.0.1", "langchain-text-splitters>=0.0.1",
"chromadb>=0.4.24", "chromadb>=0.4.24",
"qdrant-client (>=1.15.1,<2.0.0)", "qdrant-client (>=1.15.1,<2.0.0)",
"langbot-plugin==0.1.13b1", "langbot-plugin==0.2.0b1",
"asyncpg>=0.30.0", "asyncpg>=0.30.0",
"line-bot-sdk>=3.19.0", "line-bot-sdk>=3.19.0",
"tboxsdk>=0.0.10", "tboxsdk>=0.0.10",

View File

@@ -0,0 +1,61 @@
import quart
from ... import group
@group.group_class('external_knowledge_base', '/api/v1/knowledge/external-bases')
class ExternalKnowledgeBaseRouterGroup(group.RouterGroup):
async def initialize(self) -> None:
@self.route('/retrievers', methods=['GET'])
async def list_knowledge_retrievers() -> quart.Response:
"""List all available knowledge retrievers from plugins."""
retrievers = await self.ap.plugin_connector.list_knowledge_retrievers()
return self.success(data={'retrievers': retrievers})
@self.route('', methods=['POST', 'GET'])
async def handle_external_knowledge_bases() -> quart.Response:
if quart.request.method == 'GET':
external_kbs = await self.ap.external_kb_service.get_external_knowledge_bases()
return self.success(data={'bases': external_kbs})
elif quart.request.method == 'POST':
json_data = await quart.request.json
kb_uuid = await self.ap.external_kb_service.create_external_knowledge_base(json_data)
return self.success(data={'uuid': kb_uuid})
return self.http_status(405, -1, 'Method not allowed')
@self.route(
'/<kb_uuid>',
methods=['GET', 'DELETE', 'PUT'],
)
async def handle_specific_external_knowledge_base(kb_uuid: str) -> quart.Response:
if quart.request.method == 'GET':
external_kb = await self.ap.external_kb_service.get_external_knowledge_base(kb_uuid)
if external_kb is None:
return self.http_status(404, -1, 'external knowledge base not found')
return self.success(
data={
'base': external_kb,
}
)
elif quart.request.method == 'PUT':
json_data = await quart.request.json
await self.ap.external_kb_service.update_external_knowledge_base(kb_uuid, json_data)
return self.success({})
elif quart.request.method == 'DELETE':
await self.ap.external_kb_service.delete_external_knowledge_base(kb_uuid)
return self.success({})
@self.route(
'/<kb_uuid>/retrieve',
methods=['POST'],
)
async def retrieve_external_knowledge_base(kb_uuid: str) -> str:
json_data = await quart.request.json
query = json_data.get('query')
results = await self.ap.external_kb_service.retrieve_external_knowledge_base(kb_uuid, query)
return self.success(data={'results': results})

View File

@@ -0,0 +1,80 @@
from __future__ import annotations
from ....core import app
import sqlalchemy
from langbot.pkg.entity.persistence import rag as persistence_rag
import uuid
class ExternalKBService:
"""External KB service"""
ap: app.Application
def __init__(self, ap: app.Application) -> None:
self.ap = ap
# External Knowledge Base methods
async def get_external_knowledge_bases(self) -> list[dict]:
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_rag.ExternalKnowledgeBase))
external_kbs = result.all()
return [
self.ap.persistence_mgr.serialize_model(persistence_rag.ExternalKnowledgeBase, external_kb)
for external_kb in external_kbs
]
async def get_external_knowledge_base(self, kb_uuid: str) -> dict | None:
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(persistence_rag.ExternalKnowledgeBase).where(
persistence_rag.ExternalKnowledgeBase.uuid == kb_uuid
)
)
external_kb = result.first()
if external_kb is None:
return None
return self.ap.persistence_mgr.serialize_model(persistence_rag.ExternalKnowledgeBase, external_kb)
async def create_external_knowledge_base(self, kb_data: dict) -> str:
kb_data['uuid'] = str(uuid.uuid4())
await self.ap.persistence_mgr.execute_async(
sqlalchemy.insert(persistence_rag.ExternalKnowledgeBase).values(kb_data)
)
kb = await self.get_external_knowledge_base(kb_data['uuid'])
await self.ap.rag_mgr.load_external_knowledge_base(kb)
return kb_data['uuid']
async def retrieve_external_knowledge_base(self, kb_uuid: str, query: str) -> list[dict]:
"""Retrieve external knowledge base"""
runtime_kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(kb_uuid)
if runtime_kb is None:
raise Exception('Knowledge base not found')
return [
result.model_dump() for result in await runtime_kb.retrieve(query, 5)
] # top_k is just a placeholder for external knowledge base
async def update_external_knowledge_base(self, kb_uuid: str, kb_data: dict) -> None:
if 'uuid' in kb_data:
del kb_data['uuid']
await self.ap.persistence_mgr.execute_async(
sqlalchemy.update(persistence_rag.ExternalKnowledgeBase)
.values(kb_data)
.where(persistence_rag.ExternalKnowledgeBase.uuid == kb_uuid)
)
await self.ap.rag_mgr.remove_knowledge_base_from_runtime(kb_uuid)
kb = await self.get_external_knowledge_base(kb_uuid)
await self.ap.rag_mgr.load_external_knowledge_base(kb)
async def delete_external_knowledge_base(self, kb_uuid: str) -> None:
await self.ap.rag_mgr.delete_knowledge_base(kb_uuid)
await self.ap.persistence_mgr.execute_async(
sqlalchemy.delete(persistence_rag.ExternalKnowledgeBase).where(
persistence_rag.ExternalKnowledgeBase.uuid == kb_uuid
)
)

View File

@@ -71,6 +71,9 @@ class KnowledgeService:
runtime_kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(kb_uuid) runtime_kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(kb_uuid)
if runtime_kb is None: if runtime_kb is None:
raise Exception('Knowledge base not found') raise Exception('Knowledge base not found')
# Only internal KBs support file storage
if runtime_kb.get_type() != 'internal':
raise Exception('Only internal knowledge bases support file storage')
return await runtime_kb.store_file(file_id) return await runtime_kb.store_file(file_id)
async def retrieve_knowledge_base(self, kb_uuid: str, query: str) -> list[dict]: async def retrieve_knowledge_base(self, kb_uuid: str, query: str) -> list[dict]:
@@ -78,9 +81,16 @@ class KnowledgeService:
runtime_kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(kb_uuid) runtime_kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(kb_uuid)
if runtime_kb is None: if runtime_kb is None:
raise Exception('Knowledge base not found') raise Exception('Knowledge base not found')
return [
result.model_dump() for result in await runtime_kb.retrieve(query, runtime_kb.knowledge_base_entity.top_k) # Get top_k based on KB type
] if runtime_kb.get_type() == 'internal':
top_k = runtime_kb.knowledge_base_entity.top_k
elif runtime_kb.get_type() == 'external':
top_k = runtime_kb.external_kb_entity.top_k
else:
top_k = 5 # default fallback
return [result.model_dump() for result in await runtime_kb.retrieve(query, top_k)]
async def get_files_by_knowledge_base(self, kb_uuid: str) -> list[dict]: async def get_files_by_knowledge_base(self, kb_uuid: str) -> list[dict]:
"""获取知识库文件""" """获取知识库文件"""
@@ -95,6 +105,9 @@ class KnowledgeService:
runtime_kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(kb_uuid) runtime_kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(kb_uuid)
if runtime_kb is None: if runtime_kb is None:
raise Exception('Knowledge base not found') raise Exception('Knowledge base not found')
# Only internal KBs support file deletion
if runtime_kb.get_type() != 'internal':
raise Exception('Only internal knowledge bases support file deletion')
await runtime_kb.delete_file(file_id) await runtime_kb.delete_file(file_id)
async def delete_knowledge_base(self, kb_uuid: str) -> None: async def delete_knowledge_base(self, kb_uuid: str) -> None:

View File

@@ -26,6 +26,7 @@ from ..api.http.service import knowledge as knowledge_service
from ..api.http.service import mcp as mcp_service from ..api.http.service import mcp as mcp_service
from ..api.http.service import apikey as apikey_service from ..api.http.service import apikey as apikey_service
from ..api.http.service import webhook as webhook_service from ..api.http.service import webhook as webhook_service
from ..api.http.service import external_kb as external_kb_service
from ..discover import engine as discover_engine from ..discover import engine as discover_engine
from ..storage import mgr as storagemgr from ..storage import mgr as storagemgr
from ..utils import logcache from ..utils import logcache
@@ -123,6 +124,8 @@ class Application:
knowledge_service: knowledge_service.KnowledgeService = None knowledge_service: knowledge_service.KnowledgeService = None
external_kb_service: external_kb_service.ExternalKBService = None
mcp_service: mcp_service.MCPService = None mcp_service: mcp_service.MCPService = None
apikey_service: apikey_service.ApiKeyService = None apikey_service: apikey_service.ApiKeyService = None

View File

@@ -23,6 +23,7 @@ from ...api.http.service import knowledge as knowledge_service
from ...api.http.service import mcp as mcp_service from ...api.http.service import mcp as mcp_service
from ...api.http.service import apikey as apikey_service from ...api.http.service import apikey as apikey_service
from ...api.http.service import webhook as webhook_service from ...api.http.service import webhook as webhook_service
from ...api.http.service import external_kb as external_kb_service
from ...discover import engine as discover_engine from ...discover import engine as discover_engine
from ...storage import mgr as storagemgr from ...storage import mgr as storagemgr
from ...utils import logcache from ...utils import logcache
@@ -63,14 +64,6 @@ class BuildAppStage(stage.BootingStage):
ap.persistence_mgr = persistence_mgr_inst ap.persistence_mgr = persistence_mgr_inst
await persistence_mgr_inst.initialize() await persistence_mgr_inst.initialize()
async def runtime_disconnect_callback(connector: plugin_connector.PluginRuntimeConnector) -> None:
await asyncio.sleep(3)
await plugin_connector_inst.initialize()
plugin_connector_inst = plugin_connector.PluginRuntimeConnector(ap, runtime_disconnect_callback)
await plugin_connector_inst.initialize()
ap.plugin_connector = plugin_connector_inst
cmd_mgr_inst = cmdmgr.CommandManager(ap) cmd_mgr_inst = cmdmgr.CommandManager(ap)
await cmd_mgr_inst.initialize() await cmd_mgr_inst.initialize()
ap.cmd_mgr = cmd_mgr_inst ap.cmd_mgr = cmd_mgr_inst
@@ -130,6 +123,9 @@ class BuildAppStage(stage.BootingStage):
knowledge_service_inst = knowledge_service.KnowledgeService(ap) knowledge_service_inst = knowledge_service.KnowledgeService(ap)
ap.knowledge_service = knowledge_service_inst ap.knowledge_service = knowledge_service_inst
external_kb_service_inst = external_kb_service.ExternalKBService(ap)
ap.external_kb_service = external_kb_service_inst
mcp_service_inst = mcp_service.MCPService(ap) mcp_service_inst = mcp_service.MCPService(ap)
ap.mcp_service = mcp_service_inst ap.mcp_service = mcp_service_inst
@@ -139,5 +135,13 @@ class BuildAppStage(stage.BootingStage):
webhook_service_inst = webhook_service.WebhookService(ap) webhook_service_inst = webhook_service.WebhookService(ap)
ap.webhook_service = webhook_service_inst ap.webhook_service = webhook_service_inst
async def runtime_disconnect_callback(connector: plugin_connector.PluginRuntimeConnector) -> None:
await asyncio.sleep(3)
await plugin_connector_inst.initialize()
plugin_connector_inst = plugin_connector.PluginRuntimeConnector(ap, runtime_disconnect_callback)
await plugin_connector_inst.initialize()
ap.plugin_connector = plugin_connector_inst
ctrl = controller.Controller(ap) ctrl = controller.Controller(ap)
ap.ctrl = ctrl ap.ctrl = ctrl

View File

@@ -1,20 +1,6 @@
import sqlalchemy import sqlalchemy
from .base import Base from .base import Base
# Base = declarative_base()
# DATABASE_URL = os.getenv('DATABASE_URL', 'sqlite:///./rag_knowledge.db')
# print("Using database URL:", DATABASE_URL)
# engine = create_engine(DATABASE_URL, connect_args={'check_same_thread': False})
# SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
# def create_db_and_tables():
# """Creates all database tables defined in the Base."""
# Base.metadata.create_all(bind=engine)
# print('Database tables created or already exist.')
class KnowledgeBase(Base): class KnowledgeBase(Base):
__tablename__ = 'knowledge_bases' __tablename__ = 'knowledge_bases'
@@ -43,8 +29,13 @@ class Chunk(Base):
text = sqlalchemy.Column(sqlalchemy.Text) text = sqlalchemy.Column(sqlalchemy.Text)
# class Vector(Base): class ExternalKnowledgeBase(Base):
# __tablename__ = 'knowledge_base_vectors' __tablename__ = 'external_knowledge_bases'
# uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True, unique=True) uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True, unique=True)
# chunk_id = sqlalchemy.Column(sqlalchemy.String, nullable=True) name = sqlalchemy.Column(sqlalchemy.String, index=True)
# embedding = sqlalchemy.Column(sqlalchemy.LargeBinary) description = sqlalchemy.Column(sqlalchemy.Text)
plugin_author = sqlalchemy.Column(sqlalchemy.String, nullable=False)
plugin_name = sqlalchemy.Column(sqlalchemy.String, nullable=False)
retriever_name = sqlalchemy.Column(sqlalchemy.String, nullable=False)
retriever_config = sqlalchemy.Column(sqlalchemy.JSON, nullable=False, default={})
created_at = sqlalchemy.Column(sqlalchemy.DateTime, default=sqlalchemy.func.now())

View File

@@ -1,13 +0,0 @@
from __future__ import annotations
import pydantic
from typing import Any
class RetrieveResultEntry(pydantic.BaseModel):
id: str
metadata: dict[str, Any]
distance: float

View File

@@ -7,6 +7,7 @@ import typing
import os import os
import sys import sys
import httpx import httpx
import traceback
import sqlalchemy import sqlalchemy
from async_lru import alru_cache from async_lru import alru_cache
from langbot_plugin.api.entities.builtin.pipeline.query import provider_session from langbot_plugin.api.entities.builtin.pipeline.query import provider_session
@@ -101,6 +102,12 @@ class PluginRuntimeConnector:
self.handler_task = asyncio.create_task(self.handler.run()) self.handler_task = asyncio.create_task(self.handler.run())
_ = await self.handler.ping() _ = await self.handler.ping()
self.ap.logger.info('Connected to plugin runtime.') self.ap.logger.info('Connected to plugin runtime.')
# Sync polymorphic component instances after connection
try:
await self.sync_polymorphic_component_instances()
except Exception as e:
traceback.print_exc()
self.ap.logger.error(f'Failed to sync polymorphic component instances: {e}')
await self.handler_task await self.handler_task
task: asyncio.Task | None = None task: asyncio.Task | None = None
@@ -427,6 +434,31 @@ class PluginRuntimeConnector:
yield cmd_ret yield cmd_ret
# KnowledgeRetriever methods
async def list_knowledge_retrievers(self, bound_plugins: list[str] | None = None) -> list[dict[str, Any]]:
"""List all available KnowledgeRetriever components."""
if not self.is_enable_plugin:
return []
retrievers_data = await self.handler.list_knowledge_retrievers(include_plugins=bound_plugins)
return retrievers_data
async def retrieve_knowledge(
self,
plugin_author: str,
plugin_name: str,
retriever_name: str,
instance_id: str,
retrieval_context: dict[str, Any],
) -> list[dict[str, Any]]:
"""Retrieve knowledge using a KnowledgeRetriever instance."""
if not self.is_enable_plugin:
return []
return await self.handler.retrieve_knowledge(
plugin_author, plugin_name, retriever_name, instance_id, retrieval_context
)
def dispose(self): def dispose(self):
# No need to consider the shutdown on Windows # No need to consider the shutdown on Windows
# for Windows can kill processes and subprocesses chainly # for Windows can kill processes and subprocesses chainly
@@ -438,3 +470,42 @@ class PluginRuntimeConnector:
if self.heartbeat_task is not None: if self.heartbeat_task is not None:
self.heartbeat_task.cancel() self.heartbeat_task.cancel()
self.heartbeat_task = None self.heartbeat_task = None
async def sync_polymorphic_component_instances(self) -> dict[str, Any]:
"""Sync polymorphic component instances with runtime.
This collects all external knowledge bases from database and sends to runtime
to ensure instance integrity across restarts.
"""
if not self.is_enable_plugin:
return {}
# ===== external knowledge bases =====
external_kbs = await self.ap.external_kb_service.get_external_knowledge_bases()
# Build required_instances list
required_instances = []
for kb in external_kbs:
required_instances.append(
{
'instance_id': kb['uuid'],
'plugin_author': kb['plugin_author'],
'plugin_name': kb['plugin_name'],
'component_kind': 'KnowledgeRetriever',
'component_name': kb['retriever_name'],
'config': kb['retriever_config'],
}
)
self.ap.logger.info(f'Syncing {len(required_instances)} polymorphic component instances to runtime')
# Send to runtime
sync_result = await self.handler.sync_polymorphic_component_instances(required_instances)
self.ap.logger.info(
f'Sync complete: {len(sync_result.get("success_instances", []))} succeeded, '
f'{len(sync_result.get("failed_instances", []))} failed'
)
return sync_result

View File

@@ -713,3 +713,48 @@ class RuntimeConnectionHandler(handler.Handler):
async for ret in gen: async for ret in gen:
yield ret yield ret
# KnowledgeRetriever methods
async def list_knowledge_retrievers(self, include_plugins: list[str] | None = None) -> list[dict[str, Any]]:
"""List knowledge retrievers"""
result = await self.call_action(
LangBotToRuntimeAction.LIST_KNOWLEDGE_RETRIEVERS,
{
'include_plugins': include_plugins,
},
timeout=10,
)
return result['retrievers']
async def retrieve_knowledge(
self,
plugin_author: str,
plugin_name: str,
retriever_name: str,
instance_id: str,
retrieval_context: dict[str, Any],
) -> list[dict[str, Any]]:
"""Retrieve knowledge"""
result = await self.call_action(
LangBotToRuntimeAction.RETRIEVE_KNOWLEDGE,
{
'plugin_author': plugin_author,
'plugin_name': plugin_name,
'retriever_name': retriever_name,
'instance_id': instance_id,
'retrieval_context': retrieval_context,
},
timeout=30,
)
return result['retrieval_results']
async def sync_polymorphic_component_instances(self, required_instances: list[dict[str, Any]]) -> dict[str, Any]:
"""Sync polymorphic component instances with runtime"""
result = await self.call_action(
LangBotToRuntimeAction.SYNC_POLYMORPHIC_COMPONENT_INSTANCES,
{
'required_instances': required_instances,
},
timeout=30,
)
return result

View File

@@ -6,6 +6,7 @@ import typing
from .. import runner from .. import runner
import langbot_plugin.api.entities.builtin.pipeline.query as pipeline_query import langbot_plugin.api.entities.builtin.pipeline.query as pipeline_query
import langbot_plugin.api.entities.builtin.provider.message as provider_message import langbot_plugin.api.entities.builtin.provider.message as provider_message
import langbot_plugin.api.entities.builtin.rag.context as rag_context
rag_combined_prompt_template = """ rag_combined_prompt_template = """
@@ -63,7 +64,7 @@ class LocalAgentRunner(runner.RequestRunner):
if kb_uuids and user_message_text: if kb_uuids and user_message_text:
# only support text for now # only support text for now
all_results = [] all_results: list[rag_context.RetrievalResultEntry] = []
# Retrieve from each knowledge base # Retrieve from each knowledge base
for kb_uuid in kb_uuids: for kb_uuid in kb_uuids:
@@ -73,7 +74,15 @@ class LocalAgentRunner(runner.RequestRunner):
self.ap.logger.warning(f'Knowledge base {kb_uuid} not found, skipping') self.ap.logger.warning(f'Knowledge base {kb_uuid} not found, skipping')
continue continue
result = await kb.retrieve(user_message_text, kb.knowledge_base_entity.top_k) # Get top_k based on KB type
if kb.get_type() == 'internal':
top_k = kb.knowledge_base_entity.top_k
elif kb.get_type() == 'external':
top_k = 5 # external kb's top_k is managed by plugin config
else:
top_k = 5 # default fallback
result = await kb.retrieve(user_message_text, top_k)
if result: if result:
all_results.extend(result) all_results.extend(result)
@@ -81,9 +90,14 @@ class LocalAgentRunner(runner.RequestRunner):
final_user_message_text = '' final_user_message_text = ''
if all_results: if all_results:
rag_context = '\n\n'.join( texts = []
f'[{i + 1}] {entry.metadata.get("text", "")}' for i, entry in enumerate(all_results) idx = 1
) for entry in all_results:
for content in entry.content:
if content.type == 'text' and content.text is not None:
texts.append(f'[{idx}] {content.text}')
idx += 1
rag_context = '\n\n'.join(texts)
final_user_message_text = rag_combined_prompt_template.format( final_user_message_text = rag_combined_prompt_template.format(
rag_context=rag_context, user_message=user_message_text rag_context=rag_context, user_message=user_message_text
) )

View File

@@ -0,0 +1,55 @@
"""Base classes and interfaces for knowledge bases"""
from __future__ import annotations
import abc
from langbot.pkg.core import app
from langbot_plugin.api.entities.builtin.rag import context as rag_context
class KnowledgeBaseInterface(metaclass=abc.ABCMeta):
"""Abstract interface for all knowledge base types"""
ap: app.Application
def __init__(self, ap: app.Application):
self.ap = ap
@abc.abstractmethod
async def initialize(self):
"""Initialize the knowledge base"""
pass
@abc.abstractmethod
async def retrieve(self, query: str, top_k: int) -> list[rag_context.RetrievalResultEntry]:
"""Retrieve relevant documents from the knowledge base
Args:
query: The query string
top_k: Number of top results to return
Returns:
List of retrieve result entries
"""
pass
@abc.abstractmethod
def get_uuid(self) -> str:
"""Get the UUID of the knowledge base"""
pass
@abc.abstractmethod
def get_name(self) -> str:
"""Get the name of the knowledge base"""
pass
@abc.abstractmethod
def get_type(self) -> str:
"""Get the type of knowledge base (internal/external)"""
pass
@abc.abstractmethod
async def dispose(self):
"""Clean up resources"""
pass

View File

@@ -0,0 +1,85 @@
"""External knowledge base implementation"""
from __future__ import annotations
from langbot.pkg.core import app
from langbot.pkg.entity.persistence import rag as persistence_rag
from langbot_plugin.api.entities.builtin.rag import context as rag_context
from .base import KnowledgeBaseInterface
class ExternalKnowledgeBase(KnowledgeBaseInterface):
"""External knowledge base that queries via HTTP API or plugin retriever"""
external_kb_entity: persistence_rag.ExternalKnowledgeBase
# Plugin retriever instance ID
retriever_instance_id: str | None
def __init__(self, ap: app.Application, external_kb_entity: persistence_rag.ExternalKnowledgeBase):
super().__init__(ap)
self.external_kb_entity = external_kb_entity
self.retriever_instance_id = None
async def initialize(self):
"""Initialize the external knowledge base"""
# Use KB UUID as instance ID
# Instance creation is now handled by the unified sync mechanism
# when LangBot connects to runtime
self.retriever_instance_id = self.external_kb_entity.uuid
self.ap.logger.info(
f'Initialized external KB {self.external_kb_entity.uuid}, instance will be created by sync mechanism'
)
async def retrieve(self, query: str, top_k: int = 5) -> list[rag_context.RetrievalResultEntry]:
"""Retrieve documents from external knowledge base via plugin retriever"""
if not self.retriever_instance_id:
self.ap.logger.error(f'No retriever instance for KB {self.external_kb_entity.uuid}')
return []
try:
results = await self.ap.plugin_connector.retrieve_knowledge(
self.external_kb_entity.plugin_author,
self.external_kb_entity.plugin_name,
self.external_kb_entity.retriever_name,
self.retriever_instance_id,
{'query': query},
)
# Convert plugin results to RetrievalResultEntry
retrieval_entries = []
for result in results:
retrieval_entries.append(rag_context.RetrievalResultEntry(**result))
return retrieval_entries
except Exception as e:
self.ap.logger.error(f'Plugin retriever error: {e}')
import traceback
traceback.print_exc()
return []
def get_uuid(self) -> str:
"""Get the UUID of the external knowledge base"""
return self.external_kb_entity.uuid
def get_name(self) -> str:
"""Get the name of the external knowledge base"""
return self.external_kb_entity.name
def get_type(self) -> str:
"""Get the type of knowledge base"""
return 'external'
async def dispose(self):
"""Clean up resources"""
# Trigger sync to immediately delete the instance from plugin process
# This ensures instance is cleaned up without waiting for next LangBot restart
try:
await self.ap.plugin_connector.sync_polymorphic_component_instances()
self.ap.logger.info(
f'Disposed external KB {self.external_kb_entity.uuid}, triggered sync to delete instance'
)
except Exception as e:
self.ap.logger.error(f'Failed to sync after disposing KB: {e}')

View File

@@ -10,10 +10,12 @@ from langbot.pkg.rag.knowledge.services.retriever import Retriever
import sqlalchemy import sqlalchemy
from langbot.pkg.entity.persistence import rag as persistence_rag from langbot.pkg.entity.persistence import rag as persistence_rag
from langbot.pkg.core import taskmgr from langbot.pkg.core import taskmgr
from langbot.pkg.entity.rag import retriever as retriever_entities from langbot_plugin.api.entities.builtin.rag import context as rag_context
from .base import KnowledgeBaseInterface
from .external import ExternalKnowledgeBase
class RuntimeKnowledgeBase: class RuntimeKnowledgeBase(KnowledgeBaseInterface):
ap: app.Application ap: app.Application
knowledge_base_entity: persistence_rag.KnowledgeBase knowledge_base_entity: persistence_rag.KnowledgeBase
@@ -27,7 +29,7 @@ class RuntimeKnowledgeBase:
retriever: Retriever retriever: Retriever
def __init__(self, ap: app.Application, knowledge_base_entity: persistence_rag.KnowledgeBase): def __init__(self, ap: app.Application, knowledge_base_entity: persistence_rag.KnowledgeBase):
self.ap = ap super().__init__(ap)
self.knowledge_base_entity = knowledge_base_entity self.knowledge_base_entity = knowledge_base_entity
self.parser = parser.FileParser(ap=self.ap) self.parser = parser.FileParser(ap=self.ap)
self.chunker = chunker.Chunker(ap=self.ap) self.chunker = chunker.Chunker(ap=self.ap)
@@ -187,7 +189,7 @@ class RuntimeKnowledgeBase:
return stored_file_tasks[0] if stored_file_tasks else '' return stored_file_tasks[0] if stored_file_tasks else ''
async def retrieve(self, query: str, top_k: int) -> list[retriever_entities.RetrieveResultEntry]: async def retrieve(self, query: str, top_k: int) -> list[rag_context.RetrievalResultEntry]:
embedding_model = await self.ap.model_mgr.get_embedding_model_by_uuid( embedding_model = await self.ap.model_mgr.get_embedding_model_by_uuid(
self.knowledge_base_entity.embedding_model_uuid self.knowledge_base_entity.embedding_model_uuid
) )
@@ -206,6 +208,18 @@ class RuntimeKnowledgeBase:
sqlalchemy.delete(persistence_rag.File).where(persistence_rag.File.uuid == file_id) sqlalchemy.delete(persistence_rag.File).where(persistence_rag.File.uuid == file_id)
) )
def get_uuid(self) -> str:
"""Get the UUID of the knowledge base"""
return self.knowledge_base_entity.uuid
def get_name(self) -> str:
"""Get the name of the knowledge base"""
return self.knowledge_base_entity.name
def get_type(self) -> str:
"""Get the type of knowledge base"""
return 'internal'
async def dispose(self): async def dispose(self):
await self.ap.vector_db_mgr.vector_db.delete_collection(self.knowledge_base_entity.uuid) await self.ap.vector_db_mgr.vector_db.delete_collection(self.knowledge_base_entity.uuid)
@@ -213,7 +227,7 @@ class RuntimeKnowledgeBase:
class RAGManager: class RAGManager:
ap: app.Application ap: app.Application
knowledge_bases: list[RuntimeKnowledgeBase] knowledge_bases: list[KnowledgeBaseInterface]
def __init__(self, ap: app.Application): def __init__(self, ap: app.Application):
self.ap = ap self.ap = ap
@@ -227,8 +241,8 @@ class RAGManager:
self.knowledge_bases = [] self.knowledge_bases = []
# Load internal knowledge bases
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_rag.KnowledgeBase)) result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_rag.KnowledgeBase))
knowledge_bases = result.all() knowledge_bases = result.all()
for knowledge_base in knowledge_bases: for knowledge_base in knowledge_bases:
@@ -239,6 +253,21 @@ class RAGManager:
f'Error loading knowledge base {knowledge_base.uuid}: {e}\n{traceback.format_exc()}' f'Error loading knowledge base {knowledge_base.uuid}: {e}\n{traceback.format_exc()}'
) )
# Load external knowledge bases
external_result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(persistence_rag.ExternalKnowledgeBase)
)
external_kbs = external_result.all()
for external_kb in external_kbs:
try:
# Don't trigger sync during batch loading - will sync once after LangBot connects to runtime
await self.load_external_knowledge_base(external_kb, trigger_sync=False)
except Exception as e:
self.ap.logger.error(
f'Error loading external knowledge base {external_kb.uuid}: {e}\n{traceback.format_exc()}'
)
async def load_knowledge_base( async def load_knowledge_base(
self, self,
knowledge_base_entity: persistence_rag.KnowledgeBase | sqlalchemy.Row | dict, knowledge_base_entity: persistence_rag.KnowledgeBase | sqlalchemy.Row | dict,
@@ -256,21 +285,54 @@ class RAGManager:
return runtime_knowledge_base return runtime_knowledge_base
async def get_knowledge_base_by_uuid(self, kb_uuid: str) -> RuntimeKnowledgeBase | None: async def load_external_knowledge_base(
self,
external_kb_entity: persistence_rag.ExternalKnowledgeBase | sqlalchemy.Row | dict,
trigger_sync: bool = True,
) -> ExternalKnowledgeBase:
"""Load external knowledge base into runtime
Args:
external_kb_entity: External KB entity to load
trigger_sync: Whether to trigger sync after loading (default True for manual creation, False for batch loading)
"""
if isinstance(external_kb_entity, sqlalchemy.Row):
external_kb_entity = persistence_rag.ExternalKnowledgeBase(**external_kb_entity._mapping)
elif isinstance(external_kb_entity, dict):
external_kb_entity = persistence_rag.ExternalKnowledgeBase(**external_kb_entity)
external_kb = ExternalKnowledgeBase(ap=self.ap, external_kb_entity=external_kb_entity)
await external_kb.initialize()
self.knowledge_bases.append(external_kb)
# Trigger sync to create the instance immediately (for manual creation)
# Skip sync during batch loading from DB to avoid multiple sync calls
if trigger_sync:
try:
await self.ap.plugin_connector.sync_polymorphic_component_instances()
self.ap.logger.info(f'Triggered sync after loading external KB {external_kb_entity.uuid}')
except Exception as e:
self.ap.logger.error(f'Failed to sync after loading external KB: {e}')
return external_kb
async def get_knowledge_base_by_uuid(self, kb_uuid: str) -> KnowledgeBaseInterface | None:
for kb in self.knowledge_bases: for kb in self.knowledge_bases:
if kb.knowledge_base_entity.uuid == kb_uuid: if kb.get_uuid() == kb_uuid:
return kb return kb
return None return None
async def remove_knowledge_base_from_runtime(self, kb_uuid: str): async def remove_knowledge_base_from_runtime(self, kb_uuid: str):
for kb in self.knowledge_bases: for kb in self.knowledge_bases:
if kb.knowledge_base_entity.uuid == kb_uuid: if kb.get_uuid() == kb_uuid:
self.knowledge_bases.remove(kb) self.knowledge_bases.remove(kb)
return return
async def delete_knowledge_base(self, kb_uuid: str): async def delete_knowledge_base(self, kb_uuid: str):
for kb in self.knowledge_bases: for kb in self.knowledge_bases:
if kb.knowledge_base_entity.uuid == kb_uuid: if kb.get_uuid() == kb_uuid:
await kb.dispose() await kb.dispose()
self.knowledge_bases.remove(kb) self.knowledge_bases.remove(kb)
return return

View File

@@ -3,7 +3,8 @@ from __future__ import annotations
from . import base_service from . import base_service
from ....core import app from ....core import app
from ....provider.modelmgr.requester import RuntimeEmbeddingModel from ....provider.modelmgr.requester import RuntimeEmbeddingModel
from ....entity.rag import retriever as retriever_entities from langbot_plugin.api.entities.builtin.rag import context as rag_context
from langbot_plugin.api.entities.builtin.provider.message import ContentElement
class Retriever(base_service.BaseService): class Retriever(base_service.BaseService):
@@ -13,7 +14,7 @@ class Retriever(base_service.BaseService):
async def retrieve( async def retrieve(
self, kb_id: str, query: str, embedding_model: RuntimeEmbeddingModel, k: int = 5 self, kb_id: str, query: str, embedding_model: RuntimeEmbeddingModel, k: int = 5
) -> list[retriever_entities.RetrieveResultEntry]: ) -> list[rag_context.RetrievalResultEntry]:
self.ap.logger.info( self.ap.logger.info(
f"Retrieving for query: '{query[:10]}' with k={k} using {embedding_model.model_entity.uuid}" f"Retrieving for query: '{query[:10]}' with k={k} using {embedding_model.model_entity.uuid}"
) )
@@ -35,11 +36,12 @@ class Retriever(base_service.BaseService):
self.ap.logger.info('No relevant chunks found in vector database.') self.ap.logger.info('No relevant chunks found in vector database.')
return [] return []
result: list[retriever_entities.RetrieveResultEntry] = [] result: list[rag_context.RetrievalResultEntry] = []
for i, id in enumerate(matched_vector_ids): for i, id in enumerate(matched_vector_ids):
entry = retriever_entities.RetrieveResultEntry( entry = rag_context.RetrievalResultEntry(
id=id, id=id,
content=[ContentElement.from_text(vector_metadatas[i].get('text', ''))],
metadata=vector_metadatas[i], metadata=vector_metadatas[i],
distance=distances[i], distance=distances[i],
) )

View File

@@ -9,6 +9,7 @@ import {
SelectContent, SelectContent,
SelectGroup, SelectGroup,
SelectItem, SelectItem,
SelectLabel,
SelectTrigger, SelectTrigger,
SelectValue, SelectValue,
} from '@/components/ui/select'; } from '@/components/ui/select';
@@ -17,8 +18,13 @@ import { ControllerRenderProps } from 'react-hook-form';
import { Button } from '@/components/ui/button'; import { Button } from '@/components/ui/button';
import { useEffect, useState } from 'react'; import { useEffect, useState } from 'react';
import { httpClient } from '@/app/infra/http/HttpClient'; import { httpClient } from '@/app/infra/http/HttpClient';
import { LLMModel, Bot } from '@/app/infra/entities/api'; import {
import { KnowledgeBase } from '@/app/infra/entities/api'; LLMModel,
Bot,
KnowledgeBase,
ExternalKnowledgeBase,
ApiRespPluginSystemStatus,
} from '@/app/infra/entities/api';
import { toast } from 'sonner'; import { toast } from 'sonner';
import { import {
HoverCard, HoverCard,
@@ -51,10 +57,15 @@ export default function DynamicFormItemComponent({
}) { }) {
const [llmModels, setLlmModels] = useState<LLMModel[]>([]); const [llmModels, setLlmModels] = useState<LLMModel[]>([]);
const [knowledgeBases, setKnowledgeBases] = useState<KnowledgeBase[]>([]); const [knowledgeBases, setKnowledgeBases] = useState<KnowledgeBase[]>([]);
const [externalKnowledgeBases, setExternalKnowledgeBases] = useState<
ExternalKnowledgeBase[]
>([]);
const [bots, setBots] = useState<Bot[]>([]); const [bots, setBots] = useState<Bot[]>([]);
const [uploading, setUploading] = useState<boolean>(false); const [uploading, setUploading] = useState<boolean>(false);
const [kbDialogOpen, setKbDialogOpen] = useState(false); const [kbDialogOpen, setKbDialogOpen] = useState(false);
const [tempSelectedKBIds, setTempSelectedKBIds] = useState<string[]>([]); const [tempSelectedKBIds, setTempSelectedKBIds] = useState<string[]>([]);
const [pluginSystemStatus, setPluginSystemStatus] =
useState<ApiRespPluginSystemStatus | null>(null);
const { t } = useTranslation(); const { t } = useTranslation();
const handleFileUpload = async (file: File): Promise<IFileConfig | null> => { const handleFileUpload = async (file: File): Promise<IFileConfig | null> => {
@@ -113,9 +124,37 @@ export default function DynamicFormItemComponent({
.catch((err) => { .catch((err) => {
toast.error('Failed to get knowledge base list: ' + err.message); toast.error('Failed to get knowledge base list: ' + err.message);
}); });
// Fetch plugin system status
httpClient
.getPluginSystemStatus()
.then((status) => {
setPluginSystemStatus(status);
})
.catch((err) => {
console.error('Failed to get plugin system status:', err);
});
} }
}, [config.type]); }, [config.type]);
useEffect(() => {
if (
(config.type === DynamicFormItemType.KNOWLEDGE_BASE_SELECTOR ||
config.type === DynamicFormItemType.KNOWLEDGE_BASE_MULTI_SELECTOR) &&
pluginSystemStatus?.is_enable &&
pluginSystemStatus?.is_connected
) {
httpClient
.getExternalKnowledgeBases()
.then((resp) => {
setExternalKnowledgeBases(resp.bases);
})
.catch((err) => {
console.error('Failed to get external knowledge base list:', err);
});
}
}, [config.type, pluginSystemStatus]);
useEffect(() => { useEffect(() => {
if (config.type === DynamicFormItemType.BOT_SELECTOR) { if (config.type === DynamicFormItemType.BOT_SELECTOR) {
httpClient httpClient
@@ -340,12 +379,39 @@ export default function DynamicFormItemComponent({
<SelectContent> <SelectContent>
<SelectGroup> <SelectGroup>
<SelectItem value="__none__">{t('knowledge.empty')}</SelectItem> <SelectItem value="__none__">{t('knowledge.empty')}</SelectItem>
{knowledgeBases.map((base) => (
<SelectItem key={base.uuid} value={base.uuid ?? ''}>
{base.name}
</SelectItem>
))}
</SelectGroup> </SelectGroup>
{knowledgeBases.length > 0 && (
<SelectGroup>
<SelectLabel>{t('knowledge.builtIn')}</SelectLabel>
{knowledgeBases.map((base) => (
<SelectItem key={base.uuid} value={base.uuid ?? ''}>
{base.name}
</SelectItem>
))}
</SelectGroup>
)}
{externalKnowledgeBases.length > 0 && (
<SelectGroup>
<SelectLabel>{t('knowledge.external')}</SelectLabel>
{externalKnowledgeBases.map((base) => (
<SelectItem key={base.uuid} value={base.uuid ?? ''}>
<div className="flex items-center gap-2">
<img
src={httpClient.getPluginIconURL(
base.plugin_author,
base.plugin_name,
)}
alt="plugin icon"
className="w-4 h-4 rounded-[8%] flex-shrink-0"
/>
<span>{base.name}</span>
</div>
</SelectItem>
))}
</SelectGroup>
)}
</SelectContent> </SelectContent>
</Select> </Select>
); );
@@ -358,19 +424,36 @@ export default function DynamicFormItemComponent({
<div className="space-y-2"> <div className="space-y-2">
{field.value.map((kbId: string) => { {field.value.map((kbId: string) => {
const kb = knowledgeBases.find((base) => base.uuid === kbId); const kb = knowledgeBases.find((base) => base.uuid === kbId);
if (!kb) return null; const externalKb = externalKnowledgeBases.find(
(base) => base.uuid === kbId,
);
const currentKb = kb || externalKb;
if (!currentKb) return null;
return ( return (
<div <div
key={kbId} key={kbId}
className="flex items-center justify-between rounded-lg border p-3 hover:bg-accent" className="flex items-center justify-between rounded-lg border p-3 hover:bg-accent"
> >
<div className="flex-1"> <div className="flex items-center gap-2 flex-1">
<div className="font-medium">{kb.name}</div> {externalKb && (
{kb.description && ( <img
<div className="text-sm text-muted-foreground"> src={httpClient.getPluginIconURL(
{kb.description} externalKb.plugin_author,
</div> externalKb.plugin_name,
)}
alt="plugin icon"
className="w-8 h-8 rounded-[8%] flex-shrink-0"
/>
)} )}
<div className="flex-1 min-w-0">
<div className="font-medium">{currentKb.name}</div>
{currentKb.description && (
<div className="text-sm text-muted-foreground">
{currentKb.description}
</div>
)}
</div>
</div> </div>
<Button <Button
type="button" type="button"
@@ -417,39 +500,96 @@ export default function DynamicFormItemComponent({
<DialogHeader> <DialogHeader>
<DialogTitle>{t('knowledge.selectKnowledgeBases')}</DialogTitle> <DialogTitle>{t('knowledge.selectKnowledgeBases')}</DialogTitle>
</DialogHeader> </DialogHeader>
<div className="flex-1 overflow-y-auto space-y-2 pr-2"> <div className="flex-1 overflow-y-auto space-y-4 pr-2">
{knowledgeBases.map((base) => { {/* Built-in Knowledge Bases */}
const isSelected = tempSelectedKBIds.includes( {knowledgeBases.length > 0 && (
base.uuid ?? '', <div className="space-y-2">
); <div className="text-sm font-semibold text-muted-foreground px-2">
return ( {t('knowledge.builtIn')}
<div
key={base.uuid}
className="flex items-center gap-3 rounded-lg border p-3 hover:bg-accent cursor-pointer"
onClick={() => {
const kbId = base.uuid ?? '';
setTempSelectedKBIds((prev) =>
prev.includes(kbId)
? prev.filter((id) => id !== kbId)
: [...prev, kbId],
);
}}
>
<Checkbox
checked={isSelected}
aria-label={`Select ${base.name}`}
/>
<div className="flex-1">
<div className="font-medium">{base.name}</div>
{base.description && (
<div className="text-sm text-muted-foreground">
{base.description}
</div>
)}
</div>
</div> </div>
); {knowledgeBases.map((base) => {
})} const isSelected = tempSelectedKBIds.includes(
base.uuid ?? '',
);
return (
<div
key={base.uuid}
className="flex items-center gap-3 rounded-lg border p-3 hover:bg-accent cursor-pointer"
onClick={() => {
const kbId = base.uuid ?? '';
setTempSelectedKBIds((prev) =>
prev.includes(kbId)
? prev.filter((id) => id !== kbId)
: [...prev, kbId],
);
}}
>
<Checkbox
checked={isSelected}
aria-label={`Select ${base.name}`}
/>
<div className="flex-1">
<div className="font-medium">{base.name}</div>
{base.description && (
<div className="text-sm text-muted-foreground">
{base.description}
</div>
)}
</div>
</div>
);
})}
</div>
)}
{/* External Knowledge Bases */}
{externalKnowledgeBases.length > 0 && (
<div className="space-y-2">
<div className="text-sm font-semibold text-muted-foreground px-2">
{t('knowledge.external')}
</div>
{externalKnowledgeBases.map((base) => {
const isSelected = tempSelectedKBIds.includes(
base.uuid ?? '',
);
return (
<div
key={base.uuid}
className="flex items-center gap-3 rounded-lg border p-3 hover:bg-accent cursor-pointer"
onClick={() => {
const kbId = base.uuid ?? '';
setTempSelectedKBIds((prev) =>
prev.includes(kbId)
? prev.filter((id) => id !== kbId)
: [...prev, kbId],
);
}}
>
<Checkbox
checked={isSelected}
aria-label={`Select ${base.name}`}
/>
<img
src={httpClient.getPluginIconURL(
base.plugin_author,
base.plugin_name,
)}
alt="plugin icon"
className="w-8 h-8 rounded-[8%] flex-shrink-0"
/>
<div className="flex-1">
<div className="font-medium">{base.name}</div>
{base.description && (
<div className="text-sm text-muted-foreground">
{base.description}
</div>
)}
</div>
</div>
);
})}
</div>
)}
</div> </div>
<DialogFooter> <DialogFooter>
<Button <Button

View File

@@ -25,11 +25,14 @@ import { httpClient } from '@/app/infra/http/HttpClient';
import KBForm from '@/app/home/knowledge/components/kb-form/KBForm'; import KBForm from '@/app/home/knowledge/components/kb-form/KBForm';
import KBDoc from '@/app/home/knowledge/components/kb-docs/KBDoc'; import KBDoc from '@/app/home/knowledge/components/kb-docs/KBDoc';
import KBRetrieve from '@/app/home/knowledge/components/kb-retrieve/KBRetrieve'; import KBRetrieve from '@/app/home/knowledge/components/kb-retrieve/KBRetrieve';
import ExternalKBForm from '@/app/home/knowledge/components/external-kb-form/ExternalKBForm';
import ExternalKBRetrieve from '@/app/home/knowledge/components/kb-retrieve/ExternalKBRetrieve';
interface KBDetailDialogProps { interface KBDetailDialogProps {
open: boolean; open: boolean;
onOpenChange: (open: boolean) => void; onOpenChange: (open: boolean) => void;
kbId?: string; kbId?: string;
kbType: 'builtin' | 'external';
onFormCancel: () => void; onFormCancel: () => void;
onKbDeleted: () => void; onKbDeleted: () => void;
onNewKbCreated: (kbId: string) => void; onNewKbCreated: (kbId: string) => void;
@@ -40,6 +43,7 @@ export default function KBDetailDialog({
open, open,
onOpenChange, onOpenChange,
kbId: propKbId, kbId: propKbId,
kbType,
onFormCancel, onFormCancel,
onKbDeleted, onKbDeleted,
onNewKbCreated, onNewKbCreated,
@@ -55,6 +59,7 @@ export default function KBDetailDialog({
setActiveMenu('metadata'); setActiveMenu('metadata');
}, [propKbId, open]); }, [propKbId, open]);
// Build menu based on KB type
const menu = [ const menu = [
{ {
key: 'metadata', key: 'metadata',
@@ -69,19 +74,24 @@ export default function KBDetailDialog({
</svg> </svg>
), ),
}, },
{ // Only show documents for builtin KB
key: 'documents', ...(kbType === 'builtin'
label: t('knowledge.documents'), ? [
icon: ( {
<svg key: 'documents',
xmlns="http://www.w3.org/2000/svg" label: t('knowledge.documents'),
viewBox="0 0 24 24" icon: (
fill="currentColor" <svg
> xmlns="http://www.w3.org/2000/svg"
<path d="M21 8V20.9932C21 21.5501 20.5552 22 20.0066 22H3.9934C3.44495 22 3 21.556 3 21.0082V2.9918C3 2.45531 3.4487 2 4.00221 2H14.9968L21 8ZM19 9H14V4H5V20H19V9ZM8 7H11V9H8V7ZM8 11H16V13H8V11ZM8 15H16V17H8V15Z"></path> viewBox="0 0 24 24"
</svg> fill="currentColor"
), >
}, <path d="M21 8V20.9932C21 21.5501 20.5552 22 20.0066 22H3.9934C3.44495 22 3 21.556 3 21.0082V2.9918C3 2.45531 3.4487 2 4.00221 2H14.9968L21 8ZM19 9H14V4H5V20H19V9ZM8 7H11V9H8V7ZM8 11H16V13H8V11ZM8 15H16V17H8V15Z"></path>
</svg>
),
},
]
: []),
{ {
key: 'retrieve', key: 'retrieve',
label: t('knowledge.retrieve'), label: t('knowledge.retrieve'),
@@ -98,7 +108,12 @@ export default function KBDetailDialog({
]; ];
const confirmDelete = () => { const confirmDelete = () => {
httpClient.deleteKnowledgeBase(kbId ?? '').then(() => { const deletePromise =
kbType === 'builtin'
? httpClient.deleteKnowledgeBase(kbId ?? '')
: httpClient.deleteExternalKnowledgeBase(kbId ?? '');
deletePromise.then(() => {
onKbDeleted(); onKbDeleted();
}); });
setShowDeleteConfirm(false); setShowDeleteConfirm(false);
@@ -111,22 +126,35 @@ export default function KBDetailDialog({
<DialogContent className="overflow-hidden p-0 !max-w-[40vw] max-h-[70vh] flex"> <DialogContent className="overflow-hidden p-0 !max-w-[40vw] max-h-[70vh] flex">
<main className="flex flex-1 flex-col h-[70vh]"> <main className="flex flex-1 flex-col h-[70vh]">
<DialogHeader className="px-6 pt-6 pb-4 shrink-0"> <DialogHeader className="px-6 pt-6 pb-4 shrink-0">
<DialogTitle>{t('knowledge.createKnowledgeBase')}</DialogTitle> <DialogTitle>
{kbType === 'builtin'
? t('knowledge.createKnowledgeBase')
: t('knowledge.addExternal')}
</DialogTitle>
</DialogHeader> </DialogHeader>
<div className="flex-1 overflow-y-auto px-6 pb-6"> <div className="flex-1 overflow-y-auto px-6 pb-6">
{activeMenu === 'metadata' && ( {kbType === 'builtin' ? (
<KBForm <KBForm
initKbId={undefined} initKbId={undefined}
onNewKbCreated={onNewKbCreated} onNewKbCreated={onNewKbCreated}
onKbUpdated={onKbUpdated} onKbUpdated={onKbUpdated}
/> />
) : (
<ExternalKBForm
initKBId={undefined}
onFormSubmit={() => onOpenChange(false)}
onKBDeleted={() => {}}
onNewKBCreated={onNewKbCreated}
/>
)} )}
{activeMenu === 'documents' && <div>documents</div>}
</div> </div>
{activeMenu === 'metadata' && ( {activeMenu === 'metadata' && (
<DialogFooter className="px-6 py-4 border-t shrink-0"> <DialogFooter className="px-6 py-4 border-t shrink-0">
<div className="flex justify-end gap-2"> <div className="flex justify-end gap-2">
<Button type="submit" form="kb-form"> <Button
type="submit"
form={kbType === 'builtin' ? 'kb-form' : 'external-kb-form'}
>
{t('common.save')} {t('common.save')}
</Button> </Button>
<Button <Button
@@ -188,15 +216,33 @@ export default function KBDetailDialog({
</DialogTitle> </DialogTitle>
</DialogHeader> </DialogHeader>
<div className="flex-1 overflow-y-auto px-6 pb-6"> <div className="flex-1 overflow-y-auto px-6 pb-6">
{activeMenu === 'metadata' && ( {activeMenu === 'metadata' &&
<KBForm (kbType === 'builtin' ? (
initKbId={kbId} <KBForm
onNewKbCreated={onNewKbCreated} initKbId={kbId}
onKbUpdated={onKbUpdated} onNewKbCreated={onNewKbCreated}
/> onKbUpdated={onKbUpdated}
/>
) : (
<ExternalKBForm
initKBId={kbId}
onFormSubmit={() => onOpenChange(false)}
onKBDeleted={() => {
onKbDeleted();
onOpenChange(false);
}}
onNewKBCreated={onNewKbCreated}
/>
))}
{activeMenu === 'documents' && kbType === 'builtin' && (
<KBDoc kbId={kbId} />
)} )}
{activeMenu === 'documents' && <KBDoc kbId={kbId} />} {activeMenu === 'retrieve' &&
{activeMenu === 'retrieve' && <KBRetrieve kbId={kbId} />} (kbType === 'builtin' ? (
<KBRetrieve kbId={kbId} />
) : (
<ExternalKBRetrieve kbId={kbId} />
))}
</div> </div>
{activeMenu === 'metadata' && ( {activeMenu === 'metadata' && (
<DialogFooter className="px-6 py-4 border-t shrink-0"> <DialogFooter className="px-6 py-4 border-t shrink-0">
@@ -208,7 +254,12 @@ export default function KBDetailDialog({
> >
{t('common.delete')} {t('common.delete')}
</Button> </Button>
<Button type="submit" form="kb-form"> <Button
type="submit"
form={
kbType === 'builtin' ? 'kb-form' : 'external-kb-form'
}
>
{t('common.save')} {t('common.save')}
</Button> </Button>
<Button <Button

View File

@@ -0,0 +1,74 @@
import { ExternalKBCardVO } from '@/app/home/knowledge/components/external-kb-card/ExternalKBCardVO';
import { useTranslation } from 'react-i18next';
import styles from '../kb-card/KBCard.module.css';
import { httpClient } from '@/app/infra/http/HttpClient';
export default function ExternalKBCard({
kbCardVO,
}: {
kbCardVO: ExternalKBCardVO;
}) {
const { t } = useTranslation();
return (
<div className={`${styles.cardContainer}`}>
<div className="w-full h-full flex flex-row items-start gap-3">
{/* Icon */}
<img
src={httpClient.getPluginIconURL(
kbCardVO.pluginAuthor,
kbCardVO.pluginName,
)}
alt="plugin icon"
className="w-16 h-16 mt-1 rounded-[8%] flex-shrink-0"
/>
{/* Info Column */}
<div className="flex flex-col flex-1 min-w-0 h-full">
{/* Top section: Name, Description and Plugin Info */}
<div className="flex flex-col gap-0">
{/* Name and Description */}
<div className={`${styles.basicInfoNameContainer}`}>
<div className={`${styles.basicInfoNameText} ${styles.bigText}`}>
{kbCardVO.name}
</div>
<div className={`${styles.basicInfoDescriptionText}`}>
{kbCardVO.description}
</div>
</div>
{/* Plugin Info */}
<div className="flex flex-row gap-2 items-center mt-1">
<svg
className="w-5 h-5 text-gray-500 dark:text-gray-400"
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
>
<path d="M7 5C7 2.79086 8.79086 1 11 1C13.2091 1 15 2.79086 15 5H18C18.5523 5 19 5.44772 19 6V9C21.2091 9 23 10.7909 23 13C23 15.2091 21.2091 17 19 17V20C19 20.5523 18.5523 21 18 21H4C3.44772 21 3 20.5523 3 20V6C3 5.44772 3.44772 5 4 5H7ZM11 3C9.89543 3 9 3.89543 9 5C9 5.23554 9.0403 5.45952 9.11355 5.66675C9.22172 5.97282 9.17461 6.31235 8.98718 6.57739C8.79974 6.84243 8.49532 7 8.17071 7H5V19H17V15.8293C17 15.5047 17.1576 15.2003 17.4226 15.0128C17.6877 14.8254 18.0272 14.7783 18.3332 14.8865C18.5405 14.9597 18.7645 15 19 15C20.1046 15 21 14.1046 21 13C21 11.8954 20.1046 11 19 11C18.7645 11 18.5405 11.0403 18.3332 11.1135C18.0272 11.2217 17.6877 11.1746 17.4226 10.9872C17.1576 10.7997 17 10.4953 17 10.1707V7H13.8293C13.5047 7 13.2003 6.84243 13.0128 6.57739C12.8254 6.31235 12.7783 5.97282 12.8865 5.66675C12.9597 5.45952 13 5.23555 13 5C13 3.89543 12.1046 3 11 3Z"></path>
</svg>
<span className="text-sm font-medium text-gray-600 dark:text-gray-400">
{kbCardVO.pluginAuthor} / {kbCardVO.pluginName}
</span>
</div>
</div>
{/* Bottom section: Update Time */}
<div className="flex flex-row gap-2 items-center mt-auto">
<svg
className="w-5 h-5 text-gray-500 dark:text-gray-400"
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
>
<path d="M12 22C6.47715 22 2 17.5228 2 12C2 6.47715 6.47715 2 12 2C17.5228 2 22 6.47715 22 12C22 17.5228 17.5228 22 12 22ZM12 20C16.4183 20 20 16.4183 20 12C20 7.58172 16.4183 4 12 4C7.58172 4 4 7.58172 4 12C4 16.4183 7.58172 20 12 20ZM13 12H17V14H11V7H13V12Z"></path>
</svg>
<span className="text-sm font-medium text-gray-600 dark:text-gray-400">
{t('knowledge.updateTime')}
{kbCardVO.lastUpdatedTimeAgo}
</span>
</div>
</div>
</div>
</div>
);
}

View File

@@ -0,0 +1,39 @@
export class ExternalKBCardVO {
id: string;
name: string;
description: string;
retrieverName: string;
retrieverConfig: Record<string, unknown>;
lastUpdatedTimeAgo: string;
pluginAuthor: string;
pluginName: string;
constructor({
id,
name,
description,
retrieverName,
retrieverConfig,
lastUpdatedTimeAgo,
pluginAuthor,
pluginName,
}: {
id: string;
name: string;
description: string;
retrieverName: string;
retrieverConfig: Record<string, unknown>;
lastUpdatedTimeAgo: string;
pluginAuthor: string;
pluginName: string;
}) {
this.id = id;
this.name = name;
this.description = description;
this.retrieverName = retrieverName;
this.retrieverConfig = retrieverConfig;
this.lastUpdatedTimeAgo = lastUpdatedTimeAgo;
this.pluginAuthor = pluginAuthor;
this.pluginName = pluginName;
}
}

View File

@@ -0,0 +1,558 @@
import { useEffect, useState } from 'react';
import { zodResolver } from '@hookform/resolvers/zod';
import { useForm } from 'react-hook-form';
import { z } from 'zod';
import { toast } from 'sonner';
import { useTranslation } from 'react-i18next';
import { UUID } from 'uuidjs';
import {
DynamicFormItemConfig,
getDefaultValues,
parseDynamicFormItemType,
} from '@/app/home/components/dynamic-form/DynamicFormItemConfig';
import { IDynamicFormItemSchema } from '@/app/infra/entities/form/dynamic';
import DynamicFormComponent from '@/app/home/components/dynamic-form/DynamicFormComponent';
import { httpClient } from '@/app/infra/http/HttpClient';
import { ExternalKnowledgeBase } from '@/app/infra/entities/api';
import {
Dialog,
DialogContent,
DialogDescription,
DialogHeader,
DialogTitle,
DialogFooter,
} from '@/components/ui/dialog';
import { Button } from '@/components/ui/button';
import {
Form,
FormControl,
FormField,
FormItem,
FormLabel,
FormMessage,
} from '@/components/ui/form';
import { Input } from '@/components/ui/input';
import {
Select,
SelectContent,
SelectGroup,
SelectItem,
SelectTrigger,
SelectValue,
} from '@/components/ui/select';
import {
HoverCard,
HoverCardContent,
HoverCardTrigger,
} from '@/components/ui/hover-card';
import { extractI18nObject } from '@/i18n/I18nProvider';
import { I18nObject } from '@/app/infra/entities/common';
// Form schema
const getFormSchema = (t: (key: string) => string) =>
z.object({
name: z.string().min(1, { message: t('knowledge.nameRequired') }),
description: z.string().optional(),
plugin_author: z.string().min(1, { message: 'Please select a retriever' }),
plugin_name: z.string().min(1, { message: 'Please select a retriever' }),
retriever_name: z.string().min(1, { message: 'Please select a retriever' }),
retriever_config: z.record(z.string(), z.any()),
});
// Retriever information interface
interface RetrieverInfo {
plugin_author: string;
plugin_name: string;
retriever_name: string;
retriever_description: I18nObject;
manifest: {
manifest?: {
metadata?: {
label?: I18nObject;
description?: I18nObject;
};
spec?: {
config?: IDynamicFormItemSchema[];
};
};
};
}
interface ExternalKBFormProps {
initKBId?: string;
onFormSubmit: (value: z.infer<ReturnType<typeof getFormSchema>>) => void;
onKBDeleted: () => void;
onNewKBCreated: (kbId: string) => void;
}
export default function ExternalKBForm({
initKBId,
onFormSubmit,
onKBDeleted,
onNewKBCreated,
}: ExternalKBFormProps) {
const { t } = useTranslation();
const formSchema = getFormSchema(t);
// Form setup
const form = useForm<z.infer<typeof formSchema>>({
resolver: zodResolver(formSchema),
defaultValues: {
name: '',
description: '',
plugin_author: '',
plugin_name: '',
retriever_name: '',
retriever_config: {},
},
});
// State management
const [showDeleteConfirmModal, setShowDeleteConfirmModal] = useState(false);
const [availableRetrievers, setAvailableRetrievers] = useState<
RetrieverInfo[]
>([]);
const [retrieverNameToConfigMap, setRetrieverNameToConfigMap] = useState(
new Map<string, IDynamicFormItemSchema[]>(),
);
const [showDynamicForm, setShowDynamicForm] = useState<boolean>(false);
const [dynamicFormConfigList, setDynamicFormConfigList] = useState<
IDynamicFormItemSchema[]
>([]);
// Initialize form when initKBId changes
useEffect(() => {
loadFormData();
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [initKBId]);
/**
* Load form data: initialize retrievers list and load KB config if editing
*/
async function loadFormData() {
const configMap = await loadAvailableRetrievers();
if (initKBId) {
// Edit mode: load existing KB configuration
try {
const kbConfig = await loadKBConfig(initKBId);
// Set form values
form.setValue('name', kbConfig.name);
form.setValue('description', kbConfig.description || '');
form.setValue('plugin_author', kbConfig.plugin_author);
form.setValue('plugin_name', kbConfig.plugin_name);
form.setValue('retriever_name', kbConfig.retriever_name);
form.setValue('retriever_config', kbConfig.retriever_config);
// Load dynamic form for the selected retriever
const fullName = `${kbConfig.plugin_author}/${kbConfig.plugin_name}/${kbConfig.retriever_name}`;
loadDynamicFormConfig(fullName, configMap);
} catch (err) {
toast.error('Failed to load KB config: ' + (err as Error).message);
}
} else {
// Create mode: reset form
form.reset();
}
}
/**
* Load available retrievers from API and build config map
*/
async function loadAvailableRetrievers(): Promise<
Map<string, IDynamicFormItemSchema[]>
> {
const retrieversRes = await httpClient.listKnowledgeRetrievers();
setAvailableRetrievers((retrieversRes.retrievers || []) as RetrieverInfo[]);
// Build retriever name to config map
const configMap = new Map<string, IDynamicFormItemSchema[]>();
((retrieversRes.retrievers || []) as RetrieverInfo[]).forEach(
(retriever) => {
const fullName = `${retriever.plugin_author}/${retriever.plugin_name}/${retriever.retriever_name}`;
const configSchema = retriever.manifest?.manifest?.spec?.config || [];
configMap.set(
fullName,
configSchema.map(
(item) =>
new DynamicFormItemConfig({
default: item.default,
id: UUID.generate(),
label: item.label,
description: item.description,
name: item.name,
required: item.required,
type: parseDynamicFormItemType(item.type),
options: item.options,
}),
),
);
},
);
setRetrieverNameToConfigMap(configMap);
return configMap;
}
/**
* Load KB configuration from API
*/
async function loadKBConfig(
kbId: string,
): Promise<z.infer<typeof formSchema>> {
const res = await httpClient.getExternalKnowledgeBase(kbId);
const kb = res.base;
return {
name: kb.name,
description: kb.description,
plugin_author: kb.plugin_author,
plugin_name: kb.plugin_name,
retriever_name: kb.retriever_name,
retriever_config: kb.retriever_config || {},
};
}
/**
* Load dynamic form configuration for selected retriever
* @param fullRetrieverName - Full retriever name in format: plugin_author/plugin_name/retriever_name
* @param configMapOverride - Optional config map to use (for initial load)
*/
function loadDynamicFormConfig(
fullRetrieverName: string,
configMapOverride?: Map<string, IDynamicFormItemSchema[]>,
) {
if (!fullRetrieverName) {
setShowDynamicForm(false);
return;
}
// Use provided config map or fall back to state
const configMap = configMapOverride || retrieverNameToConfigMap;
const configList = configMap.get(fullRetrieverName);
if (configList && configList.length > 0) {
setDynamicFormConfigList(configList);
setShowDynamicForm(true);
// Only reset to default values when manually selecting (not initial load)
if (!configMapOverride) {
form.setValue('retriever_config', getDefaultValues(configList));
}
} else {
setShowDynamicForm(false);
if (!configMapOverride) {
form.setValue('retriever_config', {});
}
}
}
/**
* Handle retriever selection change
*/
function handleRetrieverSelect(fullRetrieverName: string) {
if (!fullRetrieverName) {
setShowDynamicForm(false);
return;
}
// Parse and update form fields
const parts = fullRetrieverName.split('/');
if (parts.length === 3) {
form.setValue('plugin_author', parts[0]);
form.setValue('plugin_name', parts[1]);
form.setValue('retriever_name', parts[2]);
}
// Load dynamic form configuration
loadDynamicFormConfig(fullRetrieverName);
}
/**
* Handle form submission (create or update)
*/
function handleFormSubmit() {
const formData: ExternalKnowledgeBase = {
name: form.getValues().name,
description: form.getValues().description || '',
plugin_author: form.getValues().plugin_author,
plugin_name: form.getValues().plugin_name,
retriever_name: form.getValues().retriever_name,
retriever_config: form.getValues().retriever_config,
};
if (initKBId) {
// Update existing KB
httpClient
.updateExternalKnowledgeBase(initKBId, { ...formData, uuid: initKBId })
.then(() => {
onFormSubmit(form.getValues());
toast.success(t('knowledge.updateExternalSuccess'));
})
.catch((err) => {
toast.error('Failed to update KB: ' + err.message);
});
} else {
// Create new KB
httpClient
.createExternalKnowledgeBase(formData)
.then((res) => {
toast.success(t('knowledge.createExternalSuccess'));
onNewKBCreated(res.uuid);
form.reset();
})
.catch((err) => {
toast.error('Failed to create KB: ' + err.message);
});
}
}
/**
* Handle KB deletion
*/
function handleDelete() {
if (!initKBId) return;
httpClient
.deleteExternalKnowledgeBase(initKBId)
.then(() => {
onKBDeleted();
toast.success(t('knowledge.deleteExternalSuccess'));
})
.catch((err) => {
toast.error('Failed to delete KB: ' + err.message);
});
}
/**
* Get retriever label with i18n support
*/
function getRetrieverLabel(fullName: string): string {
const retriever = availableRetrievers.find(
(r) =>
`${r.plugin_author}/${r.plugin_name}/${r.retriever_name}` === fullName,
);
return retriever?.manifest?.manifest?.metadata?.label
? extractI18nObject(retriever.manifest.manifest.metadata.label)
: fullName;
}
// Compute full retriever name for display
const currentRetrieverFullName =
form.watch('plugin_author') &&
form.watch('plugin_name') &&
form.watch('retriever_name')
? `${form.watch('plugin_author')}/${form.watch(
'plugin_name',
)}/${form.watch('retriever_name')}`
: '';
return (
<div>
{/* Delete Confirmation Dialog */}
<Dialog
open={showDeleteConfirmModal}
onOpenChange={setShowDeleteConfirmModal}
>
<DialogContent>
<DialogHeader>
<DialogTitle>{t('common.confirmDelete')}</DialogTitle>
</DialogHeader>
<DialogDescription>
{t('knowledge.deleteConfirmation')}
</DialogDescription>
<DialogFooter>
<Button
variant="outline"
onClick={() => setShowDeleteConfirmModal(false)}
>
{t('common.cancel')}
</Button>
<Button
variant="destructive"
onClick={() => {
handleDelete();
setShowDeleteConfirmModal(false);
}}
>
{t('common.confirmDelete')}
</Button>
</DialogFooter>
</DialogContent>
</Dialog>
{/* Main Form */}
<Form {...form}>
<form
id="external-kb-form"
onSubmit={form.handleSubmit(handleFormSubmit)}
className="space-y-8"
>
<div className="space-y-4">
{/* KB Name */}
<FormField
control={form.control}
name="name"
render={({ field }) => (
<FormItem>
<FormLabel>
{t('knowledge.kbName')}
<span className="text-red-500">*</span>
</FormLabel>
<FormControl>
<Input {...field} />
</FormControl>
<FormMessage />
</FormItem>
)}
/>
{/* KB Description */}
<FormField
control={form.control}
name="description"
render={({ field }) => (
<FormItem>
<FormLabel>{t('knowledge.kbDescription')}</FormLabel>
<FormControl>
<Input {...field} />
</FormControl>
<FormMessage />
</FormItem>
)}
/>
{/* Retriever Selector */}
<FormField
control={form.control}
name="retriever_name"
render={() => (
<FormItem>
<FormLabel>
{t('knowledge.retriever')}
<span className="text-red-500">*</span>
</FormLabel>
<FormControl>
<Select
onValueChange={handleRetrieverSelect}
value={currentRetrieverFullName}
>
<SelectTrigger className="w-full bg-[#ffffff] dark:bg-[#2a2a2e]">
<SelectValue
placeholder={t('knowledge.selectRetriever')}
/>
</SelectTrigger>
<SelectContent className="fixed z-[1000]">
<SelectGroup>
{availableRetrievers.map((retriever) => {
const fullName = `${retriever.plugin_author}/${retriever.plugin_name}/${retriever.retriever_name}`;
const label = retriever.manifest?.manifest?.metadata
?.label
? extractI18nObject(
retriever.manifest.manifest.metadata.label,
)
: retriever.retriever_name;
const description = extractI18nObject(
retriever.retriever_description,
);
return (
<HoverCard
key={fullName}
openDelay={0}
closeDelay={0}
>
<HoverCardTrigger asChild>
<SelectItem value={fullName}>
{label}
</SelectItem>
</HoverCardTrigger>
<HoverCardContent
className="w-80 data-[state=open]:animate-none"
align="end"
side="right"
sideOffset={10}
>
<div className="space-y-2">
<div className="flex items-start gap-3">
<img
src={httpClient.getPluginIconURL(
retriever.plugin_author,
retriever.plugin_name,
)}
alt="plugin icon"
className="w-10 h-10 rounded-[8%] flex-shrink-0"
/>
<div className="flex flex-col gap-1 flex-1 min-w-0">
<h4 className="font-medium text-sm">
{label}
</h4>
<p className="text-xs text-muted-foreground">
{retriever.plugin_author} /{' '}
{retriever.plugin_name}
</p>
</div>
</div>
{description && (
<p className="text-sm text-muted-foreground">
{description}
</p>
)}
</div>
</HoverCardContent>
</HoverCard>
);
})}
</SelectGroup>
</SelectContent>
</Select>
</FormControl>
<FormMessage />
</FormItem>
)}
/>
{/* Selected Retriever Card */}
{currentRetrieverFullName && (
<div className="flex items-start gap-3 p-4 rounded-lg border">
<img
src={httpClient.getPluginIconURL(
form.watch('plugin_author'),
form.watch('plugin_name'),
)}
alt="plugin icon"
className="w-12 h-12 rounded-[8%] flex-shrink-0"
/>
<div className="flex flex-col gap-1">
<div className="font-medium">
{getRetrieverLabel(currentRetrieverFullName)}
</div>
<div className="text-sm text-gray-500">
{form.watch('plugin_author')} / {form.watch('plugin_name')}
</div>
</div>
</div>
)}
{/* Dynamic Retriever Configuration Form */}
{showDynamicForm && dynamicFormConfigList.length > 0 && (
<div className="space-y-4">
<div className="text-lg font-medium">
{t('knowledge.retrieverConfiguration')}
</div>
<DynamicFormComponent
itemConfigList={dynamicFormConfigList}
initialValues={form.watch('retriever_config')}
onSubmit={(values) => {
form.setValue('retriever_config', values);
}}
/>
</div>
)}
</div>
</form>
</Form>
</div>
);
}

View File

@@ -0,0 +1,35 @@
'use client';
import React from 'react';
import { httpClient } from '@/app/infra/http/HttpClient';
import { RetrieveResult } from '@/app/infra/entities/api';
import KBRetrieveGeneric from './KBRetrieveGeneric';
interface ExternalKBRetrieveProps {
kbId: string;
}
/**
* External knowledge base retrieve component
* Uses the generic retrieve component with external KB API
*/
export default function ExternalKBRetrieve({ kbId }: ExternalKBRetrieveProps) {
const getResultTitle = (result: RetrieveResult): string => {
// For external KB, try to get document_name or use a generic title
return (
(result.metadata.document_name as string) ||
(result.metadata.source as string) ||
result.id
);
};
return (
<KBRetrieveGeneric
kbId={kbId}
retrieveFunction={httpClient.retrieveExternalKnowledgeBase.bind(
httpClient,
)}
getResultTitle={getResultTitle}
/>
);
}

View File

@@ -48,11 +48,36 @@ export default function KBRetrieve({ kbId }: KBRetrieveProps) {
} }
}; };
const getFileName = (fileId: string) => { const getFileName = (fileId?: string) => {
if (!fileId) return '';
const file = files.find((f) => f.uuid === fileId); const file = files.find((f) => f.uuid === fileId);
return file?.file_name || fileId; return file?.file_name || fileId;
}; };
/**
* Extract text content from the content array
* The content array may contain multiple items with type 'text'
*/
const extractTextFromContent = (result: RetrieveResult): string => {
// First try to get content from the new format
if (result.content && Array.isArray(result.content)) {
const textParts = result.content
.filter((item) => item.type === 'text' && item.text)
.map((item) => item.text);
if (textParts.length > 0) {
return textParts.join('\n\n');
}
}
// Fallback to metadata.text for backward compatibility
if (result.metadata?.text) {
return result.metadata.text as string;
}
return '';
};
return ( return (
<div className="space-y-4"> <div className="space-y-4">
<div className="flex gap-2"> <div className="flex gap-2">
@@ -87,7 +112,7 @@ export default function KBRetrieve({ kbId }: KBRetrieveProps) {
</CardHeader> </CardHeader>
<CardContent> <CardContent>
<p className="text-sm whitespace-pre-wrap"> <p className="text-sm whitespace-pre-wrap">
{result.metadata.text} {extractTextFromContent(result)}
</p> </p>
</CardContent> </CardContent>
</Card> </Card>

View File

@@ -0,0 +1,124 @@
'use client';
import React, { useState } from 'react';
import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card';
import { Button } from '@/components/ui/button';
import { Input } from '@/components/ui/input';
import { useTranslation } from 'react-i18next';
import { RetrieveResult } from '@/app/infra/entities/api';
import { toast } from 'sonner';
interface KBRetrieveGenericProps {
kbId: string;
retrieveFunction: (
kbId: string,
query: string,
) => Promise<{ results: RetrieveResult[] }>;
getResultTitle?: (result: RetrieveResult) => string;
}
/**
* Generic knowledge base retrieve component
* Supports both builtin and external knowledge bases
*/
export default function KBRetrieveGeneric({
kbId,
retrieveFunction,
getResultTitle,
}: KBRetrieveGenericProps) {
const { t } = useTranslation();
const [query, setQuery] = useState('');
const [results, setResults] = useState<RetrieveResult[]>([]);
const [loading, setLoading] = useState(false);
const handleRetrieve = async () => {
if (!query.trim()) return;
setLoading(true);
try {
setResults([]);
const response = await retrieveFunction(kbId, query);
setResults(response.results);
} catch (error) {
console.error('Retrieve failed:', error);
toast.error(t('knowledge.retrieveError'));
} finally {
setLoading(false);
}
};
const getTitle = (result: RetrieveResult): string => {
if (getResultTitle) {
return getResultTitle(result);
}
// Default: use file_id or document_name from metadata
return (
(result.metadata.file_id as string) ||
(result.metadata.document_name as string) ||
result.id
);
};
/**
* Extract text content from the content array
* The content array may contain multiple items with type 'text'
*/
const extractTextFromContent = (result: RetrieveResult): string => {
// First try to get content from the new format
if (result.content && Array.isArray(result.content)) {
const textParts = result.content
.filter((item) => item.type === 'text' && item.text)
.map((item) => item.text);
if (textParts.length > 0) {
return textParts.join('\n\n');
}
}
return '';
};
return (
<div className="space-y-4">
<div className="flex gap-2">
<Input
value={query}
onChange={(e) => setQuery(e.target.value)}
placeholder={t('knowledge.queryPlaceholder')}
onKeyPress={(e) => e.key === 'Enter' && handleRetrieve()}
/>
<Button onClick={handleRetrieve} disabled={loading || !query.trim()}>
{t('knowledge.query')}
</Button>
</div>
<div className="space-y-3">
{results.length === 0 && !loading && (
<p className="text-muted-foreground">{t('knowledge.noResults')}</p>
)}
{loading ? (
<p className="text-muted-foreground">{t('common.loading')}</p>
) : (
results.map((result) => (
<Card key={result.id} className="w-full">
<CardHeader className="pb-3">
<CardTitle className="text-sm font-medium flex justify-between items-center">
<span>{getTitle(result)}</span>
<span className="text-xs text-muted-foreground">
{t('knowledge.distance')}: {result.distance.toFixed(4)}
</span>
</CardTitle>
</CardHeader>
<CardContent>
<p className="text-sm whitespace-pre-wrap">
{extractTextFromContent(result)}
</p>
</CardContent>
</Card>
))
)}
</div>
</div>
);
}

View File

@@ -5,6 +5,7 @@
.knowledgeListContainer { .knowledgeListContainer {
width: 100%; width: 100%;
margin-top: 2rem;
padding-left: 0.8rem; padding-left: 0.8rem;
padding-right: 0.8rem; padding-right: 0.8rem;
display: grid; display: grid;

View File

@@ -5,23 +5,48 @@ import styles from './knowledgeBase.module.css';
import { useTranslation } from 'react-i18next'; import { useTranslation } from 'react-i18next';
import { useEffect, useState } from 'react'; import { useEffect, useState } from 'react';
import { KnowledgeBaseVO } from '@/app/home/knowledge/components/kb-card/KBCardVO'; import { KnowledgeBaseVO } from '@/app/home/knowledge/components/kb-card/KBCardVO';
import { ExternalKBCardVO } from '@/app/home/knowledge/components/external-kb-card/ExternalKBCardVO';
import KBCard from '@/app/home/knowledge/components/kb-card/KBCard'; import KBCard from '@/app/home/knowledge/components/kb-card/KBCard';
import ExternalKBCard from '@/app/home/knowledge/components/external-kb-card/ExternalKBCard';
import KBDetailDialog from '@/app/home/knowledge/KBDetailDialog'; import KBDetailDialog from '@/app/home/knowledge/KBDetailDialog';
import { httpClient } from '@/app/infra/http/HttpClient'; import { httpClient } from '@/app/infra/http/HttpClient';
import { KnowledgeBase } from '@/app/infra/entities/api'; import {
KnowledgeBase,
ExternalKnowledgeBase,
ApiRespPluginSystemStatus,
} from '@/app/infra/entities/api';
import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs';
export default function KnowledgePage() { export default function KnowledgePage() {
const { t } = useTranslation(); const { t } = useTranslation();
const [activeTab, setActiveTab] = useState('builtin');
const [knowledgeBaseList, setKnowledgeBaseList] = useState<KnowledgeBaseVO[]>( const [knowledgeBaseList, setKnowledgeBaseList] = useState<KnowledgeBaseVO[]>(
[], [],
); );
const [externalKBList, setExternalKBList] = useState<ExternalKBCardVO[]>([]);
const [selectedKbId, setSelectedKbId] = useState<string>(''); const [selectedKbId, setSelectedKbId] = useState<string>('');
const [selectedKbType, setSelectedKbType] = useState<'builtin' | 'external'>(
'builtin',
);
const [detailDialogOpen, setDetailDialogOpen] = useState(false); const [detailDialogOpen, setDetailDialogOpen] = useState(false);
const [pluginSystemStatus, setPluginSystemStatus] =
useState<ApiRespPluginSystemStatus | null>(null);
useEffect(() => { useEffect(() => {
getKnowledgeBaseList(); getKnowledgeBaseList();
getExternalKBList();
fetchPluginSystemStatus();
}, []); }, []);
async function fetchPluginSystemStatus() {
try {
const status = await httpClient.getPluginSystemStatus();
setPluginSystemStatus(status);
} catch (error) {
console.error('Failed to fetch plugin system status:', error);
}
}
async function getKnowledgeBaseList() { async function getKnowledgeBaseList() {
const resp = await httpClient.getKnowledgeBases(); const resp = await httpClient.getKnowledgeBases();
setKnowledgeBaseList( setKnowledgeBaseList(
@@ -53,13 +78,64 @@ export default function KnowledgePage() {
); );
} }
async function getExternalKBList() {
try {
const resp = await httpClient.getExternalKnowledgeBases();
setExternalKBList(
resp.bases.map((kb: ExternalKnowledgeBase) => {
const currentTime = new Date();
const lastUpdatedTimeAgo = Math.floor(
(currentTime.getTime() -
new Date(kb.created_at ?? currentTime.getTime()).getTime()) /
1000 /
60 /
60 /
24,
);
const lastUpdatedTimeAgoText =
lastUpdatedTimeAgo > 0
? ` ${lastUpdatedTimeAgo} ${t('knowledge.daysAgo')}`
: t('knowledge.today');
return new ExternalKBCardVO({
id: kb.uuid || '',
name: kb.name,
description: kb.description,
retrieverName: `${kb.plugin_author}/${kb.plugin_name}/${kb.retriever_name}`,
retrieverConfig: kb.retriever_config || {},
lastUpdatedTimeAgo: lastUpdatedTimeAgoText,
pluginAuthor: kb.plugin_author,
pluginName: kb.plugin_name,
});
}),
);
} catch (error) {
console.error('Failed to load external knowledge bases:', error);
}
}
const handleKBCardClick = (kbId: string) => { const handleKBCardClick = (kbId: string) => {
setSelectedKbId(kbId); setSelectedKbId(kbId);
setSelectedKbType('builtin');
setDetailDialogOpen(true); setDetailDialogOpen(true);
}; };
const handleCreateKBClick = () => { const handleCreateKBClick = () => {
setSelectedKbId(''); setSelectedKbId('');
setSelectedKbType('builtin');
setDetailDialogOpen(true);
};
const handleExternalKBCardClick = (kbId: string) => {
setSelectedKbId(kbId);
setSelectedKbType('external');
setDetailDialogOpen(true);
};
const handleCreateExternalKB = () => {
setSelectedKbId('');
setSelectedKbType('external');
setDetailDialogOpen(true); setDetailDialogOpen(true);
}; };
@@ -68,18 +144,30 @@ export default function KnowledgePage() {
}; };
const handleKbDeleted = () => { const handleKbDeleted = () => {
getKnowledgeBaseList(); if (selectedKbType === 'builtin') {
getKnowledgeBaseList();
} else {
getExternalKBList();
}
setDetailDialogOpen(false); setDetailDialogOpen(false);
}; };
const handleNewKbCreated = (newKbId: string) => { const handleNewKbCreated = (newKbId: string) => {
getKnowledgeBaseList(); if (selectedKbType === 'builtin') {
getKnowledgeBaseList();
} else {
getExternalKBList();
}
setSelectedKbId(newKbId); setSelectedKbId(newKbId);
setDetailDialogOpen(true); setDetailDialogOpen(true);
}; };
const handleKbUpdated = () => { const handleKbUpdated = () => {
getKnowledgeBaseList(); if (selectedKbType === 'builtin') {
getKnowledgeBaseList();
} else {
getExternalKBList();
}
}; };
return ( return (
@@ -88,28 +176,73 @@ export default function KnowledgePage() {
open={detailDialogOpen} open={detailDialogOpen}
onOpenChange={setDetailDialogOpen} onOpenChange={setDetailDialogOpen}
kbId={selectedKbId || undefined} kbId={selectedKbId || undefined}
kbType={selectedKbType}
onFormCancel={handleFormCancel} onFormCancel={handleFormCancel}
onKbDeleted={handleKbDeleted} onKbDeleted={handleKbDeleted}
onNewKbCreated={handleNewKbCreated} onNewKbCreated={handleNewKbCreated}
onKbUpdated={handleKbUpdated} onKbUpdated={handleKbUpdated}
/> />
<div className={styles.knowledgeListContainer}> <Tabs value={activeTab} onValueChange={setActiveTab} className="w-full">
<CreateCardComponent <div className="flex flex-row justify-between items-center px-[0.8rem]">
width={'100%'} <TabsList className="shadow-md py-5 bg-[#f0f0f0] dark:bg-[#2a2a2e]">
height={'10rem'} <TabsTrigger value="builtin" className="px-6 py-4 cursor-pointer">
plusSize={'90px'} {t('knowledge.builtIn')}
onClick={handleCreateKBClick} </TabsTrigger>
/> {/* Only show external tab if plugin system is enabled and connected */}
{pluginSystemStatus?.is_enable &&
pluginSystemStatus?.is_connected && (
<TabsTrigger
value="external"
className="px-6 py-4 cursor-pointer"
>
{t('knowledge.external')}
</TabsTrigger>
)}
</TabsList>
</div>
{knowledgeBaseList.map((kb) => { <TabsContent value="builtin">
return ( <div className={styles.knowledgeListContainer}>
<div key={kb.id} onClick={() => handleKBCardClick(kb.id)}> <CreateCardComponent
<KBCard kbCardVO={kb} /> width={'100%'}
</div> height={'10rem'}
); plusSize={'90px'}
})} onClick={handleCreateKBClick}
</div> />
{knowledgeBaseList.map((kb) => {
return (
<div key={kb.id} onClick={() => handleKBCardClick(kb.id)}>
<KBCard kbCardVO={kb} />
</div>
);
})}
</div>
</TabsContent>
<TabsContent value="external">
<div className={styles.knowledgeListContainer}>
<CreateCardComponent
width={'100%'}
height={'10rem'}
plusSize={'90px'}
onClick={handleCreateExternalKB}
/>
{externalKBList.map((kb) => {
return (
<div
key={kb.id}
onClick={() => handleExternalKBCardClick(kb.id)}
>
<ExternalKBCard kbCardVO={kb} />
</div>
);
})}
</div>
</TabsContent>
</Tabs>
</div> </div>
); );
} }

View File

@@ -1,5 +1,5 @@
import { TFunction } from 'i18next'; import { TFunction } from 'i18next';
import { Wrench, AudioWaveform, Hash } from 'lucide-react'; import { Wrench, AudioWaveform, Hash, Book } from 'lucide-react';
import { Badge } from '@/components/ui/badge'; import { Badge } from '@/components/ui/badge';
export default function PluginComponentList({ export default function PluginComponentList({
@@ -21,6 +21,7 @@ export default function PluginComponentList({
Tool: <Wrench className="w-5 h-5" />, Tool: <Wrench className="w-5 h-5" />,
EventListener: <AudioWaveform className="w-5 h-5" />, EventListener: <AudioWaveform className="w-5 h-5" />,
Command: <Hash className="w-5 h-5" />, Command: <Hash className="w-5 h-5" />,
KnowledgeRetriever: <Book className="w-5 h-5" />,
}; };
const componentKindList = Object.keys(components || {}); const componentKindList = Object.keys(components || {});

View File

@@ -162,6 +162,25 @@ export interface KnowledgeBase {
updated_at?: string; updated_at?: string;
} }
export interface ExternalKnowledgeBase {
uuid?: string;
name: string;
description: string;
created_at?: string;
plugin_author: string;
plugin_name: string;
retriever_name: string;
retriever_config?: Record<string, unknown>;
}
export interface ApiRespExternalKnowledgeBases {
bases: ExternalKnowledgeBase[];
}
export interface ApiRespExternalKnowledgeBase {
base: ExternalKnowledgeBase;
}
export interface ApiRespKnowledgeBaseFiles { export interface ApiRespKnowledgeBaseFiles {
files: KnowledgeBaseFile[]; files: KnowledgeBaseFile[];
} }
@@ -295,12 +314,22 @@ export interface ApiRespWebChatMessages {
messages: Message[]; messages: Message[];
} }
export interface RetrieveResultContent {
type: 'text' | 'image_url' | 'image_base64' | 'file_url';
text?: string;
file_name?: string;
file_url?: string;
image_url?: string;
image_base64?: string;
}
export interface RetrieveResult { export interface RetrieveResult {
id: string; id: string;
content?: RetrieveResultContent[];
metadata: { metadata: {
file_id: string; file_id?: string;
text: string; text?: string;
uuid: string; uuid?: string;
[key: string]: unknown; [key: string]: unknown;
}; };
distance: number; distance: number;

View File

@@ -36,6 +36,9 @@ import {
ApiRespMCPServers, ApiRespMCPServers,
ApiRespMCPServer, ApiRespMCPServer,
MCPServer, MCPServer,
ExternalKnowledgeBase,
ApiRespExternalKnowledgeBases,
ApiRespExternalKnowledgeBase,
} from '@/app/infra/entities/api'; } from '@/app/infra/entities/api';
import { Plugin } from '@/app/infra/entities/plugin'; import { Plugin } from '@/app/infra/entities/plugin';
import { GetBotLogsRequest } from '@/app/infra/http/requestParam/bots/GetBotLogsRequest'; import { GetBotLogsRequest } from '@/app/infra/http/requestParam/bots/GetBotLogsRequest';
@@ -445,6 +448,47 @@ export class BackendClient extends BaseHttpClient {
return this.post(`/api/v1/knowledge/bases/${uuid}/retrieve`, { query }); return this.post(`/api/v1/knowledge/bases/${uuid}/retrieve`, { query });
} }
// ============ External Knowledge Base API ============
public getExternalKnowledgeBases(): Promise<ApiRespExternalKnowledgeBases> {
return this.get('/api/v1/knowledge/external-bases');
}
public getExternalKnowledgeBase(
uuid: string,
): Promise<ApiRespExternalKnowledgeBase> {
return this.get(`/api/v1/knowledge/external-bases/${uuid}`);
}
public createExternalKnowledgeBase(
base: ExternalKnowledgeBase,
): Promise<{ uuid: string }> {
return this.post('/api/v1/knowledge/external-bases', base);
}
public updateExternalKnowledgeBase(
uuid: string,
base: ExternalKnowledgeBase,
): Promise<{ uuid: string }> {
return this.put(`/api/v1/knowledge/external-bases/${uuid}`, base);
}
public deleteExternalKnowledgeBase(uuid: string): Promise<object> {
return this.delete(`/api/v1/knowledge/external-bases/${uuid}`);
}
public retrieveExternalKnowledgeBase(
uuid: string,
query: string,
): Promise<ApiRespKnowledgeBaseRetrieve> {
return this.post(`/api/v1/knowledge/external-bases/${uuid}/retrieve`, {
query,
});
}
public listKnowledgeRetrievers(): Promise<{ retrievers: unknown[] }> {
return this.get('/api/v1/knowledge/external-bases/retrievers');
}
// ============ Plugins API ============ // ============ Plugins API ============
public getPlugins(): Promise<ApiRespPlugins> { public getPlugins(): Promise<ApiRespPlugins> {
return this.get('/api/v1/plugins'); return this.get('/api/v1/plugins');

View File

@@ -587,6 +587,15 @@ const enUS = {
fileName: 'File Name', fileName: 'File Name',
noResults: 'No results', noResults: 'No results',
retrieveError: 'Retrieve failed', retrieveError: 'Retrieve failed',
builtIn: 'Built-in',
external: 'External',
addExternal: 'Add External Knowledge Base',
createExternalSuccess: 'External knowledge base created successfully',
updateExternalSuccess: 'External knowledge base updated successfully',
deleteExternalSuccess: 'External knowledge base deleted successfully',
retriever: 'Retriever',
selectRetriever: 'Select a retriever...',
retrieverConfiguration: 'Retriever Configuration',
}, },
register: { register: {
title: 'Initialize LangBot 👋', title: 'Initialize LangBot 👋',

View File

@@ -591,6 +591,15 @@ const jaJP = {
fileName: 'ファイル名', fileName: 'ファイル名',
noResults: '検索結果がありません', noResults: '検索結果がありません',
retrieveError: '検索に失敗しました', retrieveError: '検索に失敗しました',
builtIn: '内蔵',
external: '外部ナレッジベース',
addExternal: '外部ナレッジベースを追加',
createExternalSuccess: '外部ナレッジベースが正常に作成されました',
updateExternalSuccess: '外部ナレッジベースが正常に更新されました',
deleteExternalSuccess: '外部ナレッジベースが正常に削除されました',
retriever: '検索器',
selectRetriever: '検索器を選択...',
retrieverConfiguration: '検索器設定',
}, },
register: { register: {
title: 'LangBot を初期化 👋', title: 'LangBot を初期化 👋',

View File

@@ -564,6 +564,15 @@ const zhHans = {
fileName: '文件名', fileName: '文件名',
noResults: '暂无结果', noResults: '暂无结果',
retrieveError: '检索失败', retrieveError: '检索失败',
builtIn: '内置',
external: '外部知识库',
addExternal: '添加外部知识库',
createExternalSuccess: '外部知识库创建成功',
updateExternalSuccess: '外部知识库更新成功',
deleteExternalSuccess: '外部知识库删除成功',
retriever: '检索器',
selectRetriever: '选择一个检索器...',
retrieverConfiguration: '检索器配置',
}, },
register: { register: {
title: '初始化 LangBot 👋', title: '初始化 LangBot 👋',

View File

@@ -561,6 +561,15 @@ const zhHant = {
fileName: '文檔名稱', fileName: '文檔名稱',
noResults: '暫無結果', noResults: '暫無結果',
retrieveError: '檢索失敗', retrieveError: '檢索失敗',
builtIn: '內置',
external: '外部知識庫',
addExternal: '添加外部知識庫',
createExternalSuccess: '外部知識庫創建成功',
updateExternalSuccess: '外部知識庫更新成功',
deleteExternalSuccess: '外部知識庫刪除成功',
retriever: '檢索器',
selectRetriever: '選擇一個檢索器...',
retrieverConfiguration: '檢索器配置',
}, },
register: { register: {
title: '初始化 LangBot 👋', title: '初始化 LangBot 👋',