Merge branch 'rc/new-plugin' into refactor/new-plugin-system

pkg/core/app.py

@@ -21,15 +21,18 @@ from ..api.http.service import user as user_service
 from ..api.http.service import model as model_service
 from ..api.http.service import pipeline as pipeline_service
 from ..api.http.service import bot as bot_service
+from ..api.http.service import knowledge as knowledge_service
 from ..discover import engine as discover_engine
 from ..storage import mgr as storagemgr
 from ..utils import logcache
 from . import taskmgr
 from . import entities as core_entities
+from ..rag.knowledge import kbmgr as rag_mgr
+from ..vector import mgr as vectordb_mgr
 
 
 class Application:
-    """运行时应用对象和上下文"""
+    """Runtime application object and context"""
 
     event_loop: asyncio.AbstractEventLoop = None
 
@@ -46,10 +49,12 @@ class Application:
 
     model_mgr: llm_model_mgr.ModelManager = None
 
-    # TODO 移动到 pipeline 里
+    rag_mgr: rag_mgr.RAGManager = None
+
+    # TODO move to pipeline
     tool_mgr: llm_tool_mgr.ToolManager = None
 
-    # ======= 配置管理器 =======
+    # ======= Config manager =======
 
     command_cfg: config_mgr.ConfigManager = None  # deprecated
 
@@ -63,7 +68,7 @@ class Application:
 
     instance_config: config_mgr.ConfigManager = None
 
-    # ======= 元数据配置管理器 =======
+    # ======= Metadata config manager =======
 
     sensitive_meta: config_mgr.ConfigManager = None
 
@@ -92,6 +97,8 @@ class Application:
 
     persistence_mgr: persistencemgr.PersistenceManager = None
 
+    vector_db_mgr: vectordb_mgr.VectorDBManager = None
+
     http_ctrl: http_controller.HTTPController = None
 
     log_cache: logcache.LogCache = None
@@ -102,12 +109,16 @@ class Application:
 
     user_service: user_service.UserService = None
 
-    model_service: model_service.ModelsService = None
+    llm_model_service: model_service.LLMModelsService = None
+
+    embedding_models_service: model_service.EmbeddingModelsService = None
 
     pipeline_service: pipeline_service.PipelineService = None
 
     bot_service: bot_service.BotService = None
 
+    knowledge_service: knowledge_service.KnowledgeService = None
+
     def __init__(self):
         pass
 
@@ -142,6 +153,7 @@ class Application:
                 name='http-api-controller',
                 scopes=[core_entities.LifecycleControlScope.APPLICATION],
             )
+
             self.task_mgr.create_task(
                 never_ending(),
                 name='never-ending-task',
@@ -153,14 +165,14 @@ class Application:
         except asyncio.CancelledError:
             pass
         except Exception as e:
-            self.logger.error(f'应用运行致命异常: {e}')
+            self.logger.error(f'Application runtime fatal exception: {e}')
             self.logger.debug(f'Traceback: {traceback.format_exc()}')
 
     def dispose(self):
         self.plugin_connector.dispose()
 
     async def print_web_access_info(self):
-        """打印访问 webui 的提示"""
+        """Print access webui tips"""
 
         if not os.path.exists(os.path.join('.', 'web/out')):
             self.logger.warning('WebUI 文件缺失，请根据文档部署：https://docs.langbot.app/zh')

pkg/core/boot.py

@@ -8,7 +8,7 @@ from . import app
 from . import stage
 from ..utils import constants, importutil
 
-# 引入启动阶段实现以便注册
+# Import startup stage implementation to register
 from . import stages
 
 importutil.import_modules_in_pkg(stages)
@@ -25,7 +25,7 @@ stage_order = [
 
 
 async def make_app(loop: asyncio.AbstractEventLoop) -> app.Application:
-    # 确定是否为调试模式
+    # Determine if it is debug mode
     if 'DEBUG' in os.environ and os.environ['DEBUG'] in ['true', '1']:
         constants.debug_mode = True
 
@@ -33,7 +33,7 @@ async def make_app(loop: asyncio.AbstractEventLoop) -> app.Application:
 
     ap.event_loop = loop
 
-    # 执行启动阶段
+    # Execute startup stage
     for stage_name in stage_order:
         stage_cls = stage.preregistered_stages[stage_name]
         stage_inst = stage_cls()
@@ -47,12 +47,12 @@ async def make_app(loop: asyncio.AbstractEventLoop) -> app.Application:
 
 async def main(loop: asyncio.AbstractEventLoop):
     try:
-        # 挂系统信号处理
+        # Hang system signal processing
         import signal
 
         def signal_handler(sig, frame):
             app_inst.dispose()
-            print('[Signal] 程序退出.')
+            print('[Signal] Program exit.')
             os._exit(0)
 
         signal.signal(signal.SIGINT, signal_handler)

pkg/core/bootutils/deps.py

@@ -2,8 +2,8 @@ import pip
 import os
 from ...utils import pkgmgr
 
-# 检查依赖，防止用户未安装
-# 左边为引入名称，右边为依赖名称
+# Check dependencies to prevent users from not installing
+# Left is the import name, right is the dependency name
 required_deps = {
     'requests': 'requests',
     'openai': 'openai',
@@ -65,7 +65,7 @@ async def install_deps(deps: list[str]):
 async def precheck_plugin_deps():
     print('[Startup] Prechecking plugin dependencies...')
 
-    # 只有在plugins目录存在时才执行插件依赖安装
+    # Only execute plugin dependency installation when the plugins directory exists
     if os.path.exists('plugins'):
         for dir in os.listdir('plugins'):
             subdir = os.path.join('plugins', dir)

pkg/core/bootutils/log.py

@@ -17,7 +17,7 @@ log_colors_config = {
 
 
 async def init_logging(extra_handlers: list[logging.Handler] = None) -> logging.Logger:
-    # 删除所有现有的logger
+    # Remove all existing loggers
     for handler in logging.root.handlers[:]:
         logging.root.removeHandler(handler)
 
@@ -54,13 +54,13 @@ async def init_logging(extra_handlers: list[logging.Handler] = None) -> logging.
         handler.setFormatter(color_formatter)
         qcg_logger.addHandler(handler)
 
-    qcg_logger.debug('日志初始化完成，日志级别：%s' % level)
+    qcg_logger.debug('Logging initialized, log level: %s' % level)
     logging.basicConfig(
-        level=logging.CRITICAL,  # 设置日志输出格式
+        level=logging.CRITICAL,  # Set log output format
         format='[DEPR][%(asctime)s.%(msecs)03d] %(pathname)s (%(lineno)d) - [%(levelname)s] :\n%(message)s',
-        # 日志输出的格式
-        # -8表示占位符，让输出左对齐，输出长度都为8位
-        datefmt='%Y-%m-%d %H:%M:%S',  # 时间输出的格式
+        # Log output format
+        # -8 is a placeholder, left-align the output, and output length is 8
+        datefmt='%Y-%m-%d %H:%M:%S',  # Time output format
         handlers=[logging.NullHandler()],
     )
 

pkg/core/migration.py

@@ -7,11 +7,11 @@ from . import app
 
 
 preregistered_migrations: list[typing.Type[Migration]] = []
-"""当前阶段暂不支持扩展"""
+"""Currently not supported for extension"""
 
 
 def migration_class(name: str, number: int):
-    """注册一个迁移"""
+    """Register a migration"""
 
     def decorator(cls: typing.Type[Migration]) -> typing.Type[Migration]:
         cls.name = name
@@ -23,7 +23,7 @@ def migration_class(name: str, number: int):
 
 
 class Migration(abc.ABC):
-    """一个版本的迁移"""
+    """A version migration"""
 
     name: str
 
@@ -36,10 +36,10 @@ class Migration(abc.ABC):
 
     @abc.abstractmethod
     async def need_migrate(self) -> bool:
-        """判断当前环境是否需要运行此迁移"""
+        """Determine if the current environment needs to run this migration"""
         pass
 
     @abc.abstractmethod
     async def run(self):
-        """执行迁移"""
+        """Run migration"""
         pass

pkg/core/note.py

@@ -9,7 +9,7 @@ preregistered_notes: list[typing.Type[LaunchNote]] = []
 
 
 def note_class(name: str, number: int):
-    """注册一个启动信息"""
+    """Register a launch information"""
 
     def decorator(cls: typing.Type[LaunchNote]) -> typing.Type[LaunchNote]:
         cls.name = name
@@ -21,7 +21,7 @@ def note_class(name: str, number: int):
 
 
 class LaunchNote(abc.ABC):
-    """启动信息"""
+    """Launch information"""
 
     name: str
 
@@ -34,10 +34,10 @@ class LaunchNote(abc.ABC):
 
     @abc.abstractmethod
     async def need_show(self) -> bool:
-        """判断当前环境是否需要显示此启动信息"""
+        """Determine if the current environment needs to display this launch information"""
         pass
 
     @abc.abstractmethod
     async def yield_note(self) -> typing.AsyncGenerator[typing.Tuple[str, int], None]:
-        """生成启动信息"""
+        """Generate launch information"""
         pass

pkg/core/notes/n001_classic_msgs.py

@@ -7,7 +7,7 @@ from .. import note
 
 @note.note_class('ClassicNotes', 1)
 class ClassicNotes(note.LaunchNote):
-    """经典启动信息"""
+    """Classic launch information"""
 
     async def need_show(self) -> bool:
         return True

pkg/core/notes/n002_selection_mode_on_windows.py

@@ -9,7 +9,7 @@ from .. import note
 
 @note.note_class('SelectionModeOnWindows', 2)
 class SelectionModeOnWindows(note.LaunchNote):
-    """Windows 上的选择模式提示信息"""
+    """Selection mode prompt information on Windows"""
 
     async def need_show(self) -> bool:
         return os.name == 'nt'
@@ -19,3 +19,8 @@ class SelectionModeOnWindows(note.LaunchNote):
             """您正在使用 Windows 系统，若窗口左上角显示处于”选择“模式，程序将被暂停运行，此时请右键窗口中空白区域退出选择模式。""",
             logging.INFO,
         )
+
+        yield (
+            """You are using Windows system, if the top left corner of the window displays "Selection" mode, the program will be paused running, please right-click on the blank area in the window to exit the selection mode.""",
+            logging.INFO,
+        )

pkg/core/stage.py

@@ -7,9 +7,9 @@ from . import app
 
 
 preregistered_stages: dict[str, typing.Type[BootingStage]] = {}
-"""预注册的请求处理阶段。在初始化时，所有请求处理阶段类会被注册到此字典中。
+"""Pre-registered request processing stages. All request processing stage classes are registered in this dictionary during initialization.
 
-当前阶段暂不支持扩展
+Currently not supported for extension
 """
 
 
@@ -22,11 +22,11 @@ def stage_class(name: str):
 
 
 class BootingStage(abc.ABC):
-    """启动阶段"""
+    """Booting stage"""
 
     name: str = None
 
     @abc.abstractmethod
     async def run(self, ap: app.Application):
-        """启动"""
+        """Run"""
         pass

pkg/core/stages/build_app.py

@@ -10,6 +10,7 @@ from ...command import cmdmgr
 from ...provider.session import sessionmgr as llm_session_mgr
 from ...provider.modelmgr import modelmgr as llm_model_mgr
 from ...provider.tools import toolmgr as llm_tool_mgr
+from ...rag.knowledge import kbmgr as rag_mgr
 from ...platform import botmgr as im_mgr
 from ...persistence import mgr as persistencemgr
 from ...api.http.controller import main as http_controller
@@ -17,18 +18,20 @@ from ...api.http.service import user as user_service
 from ...api.http.service import model as model_service
 from ...api.http.service import pipeline as pipeline_service
 from ...api.http.service import bot as bot_service
+from ...api.http.service import knowledge as knowledge_service
 from ...discover import engine as discover_engine
 from ...storage import mgr as storagemgr
 from ...utils import logcache
+from ...vector import mgr as vectordb_mgr
 from .. import taskmgr
 
 
 @stage.stage_class('BuildAppStage')
 class BuildAppStage(stage.BootingStage):
-    """构建应用阶段"""
+    """Build LangBot application"""
 
     async def run(self, ap: app.Application):
-        """构建app对象的各个组件对象并初始化"""
+        """Build LangBot application"""
         ap.task_mgr = taskmgr.AsyncTaskManager(ap)
 
         discover = discover_engine.ComponentDiscoveryEngine(ap)
@@ -43,7 +46,7 @@ class BuildAppStage(stage.BootingStage):
         await ver_mgr.initialize()
         ap.ver_mgr = ver_mgr
 
-        # 发送公告
+        # Send announcement
         ann_mgr = announce.AnnouncementManager(ap)
         ap.ann_mgr = ann_mgr
 
@@ -92,6 +95,15 @@ class BuildAppStage(stage.BootingStage):
         await pipeline_mgr.initialize()
         ap.pipeline_mgr = pipeline_mgr
 
+        rag_mgr_inst = rag_mgr.RAGManager(ap)
+        await rag_mgr_inst.initialize()
+        ap.rag_mgr = rag_mgr_inst
+
+        # 初始化向量数据库管理器
+        vectordb_mgr_inst = vectordb_mgr.VectorDBManager(ap)
+        await vectordb_mgr_inst.initialize()
+        ap.vector_db_mgr = vectordb_mgr_inst
+
         http_ctrl = http_controller.HTTPController(ap)
         await http_ctrl.initialize()
         ap.http_ctrl = http_ctrl
@@ -99,8 +111,11 @@ class BuildAppStage(stage.BootingStage):
         user_service_inst = user_service.UserService(ap)
         ap.user_service = user_service_inst
 
-        model_service_inst = model_service.ModelsService(ap)
-        ap.model_service = model_service_inst
+        llm_model_service_inst = model_service.LLMModelsService(ap)
+        ap.llm_model_service = llm_model_service_inst
+
+        embedding_models_service_inst = model_service.EmbeddingModelsService(ap)
+        ap.embedding_models_service = embedding_models_service_inst
 
         pipeline_service_inst = pipeline_service.PipelineService(ap)
         ap.pipeline_service = pipeline_service_inst
@@ -108,5 +123,8 @@ class BuildAppStage(stage.BootingStage):
         bot_service_inst = bot_service.BotService(ap)
         ap.bot_service = bot_service_inst
 
+        knowledge_service_inst = knowledge_service.KnowledgeService(ap)
+        ap.knowledge_service = knowledge_service_inst
+
         ctrl = controller.Controller(ap)
         ap.ctrl = ctrl

pkg/core/stages/genkeys.py

@@ -7,11 +7,18 @@ from .. import stage, app
 
 @stage.stage_class('GenKeysStage')
 class GenKeysStage(stage.BootingStage):
-    """生成密钥阶段"""
+    """Generate keys stage"""
 
     async def run(self, ap: app.Application):
-        """启动"""
+        """Generate keys"""
 
         if not ap.instance_config.data['system']['jwt']['secret']:
             ap.instance_config.data['system']['jwt']['secret'] = secrets.token_hex(16)
             await ap.instance_config.dump_config()
+
+        if 'recovery_key' not in ap.instance_config.data['system']:
+            ap.instance_config.data['system']['recovery_key'] = ''
+
+        if not ap.instance_config.data['system']['recovery_key']:
+            ap.instance_config.data['system']['recovery_key'] = secrets.token_hex(3).upper()
+            await ap.instance_config.dump_config()

pkg/core/stages/load_config.py

@@ -8,10 +8,10 @@ from ..bootutils import config
 
 @stage.stage_class('LoadConfigStage')
 class LoadConfigStage(stage.BootingStage):
-    """加载配置文件阶段"""
+    """Load config file stage"""
 
     async def run(self, ap: app.Application):
-        """启动"""
+        """Load config file"""
 
         # ======= deprecated =======
         if os.path.exists('data/config/command.json'):

pkg/core/stages/migrate.py

@@ -11,10 +11,13 @@ importutil.import_modules_in_pkg(migrations)
 
 @stage.stage_class('MigrationStage')
 class MigrationStage(stage.BootingStage):
-    """迁移阶段"""
+    """Migration stage
+
+    These migrations are legacy, only performed in version 3.x
+    """
 
     async def run(self, ap: app.Application):
-        """启动"""
+        """Run migration"""
 
         if any(
             [
@@ -29,7 +32,7 @@ class MigrationStage(stage.BootingStage):
 
         migrations = migration.preregistered_migrations
 
-        # 按照迁移号排序
+        # Sort by migration number
         migrations.sort(key=lambda x: x.number)
 
         for migration_cls in migrations:
@@ -37,4 +40,4 @@ class MigrationStage(stage.BootingStage):
 
             if await migration_instance.need_migrate():
                 await migration_instance.run()
-                print(f'已执行迁移 {migration_instance.name}')
+                print(f'Migration {migration_instance.name} executed')

pkg/core/stages/setup_logger.py

@@ -8,7 +8,7 @@ from ..bootutils import log
 
 class PersistenceHandler(logging.Handler, object):
     """
-    保存日志到数据库
+    Save logs to database
     """
 
     ap: app.Application
@@ -19,9 +19,9 @@ class PersistenceHandler(logging.Handler, object):
 
     def emit(self, record):
         """
-        emit函数为自定义handler类时必重写的函数，这里可以根据需要对日志消息做一些处理，比如发送日志到服务器
+        emit function is a required function for custom handler classes, here you can process the log messages as needed, such as sending logs to the server
 
-        发出记录(Emit a record)
+        Emit a record
         """
         try:
             msg = self.format(record)
@@ -34,10 +34,10 @@ class PersistenceHandler(logging.Handler, object):
 
 @stage.stage_class('SetupLoggerStage')
 class SetupLoggerStage(stage.BootingStage):
-    """设置日志器阶段"""
+    """Setup logger stage"""
 
     async def run(self, ap: app.Application):
-        """启动"""
+        """Setup logger"""
         persistence_handler = PersistenceHandler('LoggerHandler', ap)
 
         extra_handlers = []

pkg/core/stages/show_notes.py

@@ -12,10 +12,10 @@ importutil.import_modules_in_pkg(notes)
 
 @stage.stage_class('ShowNotesStage')
 class ShowNotesStage(stage.BootingStage):
-    """显示启动信息阶段"""
+    """Show notes stage"""
 
     async def run(self, ap: app.Application):
-        # 排序
+        # Sort
         note.preregistered_notes.sort(key=lambda x: x.number)
 
         for note_cls in note.preregistered_notes:

pkg/core/taskmgr.py

@@ -9,13 +9,13 @@ from . import entities as core_entities
 
 
 class TaskContext:
-    """任务跟踪上下文"""
+    """Task tracking context"""
 
     current_action: str
-    """当前正在执行的动作"""
+    """Current action being executed"""
 
     log: str
-    """记录日志"""
+    """Log"""
 
     def __init__(self):
         self.current_action = 'default'
@@ -58,40 +58,40 @@ placeholder_context: TaskContext | None = None
 
 
 class TaskWrapper:
-    """任务包装器"""
+    """Task wrapper"""
 
     _id_index: int = 0
-    """任务ID索引"""
+    """Task ID index"""
 
     id: int
-    """任务ID"""
+    """Task ID"""
 
-    task_type: str = 'system'  # 任务类型: system 或 user
-    """任务类型"""
+    task_type: str = 'system'  # Task type: system or user
+    """Task type"""
 
-    kind: str = 'system_task'  # 由发起者确定任务种类，通常同质化的任务种类相同
-    """任务种类"""
+    kind: str = 'system_task'  # Task type determined by the initiator, usually the same task type
+    """Task type"""
 
     name: str = ''
-    """任务唯一名称"""
+    """Task unique name"""
 
     label: str = ''
-    """任务显示名称"""
+    """Task display name"""
 
     task_context: TaskContext
-    """任务上下文"""
+    """Task context"""
 
     task: asyncio.Task
-    """任务"""
+    """Task"""
 
     task_stack: list = None
-    """任务堆栈"""
+    """Task stack"""
 
     ap: app.Application
-    """应用实例"""
+    """Application instance"""
 
     scopes: list[core_entities.LifecycleControlScope]
-    """任务所属生命周期控制范围"""
+    """Task scope"""
 
     def __init__(
         self,
@@ -165,13 +165,13 @@ class TaskWrapper:
 
 
 class AsyncTaskManager:
-    """保存app中的所有异步任务
-    包含系统级的和用户级（插件安装、更新等由用户直接发起的）的"""
+    """Save all asynchronous tasks in the app
+    Include system-level and user-level (plugin installation, update, etc. initiated by users directly)"""
 
     ap: app.Application
 
     tasks: list[TaskWrapper]
-    """所有任务"""
+    """All tasks"""
 
     def __init__(self, ap: app.Application):
         self.ap = ap