feat: 持久化和 web 接口基础架构

pkg/core/app.py

@@ -17,11 +17,18 @@ from ..plugin import manager as plugin_mgr
 from ..pipeline import pool
 from ..pipeline import controller, stagemgr
 from ..utils import version as version_mgr, proxy as proxy_mgr, announce as announce_mgr
+from ..persistence import mgr as persistencemgr
+from ..api.http.controller import main as http_controller
+from ..utils import logcache
 
 
 class Application:
     """运行时应用对象和上下文"""
 
+    event_loop: asyncio.AbstractEventLoop = None
+
+    asyncio_tasks: list[asyncio.Task] = []
+
     platform_mgr: im_mgr.PlatformManager = None
 
     cmd_mgr: cmdmgr.CommandManager = None
@@ -78,6 +85,12 @@ class Application:
 
     logger: logging.Logger = None
 
+    persistence_mgr: persistencemgr.PersistenceManager = None
+
+    http_ctrl: http_controller.HTTPController = None
+
+    log_cache: logcache.LogCache = None
+
     def __init__(self):
         pass
 
@@ -91,13 +104,21 @@ class Application:
 
         try:
    
+            # 后续可能会允许动态重启其他任务
+            # 故为了防止程序在非 Ctrl-C 情况下退出，这里创建一个不会结束的协程
+            async def never_ending():
+                while True:
+                    await asyncio.sleep(1)
+
             tasks = [
-                asyncio.create_task(self.platform_mgr.run()),
-                asyncio.create_task(self.ctrl.run())
+                asyncio.create_task(self.platform_mgr.run()),  # 消息平台
+                asyncio.create_task(self.ctrl.run()),  # 消息处理循环
+                asyncio.create_task(self.http_ctrl.run()),  # http 接口服务
+                asyncio.create_task(never_ending())
             ]
+            self.asyncio_tasks.extend(tasks)
 
-            # 挂信号处理
-
+            # 挂系统信号处理
             import signal
 
             def signal_handler(sig, frame):
@@ -109,7 +130,6 @@ class Application:
             signal.signal(signal.SIGINT, signal_handler)
 
             await asyncio.gather(*tasks, return_exceptions=True)
-
         except asyncio.CancelledError:
             pass
         except Exception as e:

pkg/core/boot.py

@@ -1,6 +1,7 @@
 from __future__ import print_function
 
 import traceback
+import asyncio
 
 from . import app
 from ..audit import identifier
@@ -19,13 +20,15 @@ stage_order = [
 ]
 
 
-async def make_app() -> app.Application:
+async def make_app(loop: asyncio.AbstractEventLoop) -> app.Application:
 
     # 生成标识符
     identifier.init()
 
     ap = app.Application()
 
+    ap.event_loop = loop
+
     # 执行启动阶段
     for stage_name in stage_order:
         stage_cls = stage.preregistered_stages[stage_name]
@@ -38,9 +41,9 @@ async def make_app() -> app.Application:
     return ap
 
 
-async def main():
+async def main(loop: asyncio.AbstractEventLoop):
     try:
-        app_inst = await make_app()
+        app_inst = await make_app(loop)
         await app_inst.run()
     except Exception as e:
         traceback.print_exc()

pkg/core/bootutils/deps.py

@@ -15,6 +15,9 @@ required_deps = {
     "psutil": "psutil",
     "async_lru": "async-lru",
     "ollama": "ollama",
+    "quart": "quart",
+    "sqlalchemy": "sqlalchemy[asyncio]",
+    "aiosqlite": "aiosqlite",
 }
 
 

pkg/core/bootutils/log.py

@@ -15,7 +15,7 @@ log_colors_config = {
 }
 
 
-async def init_logging() -> logging.Logger:
+async def init_logging(extra_handlers: list[logging.Handler] = None) -> logging.Logger:
     # 删除所有现有的logger
     for handler in logging.root.handlers[:]:
         logging.root.removeHandler(handler)
@@ -41,7 +41,8 @@ async def init_logging() -> logging.Logger:
 
     stream_handler = logging.StreamHandler(sys.stdout)
 
-    log_handlers: logging.Handler = [stream_handler, logging.FileHandler(log_file_name)]
+    log_handlers: list[logging.Handler] = [stream_handler, logging.FileHandler(log_file_name)]
+    log_handlers += extra_handlers if extra_handlers is not None else []
 
     for handler in log_handlers:
         handler.setLevel(level)

pkg/core/migrations/m013_http_api_config.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from .. import migration
+
+
+@migration.migration_class("http-api-config", 13)
+class HttpApiConfigMigration(migration.Migration):
+    """迁移"""
+
+    async def need_migrate(self) -> bool:
+        """判断当前环境是否需要运行此迁移"""
+        return 'http-api' not in self.ap.system_cfg.data or "persistence" not in self.ap.system_cfg.data
+
+    async def run(self):
+        """执行迁移"""
+        
+        self.ap.system_cfg.data['http-api'] = {
+            "enable": True,
+            "host": "0.0.0.0",
+            "port": 5300
+        }
+
+        self.ap.system_cfg.data['persistence'] = {
+            "sqlite": {
+                "path": "data/persistence.db"
+            },
+            "use": "sqlite"
+        }
+
+        await self.ap.system_cfg.dump_config()

pkg/core/stages/build_app.py

@@ -15,6 +15,10 @@ from ...provider.sysprompt import sysprompt as llm_prompt_mgr
 from ...provider.tools import toolmgr as llm_tool_mgr
 from ...provider import runnermgr
 from ...platform import manager as im_mgr
+from ...persistence import mgr as persistencemgr
+from ...api.http.controller import main as http_controller
+from ...utils import logcache
+
 
 @stage.stage_class("BuildAppStage")
 class BuildAppStage(stage.BootingStage):
@@ -58,6 +62,13 @@ class BuildAppStage(stage.BootingStage):
 
         ap.query_pool = pool.QueryPool()
 
+        log_cache = logcache.LogCache()
+        ap.log_cache = log_cache
+
+        persistence_mgr_inst = persistencemgr.PersistenceManager(ap)
+        await persistence_mgr_inst.initialize()
+        ap.persistence_mgr = persistence_mgr_inst
+
         plugin_mgr_inst = plugin_mgr.PluginManager(ap)
         await plugin_mgr_inst.initialize()
         ap.plugin_mgr = plugin_mgr_inst
@@ -95,6 +106,9 @@ class BuildAppStage(stage.BootingStage):
         await stage_mgr.initialize()
         ap.stage_mgr = stage_mgr
 
+        http_ctrl = http_controller.HTTPController(ap)
+        await http_ctrl.initialize()
+        ap.http_ctrl = http_ctrl
 
         ctrl = controller.Controller(ap)
         ap.ctrl = ctrl

pkg/core/stages/migrate.py

@@ -6,7 +6,7 @@ from .. import stage, app
 from .. import migration
 from ..migrations import m001_sensitive_word_migration, m002_openai_config_migration, m003_anthropic_requester_cfg_completion, m004_moonshot_cfg_completion
 from ..migrations import m005_deepseek_cfg_completion, m006_vision_config, m007_qcg_center_url, m008_ad_fixwin_config_migrate, m009_msg_truncator_cfg
-from ..migrations import m010_ollama_requester_config, m011_command_prefix_config, m012_runner_config
+from ..migrations import m010_ollama_requester_config, m011_command_prefix_config, m012_runner_config, m013_http_api_config
 
 
 @stage.stage_class("MigrationStage")

pkg/core/stages/setup_logger.py

@@ -1,9 +1,38 @@
 from __future__ import annotations
 
+import logging
+import asyncio
+from datetime import datetime
+
 from .. import stage, app
 from ..bootutils import log
 
 
+class PersistenceHandler(logging.Handler, object):
+    """
+    保存日志到数据库
+    """
+    ap: app.Application
+
+    def __init__(self, name, ap: app.Application):
+        logging.Handler.__init__(self)
+        self.ap = ap
+
+    def emit(self, record):
+        """
+        emit函数为自定义handler类时必重写的函数，这里可以根据需要对日志消息做一些处理，比如发送日志到服务器
+
+        发出记录(Emit a record)
+        """
+        try:
+            msg = self.format(record)
+            if self.ap.log_cache is not None:
+                self.ap.log_cache.add_log(msg)
+                
+        except Exception:
+            self.handleError(record)
+
+
 @stage.stage_class("SetupLoggerStage")
 class SetupLoggerStage(stage.BootingStage):
     """设置日志器阶段
@@ -12,4 +41,9 @@ class SetupLoggerStage(stage.BootingStage):
     async def run(self, ap: app.Application):
         """启动
         """
-        ap.logger = await log.init_logging()
+        persistence_handler = PersistenceHandler('LoggerHandler', ap)
+
+        extra_handlers = []
+        extra_handlers = [persistence_handler]
+
+        ap.logger = await log.init_logging(extra_handlers)