Merge branch 'rc/new-plugin' into refactor/new-plugin-system

pkg/api/http/service/bot.py

@@ -10,7 +10,7 @@ from ....entity.persistence import pipeline as persistence_pipeline
 
 
 class BotService:
-    """机器人服务"""
+    """Bot service"""
 
     ap: app.Application
 
@@ -63,7 +63,7 @@ class BotService:
         return persistence_bot
 
     async def create_bot(self, bot_data: dict) -> str:
-        """创建机器人"""
+        """Create bot"""
         # TODO: 检查配置信息格式
         bot_data['uuid'] = str(uuid.uuid4())
 
@@ -87,7 +87,7 @@ class BotService:
         return bot_data['uuid']
 
     async def update_bot(self, bot_uuid: str, bot_data: dict) -> None:
-        """更新机器人"""
+        """Update bot"""
         if 'uuid' in bot_data:
             del bot_data['uuid']
 
@@ -123,7 +123,7 @@ class BotService:
                 session.using_conversation = None
 
     async def delete_bot(self, bot_uuid: str) -> None:
-        """删除机器人"""
+        """Delete bot"""
         await self.ap.platform_mgr.remove_bot(bot_uuid)
         await self.ap.persistence_mgr.execute_async(
             sqlalchemy.delete(persistence_bot.Bot).where(persistence_bot.Bot.uuid == bot_uuid)

pkg/api/http/service/knowledge.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+import uuid
+import sqlalchemy
+
+from ....core import app
+from ....entity.persistence import rag as persistence_rag
+
+
+class KnowledgeService:
+    """知识库服务"""
+
+    ap: app.Application
+
+    def __init__(self, ap: app.Application) -> None:
+        self.ap = ap
+
+    async def get_knowledge_bases(self) -> list[dict]:
+        """获取所有知识库"""
+        result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_rag.KnowledgeBase))
+        knowledge_bases = result.all()
+        return [
+            self.ap.persistence_mgr.serialize_model(persistence_rag.KnowledgeBase, knowledge_base)
+            for knowledge_base in knowledge_bases
+        ]
+
+    async def get_knowledge_base(self, kb_uuid: str) -> dict | None:
+        """获取知识库"""
+        result = await self.ap.persistence_mgr.execute_async(
+            sqlalchemy.select(persistence_rag.KnowledgeBase).where(persistence_rag.KnowledgeBase.uuid == kb_uuid)
+        )
+        knowledge_base = result.first()
+        if knowledge_base is None:
+            return None
+        return self.ap.persistence_mgr.serialize_model(persistence_rag.KnowledgeBase, knowledge_base)
+
+    async def create_knowledge_base(self, kb_data: dict) -> str:
+        """创建知识库"""
+        kb_data['uuid'] = str(uuid.uuid4())
+        await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(persistence_rag.KnowledgeBase).values(kb_data))
+
+        kb = await self.get_knowledge_base(kb_data['uuid'])
+
+        await self.ap.rag_mgr.load_knowledge_base(kb)
+
+        return kb_data['uuid']
+
+    async def update_knowledge_base(self, kb_uuid: str, kb_data: dict) -> None:
+        """更新知识库"""
+        if 'uuid' in kb_data:
+            del kb_data['uuid']
+
+        if 'embedding_model_uuid' in kb_data:
+            del kb_data['embedding_model_uuid']
+
+        await self.ap.persistence_mgr.execute_async(
+            sqlalchemy.update(persistence_rag.KnowledgeBase)
+            .values(kb_data)
+            .where(persistence_rag.KnowledgeBase.uuid == kb_uuid)
+        )
+        await self.ap.rag_mgr.remove_knowledge_base_from_runtime(kb_uuid)
+
+        kb = await self.get_knowledge_base(kb_uuid)
+
+        await self.ap.rag_mgr.load_knowledge_base(kb)
+
+    async def store_file(self, kb_uuid: str, file_id: str) -> int:
+        """存储文件"""
+        # await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(persistence_rag.File).values(kb_id=kb_uuid, file_id=file_id))
+        # await self.ap.rag_mgr.store_file(file_id)
+        runtime_kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(kb_uuid)
+        if runtime_kb is None:
+            raise Exception('Knowledge base not found')
+        return await runtime_kb.store_file(file_id)
+
+    async def retrieve_knowledge_base(self, kb_uuid: str, query: str) -> list[dict]:
+        """检索知识库"""
+        runtime_kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(kb_uuid)
+        if runtime_kb is None:
+            raise Exception('Knowledge base not found')
+        return [
+            result.model_dump() for result in await runtime_kb.retrieve(query, runtime_kb.knowledge_base_entity.top_k)
+        ]
+
+    async def get_files_by_knowledge_base(self, kb_uuid: str) -> list[dict]:
+        """获取知识库文件"""
+        result = await self.ap.persistence_mgr.execute_async(
+            sqlalchemy.select(persistence_rag.File).where(persistence_rag.File.kb_id == kb_uuid)
+        )
+        files = result.all()
+        return [self.ap.persistence_mgr.serialize_model(persistence_rag.File, file) for file in files]
+
+    async def delete_file(self, kb_uuid: str, file_id: str) -> None:
+        """删除文件"""
+        runtime_kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(kb_uuid)
+        if runtime_kb is None:
+            raise Exception('Knowledge base not found')
+        await runtime_kb.delete_file(file_id)
+
+    async def delete_knowledge_base(self, kb_uuid: str) -> None:
+        """删除知识库"""
+        await self.ap.rag_mgr.delete_knowledge_base(kb_uuid)
+
+        await self.ap.persistence_mgr.execute_async(
+            sqlalchemy.delete(persistence_rag.KnowledgeBase).where(persistence_rag.KnowledgeBase.uuid == kb_uuid)
+        )
+
+        # delete files
+        files = await self.ap.persistence_mgr.execute_async(
+            sqlalchemy.select(persistence_rag.File).where(persistence_rag.File.kb_id == kb_uuid)
+        )
+        for file in files:
+            # delete chunks
+            await self.ap.persistence_mgr.execute_async(
+                sqlalchemy.delete(persistence_rag.Chunk).where(persistence_rag.Chunk.file_id == file.uuid)
+            )
+            # delete file
+            await self.ap.persistence_mgr.execute_async(
+                sqlalchemy.delete(persistence_rag.File).where(persistence_rag.File.uuid == file.uuid)
+            )

pkg/api/http/service/model.py

@@ -10,7 +10,7 @@ from ....provider.modelmgr import requester as model_requester
 from langbot_plugin.api.entities.builtin.provider import message as provider_message
 
 
-class ModelsService:
+class LLMModelsService:
     ap: app.Application
 
     def __init__(self, ap: app.Application) -> None:
@@ -109,5 +109,91 @@ class ModelsService:
             model=runtime_llm_model,
             messages=[provider_message.Message(role='user', content='Hello, world!')],
             funcs=[],
+            extra_args=model_data.get('extra_args', {}),
+        )
+
+
+class EmbeddingModelsService:
+    ap: app.Application
+
+    def __init__(self, ap: app.Application) -> None:
+        self.ap = ap
+
+    async def get_embedding_models(self) -> list[dict]:
+        result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_model.EmbeddingModel))
+
+        models = result.all()
+        return [self.ap.persistence_mgr.serialize_model(persistence_model.EmbeddingModel, model) for model in models]
+
+    async def create_embedding_model(self, model_data: dict) -> str:
+        model_data['uuid'] = str(uuid.uuid4())
+
+        await self.ap.persistence_mgr.execute_async(
+            sqlalchemy.insert(persistence_model.EmbeddingModel).values(**model_data)
+        )
+
+        embedding_model = await self.get_embedding_model(model_data['uuid'])
+
+        await self.ap.model_mgr.load_embedding_model(embedding_model)
+
+        return model_data['uuid']
+
+    async def get_embedding_model(self, model_uuid: str) -> dict | None:
+        result = await self.ap.persistence_mgr.execute_async(
+            sqlalchemy.select(persistence_model.EmbeddingModel).where(
+                persistence_model.EmbeddingModel.uuid == model_uuid
+            )
+        )
+
+        model = result.first()
+
+        if model is None:
+            return None
+
+        return self.ap.persistence_mgr.serialize_model(persistence_model.EmbeddingModel, model)
+
+    async def update_embedding_model(self, model_uuid: str, model_data: dict) -> None:
+        if 'uuid' in model_data:
+            del model_data['uuid']
+
+        await self.ap.persistence_mgr.execute_async(
+            sqlalchemy.update(persistence_model.EmbeddingModel)
+            .where(persistence_model.EmbeddingModel.uuid == model_uuid)
+            .values(**model_data)
+        )
+
+        await self.ap.model_mgr.remove_embedding_model(model_uuid)
+
+        embedding_model = await self.get_embedding_model(model_uuid)
+
+        await self.ap.model_mgr.load_embedding_model(embedding_model)
+
+    async def delete_embedding_model(self, model_uuid: str) -> None:
+        await self.ap.persistence_mgr.execute_async(
+            sqlalchemy.delete(persistence_model.EmbeddingModel).where(
+                persistence_model.EmbeddingModel.uuid == model_uuid
+            )
+        )
+
+        await self.ap.model_mgr.remove_embedding_model(model_uuid)
+
+    async def test_embedding_model(self, model_uuid: str, model_data: dict) -> None:
+        runtime_embedding_model: model_requester.RuntimeEmbeddingModel | None = None
+
+        if model_uuid != '_':
+            for model in self.ap.model_mgr.embedding_models:
+                if model.model_entity.uuid == model_uuid:
+                    runtime_embedding_model = model
+                    break
+
+            if runtime_embedding_model is None:
+                raise Exception('model not found')
+
+        else:
+            runtime_embedding_model = await self.ap.model_mgr.init_runtime_embedding_model(model_data)
+
+        await runtime_embedding_model.requester.invoke_embedding(
+            model=runtime_embedding_model,
+            input_text=['Hello, world!'],
             extra_args={},
         )

pkg/api/http/service/pipeline.py

@@ -38,9 +38,21 @@ class PipelineService:
             self.ap.pipeline_config_meta_output.data,
         ]
 
-    async def get_pipelines(self) -> list[dict]:
-        result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_pipeline.LegacyPipeline))
+    async def get_pipelines(self, sort_by: str = 'created_at', sort_order: str = 'DESC') -> list[dict]:
+        query = sqlalchemy.select(persistence_pipeline.LegacyPipeline)
 
+        if sort_by == 'created_at':
+            if sort_order == 'DESC':
+                query = query.order_by(persistence_pipeline.LegacyPipeline.created_at.desc())
+            else:
+                query = query.order_by(persistence_pipeline.LegacyPipeline.created_at.asc())
+        elif sort_by == 'updated_at':
+            if sort_order == 'DESC':
+                query = query.order_by(persistence_pipeline.LegacyPipeline.updated_at.desc())
+            else:
+                query = query.order_by(persistence_pipeline.LegacyPipeline.updated_at.asc())
+
+        result = await self.ap.persistence_mgr.execute_async(query)
         pipelines = result.all()
         return [
             self.ap.persistence_mgr.serialize_model(persistence_pipeline.LegacyPipeline, pipeline)

pkg/api/http/service/user.py

@@ -73,3 +73,27 @@ class UserService:
         jwt_secret = self.ap.instance_config.data['system']['jwt']['secret']
 
         return jwt.decode(token, jwt_secret, algorithms=['HS256'])['user']
+
+    async def reset_password(self, user_email: str, new_password: str) -> None:
+        ph = argon2.PasswordHasher()
+
+        hashed_password = ph.hash(new_password)
+
+        await self.ap.persistence_mgr.execute_async(
+            sqlalchemy.update(user.User).where(user.User.user == user_email).values(password=hashed_password)
+        )
+
+    async def change_password(self, user_email: str, current_password: str, new_password: str) -> None:
+        ph = argon2.PasswordHasher()
+
+        user_obj = await self.get_user_by_email(user_email)
+        if user_obj is None:
+            raise ValueError('User not found')
+
+        ph.verify(user_obj.password, current_password)
+
+        hashed_password = ph.hash(new_password)
+
+        await self.ap.persistence_mgr.execute_async(
+            sqlalchemy.update(user.User).where(user.User.user == user_email).values(password=hashed_password)
+        )