Merge branch 'rc/new-plugin' into refactor/new-plugin-system

pkg/api/http/controller/group.py

@@ -11,10 +11,10 @@ from ....core import app
 
 
 preregistered_groups: list[type[RouterGroup]] = []
-"""RouterGroup 的预注册列表"""
+"""Pre-registered list of RouterGroup"""
 
 
-def group_class(name: str, path: str) -> None:
+def group_class(name: str, path: str) -> typing.Callable[[typing.Type[RouterGroup]], typing.Type[RouterGroup]]:
     """注册一个 RouterGroup"""
 
     def decorator(cls: typing.Type[RouterGroup]) -> typing.Type[RouterGroup]:
@@ -27,7 +27,7 @@ def group_class(name: str, path: str) -> None:
 
 
 class AuthType(enum.Enum):
-    """认证类型"""
+    """Authentication type"""
 
     NONE = 'none'
     USER_TOKEN = 'user-token'
@@ -56,7 +56,7 @@ class RouterGroup(abc.ABC):
         auth_type: AuthType = AuthType.USER_TOKEN,
         **options: typing.Any,
     ) -> typing.Callable[[RouteCallable], RouteCallable]:  # decorator
-        """注册一个路由"""
+        """Register a route"""
 
         def decorator(f: RouteCallable) -> RouteCallable:
             nonlocal rule
@@ -64,11 +64,11 @@ class RouterGroup(abc.ABC):
 
             async def handler_error(*args, **kwargs):
                 if auth_type == AuthType.USER_TOKEN:
-                    # 从Authorization头中获取token
+                    # get token from Authorization header
                     token = quart.request.headers.get('Authorization', '').replace('Bearer ', '')
 
                     if not token:
-                        return self.http_status(401, -1, '未提供有效的用户令牌')
+                        return self.http_status(401, -1, 'No valid user token provided')
 
                     try:
                         user_email = await self.ap.user_service.verify_jwt_token(token)
@@ -76,9 +76,9 @@ class RouterGroup(abc.ABC):
                         # check if this account exists
                         user = await self.ap.user_service.get_user_by_email(user_email)
                         if not user:
-                            return self.http_status(401, -1, '用户不存在')
+                            return self.http_status(401, -1, 'User not found')
 
-                        # 检查f是否接受user_email参数
+                        # check if f accepts user_email parameter
                         if 'user_email' in f.__code__.co_varnames:
                             kwargs['user_email'] = user_email
                     except Exception as e:
@@ -86,10 +86,11 @@ class RouterGroup(abc.ABC):
 
                 try:
                     return await f(*args, **kwargs)
-                except Exception:  # 自动 500
+
+                except Exception as e:  # 自动 500
                     traceback.print_exc()
                     # return self.http_status(500, -2, str(e))
-                    return self.http_status(500, -2, 'internal server error')
+                    return self.http_status(500, -2, str(e))
 
             new_f = handler_error
             new_f.__name__ = (self.name + rule).replace('/', '__')
@@ -101,7 +102,7 @@ class RouterGroup(abc.ABC):
         return decorator
 
     def success(self, data: typing.Any = None) -> quart.Response:
-        """返回一个 200 响应"""
+        """Return a 200 response"""
         return quart.jsonify(
             {
                 'code': 0,
@@ -111,7 +112,7 @@ class RouterGroup(abc.ABC):
         )
 
     def fail(self, code: int, msg: str) -> quart.Response:
-        """返回一个异常响应"""
+        """Return an error response"""
 
         return quart.jsonify(
             {
@@ -120,6 +121,6 @@ class RouterGroup(abc.ABC):
             }
         )
 
-    def http_status(self, status: int, code: int, msg: str) -> quart.Response:
+    def http_status(self, status: int, code: int, msg: str) -> typing.Tuple[quart.Response, int]:
         """返回一个指定状态码的响应"""
-        return self.fail(code, msg), status
+        return (self.fail(code, msg), status)

pkg/api/http/controller/groups/files.py

@@ -2,6 +2,10 @@ from __future__ import annotations
 
 import quart
 import mimetypes
+import uuid
+import asyncio
+
+import quart.datastructures
 
 from .. import group
 
@@ -20,3 +24,23 @@ class FilesRouterGroup(group.RouterGroup):
                 mime_type = 'image/jpeg'
 
             return quart.Response(image_bytes, mimetype=mime_type)
+
+        @self.route('/documents', methods=['POST'], auth_type=group.AuthType.USER_TOKEN)
+        async def _() -> quart.Response:
+            request = quart.request
+            # get file bytes from 'file'
+            file = (await request.files)['file']
+            assert isinstance(file, quart.datastructures.FileStorage)
+
+            file_bytes = await asyncio.to_thread(file.stream.read)
+            extension = file.filename.split('.')[-1]
+            file_name = file.filename.split('.')[0]
+
+            file_key = file_name + '_' + str(uuid.uuid4())[:8] + '.' + extension
+            # save file to storage
+            await self.ap.storage_mgr.storage_provider.save(file_key, file_bytes)
+            return self.success(
+                data={
+                    'file_id': file_key,
+                }
+            )

pkg/api/http/controller/groups/knowledge/base.py

@@ -0,0 +1,90 @@
+import quart
+from ... import group
+
+
+@group.group_class('knowledge_base', '/api/v1/knowledge/bases')
+class KnowledgeBaseRouterGroup(group.RouterGroup):
+    async def initialize(self) -> None:
+        @self.route('', methods=['POST', 'GET'])
+        async def handle_knowledge_bases() -> quart.Response:
+            if quart.request.method == 'GET':
+                knowledge_bases = await self.ap.knowledge_service.get_knowledge_bases()
+                return self.success(data={'bases': knowledge_bases})
+
+            elif quart.request.method == 'POST':
+                json_data = await quart.request.json
+                knowledge_base_uuid = await self.ap.knowledge_service.create_knowledge_base(json_data)
+                return self.success(data={'uuid': knowledge_base_uuid})
+
+            return self.http_status(405, -1, 'Method not allowed')
+
+        @self.route(
+            '/<knowledge_base_uuid>',
+            methods=['GET', 'DELETE', 'PUT'],
+        )
+        async def handle_specific_knowledge_base(knowledge_base_uuid: str) -> quart.Response:
+            if quart.request.method == 'GET':
+                knowledge_base = await self.ap.knowledge_service.get_knowledge_base(knowledge_base_uuid)
+
+                if knowledge_base is None:
+                    return self.http_status(404, -1, 'knowledge base not found')
+
+                return self.success(
+                    data={
+                        'base': knowledge_base,
+                    }
+                )
+
+            elif quart.request.method == 'PUT':
+                json_data = await quart.request.json
+                await self.ap.knowledge_service.update_knowledge_base(knowledge_base_uuid, json_data)
+                return self.success({})
+
+            elif quart.request.method == 'DELETE':
+                await self.ap.knowledge_service.delete_knowledge_base(knowledge_base_uuid)
+                return self.success({})
+
+        @self.route(
+            '/<knowledge_base_uuid>/files',
+            methods=['GET', 'POST'],
+        )
+        async def get_knowledge_base_files(knowledge_base_uuid: str) -> str:
+            if quart.request.method == 'GET':
+                files = await self.ap.knowledge_service.get_files_by_knowledge_base(knowledge_base_uuid)
+                return self.success(
+                    data={
+                        'files': files,
+                    }
+                )
+
+            elif quart.request.method == 'POST':
+                json_data = await quart.request.json
+                file_id = json_data.get('file_id')
+                if not file_id:
+                    return self.http_status(400, -1, 'File ID is required')
+
+                # 调用服务层方法将文件与知识库关联
+                task_id = await self.ap.knowledge_service.store_file(knowledge_base_uuid, file_id)
+                return self.success(
+                    {
+                        'task_id': task_id,
+                    }
+                )
+
+        @self.route(
+            '/<knowledge_base_uuid>/files/<file_id>',
+            methods=['DELETE'],
+        )
+        async def delete_specific_file_in_kb(file_id: str, knowledge_base_uuid: str) -> str:
+            await self.ap.knowledge_service.delete_file(knowledge_base_uuid, file_id)
+            return self.success({})
+
+        @self.route(
+            '/<knowledge_base_uuid>/retrieve',
+            methods=['POST'],
+        )
+        async def retrieve_knowledge_base(knowledge_base_uuid: str) -> str:
+            json_data = await quart.request.json
+            query = json_data.get('query')
+            results = await self.ap.knowledge_service.retrieve_knowledge_base(knowledge_base_uuid, query)
+            return self.success(data={'results': results})

pkg/api/http/controller/groups/pipelines/pipelines.py

@@ -11,7 +11,11 @@ class PipelinesRouterGroup(group.RouterGroup):
         @self.route('', methods=['GET', 'POST'])
         async def _() -> str:
             if quart.request.method == 'GET':
-                return self.success(data={'pipelines': await self.ap.pipeline_service.get_pipelines()})
+                sort_by = quart.request.args.get('sort_by', 'created_at')
+                sort_order = quart.request.args.get('sort_order', 'DESC')
+                return self.success(
+                    data={'pipelines': await self.ap.pipeline_service.get_pipelines(sort_by, sort_order)}
+                )
             elif quart.request.method == 'POST':
                 json_data = await quart.request.json
 

pkg/api/http/controller/groups/pipelines/webchat.py

@@ -1,3 +1,5 @@
+import json
+
 import quart
 
 from ... import group
@@ -8,11 +10,19 @@ class WebChatDebugRouterGroup(group.RouterGroup):
     async def initialize(self) -> None:
         @self.route('/send', methods=['POST'])
         async def send_message(pipeline_uuid: str) -> str:
-            """发送调试消息到流水线"""
+            """Send a message to the pipeline for debugging"""
+
+            async def stream_generator(generator):
+                yield 'data: {"type": "start"}\n\n'
+                async for message in generator:
+                    yield f'data: {json.dumps({"message": message})}\n\n'
+                yield 'data: {"type": "end"}\n\n'
+
             try:
                 data = await quart.request.get_json()
                 session_type = data.get('session_type', 'person')
                 message_chain_obj = data.get('message', [])
+                is_stream = data.get('is_stream', False)
 
                 if not message_chain_obj:
                     return self.http_status(400, -1, 'message is required')
@@ -25,20 +35,40 @@ class WebChatDebugRouterGroup(group.RouterGroup):
                 if not webchat_adapter:
                     return self.http_status(404, -1, 'WebChat adapter not found')
 
-                result = await webchat_adapter.send_webchat_message(pipeline_uuid, session_type, message_chain_obj)
-
-                return self.success(
-                    data={
-                        'message': result,
+                if is_stream:
+                    generator = webchat_adapter.send_webchat_message(
+                        pipeline_uuid, session_type, message_chain_obj, is_stream
+                    )
+                    # 设置正确的响应头
+                    headers = {
+                        'Content-Type': 'text/event-stream',
+                        'Transfer-Encoding': 'chunked',
+                        'Cache-Control': 'no-cache',
+                        'Connection': 'keep-alive',
                     }
-                )
+                    return quart.Response(stream_generator(generator), mimetype='text/event-stream', headers=headers)
+
+                else:  # non-stream
+                    result = None
+                    async for message in webchat_adapter.send_webchat_message(
+                        pipeline_uuid, session_type, message_chain_obj
+                    ):
+                        result = message
+                    if result is not None:
+                        return self.success(
+                            data={
+                                'message': result,
+                            }
+                        )
+                    else:
+                        return self.http_status(400, -1, 'message is required')
 
             except Exception as e:
                 return self.http_status(500, -1, f'Internal server error: {str(e)}')
 
         @self.route('/messages/<session_type>', methods=['GET'])
         async def get_messages(pipeline_uuid: str, session_type: str) -> str:
-            """获取调试消息历史"""
+            """Get the message history of the pipeline for debugging"""
             try:
                 if session_type not in ['person', 'group']:
                     return self.http_status(400, -1, 'session_type must be person or group')
@@ -57,7 +87,7 @@ class WebChatDebugRouterGroup(group.RouterGroup):
 
         @self.route('/reset/<session_type>', methods=['POST'])
         async def reset_session(session_type: str) -> str:
-            """重置调试会话"""
+            """Reset the debug session"""
             try:
                 if session_type not in ['person', 'group']:
                     return self.http_status(400, -1, 'session_type must be person or group')

pkg/api/http/controller/groups/provider/models.py

@@ -9,18 +9,18 @@ class LLMModelsRouterGroup(group.RouterGroup):
         @self.route('', methods=['GET', 'POST'])
         async def _() -> str:
             if quart.request.method == 'GET':
-                return self.success(data={'models': await self.ap.model_service.get_llm_models()})
+                return self.success(data={'models': await self.ap.llm_model_service.get_llm_models()})
             elif quart.request.method == 'POST':
                 json_data = await quart.request.json
 
-                model_uuid = await self.ap.model_service.create_llm_model(json_data)
+                model_uuid = await self.ap.llm_model_service.create_llm_model(json_data)
 
                 return self.success(data={'uuid': model_uuid})
 
         @self.route('/<model_uuid>', methods=['GET', 'PUT', 'DELETE'])
         async def _(model_uuid: str) -> str:
             if quart.request.method == 'GET':
-                model = await self.ap.model_service.get_llm_model(model_uuid)
+                model = await self.ap.llm_model_service.get_llm_model(model_uuid)
 
                 if model is None:
                     return self.http_status(404, -1, 'model not found')
@@ -29,11 +29,11 @@ class LLMModelsRouterGroup(group.RouterGroup):
             elif quart.request.method == 'PUT':
                 json_data = await quart.request.json
 
-                await self.ap.model_service.update_llm_model(model_uuid, json_data)
+                await self.ap.llm_model_service.update_llm_model(model_uuid, json_data)
 
                 return self.success()
             elif quart.request.method == 'DELETE':
-                await self.ap.model_service.delete_llm_model(model_uuid)
+                await self.ap.llm_model_service.delete_llm_model(model_uuid)
 
                 return self.success()
 
@@ -41,6 +41,49 @@ class LLMModelsRouterGroup(group.RouterGroup):
         async def _(model_uuid: str) -> str:
             json_data = await quart.request.json
 
-            await self.ap.model_service.test_llm_model(model_uuid, json_data)
+            await self.ap.llm_model_service.test_llm_model(model_uuid, json_data)
+
+            return self.success()
+
+
+@group.group_class('models/embedding', '/api/v1/provider/models/embedding')
+class EmbeddingModelsRouterGroup(group.RouterGroup):
+    async def initialize(self) -> None:
+        @self.route('', methods=['GET', 'POST'])
+        async def _() -> str:
+            if quart.request.method == 'GET':
+                return self.success(data={'models': await self.ap.embedding_models_service.get_embedding_models()})
+            elif quart.request.method == 'POST':
+                json_data = await quart.request.json
+
+                model_uuid = await self.ap.embedding_models_service.create_embedding_model(json_data)
+
+                return self.success(data={'uuid': model_uuid})
+
+        @self.route('/<model_uuid>', methods=['GET', 'PUT', 'DELETE'])
+        async def _(model_uuid: str) -> str:
+            if quart.request.method == 'GET':
+                model = await self.ap.embedding_models_service.get_embedding_model(model_uuid)
+
+                if model is None:
+                    return self.http_status(404, -1, 'model not found')
+
+                return self.success(data={'model': model})
+            elif quart.request.method == 'PUT':
+                json_data = await quart.request.json
+
+                await self.ap.embedding_models_service.update_embedding_model(model_uuid, json_data)
+
+                return self.success()
+            elif quart.request.method == 'DELETE':
+                await self.ap.embedding_models_service.delete_embedding_model(model_uuid)
+
+                return self.success()
+
+        @self.route('/<model_uuid>/test', methods=['POST'])
+        async def _(model_uuid: str) -> str:
+            json_data = await quart.request.json
+
+            await self.ap.embedding_models_service.test_embedding_model(model_uuid, json_data)
 
             return self.success()

pkg/api/http/controller/groups/provider/requesters.py

@@ -8,7 +8,8 @@ class RequestersRouterGroup(group.RouterGroup):
     async def initialize(self) -> None:
         @self.route('', methods=['GET'])
         async def _() -> quart.Response:
-            return self.success(data={'requesters': self.ap.model_mgr.get_available_requesters_info()})
+            model_type = quart.request.args.get('type', '')
+            return self.success(data={'requesters': self.ap.model_mgr.get_available_requesters_info(model_type)})
 
         @self.route('/<requester_name>', methods=['GET'])
         async def _(requester_name: str) -> quart.Response:

pkg/api/http/controller/groups/user.py

@@ -1,5 +1,6 @@
 import quart
 import argon2
+import asyncio
 
 from .. import group
 
@@ -13,7 +14,7 @@ class UserRouterGroup(group.RouterGroup):
                 return self.success(data={'initialized': await self.ap.user_service.is_initialized()})
 
             if await self.ap.user_service.is_initialized():
-                return self.fail(1, '系统已初始化')
+                return self.fail(1, 'System already initialized')
 
             json_data = await quart.request.json
 
@@ -31,7 +32,7 @@ class UserRouterGroup(group.RouterGroup):
             try:
                 token = await self.ap.user_service.authenticate(json_data['user'], json_data['password'])
             except argon2.exceptions.VerifyMismatchError:
-                return self.fail(1, '用户名或密码错误')
+                return self.fail(1, 'Invalid username or password')
 
             return self.success(data={'token': token})
 
@@ -40,3 +41,45 @@ class UserRouterGroup(group.RouterGroup):
             token = await self.ap.user_service.generate_jwt_token(user_email)
 
             return self.success(data={'token': token})
+
+        @self.route('/reset-password', methods=['POST'], auth_type=group.AuthType.NONE)
+        async def _() -> str:
+            json_data = await quart.request.json
+
+            user_email = json_data['user']
+            recovery_key = json_data['recovery_key']
+            new_password = json_data['new_password']
+
+            # hard sleep 3s for security
+            await asyncio.sleep(3)
+
+            if not await self.ap.user_service.is_initialized():
+                return self.http_status(400, -1, 'System not initialized')
+
+            user_obj = await self.ap.user_service.get_user_by_email(user_email)
+
+            if user_obj is None:
+                return self.http_status(400, -1, 'User not found')
+
+            if recovery_key != self.ap.instance_config.data['system']['recovery_key']:
+                return self.http_status(403, -1, 'Invalid recovery key')
+
+            await self.ap.user_service.reset_password(user_email, new_password)
+
+            return self.success(data={'user': user_email})
+
+        @self.route('/change-password', methods=['POST'], auth_type=group.AuthType.USER_TOKEN)
+        async def _(user_email: str) -> str:
+            json_data = await quart.request.json
+
+            current_password = json_data['current_password']
+            new_password = json_data['new_password']
+
+            try:
+                await self.ap.user_service.change_password(user_email, current_password, new_password)
+            except argon2.exceptions.VerifyMismatchError:
+                return self.http_status(400, -1, 'Current password is incorrect')
+            except ValueError as e:
+                return self.http_status(400, -1, str(e))
+
+            return self.success(data={'user': user_email})

pkg/api/http/controller/main.py

@@ -14,11 +14,13 @@ from . import group
 from .groups import provider as groups_provider
 from .groups import platform as groups_platform
 from .groups import pipelines as groups_pipelines
+from .groups import knowledge as groups_knowledge
 
 importutil.import_modules_in_pkg(groups)
 importutil.import_modules_in_pkg(groups_provider)
 importutil.import_modules_in_pkg(groups_platform)
 importutil.import_modules_in_pkg(groups_pipelines)
+importutil.import_modules_in_pkg(groups_knowledge)
 
 
 class HTTPController:
@@ -45,7 +47,7 @@ class HTTPController:
                 try:
                     await self.quart_app.run_task(*args, **kwargs)
                 except Exception as e:
-                    self.ap.logger.error(f'启动 HTTP 服务失败： {e}')
+                    self.ap.logger.error(f'Failed to start HTTP service: {e}')
 
             self.ap.task_mgr.create_task(
                 exception_handler(