Merge branch 'feat/streaming' of github.com:fdc310/LangBot into streaming_feature

pkg/provider/modelmgr/modelmgr.py

@@ -1,12 +1,14 @@
 from __future__ import annotations
 
 import sqlalchemy
+import traceback
 
 from . import entities, requester
 from ...core import app
 from ...discover import engine
 from . import token
 from ...entity.persistence import model as persistence_model
+from ...entity.errors import provider as provider_errors
 
 FETCH_MODEL_LIST_URL = 'https://api.qchatgpt.rockchin.top/api/v2/fetch/model_list'
 
@@ -64,7 +66,12 @@ class ModelManager:
 
         # load models
         for llm_model in llm_models:
-            await self.load_llm_model(llm_model)
+            try:
+                await self.load_llm_model(llm_model)
+            except provider_errors.RequesterNotFoundError as e:
+                self.ap.logger.warning(f'Requester {e.requester_name} not found, skipping model {llm_model.uuid}')
+            except Exception as e:
+                self.ap.logger.error(f'Failed to load model {llm_model.uuid}: {e}\n{traceback.format_exc()}')
 
     async def init_runtime_llm_model(
         self,
@@ -76,6 +83,9 @@ class ModelManager:
         elif isinstance(model_info, dict):
             model_info = persistence_model.LLMModel(**model_info)
 
+        if model_info.requester not in self.requester_dict:
+            raise provider_errors.RequesterNotFoundError(model_info.requester)
+
         requester_inst = self.requester_dict[model_info.requester](ap=self.ap, config=model_info.requester_config)
 
         await requester_inst.initialize()

pkg/provider/modelmgr/requesters/302aichatcmpl.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+import typing
+import openai
+
+from . import chatcmpl
+
+
+class AI302ChatCompletions(chatcmpl.OpenAIChatCompletions):
+    """302 AI ChatCompletion API 请求器"""
+
+    client: openai.AsyncClient
+
+    default_config: dict[str, typing.Any] = {
+        'base_url': 'https://api.302.ai/v1',
+        'timeout': 120,
+    }

pkg/provider/modelmgr/requesters/302aichatcmpl.yaml

@@ -0,0 +1,28 @@
+apiVersion: v1
+kind: LLMAPIRequester
+metadata:
+  name: 302-ai-chat-completions
+  label:
+    en_US: 302 AI
+    zh_Hans: 302 AI
+  icon: 302ai.png
+spec:
+  config:
+    - name: base_url
+      label:
+        en_US: Base URL
+        zh_Hans: 基础 URL
+      type: string
+      required: true
+      default: "https://api.302.ai/v1"
+    - name: timeout
+      label:
+        en_US: Timeout
+        zh_Hans: 超时时间
+      type: integer
+      required: true
+      default: 120
+execution:
+  python:
+    path: ./302aichatcmpl.py
+    attr: AI302ChatCompletions

pkg/provider/runners/difysvapi.py

@@ -108,7 +108,13 @@ class DifyServiceAPIRunner(runner.RequestRunner):
 
         mode = 'basic'  # 标记是基础编排还是工作流编排
 
-        basic_mode_pending_chunk = ''
+        stream_output_pending_chunk = ''
+
+        batch_pending_max_size = self.pipeline_config['ai']['dify-service-api'].get(
+            'output-batch-size', 0
+        )  # 积累一定量的消息更新消息一次
+
+        batch_pending_index = 0
 
         inputs = {}
 
@@ -126,6 +132,13 @@ class DifyServiceAPIRunner(runner.RequestRunner):
         ):
             self.ap.logger.debug('dify-chat-chunk: ' + str(chunk))
 
+            # 查询异常情况
+            if chunk['event'] == 'error':
+                yield llm_entities.Message(
+                    role='assistant',
+                    content=f"查询异常: [{chunk['code']}]. {chunk['message']}.\n请重试，如果还报错，请用 <font color='red'>**!reset**</font> 命令重置对话再尝试。",
+                )
+
             if chunk['event'] == 'workflow_started':
                 mode = 'workflow'
 
@@ -136,15 +149,35 @@ class DifyServiceAPIRunner(runner.RequestRunner):
                             role='assistant',
                             content=self._try_convert_thinking(chunk['data']['outputs']['answer']),
                         )
+                elif chunk['event'] == 'message':
+                    stream_output_pending_chunk += chunk['answer']
+                    if self.pipeline_config['ai']['dify-service-api'].get('enable-streaming', False):
+                        # 消息数超过量就输出，从而达到streaming的效果
+                        batch_pending_index += 1
+                        if batch_pending_index >= batch_pending_max_size:
+                            yield llm_entities.Message(
+                                role='assistant',
+                                content=self._try_convert_thinking(stream_output_pending_chunk),
+                            )
+                            batch_pending_index = 0
             elif mode == 'basic':
                 if chunk['event'] == 'message':
-                    basic_mode_pending_chunk += chunk['answer']
+                    stream_output_pending_chunk += chunk['answer']
+                    if self.pipeline_config['ai']['dify-service-api'].get('enable-streaming', False):
+                        # 消息数超过量就输出，从而达到streaming的效果
+                        batch_pending_index += 1
+                        if batch_pending_index >= batch_pending_max_size:
+                            yield llm_entities.Message(
+                                role='assistant',
+                                content=self._try_convert_thinking(stream_output_pending_chunk),
+                            )
+                            batch_pending_index = 0
                 elif chunk['event'] == 'message_end':
                     yield llm_entities.Message(
                         role='assistant',
-                        content=self._try_convert_thinking(basic_mode_pending_chunk),
+                        content=self._try_convert_thinking(stream_output_pending_chunk),
                     )
-                    basic_mode_pending_chunk = ''
+                    stream_output_pending_chunk = ''
 
         if chunk is None:
             raise errors.DifyAPIError('Dify API 没有返回任何响应，请检查网络连接和API配置')