diff --git a/README.md b/README.md index f7ef74e6..74def45c 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,7 @@ | [阿里云百炼](https://bailian.console.aliyun.com/) | ✅ | 大模型聚合平台, LLMOps 平台 | | [火山方舟](https://console.volcengine.com/ark/region:ark+cn-beijing/model?vendor=Bytedance&view=LIST_VIEW) | ✅ | 大模型聚合平台, LLMOps 平台 | | [MCP](https://modelcontextprotocol.io/) | ✅ | 支持通过 MCP 协议获取工具 | +| [ModelScope](https://modelscope.cn/docs/model-service/API-Inference/intro) | ✅ | | ### TTS diff --git a/README_EN.md b/README_EN.md index 77338219..894ce5fe 100644 --- a/README_EN.md +++ b/README_EN.md @@ -114,6 +114,7 @@ Directly use the released version to run, see the [Manual Deployment](https://do | [Aliyun Bailian](https://bailian.console.aliyun.com/) | ✅ | LLM gateway(MaaS), LLMOps platform | | [Volc Engine Ark](https://console.volcengine.com/ark/region:ark+cn-beijing/model?vendor=Bytedance&view=LIST_VIEW) | ✅ | LLM gateway(MaaS), LLMOps platform | | [MCP](https://modelcontextprotocol.io/) | ✅ | Support tool access through MCP protocol | +| [ModelScope](https://modelscope.cn/docs/model-service/API-Inference/intro) | ✅ | | ## 🤝 Community Contribution diff --git a/README_JP.md b/README_JP.md index 440134a2..5589ad79 100644 --- a/README_JP.md +++ b/README_JP.md @@ -113,6 +113,7 @@ LangBotはBTPanelにリストされています。BTPanelをインストール | [Aliyun Bailian](https://bailian.console.aliyun.com/) | ✅ | LLMゲートウェイ(MaaS), LLMOpsプラットフォーム | | [Volc Engine Ark](https://console.volcengine.com/ark/region:ark+cn-beijing/model?vendor=Bytedance&view=LIST_VIEW) | ✅ | LLMゲートウェイ(MaaS), LLMOpsプラットフォーム | | [MCP](https://modelcontextprotocol.io/) | ✅ | MCPプロトコルをサポート | +| [ModelScope](https://modelscope.cn/docs/model-service/API-Inference/intro) | ✅ | | ## 🤝 コミュニティ貢献 diff --git a/pkg/core/migrations/m039_modelscope_cfg_completion.py b/pkg/core/migrations/m039_modelscope_cfg_completion.py new file mode 100644 index 00000000..ef1a82fd --- /dev/null +++ b/pkg/core/migrations/m039_modelscope_cfg_completion.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from .. import migration + + +@migration.migration_class("modelscope-config-completion", 4) +class ModelScopeConfigCompletionMigration(migration.Migration): + """OpenAI配置迁移 + """ + + async def need_migrate(self) -> bool: + """判断当前环境是否需要运行此迁移 + """ + return 'modelscope-chat-completions' not in self.ap.provider_cfg.data['requester'] \ + or 'modelscope' not in self.ap.provider_cfg.data['keys'] + + async def run(self): + """执行迁移 + """ + if 'modelscope-chat-completions' not in self.ap.provider_cfg.data['requester']: + self.ap.provider_cfg.data['requester']['modelscope-chat-completions'] = { + 'base-url': 'https://api.modelscope.cn/v1', + 'args': {}, + 'timeout': 120, + } + + if 'modelscope' not in self.ap.provider_cfg.data['keys']: + self.ap.provider_cfg.data['keys']['modelscope'] = [] + + await self.ap.provider_cfg.dump_config() diff --git a/pkg/core/stages/migrate.py b/pkg/core/stages/migrate.py index a12129f2..f8e6965d 100644 --- a/pkg/core/stages/migrate.py +++ b/pkg/core/stages/migrate.py @@ -12,7 +12,7 @@ from ..migrations import m020_wecom_config, m021_lark_config, m022_lmstudio_conf from ..migrations import m026_qqofficial_config, m027_wx_official_account_config, m028_aliyun_requester_config from ..migrations import m029_dashscope_app_api_config, m030_lark_config_cmpl, m031_dingtalk_config, m032_volcark_config from ..migrations import m033_dify_thinking_config, m034_gewechat_file_url_config, m035_wxoa_mode, m036_wxoa_loading_message -from ..migrations import m037_mcp_config, m038_tg_dingtalk_markdown +from ..migrations import m037_mcp_config, m038_tg_dingtalk_markdown, m039_modelscope_cfg_completion @stage.stage_class("MigrationStage") diff --git a/pkg/provider/modelmgr/modelmgr.py b/pkg/provider/modelmgr/modelmgr.py index e8c231c5..a5ffe6bc 100644 --- a/pkg/provider/modelmgr/modelmgr.py +++ b/pkg/provider/modelmgr/modelmgr.py @@ -6,7 +6,7 @@ from . import entities, requester from ...core import app from ...discover import engine from . import token -from .requesters import bailianchatcmpl, chatcmpl, anthropicmsgs, moonshotchatcmpl, deepseekchatcmpl, ollamachat, giteeaichatcmpl, volcarkchatcmpl, xaichatcmpl, zhipuaichatcmpl, lmstudiochatcmpl, siliconflowchatcmpl, volcarkchatcmpl +from .requesters import bailianchatcmpl, chatcmpl, anthropicmsgs, moonshotchatcmpl, deepseekchatcmpl, ollamachat, giteeaichatcmpl, volcarkchatcmpl, xaichatcmpl, zhipuaichatcmpl, lmstudiochatcmpl, siliconflowchatcmpl, volcarkchatcmpl, modelscopechatcmpl FETCH_MODEL_LIST_URL = "https://api.qchatgpt.rockchin.top/api/v2/fetch/model_list" diff --git a/pkg/provider/modelmgr/requesters/modelscopechatcmpl.py b/pkg/provider/modelmgr/requesters/modelscopechatcmpl.py new file mode 100644 index 00000000..8f51241e --- /dev/null +++ b/pkg/provider/modelmgr/requesters/modelscopechatcmpl.py @@ -0,0 +1,207 @@ +from __future__ import annotations + +import asyncio +import typing +import json +import base64 +from typing import AsyncGenerator + +import openai +import openai.types.chat.chat_completion as chat_completion +import openai.types.chat.chat_completion_message_tool_call as chat_completion_message_tool_call +import httpx +import aiohttp +import async_lru + +from .. import entities, errors, requester +from ....core import entities as core_entities, app +from ... import entities as llm_entities +from ...tools import entities as tools_entities +from ....utils import image + + +class ModelScopeChatCompletions(requester.LLMAPIRequester): + """ModelScope ChatCompletion API 请求器""" + + client: openai.AsyncClient + + requester_cfg: dict + + def __init__(self, ap: app.Application): + self.ap = ap + + self.requester_cfg = self.ap.provider_cfg.data['requester']['modelscope-chat-completions'] + + async def initialize(self): + + self.client = openai.AsyncClient( + api_key="", + base_url=self.requester_cfg['base-url'], + timeout=self.requester_cfg['timeout'], + http_client=httpx.AsyncClient( + trust_env=True, + timeout=self.requester_cfg['timeout'] + ) + ) + + async def _req( + self, + args: dict, + ) -> chat_completion.ChatCompletion: + args["stream"] = True + + chunk = None + + pending_content = "" + + tool_calls = [] + + resp_gen: openai.AsyncStream = await self.client.chat.completions.create(**args) + + async for chunk in resp_gen: + # print(chunk) + if not chunk or not chunk.id or not chunk.choices or not chunk.choices[0] or not chunk.choices[0].delta: + continue + + if chunk.choices[0].delta.content is not None: + pending_content += chunk.choices[0].delta.content + + if chunk.choices[0].delta.tool_calls is not None: + for tool_call in chunk.choices[0].delta.tool_calls: + for tc in tool_calls: + if tc.index == tool_call.index: + tc.function.arguments += tool_call.function.arguments + break + else: + tool_calls.append(tool_call) + + if chunk.choices[0].finish_reason is not None: + break + + real_tool_calls = [] + + for tc in tool_calls: + function = chat_completion_message_tool_call.Function( + name=tc.function.name, + arguments=tc.function.arguments + ) + real_tool_calls.append(chat_completion_message_tool_call.ChatCompletionMessageToolCall( + id=tc.id, + function=function, + type="function" + )) + + return chat_completion.ChatCompletion( + id=chunk.id, + object="chat.completion", + created=chunk.created, + choices=[ + chat_completion.Choice( + index=0, + message=chat_completion.ChatCompletionMessage( + role="assistant", + content=pending_content, + tool_calls=real_tool_calls if len(real_tool_calls) > 0 else None + ), + finish_reason=chunk.choices[0].finish_reason if hasattr(chunk.choices[0], 'finish_reason') and chunk.choices[0].finish_reason is not None else 'stop', + logprobs=chunk.choices[0].logprobs, + ) + ], + model=chunk.model, + service_tier=chunk.service_tier if hasattr(chunk, 'service_tier') else None, + system_fingerprint=chunk.system_fingerprint if hasattr(chunk, 'system_fingerprint') else None, + usage=chunk.usage if hasattr(chunk, 'usage') else None + ) if chunk else None + return await self.client.chat.completions.create(**args) + + async def _make_msg( + self, + chat_completion: chat_completion.ChatCompletion, + ) -> llm_entities.Message: + chatcmpl_message = chat_completion.choices[0].message.dict() + + # 确保 role 字段存在且不为 None + if 'role' not in chatcmpl_message or chatcmpl_message['role'] is None: + chatcmpl_message['role'] = 'assistant' + + message = llm_entities.Message(**chatcmpl_message) + + return message + + async def _closure( + self, + query: core_entities.Query, + req_messages: list[dict], + use_model: entities.LLMModelInfo, + use_funcs: list[tools_entities.LLMFunction] = None, + ) -> llm_entities.Message: + self.client.api_key = use_model.token_mgr.get_token() + + args = self.requester_cfg['args'].copy() + args["model"] = use_model.name if use_model.model_name is None else use_model.model_name + + if use_funcs: + tools = await self.ap.tool_mgr.generate_tools_for_openai(use_funcs) + + if tools: + args["tools"] = tools + + # 设置此次请求中的messages + messages = req_messages.copy() + + # 检查vision + for msg in messages: + if 'content' in msg and isinstance(msg["content"], list): + for me in msg["content"]: + if me["type"] == "image_base64": + me["image_url"] = { + "url": me["image_base64"] + } + me["type"] = "image_url" + del me["image_base64"] + + args["messages"] = messages + + # 发送请求 + resp = await self._req(args) + + # 处理请求结果 + message = await self._make_msg(resp) + + return message + + async def call( + self, + query: core_entities.Query, + model: entities.LLMModelInfo, + messages: typing.List[llm_entities.Message], + funcs: typing.List[tools_entities.LLMFunction] = None, + ) -> llm_entities.Message: + req_messages = [] # req_messages 仅用于类内，外部同步由 query.messages 进行 + for m in messages: + msg_dict = m.dict(exclude_none=True) + content = msg_dict.get("content") + if isinstance(content, list): + # 检查 content 列表中是否每个部分都是文本 + if all(isinstance(part, dict) and part.get("type") == "text" for part in content): + # 将所有文本部分合并为一个字符串 + msg_dict["content"] = "\n".join(part["text"] for part in content) + req_messages.append(msg_dict) + + try: + return await self._closure(query=query, req_messages=req_messages, use_model=model, use_funcs=funcs) + except asyncio.TimeoutError: + raise errors.RequesterError('请求超时') + except openai.BadRequestError as e: + if 'context_length_exceeded' in e.message: + raise errors.RequesterError(f'上文过长，请重置会话: {e.message}') + else: + raise errors.RequesterError(f'请求参数错误: {e.message}') + except openai.AuthenticationError as e: + raise errors.RequesterError(f'无效的 api-key: {e.message}') + except openai.NotFoundError as e: + raise errors.RequesterError(f'请求路径错误: {e.message}') + except openai.RateLimitError as e: + raise errors.RequesterError(f'请求过于频繁或余额不足: {e.message}') + except openai.APIError as e: + raise errors.RequesterError(f'请求错误: {e.message}') \ No newline at end of file diff --git a/pkg/provider/modelmgr/requesters/modelscopechatcmpl.yaml b/pkg/provider/modelmgr/requesters/modelscopechatcmpl.yaml new file mode 100644 index 00000000..eb8c70ec --- /dev/null +++ b/pkg/provider/modelmgr/requesters/modelscopechatcmpl.yaml @@ -0,0 +1,34 @@ +apiVersion: v1 +kind: LLMAPIRequester +metadata: + name: modelscope-chat-completions + label: + en_US: ModelScope + zh_CN: 魔搭社区 +spec: + config: + - name: base-url + label: + en_US: Base URL + zh_CN: 基础 URL + type: string + required: true + default: "https://api-inference.modelscope.cn/v1" + - name: args + label: + en_US: Args + zh_CN: 附加参数 + type: object + required: true + default: {} + - name: timeout + label: + en_US: Timeout + zh_CN: 超时时间 + type: int + required: true + default: 120 +execution: + python: + path: ./modelscopechatcmpl.py + attr: ModelScopeChatCompletions diff --git a/templates/metadata/llm-models.json b/templates/metadata/llm-models.json index a12c7687..66163064 100644 --- a/templates/metadata/llm-models.json +++ b/templates/metadata/llm-models.json @@ -232,6 +232,96 @@ "token_mgr": "zhipuai", "vision_supported": true, "tool_call_supported": true + }, + { + "name": "Qwen/Qwen2.5-Coder-32B-Instruct", + "requester": "modelscope-chat-completions", + "token_mgr": "modelscope", + "tool_call_supported": true + }, + { + "name": "Qwen/Qwen2.5-Coder-14B-Instruct", + "requester": "modelscope-chat-completions", + "token_mgr": "modelscope", + "tool_call_supported": true + }, + { + "name": "Qwen/Qwen2.5-Coder-7B-Instruct", + "requester": "modelscope-chat-completions", + "token_mgr": "modelscope", + "tool_call_supported": true + }, + { + "name": "Qwen/Qwen2.5-72B-Instruct", + "requester": "modelscope-chat-completions", + "token_mgr": "modelscope", + "tool_call_supported": true + }, + { + "name": "Qwen/Qwen2.5-32B-Instruct", + "requester": "modelscope-chat-completions", + "token_mgr": "modelscope", + "tool_call_supported": true + }, + { + "name": "Qwen/Qwen2.5-14B-Instruct", + "requester": "modelscope-chat-completions", + "token_mgr": "modelscope", + "tool_call_supported": true + }, + { + "name": "Qwen/Qwen2.5-7B-Instruct", + "requester": "modelscope-chat-completions", + "token_mgr": "modelscope", + "tool_call_supported": true + }, + { + "name": "Qwen/QwQ-32B-Preview", + "requester": "modelscope-chat-completions", + "token_mgr": "modelscope", + "tool_call_supported": true + }, + { + "name": "Qwen/QwQ-32B", + "requester": "modelscope-chat-completions", + "token_mgr": "modelscope", + "tool_call_supported": true + }, + { + "name": "LLM-Research/Llama-3.3-70B-Instruct", + "requester": "modelscope-chat-completions", + "token_mgr": "modelscope", + "tool_call_supported": true + }, + { + "name": "LLM-Research/Meta-Llama-3.1-405B-Instruct", + "requester": "modelscope-chat-completions", + "token_mgr": "modelscope", + "tool_call_supported": true + }, + { + "name": "LLM-Research/Meta-Llama-3.1-8B-Instruct", + "requester": "modelscope-chat-completions", + "token_mgr": "modelscope", + "tool_call_supported": true + }, + { + "name": "LLM-Research/Meta-Llama-3.1-70B-Instruct", + "requester": "modelscope-chat-completions", + "token_mgr": "modelscope", + "tool_call_supported": true + }, + { + "name": "mistralai/Ministral-8B-Instruct-2410", + "requester": "modelscope-chat-completions", + "token_mgr": "modelscope", + "tool_call_supported": true + }, + { + "name": "deepseek-ai/DeepSeek-V3-0324", + "requester": "modelscope-chat-completions", + "token_mgr": "modelscope", + "tool_call_supported": true } ] } \ No newline at end of file diff --git a/templates/provider.json b/templates/provider.json index 6b9ce010..135c086d 100644 --- a/templates/provider.json +++ b/templates/provider.json @@ -31,6 +31,9 @@ ], "volcark": [ "xxxxxxxx" + ], + "modelscope": [ + "xxxxxxxx" ] }, "requester": { @@ -95,6 +98,11 @@ "args": {}, "base-url": "https://ark.cn-beijing.volces.com/api/v3", "timeout": 120 + }, + "modelscope-chat-completions": { + "base-url": "https://api-inference.modelscope.cn/v1", + "args": {}, + "timeout": 120 } }, "model": "gpt-4o",