From 37ef1c9fab1a390e2f89f2a2b340cff789367dfc Mon Sep 17 00:00:00 2001 From: RockChinQ <1010553892@qq.com> Date: Thu, 16 May 2024 20:32:30 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=88=A0=E9=99=A4oss=E7=9B=B8=E5=85=B3?= =?UTF-8?q?=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../migrations/m006_vision_and_oss_config.py | 35 -------- pkg/config/migrations/m006_vision_config.py | 19 +++++ pkg/core/app.py | 3 - pkg/core/bootutils/deps.py | 1 - pkg/core/stages/build_app.py | 6 -- pkg/core/stages/migrate.py | 4 +- pkg/oss/__init__.py | 0 pkg/oss/oss.py | 85 ------------------- pkg/oss/service.py | 67 --------------- pkg/oss/services/__init__.py | 0 pkg/oss/services/aliyun.py | 48 ----------- pkg/provider/modelmgr/apis/chatcmpl.py | 35 ++------ pkg/provider/modelmgr/modelmgr.py | 4 - requirements.txt | 2 - templates/system.json | 12 --- 15 files changed, 26 insertions(+), 295 deletions(-) delete mode 100644 pkg/config/migrations/m006_vision_and_oss_config.py create mode 100644 pkg/config/migrations/m006_vision_config.py delete mode 100644 pkg/oss/__init__.py delete mode 100644 pkg/oss/oss.py delete mode 100644 pkg/oss/service.py delete mode 100644 pkg/oss/services/__init__.py delete mode 100644 pkg/oss/services/aliyun.py diff --git a/pkg/config/migrations/m006_vision_and_oss_config.py b/pkg/config/migrations/m006_vision_and_oss_config.py deleted file mode 100644 index 2d05b975..00000000 --- a/pkg/config/migrations/m006_vision_and_oss_config.py +++ /dev/null @@ -1,35 +0,0 @@ -from __future__ import annotations - -from .. import migration - - -@migration.migration_class("vision-and-oss-config", 6) -class VisionAndOSSConfigMigration(migration.Migration): - """迁移""" - - async def need_migrate(self) -> bool: - """判断当前环境是否需要运行此迁移""" - return "enable-vision" not in self.ap.provider_cfg.data \ - or "oss" not in self.ap.system_cfg.data - - async def run(self): - """执行迁移""" - if "enable-vision" not in self.ap.provider_cfg.data: - self.ap.provider_cfg.data["enable-vision"] = False - - if "oss" not in self.ap.system_cfg.data: - self.ap.system_cfg.data["oss"] = [ - { - "type": "aliyun", - "endpoint": "https://oss-cn-hangzhou.aliyuncs.com", - "public-read-base-url": "https://qchatgpt.oss-cn-hangzhou.aliyuncs.com", - "access-key-id": "LTAI5tJ5Q5J8J6J5J5J5J5J5", - "access-key-secret": "xxxxxx", - "bucket": "qchatgpt", - "prefix": "qchatgpt", - "enable": False, - } - ] - - await self.ap.provider_cfg.dump_config() - await self.ap.system_cfg.dump_config() diff --git a/pkg/config/migrations/m006_vision_config.py b/pkg/config/migrations/m006_vision_config.py new file mode 100644 index 00000000..8084611e --- /dev/null +++ b/pkg/config/migrations/m006_vision_config.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from .. import migration + + +@migration.migration_class("vision-config", 6) +class VisionConfigMigration(migration.Migration): + """迁移""" + + async def need_migrate(self) -> bool: + """判断当前环境是否需要运行此迁移""" + return "enable-vision" not in self.ap.provider_cfg.data + + async def run(self): + """执行迁移""" + if "enable-vision" not in self.ap.provider_cfg.data: + self.ap.provider_cfg.data["enable-vision"] = False + + await self.ap.provider_cfg.dump_config() diff --git a/pkg/core/app.py b/pkg/core/app.py index 1705e299..1ed53042 100644 --- a/pkg/core/app.py +++ b/pkg/core/app.py @@ -15,7 +15,6 @@ from ..command import cmdmgr from ..plugin import manager as plugin_mgr from ..pipeline import pool from ..pipeline import controller, stagemgr -from ..oss import oss from ..utils import version as version_mgr, proxy as proxy_mgr @@ -72,8 +71,6 @@ class Application: proxy_mgr: proxy_mgr.ProxyManager = None - oss_mgr: oss.OSSServiceManager = None - logger: logging.Logger = None def __init__(self): diff --git a/pkg/core/bootutils/deps.py b/pkg/core/bootutils/deps.py index ab407048..4adf1323 100644 --- a/pkg/core/bootutils/deps.py +++ b/pkg/core/bootutils/deps.py @@ -14,7 +14,6 @@ required_deps = { "yaml": "pyyaml", "aiohttp": "aiohttp", "psutil": "psutil", - "oss2": "oss2", } diff --git a/pkg/core/stages/build_app.py b/pkg/core/stages/build_app.py index ff8c1e74..39ecc022 100644 --- a/pkg/core/stages/build_app.py +++ b/pkg/core/stages/build_app.py @@ -14,8 +14,6 @@ from ...provider.modelmgr import modelmgr as llm_model_mgr from ...provider.sysprompt import sysprompt as llm_prompt_mgr from ...provider.tools import toolmgr as llm_tool_mgr from ...platform import manager as im_mgr -from ...oss import oss as oss_mgr - @stage.stage_class("BuildAppStage") class BuildAppStage(stage.BootingStage): @@ -69,10 +67,6 @@ class BuildAppStage(stage.BootingStage): await cmd_mgr_inst.initialize() ap.cmd_mgr = cmd_mgr_inst - oss_mgr_inst = oss_mgr.OSSServiceManager(ap) - await oss_mgr_inst.initialize() - ap.oss_mgr = oss_mgr_inst - llm_model_mgr_inst = llm_model_mgr.ModelManager(ap) await llm_model_mgr_inst.initialize() ap.model_mgr = llm_model_mgr_inst diff --git a/pkg/core/stages/migrate.py b/pkg/core/stages/migrate.py index 44102fea..4d5b8d8c 100644 --- a/pkg/core/stages/migrate.py +++ b/pkg/core/stages/migrate.py @@ -4,8 +4,8 @@ import importlib from .. import stage, app from ...config import migration -from ...config.migrations import m001_sensitive_word_migration, m002_openai_config_migration, m003_anthropic_requester_cfg_completion, m004_moonshot_cfg_completion -from ...config.migrations import m005_deepseek_cfg_completion, m006_vision_and_oss_config +from ...config.migrations import m001_sensitive_word_migration, m002_openai_config_migration, m003_anthropic_requester_cfg_completion, m004_moonshot_cfg_completion, m006_vision_config +from ...config.migrations import m005_deepseek_cfg_completion @stage.stage_class("MigrationStage") diff --git a/pkg/oss/__init__.py b/pkg/oss/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/pkg/oss/oss.py b/pkg/oss/oss.py deleted file mode 100644 index 5474ed39..00000000 --- a/pkg/oss/oss.py +++ /dev/null @@ -1,85 +0,0 @@ -from __future__ import annotations - -import aiohttp -import typing -from urllib.parse import urlparse, parse_qs -import ssl - -from . import service as osssv -from ..core import app -from .services import aliyun - - -class OSSServiceManager: - - ap: app.Application - - service: osssv.OSSService = None - - def __init__(self, ap: app.Application): - self.ap = ap - - async def initialize(self): - """初始化 - """ - - mapping = {} - - for svcls in osssv.preregistered_services: - mapping[svcls.name] = svcls - - for sv in self.ap.system_cfg.data['oss']: - if sv['enable']: - - if sv['type'] not in mapping: - raise Exception(f"未知的OSS服务类型: {sv['type']}") - - self.service = mapping[sv['type']](self.ap, sv) - await self.service.initialize() - break - - def available(self) -> bool: - """是否可用 - - Returns: - bool: 是否可用 - """ - return self.service is not None - - async def fetch_image(self, image_url: str) -> bytes: - parsed = urlparse(image_url) - query = parse_qs(parsed.query) - - # Flatten the query dictionary - query = {k: v[0] for k, v in query.items()} - - ssl_context = ssl.create_default_context() - ssl_context.check_hostname = False - ssl_context.verify_mode = ssl.CERT_NONE - - async with aiohttp.ClientSession(trust_env=False) as session: - async with session.get( - f"http://{parsed.netloc}{parsed.path}", - params=query, - ssl=ssl_context - ) as resp: - resp.raise_for_status() # 检查HTTP错误 - file_bytes = await resp.read() - return file_bytes - - async def upload_url_image( - self, - image_url: str, - ) -> str: - """上传URL图片 - - Args: - image_url (str): 图片URL - - Returns: - str: 文件URL - """ - - file_bytes = await self.fetch_image(image_url) - - return await self.service.upload(file_bytes=file_bytes, ext=".jpg") \ No newline at end of file diff --git a/pkg/oss/service.py b/pkg/oss/service.py deleted file mode 100644 index a8228447..00000000 --- a/pkg/oss/service.py +++ /dev/null @@ -1,67 +0,0 @@ -from __future__ import annotations - -import typing -import abc - -from ..core import app - - -preregistered_services: list[typing.Type[OSSService]] = [] - -def service_class( - name: str -) -> typing.Callable[[typing.Type[OSSService]], typing.Type[OSSService]]: - """OSS服务类装饰器 - - Args: - name (str): 服务名称 - - Returns: - typing.Callable[[typing.Type[OSSService]], typing.Type[OSSService]]: 装饰器 - """ - def decorator(cls: typing.Type[OSSService]) -> typing.Type[OSSService]: - assert issubclass(cls, OSSService) - - cls.name = name - - preregistered_services.append(cls) - - return cls - - return decorator - - -class OSSService(metaclass=abc.ABCMeta): - """OSS抽象类""" - - name: str - - ap: app.Application - - cfg: dict - - def __init__(self, ap: app.Application, cfg: dict) -> None: - self.ap = ap - self.cfg = cfg - - async def initialize(self): - pass - - @abc.abstractmethod - async def upload( - self, - local_file: str=None, - file_bytes: bytes=None, - ext: str=None, - ) -> str: - """上传文件 - - Args: - local_file (str, optional): 本地文件路径. Defaults to None. - file_bytes (bytes, optional): 文件字节. Defaults to None. - ext (str, optional): 文件扩展名. Defaults to None. - - Returns: - str: 文件URL - """ - pass diff --git a/pkg/oss/services/__init__.py b/pkg/oss/services/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/pkg/oss/services/aliyun.py b/pkg/oss/services/aliyun.py deleted file mode 100644 index d30ac895..00000000 --- a/pkg/oss/services/aliyun.py +++ /dev/null @@ -1,48 +0,0 @@ -from __future__ import annotations - -import uuid - -import oss2 - -from .. import service as osssv - - -@osssv.service_class('aliyun') -class AliyunOSSService(osssv.OSSService): - """阿里云OSS服务""" - - auth: oss2.Auth - - bucket: oss2.Bucket - - async def initialize(self): - self.auth = oss2.Auth( - self.cfg['access-key-id'], - self.cfg['access-key-secret'] - ) - - self.bucket = oss2.Bucket( - self.auth, - self.cfg['endpoint'], - self.cfg['bucket'] - ) - - async def upload( - self, - local_file: str=None, - file_bytes: bytes=None, - ext: str=None, - ) -> str: - if local_file is not None: - with open(local_file, 'rb') as f: - file_bytes = f.read() - - if file_bytes is None: - raise Exception("缺少文件内容") - - name = str(uuid.uuid1()) - - key = f"{self.cfg['prefix']}/{name}{ext}" - self.bucket.put_object(key, file_bytes) - - return f"{self.cfg['public-read-base-url']}/{key}" diff --git a/pkg/provider/modelmgr/apis/chatcmpl.py b/pkg/provider/modelmgr/apis/chatcmpl.py index c2242a9a..709a65ac 100644 --- a/pkg/provider/modelmgr/apis/chatcmpl.py +++ b/pkg/provider/modelmgr/apis/chatcmpl.py @@ -26,18 +26,9 @@ class OpenAIChatCompletions(api.LLMAPIRequester): requester_cfg: dict - cached_image_oss_url: dict[str, str] = {} - """缓存的OSS服务的图片URL - - key: 前文message中的原图片URL（QQ图片） - value: OSS服务的图片URL - """ - def __init__(self, ap: app.Application): self.ap = ap - self.cached_image_oss_url = {} - self.requester_cfg = self.ap.provider_cfg.data['requester']['openai-chat-completions'] async def initialize(self): @@ -89,13 +80,11 @@ class OpenAIChatCompletions(api.LLMAPIRequester): messages = req_messages.copy() # 检查vision - if self.ap.oss_mgr.available(): - for msg in messages: - if 'content' in msg and isinstance(msg["content"], list): - for me in msg["content"]: - if me["type"] == "image_url": - # me["image_url"]['url'] = await self.get_oss_url(me["image_url"]['url']) - me["image_url"]['url'] = await self.get_base64_str(me["image_url"]['url']) + for msg in messages: + if 'content' in msg and isinstance(msg["content"], list): + for me in msg["content"]: + if me["type"] == "image_url": + me["image_url"]['url'] = await self.get_base64_str(me["image_url"]['url']) args["messages"] = messages @@ -135,20 +124,6 @@ class OpenAIChatCompletions(api.LLMAPIRequester): except openai.APIError as e: raise errors.RequesterError(f'请求错误: {e.message}') - async def get_oss_url( - self, - original_url: str, - ) -> str: - - if original_url in self.cached_image_oss_url: - return self.cached_image_oss_url[original_url] - - oss_url = await self.ap.oss_mgr.upload_url_image(original_url) - - self.cached_image_oss_url[original_url] = oss_url - - return oss_url - async def get_base64_str( self, original_url: str, diff --git a/pkg/provider/modelmgr/modelmgr.py b/pkg/provider/modelmgr/modelmgr.py index 85bf8e98..79e467a5 100644 --- a/pkg/provider/modelmgr/modelmgr.py +++ b/pkg/provider/modelmgr/modelmgr.py @@ -38,10 +38,6 @@ class ModelManager: async def initialize(self): - # 检查是否启用了vision但是没有配置oss - if self.ap.provider_cfg.data['enable-vision'] and not self.ap.oss_mgr.available(): - self.ap.logger.warn("启用了视觉但是没有配置可用的oss服务，基于 URL 传递图片的视觉 API 将无法正常使用") - # 初始化token_mgr, requester for k, v in self.ap.provider_cfg.data['keys'].items(): self.token_mgrs[k] = token.TokenManager(k, v) diff --git a/requirements.txt b/requirements.txt index 63996e0f..e43f9f0b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,5 +14,3 @@ pydantic websockets urllib3 psutil - -oss2 \ No newline at end of file diff --git a/templates/system.json b/templates/system.json index 906640c7..72d29b98 100644 --- a/templates/system.json +++ b/templates/system.json @@ -1,17 +1,5 @@ { "admin-sessions": [], - "oss": [ - { - "type": "aliyun", - "endpoint": "https://oss-cn-hangzhou.aliyuncs.com", - "public-read-base-url": "https://qchatgpt.oss-cn-hangzhou.aliyuncs.com", - "access-key-id": "LTAI5tJ5Q5J8J6J5J5J5J5J5", - "access-key-secret": "xxxxxx", - "bucket": "qchatgpt", - "prefix": "qchatgpt", - "enable": false - } - ], "network-proxies": { "http": null, "https": null