mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-07-20 03:16:14 +00:00
feat: 不再预先计算前文token数而是在报错时提醒用户重置
This commit is contained in:
RockChinQ
@@ -0,0 +1,43 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import abc
|
||||
import typing
|
||||
|
||||
from ...core import app
|
||||
from ...core import entities as core_entities
|
||||
from .. import entities as llm_entities
|
||||
|
||||
|
||||
preregistered_requesters: list[typing.Type[LLMAPIRequester]] = []
|
||||
|
||||
def requester_class(name: str):
|
||||
|
||||
def decorator(cls: typing.Type[LLMAPIRequester]) -> typing.Type[LLMAPIRequester]:
|
||||
cls.name = name
|
||||
preregistered_requesters.append(cls)
|
||||
return cls
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
class LLMAPIRequester(metaclass=abc.ABCMeta):
|
||||
"""LLM API请求器
|
||||
"""
|
||||
name: str = None
|
||||
|
||||
ap: app.Application
|
||||
|
||||
def __init__(self, ap: app.Application):
|
||||
self.ap = ap
|
||||
|
||||
async def initialize(self):
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
async def request(
|
||||
self,
|
||||
query: core_entities.Query,
|
||||
) -> typing.AsyncGenerator[llm_entities.Message, None]:
|
||||
"""请求
|
||||
"""
|
||||
raise NotImplementedError
|
||||
@@ -0,0 +1,148 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import typing
|
||||
import json
|
||||
from typing import AsyncGenerator
|
||||
|
||||
import openai
|
||||
import openai.types.chat.chat_completion as chat_completion
|
||||
import httpx
|
||||
|
||||
from pkg.provider.entities import Message
|
||||
|
||||
from .. import api, entities, errors
|
||||
from ....core import entities as core_entities
|
||||
from ... import entities as llm_entities
|
||||
from ...tools import entities as tools_entities
|
||||
|
||||
|
||||
@api.requester_class("openai-chat-completion")
|
||||
class OpenAIChatCompletion(api.LLMAPIRequester):
|
||||
"""OpenAI ChatCompletion API 请求器"""
|
||||
|
||||
client: openai.AsyncClient
|
||||
|
||||
async def initialize(self):
|
||||
self.client = openai.AsyncClient(
|
||||
api_key="",
|
||||
base_url=self.ap.provider_cfg.data['openai-config']['base_url'],
|
||||
timeout=self.ap.provider_cfg.data['openai-config']['request-timeout'],
|
||||
http_client=httpx.AsyncClient(
|
||||
proxies=self.ap.proxy_mgr.get_forward_proxies()
|
||||
)
|
||||
)
|
||||
|
||||
async def _req(
|
||||
self,
|
||||
args: dict,
|
||||
) -> chat_completion.ChatCompletion:
|
||||
self.ap.logger.debug(f"req chat_completion with args {args}")
|
||||
return await self.client.chat.completions.create(**args)
|
||||
|
||||
async def _make_msg(
|
||||
self,
|
||||
chat_completion: chat_completion.ChatCompletion,
|
||||
) -> llm_entities.Message:
|
||||
chatcmpl_message = chat_completion.choices[0].message.dict()
|
||||
|
||||
message = llm_entities.Message(**chatcmpl_message)
|
||||
|
||||
return message
|
||||
|
||||
async def _closure(
|
||||
self,
|
||||
req_messages: list[dict],
|
||||
use_model: entities.LLMModelInfo,
|
||||
use_funcs: list[tools_entities.LLMFunction] = None,
|
||||
) -> llm_entities.Message:
|
||||
self.client.api_key = use_model.token_mgr.get_token()
|
||||
|
||||
args = self.ap.provider_cfg.data['openai-config']['chat-completions-params'].copy()
|
||||
args["model"] = use_model.name if use_model.model_name is None else use_model.model_name
|
||||
|
||||
if use_model.tool_call_supported:
|
||||
tools = await self.ap.tool_mgr.generate_tools_for_openai(use_funcs)
|
||||
|
||||
if tools:
|
||||
args["tools"] = tools
|
||||
|
||||
# 设置此次请求中的messages
|
||||
messages = req_messages
|
||||
args["messages"] = messages
|
||||
|
||||
# 发送请求
|
||||
resp = await self._req(args)
|
||||
|
||||
# 处理请求结果
|
||||
message = await self._make_msg(resp)
|
||||
|
||||
return message
|
||||
|
||||
async def _request(
|
||||
self, query: core_entities.Query
|
||||
) -> typing.AsyncGenerator[llm_entities.Message, None]:
|
||||
"""请求"""
|
||||
|
||||
pending_tool_calls = []
|
||||
|
||||
req_messages = [ # req_messages 仅用于类内，外部同步由 query.messages 进行
|
||||
m.dict(exclude_none=True) for m in query.prompt.messages
|
||||
] + [m.dict(exclude_none=True) for m in query.messages]
|
||||
|
||||
# req_messages.append({"role": "user", "content": str(query.message_chain)})
|
||||
|
||||
msg = await self._closure(req_messages, query.use_model, query.use_funcs)
|
||||
|
||||
yield msg
|
||||
|
||||
pending_tool_calls = msg.tool_calls
|
||||
|
||||
req_messages.append(msg.dict(exclude_none=True))
|
||||
|
||||
while pending_tool_calls:
|
||||
for tool_call in pending_tool_calls:
|
||||
func = tool_call.function
|
||||
|
||||
parameters = json.loads(func.arguments)
|
||||
|
||||
func_ret = await self.ap.tool_mgr.execute_func_call(
|
||||
query, func.name, parameters
|
||||
)
|
||||
|
||||
msg = llm_entities.Message(
|
||||
role="tool", content=json.dumps(func_ret, ensure_ascii=False), tool_call_id=tool_call.id
|
||||
)
|
||||
|
||||
yield msg
|
||||
|
||||
req_messages.append(msg.dict(exclude_none=True))
|
||||
|
||||
# 处理完所有调用，继续请求
|
||||
msg = await self._closure(req_messages, query.use_model, query.use_funcs)
|
||||
|
||||
yield msg
|
||||
|
||||
pending_tool_calls = msg.tool_calls
|
||||
|
||||
req_messages.append(msg.dict(exclude_none=True))
|
||||
|
||||
async def request(self, query: core_entities.Query) -> AsyncGenerator[Message, None]:
|
||||
try:
|
||||
async for msg in self._request(query):
|
||||
yield msg
|
||||
except asyncio.TimeoutError:
|
||||
raise errors.RequesterError('请求超时')
|
||||
except openai.BadRequestError as e:
|
||||
if 'context_length_exceeded' in e.message:
|
||||
raise errors.RequesterError(f'上文过长，请重置会话: {e.message}')
|
||||
else:
|
||||
raise errors.RequesterError(f'请求参数错误: {e.message}')
|
||||
except openai.AuthenticationError as e:
|
||||
raise errors.RequesterError(f'无效的 api-key: {e.message}')
|
||||
except openai.NotFoundError as e:
|
||||
raise errors.RequesterError(f'请求路径错误: {e.message}')
|
||||
except openai.RateLimitError as e:
|
||||
raise errors.RequesterError(f'请求过于频繁: {e.message}')
|
||||
except openai.APIError as e:
|
||||
raise errors.RequesterError(f'请求错误: {e.message}')
|
||||
@@ -0,0 +1,25 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import typing
|
||||
|
||||
import pydantic
|
||||
|
||||
from . import api
|
||||
from . import token
|
||||
|
||||
|
||||
class LLMModelInfo(pydantic.BaseModel):
|
||||
"""模型"""
|
||||
|
||||
name: str
|
||||
|
||||
model_name: typing.Optional[str] = None
|
||||
|
||||
token_mgr: token.TokenManager
|
||||
|
||||
requester: api.LLMAPIRequester
|
||||
|
||||
tool_call_supported: typing.Optional[bool] = False
|
||||
|
||||
class Config:
|
||||
arbitrary_types_allowed = True
|
||||
@@ -0,0 +1,5 @@
|
||||
class RequesterError(Exception):
|
||||
"""Base class for all Requester errors."""
|
||||
|
||||
def __init__(self, message: str):
|
||||
super().__init__("模型请求失败: "+message)
|
||||
@@ -0,0 +1,194 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from . import entities
|
||||
from ...core import app
|
||||
|
||||
from . import token
|
||||
from .apis import chatcmpl
|
||||
|
||||
|
||||
class ModelManager:
|
||||
"""模型管理器"""
|
||||
|
||||
ap: app.Application
|
||||
|
||||
model_list: list[entities.LLMModelInfo]
|
||||
|
||||
def __init__(self, ap: app.Application):
|
||||
self.ap = ap
|
||||
self.model_list = []
|
||||
|
||||
async def get_model_by_name(self, name: str) -> entities.LLMModelInfo:
|
||||
"""通过名称获取模型
|
||||
"""
|
||||
for model in self.model_list:
|
||||
if model.name == name:
|
||||
return model
|
||||
raise ValueError(f"不支持模型: {name} , 请检查配置文件")
|
||||
|
||||
async def initialize(self):
|
||||
openai_chat_completion = chatcmpl.OpenAIChatCompletion(self.ap)
|
||||
await openai_chat_completion.initialize()
|
||||
openai_token_mgr = token.TokenManager("openai", list(self.ap.provider_cfg.data['openai-config']['api-keys']))
|
||||
|
||||
model_list = [
|
||||
entities.LLMModelInfo(
|
||||
name="gpt-3.5-turbo",
|
||||
token_mgr=openai_token_mgr,
|
||||
requester=openai_chat_completion,
|
||||
tool_call_supported=True,
|
||||
),
|
||||
entities.LLMModelInfo(
|
||||
name="gpt-3.5-turbo-1106",
|
||||
token_mgr=openai_token_mgr,
|
||||
requester=openai_chat_completion,
|
||||
tool_call_supported=True,
|
||||
),
|
||||
entities.LLMModelInfo(
|
||||
name="gpt-3.5-turbo-16k",
|
||||
token_mgr=openai_token_mgr,
|
||||
requester=openai_chat_completion,
|
||||
tool_call_supported=True,
|
||||
),
|
||||
entities.LLMModelInfo(
|
||||
name="gpt-3.5-turbo-0613",
|
||||
token_mgr=openai_token_mgr,
|
||||
requester=openai_chat_completion,
|
||||
tool_call_supported=True,
|
||||
),
|
||||
entities.LLMModelInfo(
|
||||
name="gpt-3.5-turbo-16k-0613",
|
||||
token_mgr=openai_token_mgr,
|
||||
requester=openai_chat_completion,
|
||||
tool_call_supported=True,
|
||||
),
|
||||
entities.LLMModelInfo(
|
||||
name="gpt-3.5-turbo-0301",
|
||||
token_mgr=openai_token_mgr,
|
||||
requester=openai_chat_completion,
|
||||
tool_call_supported=True,
|
||||
)
|
||||
]
|
||||
|
||||
self.model_list.extend(model_list)
|
||||
|
||||
gpt4_model_list = [
|
||||
entities.LLMModelInfo(
|
||||
name="gpt-4-0125-preview",
|
||||
token_mgr=openai_token_mgr,
|
||||
requester=openai_chat_completion,
|
||||
tool_call_supported=True,
|
||||
),
|
||||
entities.LLMModelInfo(
|
||||
name="gpt-4-turbo-preview",
|
||||
token_mgr=openai_token_mgr,
|
||||
requester=openai_chat_completion,
|
||||
tool_call_supported=True,
|
||||
),
|
||||
entities.LLMModelInfo(
|
||||
name="gpt-4-1106-preview",
|
||||
token_mgr=openai_token_mgr,
|
||||
requester=openai_chat_completion,
|
||||
tool_call_supported=True,
|
||||
),
|
||||
entities.LLMModelInfo(
|
||||
name="gpt-4-vision-preview",
|
||||
token_mgr=openai_token_mgr,
|
||||
requester=openai_chat_completion,
|
||||
tool_call_supported=True,
|
||||
),
|
||||
entities.LLMModelInfo(
|
||||
name="gpt-4",
|
||||
token_mgr=openai_token_mgr,
|
||||
requester=openai_chat_completion,
|
||||
tool_call_supported=True,
|
||||
),
|
||||
entities.LLMModelInfo(
|
||||
name="gpt-4-0613",
|
||||
token_mgr=openai_token_mgr,
|
||||
requester=openai_chat_completion,
|
||||
tool_call_supported=True,
|
||||
),
|
||||
entities.LLMModelInfo(
|
||||
name="gpt-4-32k",
|
||||
token_mgr=openai_token_mgr,
|
||||
requester=openai_chat_completion,
|
||||
tool_call_supported=True,
|
||||
),
|
||||
entities.LLMModelInfo(
|
||||
name="gpt-4-32k-0613",
|
||||
token_mgr=openai_token_mgr,
|
||||
requester=openai_chat_completion,
|
||||
tool_call_supported=True,
|
||||
)
|
||||
]
|
||||
|
||||
self.model_list.extend(gpt4_model_list)
|
||||
|
||||
one_api_model_list = [
|
||||
entities.LLMModelInfo(
|
||||
name="OneAPI/SparkDesk",
|
||||
model_name='SparkDesk',
|
||||
token_mgr=openai_token_mgr,
|
||||
requester=openai_chat_completion,
|
||||
tool_call_supported=False,
|
||||
),
|
||||
entities.LLMModelInfo(
|
||||
name="OneAPI/chatglm_pro",
|
||||
model_name='chatglm_pro',
|
||||
token_mgr=openai_token_mgr,
|
||||
requester=openai_chat_completion,
|
||||
tool_call_supported=False,
|
||||
),
|
||||
entities.LLMModelInfo(
|
||||
name="OneAPI/chatglm_std",
|
||||
model_name='chatglm_std',
|
||||
token_mgr=openai_token_mgr,
|
||||
requester=openai_chat_completion,
|
||||
tool_call_supported=False,
|
||||
),
|
||||
entities.LLMModelInfo(
|
||||
name="OneAPI/chatglm_lite",
|
||||
model_name='chatglm_lite',
|
||||
token_mgr=openai_token_mgr,
|
||||
requester=openai_chat_completion,
|
||||
tool_call_supported=False,
|
||||
),
|
||||
entities.LLMModelInfo(
|
||||
name="OneAPI/qwen-v1",
|
||||
model_name='qwen-v1',
|
||||
token_mgr=openai_token_mgr,
|
||||
requester=openai_chat_completion,
|
||||
tool_call_supported=False,
|
||||
),
|
||||
entities.LLMModelInfo(
|
||||
name="OneAPI/qwen-plus-v1",
|
||||
model_name='qwen-plus-v1',
|
||||
token_mgr=openai_token_mgr,
|
||||
requester=openai_chat_completion,
|
||||
tool_call_supported=False,
|
||||
),
|
||||
entities.LLMModelInfo(
|
||||
name="OneAPI/ERNIE-Bot",
|
||||
model_name='ERNIE-Bot',
|
||||
token_mgr=openai_token_mgr,
|
||||
requester=openai_chat_completion,
|
||||
tool_call_supported=False,
|
||||
),
|
||||
entities.LLMModelInfo(
|
||||
name="OneAPI/ERNIE-Bot-turbo",
|
||||
model_name='ERNIE-Bot-turbo',
|
||||
token_mgr=openai_token_mgr,
|
||||
requester=openai_chat_completion,
|
||||
tool_call_supported=False,
|
||||
),
|
||||
entities.LLMModelInfo(
|
||||
name="OneAPI/gemini-pro",
|
||||
model_name='gemini-pro',
|
||||
token_mgr=openai_token_mgr,
|
||||
requester=openai_chat_completion,
|
||||
tool_call_supported=False,
|
||||
),
|
||||
]
|
||||
|
||||
self.model_list.extend(one_api_model_list)
|
||||
@@ -0,0 +1,27 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import typing
|
||||
|
||||
import pydantic
|
||||
|
||||
|
||||
class TokenManager():
|
||||
"""鉴权 Token 管理器
|
||||
"""
|
||||
|
||||
provider: str
|
||||
|
||||
tokens: list[str]
|
||||
|
||||
using_token_index: typing.Optional[int] = 0
|
||||
|
||||
def __init__(self, provider: str, tokens: list[str]):
|
||||
self.provider = provider
|
||||
self.tokens = tokens
|
||||
self.using_token_index = 0
|
||||
|
||||
def get_token(self) -> str:
|
||||
return self.tokens[self.using_token_index]
|
||||
|
||||
def next_token(self):
|
||||
self.using_token_index = (self.using_token_index + 1) % len(self.tokens)
|
||||
Reference in New Issue
Block a user