diff --git a/.gitignore b/.gitignore index 894a1881f..ddc2fc6f6 100644 --- a/.gitignore +++ b/.gitignore @@ -45,4 +45,4 @@ dev .env *.key -*.key.pub \ No newline at end of file +*.key.pub diff --git a/Dockerfile b/Dockerfile index 008d26523..ec636429f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -74,7 +74,7 @@ EXPOSE 23001 ENV KEEP_ALIVE_TIMEOUT=30 CMD if [ -n "$PROXY_URL" ]; then \ - export HOSTNAME="127.0.0.1"; \ + export HOSTNAME="0.0.0.0"; \ protocol=$(echo $PROXY_URL | cut -d: -f1); \ host=$(echo $PROXY_URL | cut -d/ -f3 | cut -d: -f1); \ port=$(echo $PROXY_URL | cut -d: -f3); \ @@ -89,7 +89,7 @@ CMD if [ -n "$PROXY_URL" ]; then \ echo "[ProxyList]" >> $conf; \ echo "$protocol $host $port" >> $conf; \ cat /etc/proxychains.conf; \ - proxychains -f $conf node server.js --host 0.0.0.0; \ + proxychains -f $conf node server.js; \ else \ node server.js; \ fi diff --git a/README.md b/README.md index d496d68ed..24967c164 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ One-Click to get a well-designed cross-platform ChatGPT web UI, with GPT3, GPT4 [网页版](https://app.nextchat.dev/) / [客户端](https://github.com/Yidadaa/ChatGPT-Next-Web/releases) / [反馈](https://github.com/Yidadaa/ChatGPT-Next-Web/issues) -[web-url]: https://chatgpt.nextweb.fun +[web-url]: https://app.nextchat.dev/ [download-url]: https://github.com/Yidadaa/ChatGPT-Next-Web/releases [Web-image]: https://img.shields.io/badge/Web-PWA-orange?logo=microsoftedge [Windows-image]: https://img.shields.io/badge/-Windows-blue?logo=windows @@ -181,6 +181,7 @@ Specify OpenAI organization ID. ### `AZURE_URL` (optional) > Example: https://{azure-resource-url}/openai/deployments/{deploy-name} +> if you config deployment name in `CUSTOM_MODELS`, you can remove `{deploy-name}` in `AZURE_URL` Azure deploy url. @@ -212,6 +213,34 @@ anthropic claude Api version. anthropic claude Api Url. +### `BAIDU_API_KEY` (optional) + +Baidu Api Key. + +### `BAIDU_SECRET_KEY` (optional) + +Baidu Secret Key. + +### `BAIDU_URL` (optional) + +Baidu Api Url. + +### `BYTEDANCE_API_KEY` (optional) + +ByteDance Api Key. + +### `BYTEDANCE_URL` (optional) + +ByteDance Api Url. + +### `ALIBABA_API_KEY` (optional) + +Alibaba Cloud Api Key. + +### `ALIBABA_URL` (optional) + +Alibaba Cloud Api Url. + ### `HIDE_USER_API_KEY` (optional) > Default: Empty @@ -245,6 +274,16 @@ To control custom models, use `+` to add a custom model, use `-` to hide a model User `-all` to disable all default models, `+all` to enable all default models. +For Azure: use `modelName@azure=deploymentName` to customize model name and deployment name. +> Example: `+gpt-3.5-turbo@azure=gpt35` will show option `gpt35(Azure)` in model list. + +For ByteDance: use `modelName@bytedance=deploymentName` to customize model name and deployment name. +> Example: `+Doubao-lite-4k@bytedance=ep-xxxxx-xxx` will show option `Doubao-lite-4k(ByteDance)` in model list. + +### `DEFAULT_MODEL` （optional） + +Change default model + ### `WHITE_WEBDEV_ENDPOINTS` (optional) You can use this option if you want to increase the number of webdav service addresses you are allowed to access, as required by the format： diff --git a/README_CN.md b/README_CN.md index 6811102b6..5400bb276 100644 --- a/README_CN.md +++ b/README_CN.md @@ -95,6 +95,7 @@ OpenAI 接口代理 URL，如果你手动配置了 openai 接口代理，请填 ### `AZURE_URL` （可选） > 形如：https://{azure-resource-url}/openai/deployments/{deploy-name} +> 如果你已经在`CUSTOM_MODELS`中参考`displayName`的方式配置了{deploy-name}，那么可以从`AZURE_URL`中移除`{deploy-name}` Azure 部署地址。 @@ -106,26 +107,54 @@ Azure 密钥。 Azure Api 版本，你可以在这里找到：[Azure 文档](https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#chat-completions)。 -### `GOOGLE_API_KEY` (optional) +### `GOOGLE_API_KEY` (可选) Google Gemini Pro 密钥. -### `GOOGLE_URL` (optional) +### `GOOGLE_URL` (可选) Google Gemini Pro Api Url. -### `ANTHROPIC_API_KEY` (optional) +### `ANTHROPIC_API_KEY` (可选) anthropic claude Api Key. -### `ANTHROPIC_API_VERSION` (optional) +### `ANTHROPIC_API_VERSION` (可选) anthropic claude Api version. -### `ANTHROPIC_URL` (optional) +### `ANTHROPIC_URL` (可选) anthropic claude Api Url. +### `BAIDU_API_KEY` (可选) + +Baidu Api Key. + +### `BAIDU_SECRET_KEY` (可选) + +Baidu Secret Key. + +### `BAIDU_URL` (可选) + +Baidu Api Url. + +### `BYTEDANCE_API_KEY` (可选) + +ByteDance Api Key. + +### `BYTEDANCE_URL` (可选) + +ByteDance Api Url. + +### `ALIBABA_API_KEY` (可选) + +阿里云（千问）Api Key. + +### `ALIBABA_URL` (可选) + +阿里云（千问）Api Url. + ### `HIDE_USER_API_KEY` （可选） 如果你不想让用户自行填入 API Key，将此环境变量设置为 1 即可。 @@ -156,7 +185,19 @@ anthropic claude Api Url. 用来控制模型列表，使用 `+` 增加一个模型，使用 `-` 来隐藏一个模型，使用 `模型名=展示名` 来自定义模型的展示名，用英文逗号隔开。 +在Azure的模式下，支持使用`modelName@azure=deploymentName`的方式配置模型名称和部署名称(deploy-name) +> 示例：`+gpt-3.5-turbo@azure=gpt35`这个配置会在模型列表显示一个`gpt35(Azure)`的选项 + +在ByteDance的模式下，支持使用`modelName@bytedance=deploymentName`的方式配置模型名称和部署名称(deploy-name) +> 示例: `+Doubao-lite-4k@bytedance=ep-xxxxx-xxx`这个配置会在模型列表显示一个`Doubao-lite-4k(ByteDance)`的选项 + + +### `DEFAULT_MODEL` （可选） + +更改默认模型 + ### `DEFAULT_INPUT_TEMPLATE` （可选） + 自定义默认的 template，用于初始化『设置』中的『用户输入预处理』配置项 ## 开发 diff --git a/app/api/alibaba/[...path]/route.ts b/app/api/alibaba/[...path]/route.ts new file mode 100644 index 000000000..b2c42ac78 --- /dev/null +++ b/app/api/alibaba/[...path]/route.ts @@ -0,0 +1,170 @@ +import { getServerSideConfig } from "@/app/config/server"; +import { + Alibaba, + ALIBABA_BASE_URL, + ApiPath, + ModelProvider, + ServiceProvider, +} from "@/app/constant"; +import { prettyObject } from "@/app/utils/format"; +import { NextRequest, NextResponse } from "next/server"; +import { auth } from "@/app/api/auth"; +import { isModelAvailableInServer } from "@/app/utils/model"; +import type { RequestPayload } from "@/app/client/platforms/openai"; + +const serverConfig = getServerSideConfig(); + +async function handle( + req: NextRequest, + { params }: { params: { path: string[] } }, +) { + console.log("[Alibaba Route] params ", params); + + if (req.method === "OPTIONS") { + return NextResponse.json({ body: "OK" }, { status: 200 }); + } + + const authResult = auth(req, ModelProvider.Qwen); + if (authResult.error) { + return NextResponse.json(authResult, { + status: 401, + }); + } + + try { + const response = await request(req); + return response; + } catch (e) { + console.error("[Alibaba] ", e); + return NextResponse.json(prettyObject(e)); + } +} + +export const GET = handle; +export const POST = handle; + +export const runtime = "edge"; +export const preferredRegion = [ + "arn1", + "bom1", + "cdg1", + "cle1", + "cpt1", + "dub1", + "fra1", + "gru1", + "hnd1", + "iad1", + "icn1", + "kix1", + "lhr1", + "pdx1", + "sfo1", + "sin1", + "syd1", +]; + +async function request(req: NextRequest) { + const controller = new AbortController(); + + // alibaba use base url or just remove the path + let path = `${req.nextUrl.pathname}`.replaceAll(ApiPath.Alibaba, ""); + + let baseUrl = serverConfig.alibabaUrl || ALIBABA_BASE_URL; + + if (!baseUrl.startsWith("http")) { + baseUrl = `https://${baseUrl}`; + } + + if (baseUrl.endsWith("/")) { + baseUrl = baseUrl.slice(0, -1); + } + + console.log("[Proxy] ", path); + console.log("[Base Url]", baseUrl); + + const timeoutId = setTimeout( + () => { + controller.abort(); + }, + 10 * 60 * 1000, + ); + + const fetchUrl = `${baseUrl}${path}`; + + const clonedBody = await req.text(); + + const { messages, model, stream, top_p, ...rest } = JSON.parse( + clonedBody, + ) as RequestPayload; + + const requestBody = { + model, + input: { + messages, + }, + parameters: { + ...rest, + top_p: top_p === 1 ? 0.99 : top_p, // qwen top_p is should be < 1 + result_format: "message", + incremental_output: true, + }, + }; + + const fetchOptions: RequestInit = { + headers: { + "Content-Type": "application/json", + Authorization: req.headers.get("Authorization") ?? "", + "X-DashScope-SSE": stream ? "enable" : "disable", + }, + method: req.method, + body: JSON.stringify(requestBody), + redirect: "manual", + // @ts-ignore + duplex: "half", + signal: controller.signal, + }; + + // #1815 try to refuse some request to some models + if (serverConfig.customModels && req.body) { + try { + // not undefined and is false + if ( + isModelAvailableInServer( + serverConfig.customModels, + model as string, + ServiceProvider.Alibaba as string, + ) + ) { + return NextResponse.json( + { + error: true, + message: `you are not allowed to use ${model} model`, + }, + { + status: 403, + }, + ); + } + } catch (e) { + console.error(`[Alibaba] filter`, e); + } + } + try { + const res = await fetch(fetchUrl, fetchOptions); + + // to prevent browser prompt for credentials + const newHeaders = new Headers(res.headers); + newHeaders.delete("www-authenticate"); + // to disable nginx buffering + newHeaders.set("X-Accel-Buffering", "no"); + + return new Response(res.body, { + status: res.status, + statusText: res.statusText, + headers: newHeaders, + }); + } finally { + clearTimeout(timeoutId); + } +} diff --git a/app/api/anthropic/[...path]/route.ts b/app/api/anthropic/[...path]/route.ts index 4264893d9..78106efa7 100644 --- a/app/api/anthropic/[...path]/route.ts +++ b/app/api/anthropic/[...path]/route.ts @@ -4,12 +4,13 @@ import { Anthropic, ApiPath, DEFAULT_MODELS, + ServiceProvider, ModelProvider, } from "@/app/constant"; import { prettyObject } from "@/app/utils/format"; import { NextRequest, NextResponse } from "next/server"; import { auth } from "../../auth"; -import { collectModelTable } from "@/app/utils/model"; +import { isModelAvailableInServer } from "@/app/utils/model"; const ALLOWD_PATH = new Set([Anthropic.ChatPath, Anthropic.ChatPath1]); @@ -136,17 +137,19 @@ async function request(req: NextRequest) { // #1815 try to refuse some request to some models if (serverConfig.customModels && req.body) { try { - const modelTable = collectModelTable( - DEFAULT_MODELS, - serverConfig.customModels, - ); const clonedBody = await req.text(); fetchOptions.body = clonedBody; const jsonBody = JSON.parse(clonedBody) as { model?: string }; // not undefined and is false - if (modelTable[jsonBody?.model ?? ""].available === false) { + if ( + isModelAvailableInServer( + serverConfig.customModels, + jsonBody?.model as string, + ServiceProvider.Anthropic as string, + ) + ) { return NextResponse.json( { error: true, diff --git a/app/api/auth.ts b/app/api/auth.ts index 0ecb9ad70..09935f94c 100644 --- a/app/api/auth.ts +++ b/app/api/auth.ts @@ -78,9 +78,19 @@ export function auth( case ModelProvider.Claude: systemApiKey = serverConfig.anthropicApiKey; break; + case ModelProvider.Doubao: + systemApiKey = serverConfig.bytedanceApiKey; + break; + case ModelProvider.Ernie: + systemApiKey = serverConfig.baiduApiKey; + break; + case ModelProvider.Qwen: + systemApiKey = serverConfig.alibabaApiKey; + break; case ModelProvider.GPT: default: - if (isAzure) { + // isAzure + if (req.nextUrl.pathname.includes("azure/deployments")) { systemApiKey = serverConfig.azureApiKey; } else { systemApiKey = serverConfig.apiKey; diff --git a/app/api/azure/[...path]/route.ts b/app/api/azure/[...path]/route.ts new file mode 100644 index 000000000..4a17de0c8 --- /dev/null +++ b/app/api/azure/[...path]/route.ts @@ -0,0 +1,57 @@ +import { getServerSideConfig } from "@/app/config/server"; +import { ModelProvider } from "@/app/constant"; +import { prettyObject } from "@/app/utils/format"; +import { NextRequest, NextResponse } from "next/server"; +import { auth } from "../../auth"; +import { requestOpenai } from "../../common"; + +async function handle( + req: NextRequest, + { params }: { params: { path: string[] } }, +) { + console.log("[Azure Route] params ", params); + + if (req.method === "OPTIONS") { + return NextResponse.json({ body: "OK" }, { status: 200 }); + } + + const subpath = params.path.join("/"); + + const authResult = auth(req, ModelProvider.GPT); + if (authResult.error) { + return NextResponse.json(authResult, { + status: 401, + }); + } + + try { + return await requestOpenai(req); + } catch (e) { + console.error("[Azure] ", e); + return NextResponse.json(prettyObject(e)); + } +} + +export const GET = handle; +export const POST = handle; + +export const runtime = "edge"; +export const preferredRegion = [ + "arn1", + "bom1", + "cdg1", + "cle1", + "cpt1", + "dub1", + "fra1", + "gru1", + "hnd1", + "iad1", + "icn1", + "kix1", + "lhr1", + "pdx1", + "sfo1", + "sin1", + "syd1", +]; diff --git a/app/api/baidu/[...path]/route.ts b/app/api/baidu/[...path]/route.ts new file mode 100644 index 000000000..94c9963c7 --- /dev/null +++ b/app/api/baidu/[...path]/route.ts @@ -0,0 +1,169 @@ +import { getServerSideConfig } from "@/app/config/server"; +import { + BAIDU_BASE_URL, + ApiPath, + ModelProvider, + BAIDU_OATUH_URL, + ServiceProvider, +} from "@/app/constant"; +import { prettyObject } from "@/app/utils/format"; +import { NextRequest, NextResponse } from "next/server"; +import { auth } from "@/app/api/auth"; +import { isModelAvailableInServer } from "@/app/utils/model"; +import { getAccessToken } from "@/app/utils/baidu"; + +const serverConfig = getServerSideConfig(); + +async function handle( + req: NextRequest, + { params }: { params: { path: string[] } }, +) { + console.log("[Baidu Route] params ", params); + + if (req.method === "OPTIONS") { + return NextResponse.json({ body: "OK" }, { status: 200 }); + } + + const authResult = auth(req, ModelProvider.Ernie); + if (authResult.error) { + return NextResponse.json(authResult, { + status: 401, + }); + } + + if (!serverConfig.baiduApiKey || !serverConfig.baiduSecretKey) { + return NextResponse.json( + { + error: true, + message: `missing BAIDU_API_KEY or BAIDU_SECRET_KEY in server env vars`, + }, + { + status: 401, + }, + ); + } + + try { + const response = await request(req); + return response; + } catch (e) { + console.error("[Baidu] ", e); + return NextResponse.json(prettyObject(e)); + } +} + +export const GET = handle; +export const POST = handle; + +export const runtime = "edge"; +export const preferredRegion = [ + "arn1", + "bom1", + "cdg1", + "cle1", + "cpt1", + "dub1", + "fra1", + "gru1", + "hnd1", + "iad1", + "icn1", + "kix1", + "lhr1", + "pdx1", + "sfo1", + "sin1", + "syd1", +]; + +async function request(req: NextRequest) { + const controller = new AbortController(); + + let path = `${req.nextUrl.pathname}`.replaceAll(ApiPath.Baidu, ""); + + let baseUrl = serverConfig.baiduUrl || BAIDU_BASE_URL; + + if (!baseUrl.startsWith("http")) { + baseUrl = `https://${baseUrl}`; + } + + if (baseUrl.endsWith("/")) { + baseUrl = baseUrl.slice(0, -1); + } + + console.log("[Proxy] ", path); + console.log("[Base Url]", baseUrl); + + const timeoutId = setTimeout( + () => { + controller.abort(); + }, + 10 * 60 * 1000, + ); + + const { access_token } = await getAccessToken( + serverConfig.baiduApiKey as string, + serverConfig.baiduSecretKey as string, + ); + const fetchUrl = `${baseUrl}${path}?access_token=${access_token}`; + + const fetchOptions: RequestInit = { + headers: { + "Content-Type": "application/json", + }, + method: req.method, + body: req.body, + redirect: "manual", + // @ts-ignore + duplex: "half", + signal: controller.signal, + }; + + // #1815 try to refuse some request to some models + if (serverConfig.customModels && req.body) { + try { + const clonedBody = await req.text(); + fetchOptions.body = clonedBody; + + const jsonBody = JSON.parse(clonedBody) as { model?: string }; + + // not undefined and is false + if ( + isModelAvailableInServer( + serverConfig.customModels, + jsonBody?.model as string, + ServiceProvider.Baidu as string, + ) + ) { + return NextResponse.json( + { + error: true, + message: `you are not allowed to use ${jsonBody?.model} model`, + }, + { + status: 403, + }, + ); + } + } catch (e) { + console.error(`[Baidu] filter`, e); + } + } + try { + const res = await fetch(fetchUrl, fetchOptions); + + // to prevent browser prompt for credentials + const newHeaders = new Headers(res.headers); + newHeaders.delete("www-authenticate"); + // to disable nginx buffering + newHeaders.set("X-Accel-Buffering", "no"); + + return new Response(res.body, { + status: res.status, + statusText: res.statusText, + headers: newHeaders, + }); + } finally { + clearTimeout(timeoutId); + } +} diff --git a/app/api/bytedance/[...path]/route.ts b/app/api/bytedance/[...path]/route.ts new file mode 100644 index 000000000..336c837f0 --- /dev/null +++ b/app/api/bytedance/[...path]/route.ts @@ -0,0 +1,153 @@ +import { getServerSideConfig } from "@/app/config/server"; +import { + BYTEDANCE_BASE_URL, + ApiPath, + ModelProvider, + ServiceProvider, +} from "@/app/constant"; +import { prettyObject } from "@/app/utils/format"; +import { NextRequest, NextResponse } from "next/server"; +import { auth } from "@/app/api/auth"; +import { isModelAvailableInServer } from "@/app/utils/model"; + +const serverConfig = getServerSideConfig(); + +async function handle( + req: NextRequest, + { params }: { params: { path: string[] } }, +) { + console.log("[ByteDance Route] params ", params); + + if (req.method === "OPTIONS") { + return NextResponse.json({ body: "OK" }, { status: 200 }); + } + + const authResult = auth(req, ModelProvider.Doubao); + if (authResult.error) { + return NextResponse.json(authResult, { + status: 401, + }); + } + + try { + const response = await request(req); + return response; + } catch (e) { + console.error("[ByteDance] ", e); + return NextResponse.json(prettyObject(e)); + } +} + +export const GET = handle; +export const POST = handle; + +export const runtime = "edge"; +export const preferredRegion = [ + "arn1", + "bom1", + "cdg1", + "cle1", + "cpt1", + "dub1", + "fra1", + "gru1", + "hnd1", + "iad1", + "icn1", + "kix1", + "lhr1", + "pdx1", + "sfo1", + "sin1", + "syd1", +]; + +async function request(req: NextRequest) { + const controller = new AbortController(); + + let path = `${req.nextUrl.pathname}`.replaceAll(ApiPath.ByteDance, ""); + + let baseUrl = serverConfig.bytedanceUrl || BYTEDANCE_BASE_URL; + + if (!baseUrl.startsWith("http")) { + baseUrl = `https://${baseUrl}`; + } + + if (baseUrl.endsWith("/")) { + baseUrl = baseUrl.slice(0, -1); + } + + console.log("[Proxy] ", path); + console.log("[Base Url]", baseUrl); + + const timeoutId = setTimeout( + () => { + controller.abort(); + }, + 10 * 60 * 1000, + ); + + const fetchUrl = `${baseUrl}${path}`; + + const fetchOptions: RequestInit = { + headers: { + "Content-Type": "application/json", + Authorization: req.headers.get("Authorization") ?? "", + }, + method: req.method, + body: req.body, + redirect: "manual", + // @ts-ignore + duplex: "half", + signal: controller.signal, + }; + + // #1815 try to refuse some request to some models + if (serverConfig.customModels && req.body) { + try { + const clonedBody = await req.text(); + fetchOptions.body = clonedBody; + + const jsonBody = JSON.parse(clonedBody) as { model?: string }; + + // not undefined and is false + if ( + isModelAvailableInServer( + serverConfig.customModels, + jsonBody?.model as string, + ServiceProvider.ByteDance as string, + ) + ) { + return NextResponse.json( + { + error: true, + message: `you are not allowed to use ${jsonBody?.model} model`, + }, + { + status: 403, + }, + ); + } + } catch (e) { + console.error(`[ByteDance] filter`, e); + } + } + + try { + const res = await fetch(fetchUrl, fetchOptions); + + // to prevent browser prompt for credentials + const newHeaders = new Headers(res.headers); + newHeaders.delete("www-authenticate"); + // to disable nginx buffering + newHeaders.set("X-Accel-Buffering", "no"); + + return new Response(res.body, { + status: res.status, + statusText: res.statusText, + headers: newHeaders, + }); + } finally { + clearTimeout(timeoutId); + } +} diff --git a/app/api/common.ts b/app/api/common.ts index 75d4b1bb0..3c4cfd5fc 100644 --- a/app/api/common.ts +++ b/app/api/common.ts @@ -1,13 +1,20 @@ import { NextRequest, NextResponse } from "next/server"; import { getServerSideConfig } from "../config/server"; -import { DEFAULT_MODELS, OPENAI_BASE_URL, GEMINI_BASE_URL } from "../constant"; -import { collectModelTable } from "../utils/model"; -import { makeAzurePath } from "../azure"; +import { + DEFAULT_MODELS, + OPENAI_BASE_URL, + GEMINI_BASE_URL, + ServiceProvider, +} from "../constant"; + +// import { makeAzurePath } from "../azure"; import { getIP } from "@/app/api/auth"; import { getSessionName } from "@/lib/auth"; import prisma from "@/lib/prisma"; import { getTokenLength } from "@/lib/utils"; +import { isModelAvailableInServer } from "../utils/model"; + const serverConfig = getServerSideConfig(); export async function requestOpenai( @@ -18,6 +25,8 @@ export async function requestOpenai( ) { const controller = new AbortController(); + const isAzure = req.nextUrl.pathname.includes("azure/deployments"); + var authValue, authHeaderName = ""; if (isAzure) { @@ -62,6 +71,44 @@ export async function requestOpenai( 10 * 60 * 1000, ); + if (isAzure) { + const azureApiVersion = + req?.nextUrl?.searchParams?.get("api-version") || + serverConfig.azureApiVersion; + baseUrl = baseUrl.split("/deployments").shift() as string; + path = `${req.nextUrl.pathname.replaceAll( + "/api/azure/", + "", + )}?api-version=${azureApiVersion}`; + + // Forward compatibility: + // if display_name(deployment_name) not set, and '{deploy-id}' in AZURE_URL + // then using default '{deploy-id}' + if (serverConfig.customModels && serverConfig.azureUrl) { + const modelName = path.split("/")[1]; + let realDeployName = ""; + serverConfig.customModels + .split(",") + .filter((v) => !!v && !v.startsWith("-") && v.includes(modelName)) + .forEach((m) => { + const [fullName, displayName] = m.split("="); + const [_, providerName] = fullName.split("@"); + if (providerName === "azure" && !displayName) { + const [_, deployId] = (serverConfig?.azureUrl ?? "").split( + "deployments/", + ); + if (deployId) { + realDeployName = deployId; + } + } + }); + if (realDeployName) { + console.log("[Replace with DeployId", realDeployName); + path = path.replaceAll(modelName, realDeployName); + } + } + } + const fetchUrl = `${baseUrl}/${path}`; const fetchOptions: RequestInit = { headers: { @@ -84,13 +131,24 @@ export async function requestOpenai( // #1815 try to refuse some model request if (current_model) { try { - const modelTable = collectModelTable( - DEFAULT_MODELS, - serverConfig.customModels, - ); + const clonedBody = await req.text(); + fetchOptions.body = clonedBody; + + const jsonBody = JSON.parse(clonedBody) as { model?: string }; // not undefined and is false - if (!modelTable[current_model ?? ""].available) { + if ( + isModelAvailableInServer( + serverConfig.customModels, + jsonBody?.model as string, + ServiceProvider.OpenAI as string, + ) || + isModelAvailableInServer( + serverConfig.customModels, + jsonBody?.model as string, + ServiceProvider.Azure as string, + ) + ) { return NextResponse.json( { error: true, diff --git a/app/api/google/[...path]/route.ts b/app/api/google/[...path]/route.ts index ebd192891..81e50538a 100644 --- a/app/api/google/[...path]/route.ts +++ b/app/api/google/[...path]/route.ts @@ -63,7 +63,9 @@ async function handle( ); } - const fetchUrl = `${baseUrl}/${path}?key=${key}`; + const fetchUrl = `${baseUrl}/${path}?key=${key}${ + req?.nextUrl?.searchParams?.get("alt") == "sse" ? "&alt=sse" : "" + }`; const fetchOptions: RequestInit = { headers: { "Content-Type": "application/json", diff --git a/app/azure.ts b/app/azure.ts index 05d686dd7..e69de29bb 100644 --- a/app/azure.ts +++ b/app/azure.ts @@ -1,16 +0,0 @@ -export function makeAzurePath( - path: string, - apiVersion: string, - azureModel?: string, -) { - // should omit /v1 prefix - // path = path.replaceAll("v1/", ""); - path = path.replaceAll( - "v1/chat/completions", - `openai/deployments/${azureModel}/chat/completions`, - ); - // should add api-key to query string - path += `${path.includes("?") ? "&" : "?"}api-version=${apiVersion}`; - - return path; -} diff --git a/app/client/api.ts b/app/client/api.ts index b6002e85b..803b8ac34 100644 --- a/app/client/api.ts +++ b/app/client/api.ts @@ -10,6 +10,10 @@ import { ChatMessage, ModelType, useAccessStore, useChatStore } from "../store"; import { ChatGPTApi } from "./platforms/openai"; import { GeminiProApi } from "./platforms/google"; import { ClaudeApi } from "./platforms/anthropic"; +import { ErnieApi } from "./platforms/baidu"; +import { DoubaoApi } from "./platforms/bytedance"; +import { QwenApi } from "./platforms/alibaba"; + export const ROLES = ["system", "user", "assistant"] as const; export type MessageRole = (typeof ROLES)[number]; @@ -37,6 +41,7 @@ export interface RequestMessage { export interface LLMConfig { model: string; + providerName?: string; temperature?: number; top_p?: number; stream?: boolean; @@ -61,6 +66,7 @@ export interface LLMUsage { export interface LLMModel { name: string; + displayName?: string; describe: string; available: boolean; provider: LLMModelProvider; @@ -110,6 +116,15 @@ export class ClientApi { case ModelProvider.Claude: this.llm = new ClaudeApi(); break; + case ModelProvider.Ernie: + this.llm = new ErnieApi(); + break; + case ModelProvider.Doubao: + this.llm = new DoubaoApi(); + break; + case ModelProvider.Qwen: + this.llm = new QwenApi(); + break; default: this.llm = new ChatGPTApi(); } @@ -163,36 +178,82 @@ export class ClientApi { export function getHeaders(isAzure?: boolean) { const accessStore = useAccessStore.getState(); + const chatStore = useChatStore.getState(); const headers: Record = { "Content-Type": "application/json", Accept: "application/json", }; - const modelConfig = useChatStore.getState().currentSession().mask.modelConfig; - const isGoogle = modelConfig.model.startsWith("gemini"); - // const isAzure = accessStore.provider === ServiceProvider.Azure; - const authHeader = isAzure ? "api-key" : "Authorization"; - const apiKey = isGoogle - ? accessStore.googleApiKey - : isAzure - ? accessStore.azureApiKey - : accessStore.openaiApiKey; - const clientConfig = getClientConfig(); - const makeBearer = (s: string) => `${isAzure ? "" : "Bearer "}${s.trim()}`; - const validString = (x: string) => x && x.length > 0; + const clientConfig = getClientConfig(); + + function getConfig() { + const modelConfig = chatStore.currentSession().mask.modelConfig; + const isGoogle = modelConfig.providerName == ServiceProvider.Google; + // const isAzure = modelConfig.providerName === ServiceProvider.Azure; + const isAnthropic = modelConfig.providerName === ServiceProvider.Anthropic; + const isBaidu = modelConfig.providerName == ServiceProvider.Baidu; + const isByteDance = modelConfig.providerName === ServiceProvider.ByteDance; + const isAlibaba = modelConfig.providerName === ServiceProvider.Alibaba; + const isEnabledAccessControl = accessStore.enabledAccessControl(); + const apiKey = isGoogle + ? accessStore.googleApiKey + : isAzure + ? accessStore.azureApiKey + : isAnthropic + ? accessStore.anthropicApiKey + : isByteDance + ? accessStore.bytedanceApiKey + : isAlibaba + ? accessStore.alibabaApiKey + : accessStore.openaiApiKey; + return { + isGoogle, + isAzure, + isAnthropic, + isBaidu, + isByteDance, + isAlibaba, + apiKey, + isEnabledAccessControl, + }; + } + + function getAuthHeader(): string { + return isAzure ? "api-key" : isAnthropic ? "x-api-key" : "Authorization"; + } + + function getBearerToken(apiKey: string, noBearer: boolean = false): string { + return validString(apiKey) + ? `${noBearer ? "" : "Bearer "}${apiKey.trim()}` + : ""; + } + + function validString(x: string): boolean { + return x?.length > 0; + } + const { + isGoogle, + // isAzure, + isAnthropic, + isBaidu, + apiKey, + isEnabledAccessControl, + } = getConfig(); // when using google api in app, not set auth header - if (!(isGoogle && clientConfig?.isApp)) { - // use user's api key first - if (validString(apiKey)) { - headers[authHeader] = makeBearer(apiKey); - } else if ( - accessStore.enabledAccessControl() && - validString(accessStore.accessCode) - ) { - headers[authHeader] = makeBearer( - ACCESS_CODE_PREFIX + accessStore.accessCode, - ); - } + if (isGoogle && clientConfig?.isApp) return headers; + // when using baidu api in app, not set auth header + if (isBaidu && clientConfig?.isApp) return headers; + + const authHeader = getAuthHeader(); + + const bearerToken = getBearerToken(apiKey, isAzure || isAnthropic); + + if (bearerToken) { + headers[authHeader] = bearerToken; + } else if (isEnabledAccessControl && validString(accessStore.accessCode)) { + headers["Authorization"] = getBearerToken( + ACCESS_CODE_PREFIX + accessStore.accessCode, + ); } if (validString(accessStore.midjourneyProxyUrl)) { @@ -201,6 +262,23 @@ export function getHeaders(isAzure?: boolean) { return headers; } +export function getClientApi(provider: ServiceProvider): ClientApi { + switch (provider) { + case ServiceProvider.Google: + return new ClientApi(ModelProvider.GeminiPro); + case ServiceProvider.Anthropic: + return new ClientApi(ModelProvider.Claude); + case ServiceProvider.Baidu: + return new ClientApi(ModelProvider.Ernie); + case ServiceProvider.ByteDance: + return new ClientApi(ModelProvider.Doubao); + case ServiceProvider.Alibaba: + return new ClientApi(ModelProvider.Qwen); + default: + return new ClientApi(ModelProvider.GPT); + } +} + export function useGetMidjourneySelfProxyUrl(url: string) { const accessStore = useAccessStore.getState(); console.log("useMjImgSelfProxy", accessStore.useMjImgSelfProxy); diff --git a/app/client/platforms/alibaba.ts b/app/client/platforms/alibaba.ts new file mode 100644 index 000000000..72126d728 --- /dev/null +++ b/app/client/platforms/alibaba.ts @@ -0,0 +1,255 @@ +"use client"; +import { + ApiPath, + Alibaba, + ALIBABA_BASE_URL, + REQUEST_TIMEOUT_MS, +} from "@/app/constant"; +import { useAccessStore, useAppConfig, useChatStore } from "@/app/store"; + +import { + ChatOptions, + getHeaders, + LLMApi, + LLMModel, + MultimodalContent, +} from "../api"; +import Locale from "../../locales"; +import { + EventStreamContentType, + fetchEventSource, +} from "@fortaine/fetch-event-source"; +import { prettyObject } from "@/app/utils/format"; +import { getClientConfig } from "@/app/config/client"; +import { getMessageTextContent, isVisionModel } from "@/app/utils"; + +export interface OpenAIListModelResponse { + object: string; + data: Array<{ + id: string; + object: string; + root: string; + }>; +} + +interface RequestPayload { + messages: { + role: "system" | "user" | "assistant"; + content: string | MultimodalContent[]; + }[]; + stream?: boolean; + model: string; + temperature: number; + presence_penalty: number; + frequency_penalty: number; + top_p: number; + max_tokens?: number; +} + +export class QwenApi implements LLMApi { + path(path: string): string { + const accessStore = useAccessStore.getState(); + + let baseUrl = ""; + + if (accessStore.useCustomConfig) { + baseUrl = accessStore.alibabaUrl; + } + + if (baseUrl.length === 0) { + const isApp = !!getClientConfig()?.isApp; + baseUrl = isApp ? ALIBABA_BASE_URL : ApiPath.Alibaba; + } + + if (baseUrl.endsWith("/")) { + baseUrl = baseUrl.slice(0, baseUrl.length - 1); + } + if (!baseUrl.startsWith("http") && !baseUrl.startsWith(ApiPath.Alibaba)) { + baseUrl = "https://" + baseUrl; + } + + console.log("[Proxy Endpoint] ", baseUrl, path); + + return [baseUrl, path].join("/"); + } + + extractMessage(res: any) { + return res?.output?.choices?.at(0)?.message?.content ?? ""; + } + + async chat(options: ChatOptions) { + const messages = options.messages.map((v) => ({ + role: v.role, + content: getMessageTextContent(v), + })); + + const modelConfig = { + ...useAppConfig.getState().modelConfig, + ...useChatStore.getState().currentSession().mask.modelConfig, + ...{ + model: options.config.model, + }, + }; + + const requestPayload: RequestPayload = { + messages, + stream: options.config.stream, + model: modelConfig.model, + temperature: modelConfig.temperature, + presence_penalty: modelConfig.presence_penalty, + frequency_penalty: modelConfig.frequency_penalty, + top_p: modelConfig.top_p, + }; + + const shouldStream = !!options.config.stream; + const controller = new AbortController(); + options.onController?.(controller); + + try { + const chatPath = this.path(Alibaba.ChatPath); + const chatPayload = { + method: "POST", + body: JSON.stringify(requestPayload), + signal: controller.signal, + headers: getHeaders(), + }; + + // make a fetch request + const requestTimeoutId = setTimeout( + () => controller.abort(), + REQUEST_TIMEOUT_MS, + ); + + if (shouldStream) { + let responseText = ""; + let remainText = ""; + let finished = false; + + // animate response to make it looks smooth + function animateResponseText() { + if (finished || controller.signal.aborted) { + responseText += remainText; + console.log("[Response Animation] finished"); + if (responseText?.length === 0) { + options.onError?.(new Error("empty response from server")); + } + return; + } + + if (remainText.length > 0) { + const fetchCount = Math.max(1, Math.round(remainText.length / 60)); + const fetchText = remainText.slice(0, fetchCount); + responseText += fetchText; + remainText = remainText.slice(fetchCount); + options.onUpdate?.(responseText, fetchText); + } + + requestAnimationFrame(animateResponseText); + } + + // start animaion + animateResponseText(); + + const finish = () => { + if (!finished) { + finished = true; + options.onFinish(responseText + remainText); + } + }; + + controller.signal.onabort = finish; + + fetchEventSource(chatPath, { + ...chatPayload, + async onopen(res) { + clearTimeout(requestTimeoutId); + const contentType = res.headers.get("content-type"); + console.log( + "[Alibaba] request response content type: ", + contentType, + ); + + if (contentType?.startsWith("text/plain")) { + responseText = await res.clone().text(); + return finish(); + } + + if ( + !res.ok || + !res.headers + .get("content-type") + ?.startsWith(EventStreamContentType) || + res.status !== 200 + ) { + const responseTexts = [responseText]; + let extraInfo = await res.clone().text(); + try { + const resJson = await res.clone().json(); + extraInfo = prettyObject(resJson); + } catch {} + + if (res.status === 401) { + responseTexts.push(Locale.Error.Unauthorized); + } + + if (extraInfo) { + responseTexts.push(extraInfo); + } + + responseText = responseTexts.join("\n\n"); + + return finish(); + } + }, + onmessage(msg) { + if (msg.data === "[DONE]" || finished) { + return finish(); + } + const text = msg.data; + try { + const json = JSON.parse(text); + const choices = json.output.choices as Array<{ + message: { content: string }; + }>; + const delta = choices[0]?.message?.content; + if (delta) { + remainText += delta; + } + } catch (e) { + console.error("[Request] parse error", text, msg); + } + }, + onclose() { + finish(); + }, + onerror(e) { + options.onError?.(e); + throw e; + }, + openWhenHidden: true, + }); + } else { + const res = await fetch(chatPath, chatPayload); + clearTimeout(requestTimeoutId); + + const resJson = await res.json(); + const message = this.extractMessage(resJson); + options.onFinish(message); + } + } catch (e) { + console.log("[Request] failed to make a chat request", e); + options.onError?.(e as Error); + } + } + async usage() { + return { + used: 0, + total: 0, + }; + } + + async models(): Promise { + return []; + } +} +export { Alibaba }; diff --git a/app/client/platforms/anthropic.ts b/app/client/platforms/anthropic.ts index e90c8f057..460e8f0bc 100644 --- a/app/client/platforms/anthropic.ts +++ b/app/client/platforms/anthropic.ts @@ -1,5 +1,5 @@ import { ACCESS_CODE_PREFIX, Anthropic, ApiPath } from "@/app/constant"; -import { ChatOptions, LLMApi, MultimodalContent } from "../api"; +import { ChatOptions, getHeaders, LLMApi, MultimodalContent } from "../api"; import { useAccessStore, useAppConfig, useChatStore } from "@/app/store"; import { getClientConfig } from "@/app/config/client"; import { DEFAULT_API_HOST } from "@/app/constant"; @@ -190,11 +190,10 @@ export class ClaudeApi implements LLMApi { body: JSON.stringify(requestBody), signal: controller.signal, headers: { - "Content-Type": "application/json", - Accept: "application/json", - "x-api-key": accessStore.anthropicApiKey, + ...getHeaders(), // get common headers "anthropic-version": accessStore.anthropicApiVersion, - Authorization: getAuthKey(accessStore.anthropicApiKey), + // do not send `anthropicApiKey` in browser!!! + // Authorization: getAuthKey(accessStore.anthropicApiKey), }, }; @@ -389,27 +388,3 @@ function trimEnd(s: string, end = " ") { return s; } - -function bearer(value: string) { - return `Bearer ${value.trim()}`; -} - -function getAuthKey(apiKey = "") { - const accessStore = useAccessStore.getState(); - const isApp = !!getClientConfig()?.isApp; - let authKey = ""; - - if (apiKey) { - // use user's api key first - authKey = bearer(apiKey); - } else if ( - accessStore.enabledAccessControl() && - !isApp && - !!accessStore.accessCode - ) { - // or use access code - authKey = bearer(ACCESS_CODE_PREFIX + accessStore.accessCode); - } - - return authKey; -} diff --git a/app/client/platforms/baidu.ts b/app/client/platforms/baidu.ts new file mode 100644 index 000000000..188b78bf9 --- /dev/null +++ b/app/client/platforms/baidu.ts @@ -0,0 +1,273 @@ +"use client"; +import { + ApiPath, + Baidu, + BAIDU_BASE_URL, + REQUEST_TIMEOUT_MS, +} from "@/app/constant"; +import { useAccessStore, useAppConfig, useChatStore } from "@/app/store"; +import { getAccessToken } from "@/app/utils/baidu"; + +import { + ChatOptions, + getHeaders, + LLMApi, + LLMModel, + MultimodalContent, +} from "../api"; +import Locale from "../../locales"; +import { + EventStreamContentType, + fetchEventSource, +} from "@fortaine/fetch-event-source"; +import { prettyObject } from "@/app/utils/format"; +import { getClientConfig } from "@/app/config/client"; +import { getMessageTextContent } from "@/app/utils"; + +export interface OpenAIListModelResponse { + object: string; + data: Array<{ + id: string; + object: string; + root: string; + }>; +} + +interface RequestPayload { + messages: { + role: "system" | "user" | "assistant"; + content: string | MultimodalContent[]; + }[]; + stream?: boolean; + model: string; + temperature: number; + presence_penalty: number; + frequency_penalty: number; + top_p: number; + max_tokens?: number; +} + +export class ErnieApi implements LLMApi { + path(path: string): string { + const accessStore = useAccessStore.getState(); + + let baseUrl = ""; + + if (accessStore.useCustomConfig) { + baseUrl = accessStore.baiduUrl; + } + + if (baseUrl.length === 0) { + const isApp = !!getClientConfig()?.isApp; + // do not use proxy for baidubce api + baseUrl = isApp ? BAIDU_BASE_URL : ApiPath.Baidu; + } + + if (baseUrl.endsWith("/")) { + baseUrl = baseUrl.slice(0, baseUrl.length - 1); + } + if (!baseUrl.startsWith("http") && !baseUrl.startsWith(ApiPath.Baidu)) { + baseUrl = "https://" + baseUrl; + } + + console.log("[Proxy Endpoint] ", baseUrl, path); + + return [baseUrl, path].join("/"); + } + + async chat(options: ChatOptions) { + const messages = options.messages.map((v) => ({ + role: v.role, + content: getMessageTextContent(v), + })); + + // "error_code": 336006, "error_msg": "the length of messages must be an odd number", + if (messages.length % 2 === 0) { + messages.unshift({ + role: "user", + content: " ", + }); + } + + const modelConfig = { + ...useAppConfig.getState().modelConfig, + ...useChatStore.getState().currentSession().mask.modelConfig, + ...{ + model: options.config.model, + }, + }; + + const shouldStream = !!options.config.stream; + const requestPayload: RequestPayload = { + messages, + stream: shouldStream, + model: modelConfig.model, + temperature: modelConfig.temperature, + presence_penalty: modelConfig.presence_penalty, + frequency_penalty: modelConfig.frequency_penalty, + top_p: modelConfig.top_p, + }; + + console.log("[Request] Baidu payload: ", requestPayload); + + const controller = new AbortController(); + options.onController?.(controller); + + try { + let chatPath = this.path(Baidu.ChatPath(modelConfig.model)); + + // getAccessToken can not run in browser, because cors error + if (!!getClientConfig()?.isApp) { + const accessStore = useAccessStore.getState(); + if (accessStore.useCustomConfig) { + if (accessStore.isValidBaidu()) { + const { access_token } = await getAccessToken( + accessStore.baiduApiKey, + accessStore.baiduSecretKey, + ); + chatPath = `${chatPath}${ + chatPath.includes("?") ? "&" : "?" + }access_token=${access_token}`; + } + } + } + const chatPayload = { + method: "POST", + body: JSON.stringify(requestPayload), + signal: controller.signal, + headers: getHeaders(), + }; + + // make a fetch request + const requestTimeoutId = setTimeout( + () => controller.abort(), + REQUEST_TIMEOUT_MS, + ); + + if (shouldStream) { + let responseText = ""; + let remainText = ""; + let finished = false; + + // animate response to make it looks smooth + function animateResponseText() { + if (finished || controller.signal.aborted) { + responseText += remainText; + console.log("[Response Animation] finished"); + if (responseText?.length === 0) { + options.onError?.(new Error("empty response from server")); + } + return; + } + + if (remainText.length > 0) { + const fetchCount = Math.max(1, Math.round(remainText.length / 60)); + const fetchText = remainText.slice(0, fetchCount); + responseText += fetchText; + remainText = remainText.slice(fetchCount); + options.onUpdate?.(responseText, fetchText); + } + + requestAnimationFrame(animateResponseText); + } + + // start animaion + animateResponseText(); + + const finish = () => { + if (!finished) { + finished = true; + options.onFinish(responseText + remainText); + } + }; + + controller.signal.onabort = finish; + + fetchEventSource(chatPath, { + ...chatPayload, + async onopen(res) { + clearTimeout(requestTimeoutId); + const contentType = res.headers.get("content-type"); + console.log("[Baidu] request response content type: ", contentType); + + if (contentType?.startsWith("text/plain")) { + responseText = await res.clone().text(); + return finish(); + } + + if ( + !res.ok || + !res.headers + .get("content-type") + ?.startsWith(EventStreamContentType) || + res.status !== 200 + ) { + const responseTexts = [responseText]; + let extraInfo = await res.clone().text(); + try { + const resJson = await res.clone().json(); + extraInfo = prettyObject(resJson); + } catch {} + + if (res.status === 401) { + responseTexts.push(Locale.Error.Unauthorized); + } + + if (extraInfo) { + responseTexts.push(extraInfo); + } + + responseText = responseTexts.join("\n\n"); + + return finish(); + } + }, + onmessage(msg) { + if (msg.data === "[DONE]" || finished) { + return finish(); + } + const text = msg.data; + try { + const json = JSON.parse(text); + const delta = json?.result; + if (delta) { + remainText += delta; + } + } catch (e) { + console.error("[Request] parse error", text, msg); + } + }, + onclose() { + finish(); + }, + onerror(e) { + options.onError?.(e); + throw e; + }, + openWhenHidden: true, + }); + } else { + const res = await fetch(chatPath, chatPayload); + clearTimeout(requestTimeoutId); + + const resJson = await res.json(); + const message = resJson?.result; + options.onFinish(message); + } + } catch (e) { + console.log("[Request] failed to make a chat request", e); + options.onError?.(e as Error); + } + } + async usage() { + return { + used: 0, + total: 0, + }; + } + + async models(): Promise { + return []; + } +} +export { Baidu }; diff --git a/app/client/platforms/bytedance.ts b/app/client/platforms/bytedance.ts new file mode 100644 index 000000000..7677cafe1 --- /dev/null +++ b/app/client/platforms/bytedance.ts @@ -0,0 +1,255 @@ +"use client"; +import { + ApiPath, + ByteDance, + BYTEDANCE_BASE_URL, + REQUEST_TIMEOUT_MS, +} from "@/app/constant"; +import { useAccessStore, useAppConfig, useChatStore } from "@/app/store"; + +import { + ChatOptions, + getHeaders, + LLMApi, + LLMModel, + MultimodalContent, +} from "../api"; +import Locale from "../../locales"; +import { + EventStreamContentType, + fetchEventSource, +} from "@fortaine/fetch-event-source"; +import { prettyObject } from "@/app/utils/format"; +import { getClientConfig } from "@/app/config/client"; +import { getMessageTextContent } from "@/app/utils"; + +export interface OpenAIListModelResponse { + object: string; + data: Array<{ + id: string; + object: string; + root: string; + }>; +} + +interface RequestPayload { + messages: { + role: "system" | "user" | "assistant"; + content: string | MultimodalContent[]; + }[]; + stream?: boolean; + model: string; + temperature: number; + presence_penalty: number; + frequency_penalty: number; + top_p: number; + max_tokens?: number; +} + +export class DoubaoApi implements LLMApi { + path(path: string): string { + const accessStore = useAccessStore.getState(); + + let baseUrl = ""; + + if (accessStore.useCustomConfig) { + baseUrl = accessStore.bytedanceUrl; + } + + if (baseUrl.length === 0) { + const isApp = !!getClientConfig()?.isApp; + baseUrl = isApp ? BYTEDANCE_BASE_URL : ApiPath.ByteDance; + } + + if (baseUrl.endsWith("/")) { + baseUrl = baseUrl.slice(0, baseUrl.length - 1); + } + if (!baseUrl.startsWith("http") && !baseUrl.startsWith(ApiPath.ByteDance)) { + baseUrl = "https://" + baseUrl; + } + + console.log("[Proxy Endpoint] ", baseUrl, path); + + return [baseUrl, path].join("/"); + } + + extractMessage(res: any) { + return res.choices?.at(0)?.message?.content ?? ""; + } + + async chat(options: ChatOptions) { + const messages = options.messages.map((v) => ({ + role: v.role, + content: getMessageTextContent(v), + })); + + const modelConfig = { + ...useAppConfig.getState().modelConfig, + ...useChatStore.getState().currentSession().mask.modelConfig, + ...{ + model: options.config.model, + }, + }; + + const shouldStream = !!options.config.stream; + const requestPayload: RequestPayload = { + messages, + stream: shouldStream, + model: modelConfig.model, + temperature: modelConfig.temperature, + presence_penalty: modelConfig.presence_penalty, + frequency_penalty: modelConfig.frequency_penalty, + top_p: modelConfig.top_p, + }; + + const controller = new AbortController(); + options.onController?.(controller); + + try { + const chatPath = this.path(ByteDance.ChatPath); + const chatPayload = { + method: "POST", + body: JSON.stringify(requestPayload), + signal: controller.signal, + headers: getHeaders(), + }; + + // make a fetch request + const requestTimeoutId = setTimeout( + () => controller.abort(), + REQUEST_TIMEOUT_MS, + ); + + if (shouldStream) { + let responseText = ""; + let remainText = ""; + let finished = false; + + // animate response to make it looks smooth + function animateResponseText() { + if (finished || controller.signal.aborted) { + responseText += remainText; + console.log("[Response Animation] finished"); + if (responseText?.length === 0) { + options.onError?.(new Error("empty response from server")); + } + return; + } + + if (remainText.length > 0) { + const fetchCount = Math.max(1, Math.round(remainText.length / 60)); + const fetchText = remainText.slice(0, fetchCount); + responseText += fetchText; + remainText = remainText.slice(fetchCount); + options.onUpdate?.(responseText, fetchText); + } + + requestAnimationFrame(animateResponseText); + } + + // start animaion + animateResponseText(); + + const finish = () => { + if (!finished) { + finished = true; + options.onFinish(responseText + remainText); + } + }; + + controller.signal.onabort = finish; + + fetchEventSource(chatPath, { + ...chatPayload, + async onopen(res) { + clearTimeout(requestTimeoutId); + const contentType = res.headers.get("content-type"); + console.log( + "[ByteDance] request response content type: ", + contentType, + ); + + if (contentType?.startsWith("text/plain")) { + responseText = await res.clone().text(); + return finish(); + } + + if ( + !res.ok || + !res.headers + .get("content-type") + ?.startsWith(EventStreamContentType) || + res.status !== 200 + ) { + const responseTexts = [responseText]; + let extraInfo = await res.clone().text(); + try { + const resJson = await res.clone().json(); + extraInfo = prettyObject(resJson); + } catch {} + + if (res.status === 401) { + responseTexts.push(Locale.Error.Unauthorized); + } + + if (extraInfo) { + responseTexts.push(extraInfo); + } + + responseText = responseTexts.join("\n\n"); + + return finish(); + } + }, + onmessage(msg) { + if (msg.data === "[DONE]" || finished) { + return finish(); + } + const text = msg.data; + try { + const json = JSON.parse(text); + const choices = json.choices as Array<{ + delta: { content: string }; + }>; + const delta = choices[0]?.delta?.content; + if (delta) { + remainText += delta; + } + } catch (e) { + console.error("[Request] parse error", text, msg); + } + }, + onclose() { + finish(); + }, + onerror(e) { + options.onError?.(e); + throw e; + }, + openWhenHidden: true, + }); + } else { + const res = await fetch(chatPath, chatPayload); + clearTimeout(requestTimeoutId); + + const resJson = await res.json(); + const message = this.extractMessage(resJson); + options.onFinish(message); + } + } catch (e) { + console.log("[Request] failed to make a chat request", e); + options.onError?.(e as Error); + } + } + async usage() { + return { + used: 0, + total: 0, + }; + } + + async models(): Promise { + return []; + } +} +export { ByteDance }; diff --git a/app/client/platforms/google.ts b/app/client/platforms/google.ts index b6eb8d3df..828b28a0d 100644 --- a/app/client/platforms/google.ts +++ b/app/client/platforms/google.ts @@ -3,6 +3,12 @@ import { ChatOptions, getHeaders, LLMApi, LLMModel, LLMUsage } from "../api"; import { useAccessStore, useAppConfig, useChatStore } from "@/app/store"; import { getClientConfig } from "@/app/config/client"; import { DEFAULT_API_HOST } from "@/app/constant"; +import Locale from "../../locales"; +import { + EventStreamContentType, + fetchEventSource, +} from "@fortaine/fetch-event-source"; +import { prettyObject } from "@/app/utils/format"; import { getMessageTextContent, getMessageImages, @@ -20,7 +26,7 @@ export class GeminiProApi implements LLMApi { ); } async chat(options: ChatOptions): Promise { - // const apiClient = this; + const apiClient = this; let multimodal = false; const messages = options.messages.map((v) => { let parts: any[] = [{ text: getMessageTextContent(v) }]; @@ -147,10 +153,11 @@ export class GeminiProApi implements LLMApi { let remainText = ""; let finished = false; - let existingTexts: string[] = []; const finish = () => { - finished = true; - options.onFinish(existingTexts.join("")); + if (!finished) { + finished = true; + options.onFinish(responseText + remainText); + } }; // animate response to make it looks smooth @@ -175,72 +182,85 @@ export class GeminiProApi implements LLMApi { // start animaion animateResponseText(); - fetch( - baseUrl.replace("generateContent", "streamGenerateContent"), - chatPayload, - ) - .then((response) => { - const reader = response?.body?.getReader(); - const decoder = new TextDecoder(); - let partialData = ""; + controller.signal.onabort = finish; - return reader?.read().then(function processText({ - done, - value, - }): Promise { - if (done) { - if (response.status !== 200) { - try { - let data = JSON.parse(ensureProperEnding(partialData)); - if (data && data[0].error) { - options.onError?.(new Error(data[0].error.message)); - } else { - options.onError?.(new Error("Request failed")); - } - } catch (_) { - options.onError?.(new Error("Request failed")); - } - } + // https://github.com/google-gemini/cookbook/blob/main/quickstarts/rest/Streaming_REST.ipynb + const chatPath = + baseUrl.replace("generateContent", "streamGenerateContent") + + (baseUrl.indexOf("?") > -1 ? "&alt=sse" : "?alt=sse"); + fetchEventSource(chatPath, { + ...chatPayload, + async onopen(res) { + clearTimeout(requestTimeoutId); + const contentType = res.headers.get("content-type"); + console.log( + "[Gemini] request response content type: ", + contentType, + ); - console.log("Stream complete"); - // options.onFinish(responseText + remainText); - finished = true; - return Promise.resolve(); - } - - partialData += decoder.decode(value, { stream: true }); + if (contentType?.startsWith("text/plain")) { + responseText = await res.clone().text(); + return finish(); + } + if ( + !res.ok || + !res.headers + .get("content-type") + ?.startsWith(EventStreamContentType) || + res.status !== 200 + ) { + const responseTexts = [responseText]; + let extraInfo = await res.clone().text(); try { - let data = JSON.parse(ensureProperEnding(partialData)); + const resJson = await res.clone().json(); + extraInfo = prettyObject(resJson); + } catch {} - const textArray = data.reduce( - (acc: string[], item: { candidates: any[] }) => { - const texts = item.candidates.map((candidate) => - candidate.content.parts - .map((part: { text: any }) => part.text) - .join(""), - ); - return acc.concat(texts); - }, - [], - ); - - if (textArray.length > existingTexts.length) { - const deltaArray = textArray.slice(existingTexts.length); - existingTexts = textArray; - remainText += deltaArray.join(""); - } - } catch (error) { - // console.log("[Response Animation] error: ", error,partialData); - // skip error message when parsing json + if (res.status === 401) { + responseTexts.push(Locale.Error.Unauthorized); } - return reader.read().then(processText); - }); - }) - .catch((error) => { - console.error("Error:", error); - }); + if (extraInfo) { + responseTexts.push(extraInfo); + } + + responseText = responseTexts.join("\n\n"); + + return finish(); + } + }, + onmessage(msg) { + if (msg.data === "[DONE]" || finished) { + return finish(); + } + const text = msg.data; + try { + const json = JSON.parse(text); + const delta = apiClient.extractMessage(json); + + if (delta) { + remainText += delta; + } + + const blockReason = json?.promptFeedback?.blockReason; + if (blockReason) { + // being blocked + console.log(`[Google] [Safety Ratings] result:`, blockReason); + } + } catch (e) { + console.error("[Request] parse error", text, msg); + } + }, + onclose() { + finish(); + }, + onerror(e) { + options.onError?.(e); + throw e; + }, + openWhenHidden: true, + }); } else { const res = await fetch(baseUrl, chatPayload); clearTimeout(requestTimeoutId); @@ -254,7 +274,7 @@ export class GeminiProApi implements LLMApi { ), ); } - const message = this.extractMessage(resJson); + const message = apiClient.extractMessage(resJson); options.onFinish(message); } } catch (e) { diff --git a/app/client/platforms/openai.ts b/app/client/platforms/openai.ts index 76f2aeacf..29facd7bc 100644 --- a/app/client/platforms/openai.ts +++ b/app/client/platforms/openai.ts @@ -1,14 +1,17 @@ "use client"; +// azure and openai, using same models. so using same LLMApi. import { ApiPath, AZURE_MODELS, DEFAULT_API_HOST, DEFAULT_MODELS, OpenaiPath, + Azure, REQUEST_TIMEOUT_MS, ServiceProvider, } from "@/app/constant"; import { useAccessStore, useAppConfig, useChatStore } from "@/app/store"; +import { collectModelsWithDefaultModel } from "@/app/utils/model"; import { ChatOptions, @@ -25,7 +28,6 @@ import { } from "@fortaine/fetch-event-source"; import { prettyObject } from "@/app/utils/format"; import { getClientConfig } from "@/app/config/client"; -import { makeAzurePath } from "@/app/azure"; import { getMessageTextContent, getMessageImages, @@ -41,7 +43,7 @@ export interface OpenAIListModelResponse { }>; } -interface RequestPayload { +export interface RequestPayload { messages: { role: "system" | "user" | "assistant"; content: string | MultimodalContent[]; @@ -63,33 +65,31 @@ export class ChatGPTApi implements LLMApi { let baseUrl = ""; + // const isAzure = path.includes("deployments"); if (accessStore.useCustomConfig) { - // const isAzure = accessStore.provider === ServiceProvider.Azure; - if (isAzure && !accessStore.isValidAzure()) { throw Error( "incomplete azure config, please check it in your settings page", ); } - if (isAzure) { - path = makeAzurePath(path, accessStore.azureApiVersion); - } - baseUrl = isAzure ? accessStore.azureUrl : accessStore.openaiUrl; } if (baseUrl.length === 0) { const isApp = !!getClientConfig()?.isApp; - baseUrl = isApp - ? DEFAULT_API_HOST + "/proxy" + ApiPath.OpenAI - : ApiPath.OpenAI; + const apiPath = isAzure ? ApiPath.Azure : ApiPath.OpenAI; + baseUrl = isApp ? DEFAULT_API_HOST + "/proxy" + apiPath : apiPath; } if (baseUrl.endsWith("/")) { baseUrl = baseUrl.slice(0, baseUrl.length - 1); } - if (!baseUrl.startsWith("http") && !baseUrl.startsWith(ApiPath.OpenAI)) { + if ( + !baseUrl.startsWith("http") && + !isAzure && + !baseUrl.startsWith(ApiPath.OpenAI) + ) { baseUrl = "https://" + baseUrl; } @@ -118,6 +118,7 @@ export class ChatGPTApi implements LLMApi { ...useChatStore.getState().currentSession().mask.modelConfig, ...{ model: options.config.model, + providerName: options.config.providerName, }, }; const is_azure = AZURE_MODELS.includes(modelConfig.model); @@ -146,11 +147,37 @@ export class ChatGPTApi implements LLMApi { options.onController?.(controller); try { - let chatPath = this.path( - OpenaiPath.ChatPath, - is_azure, - modelConfig.model, - ); + let chatPath = ""; + if (modelConfig.providerName === ServiceProvider.Azure) { + // find model, and get displayName as deployName + const { models: configModels, customModels: configCustomModels } = + useAppConfig.getState(); + const { + defaultModel, + customModels: accessCustomModels, + useCustomConfig, + } = useAccessStore.getState(); + const models = collectModelsWithDefaultModel( + configModels, + [configCustomModels, accessCustomModels].join(","), + defaultModel, + ); + const model = models.find( + (model) => + model.name === modelConfig.model && + model?.provider?.providerName === ServiceProvider.Azure, + ); + chatPath = this.path( + Azure.ChatPath( + (model?.displayName ?? model?.name) as string, + useCustomConfig ? useAccessStore.getState().azureApiVersion : "", + ), + is_azure, + modelConfig.model, + ); + } else { + chatPath = this.path(OpenaiPath.ChatPath); + } const chatPayload = { method: "POST", body: JSON.stringify(requestPayload), diff --git a/app/components/chat.tsx b/app/components/chat.tsx index 6c08f147f..0bfc684c5 100644 --- a/app/components/chat.tsx +++ b/app/components/chat.tsx @@ -92,6 +92,7 @@ import { Path, REQUEST_TIMEOUT_MS, UNFINISHED_INPUT, + ServiceProvider, } from "../constant"; import { Avatar } from "./emoji"; // import { ContextPrompts, MaskAvatar, MaskConfig } from "./mask"; @@ -471,6 +472,9 @@ export function ChatActions(props: { // switch model const currentModel = chatStore.currentSession().mask.modelConfig.model; + const currentProviderName = + chatStore.currentSession().mask.modelConfig?.providerName || + ServiceProvider.OpenAI; const allModels = useAllModels(); const models = useMemo(() => { const filteredModels = allModels.filter((m) => m.available); @@ -486,6 +490,14 @@ export function ChatActions(props: { return filteredModels; } }, [allModels]); + const currentModelName = useMemo(() => { + const model = models.find( + (m) => + m.name == currentModel && + m?.provider?.providerName == currentProviderName, + ); + return model?.displayName ?? ""; + }, [models, currentModel, currentProviderName]); const [showModelSelector, setShowModelSelector] = useState(false); const [showUploadImage, setShowUploadImage] = useState(false); const current_day_token = localStorage.getItem("current_day_token") ?? ""; @@ -503,13 +515,17 @@ export function ChatActions(props: { const isUnavaliableModel = !models.some((m) => m.name === currentModel); if (isUnavaliableModel && models.length > 0) { // show next model to default model if exist - let nextModel: ModelType = ( - models.find((model) => model.isDefault) || models[0] - ).name as ModelType; - chatStore.updateCurrentSession( - (session) => (session.mask.modelConfig.model = nextModel), + let nextModel = models.find((model) => model.isDefault) || models[0]; + chatStore.updateCurrentSession((session) => { + session.mask.modelConfig.model = nextModel.name; + session.mask.modelConfig.providerName = nextModel?.provider + ?.providerName as ServiceProvider; + }); + showToast( + nextModel?.provider?.providerName == "ByteDance" + ? nextModel.displayName + : nextModel.name, ); - showToast(nextModel); } // eslint-disable-next-line react-hooks/exhaustive-deps }, [chatStore, currentModel, models]); @@ -592,7 +608,7 @@ export function ChatActions(props: { setShowModelSelector(true)} - text={currentModel} + text={currentModelName} icon={} /> @@ -610,21 +626,36 @@ export function ChatActions(props: { {/*/>*/} {showModelSelector && ( - ({ - title: m.displayName, + title: `${m.displayName}${ + m?.provider?.providerName + ? "(" + m?.provider?.providerName + ")" + : "" + }`, subTitle: m.describe, - value: m.name, + value: `${m.name}@${m?.provider?.providerName}`, }))} onClose={() => setShowModelSelector(false)} onSelection={(s) => { if (s.length === 0) return; + const [model, providerName] = s[0].split("@"); chatStore.updateCurrentSession((session) => { - session.mask.modelConfig.model = s[0] as ModelType; + session.mask.modelConfig.model = model as ModelType; + session.mask.modelConfig.providerName = + providerName as ServiceProvider; session.mask.syncGlobalConfig = false; }); - showToast(s[0]); + if (providerName == "ByteDance") { + const selectedModel = models.find( + (m) => + m.name == model && m?.provider?.providerName == providerName, + ); + showToast(selectedModel?.displayName ?? ""); + } else { + showToast(model); + } }} /> )} diff --git a/app/components/exporter.tsx b/app/components/exporter.tsx index 20e240d93..948807d4c 100644 --- a/app/components/exporter.tsx +++ b/app/components/exporter.tsx @@ -36,11 +36,10 @@ import { toBlob, toPng } from "html-to-image"; import { DEFAULT_MASK_AVATAR } from "../store/mask"; import { prettyObject } from "../utils/format"; -import { EXPORT_MESSAGE_CLASS_NAME, ModelProvider } from "../constant"; +import { EXPORT_MESSAGE_CLASS_NAME } from "../constant"; import { getClientConfig } from "../config/client"; -import { ClientApi } from "../client/api"; +import { type ClientApi, getClientApi } from "../client/api"; import { getMessageTextContent } from "../utils"; -import { identifyDefaultClaudeModel } from "../utils/checkers"; const Markdown = dynamic(async () => (await import("./markdown")).Markdown, { loading: () => , @@ -313,14 +312,7 @@ export function PreviewActions(props: { const onRenderMsgs = (msgs: ChatMessage[]) => { setShouldExport(false); - var api: ClientApi; - if (config.modelConfig.model.startsWith("gemini")) { - api = new ClientApi(ModelProvider.GeminiPro); - } else if (identifyDefaultClaudeModel(config.modelConfig.model)) { - api = new ClientApi(ModelProvider.Claude); - } else { - api = new ClientApi(ModelProvider.GPT); - } + const api: ClientApi = getClientApi(config.modelConfig.providerName); api .share(msgs) diff --git a/app/components/home.tsx b/app/components/home.tsx index ca1841142..19f83e5df 100644 --- a/app/components/home.tsx +++ b/app/components/home.tsx @@ -12,7 +12,7 @@ import LoadingIcon from "../icons/three-dots.svg"; import { getCSSVar, useMobileScreen } from "../utils"; import dynamic from "next/dynamic"; -import { ModelProvider, Path, SlotID } from "../constant"; +import { Path, SlotID } from "../constant"; import { ErrorBoundary } from "./error"; import { getISOLang, getLang } from "../locales"; @@ -27,9 +27,8 @@ import { SideBar } from "./sidebar"; import { useAppConfig } from "@/app/store"; import { AuthPage } from "./auth"; import { getClientConfig } from "../config/client"; -import { ClientApi } from "../client/api"; +import { type ClientApi, getClientApi } from "../client/api"; import { useAccessStore } from "../store"; -import { identifyDefaultClaudeModel } from "../utils/checkers"; export function Loading(props: { noLogo?: boolean }) { return ( @@ -174,14 +173,8 @@ function Screen() { export function useLoadData() { const config = useAppConfig(); - var api: ClientApi; - if (config.modelConfig.model.startsWith("gemini")) { - api = new ClientApi(ModelProvider.GeminiPro); - } else if (identifyDefaultClaudeModel(config.modelConfig.model)) { - api = new ClientApi(ModelProvider.Claude); - } else { - api = new ClientApi(ModelProvider.GPT); - } + const api: ClientApi = getClientApi(config.modelConfig.providerName); + useEffect(() => { (async () => { const models = await api.llm.models(); diff --git a/app/components/model-config.tsx b/app/components/model-config.tsx index e46a018f4..346fd3a71 100644 --- a/app/components/model-config.tsx +++ b/app/components/model-config.tsx @@ -1,3 +1,4 @@ +import { ServiceProvider } from "@/app/constant"; import { ModalConfigValidator, ModelConfig } from "../store"; import Locale from "../locales"; @@ -10,25 +11,25 @@ export function ModelConfigList(props: { updateConfig: (updater: (config: ModelConfig) => void) => void; }) { const allModels = useAllModels(); + const value = `${props.modelConfig.model}@${props.modelConfig?.providerName}`; return ( <>