diff --git a/README.md b/README.md index 9696899ff..ab2d86e61 100644 --- a/README.md +++ b/README.md @@ -352,6 +352,22 @@ For ByteDance: use `modelName@bytedance=deploymentName` to customize model name 配置所有模型都使用 OpenAI 路由，在使用类似 `one-api` 的中转项目时会很有用 将此环境变量设置为 1 即可 +### `DEEPSEEK_API_KEY` (可选) + +DeepSeek Api Key. + +### `DEEPSEEK_URL` (可选) + +DeepSeek Api Url. + +### `SILICONFLOW_API_KEY` （可选） + +硅基流动 API Key. + +### `SILICONFLOW_URL` （可选） + +硅基流动 API URL. + ## 部署 ### 容器部署 （推荐） diff --git a/app/api/[provider]/[...path]/route.ts b/app/api/[provider]/[...path]/route.ts index 3017fd371..8975bf971 100644 --- a/app/api/[provider]/[...path]/route.ts +++ b/app/api/[provider]/[...path]/route.ts @@ -10,6 +10,8 @@ import { handle as alibabaHandler } from "../../alibaba"; import { handle as moonshotHandler } from "../../moonshot"; import { handle as stabilityHandler } from "../../stability"; import { handle as iflytekHandler } from "../../iflytek"; +import { handle as deepseekHandler } from "../../deepseek"; +import { handle as siliconflowHandler } from "../../siliconflow"; import { handle as xaiHandler } from "../../xai"; import { handle as chatglmHandler } from "../../glm"; import { handle as proxyHandler } from "../../proxy"; @@ -40,10 +42,14 @@ async function handle( return stabilityHandler(req, { params }); case ApiPath.Iflytek: return iflytekHandler(req, { params }); + case ApiPath.DeepSeek: + return deepseekHandler(req, { params }); case ApiPath.XAI: return xaiHandler(req, { params }); case ApiPath.ChatGLM: return chatglmHandler(req, { params }); + case ApiPath.SiliconFlow: + return siliconflowHandler(req, { params }); case ApiPath.OpenAI: return openaiHandler(req, { params }); default: diff --git a/app/api/auth.ts b/app/api/auth.ts index 0383ffe15..8c78c70c8 100644 --- a/app/api/auth.ts +++ b/app/api/auth.ts @@ -28,12 +28,7 @@ export function auth(req: NextRequest, modelProvider: ModelProvider) { const authToken = req.headers.get("Authorization") ?? ""; // check if it is openai api key or user token - let { accessCode, apiKey } = parseApiKey(authToken); - - if (modelProvider === ModelProvider.GeminiPro) { - const googleAuthToken = req.headers.get("x-goog-api-key") ?? ""; - apiKey = googleAuthToken.trim().replaceAll("Bearer ", "").trim(); - } + const { accessCode, apiKey } = parseApiKey(authToken); const hashedCode = md5.hash(accessCode ?? "").trim(); @@ -72,6 +67,9 @@ export function auth(req: NextRequest, modelProvider: ModelProvider) { let systemApiKey: string | undefined; switch (modelProvider) { + case ModelProvider.Stability: + systemApiKey = serverConfig.stabilityApiKey; + break; case ModelProvider.GeminiPro: systemApiKey = serverConfig.googleApiKey; break; @@ -87,6 +85,25 @@ export function auth(req: NextRequest, modelProvider: ModelProvider) { case ModelProvider.Qwen: systemApiKey = serverConfig.alibabaApiKey; break; + case ModelProvider.Moonshot: + systemApiKey = serverConfig.moonshotApiKey; + break; + case ModelProvider.Iflytek: + systemApiKey = + serverConfig.iflytekApiKey + ":" + serverConfig.iflytekApiSecret; + break; + case ModelProvider.DeepSeek: + systemApiKey = serverConfig.deepseekApiKey; + break; + case ModelProvider.XAI: + systemApiKey = serverConfig.xaiApiKey; + break; + case ModelProvider.ChatGLM: + systemApiKey = serverConfig.chatglmApiKey; + break; + case ModelProvider.SiliconFlow: + systemApiKey = serverConfig.siliconFlowApiKey; + break; case ModelProvider.GPT: default: if (req.nextUrl.pathname.includes("azure/deployments")) { @@ -110,54 +127,3 @@ export function auth(req: NextRequest, modelProvider: ModelProvider) { error: false, }; } - -export function googleAuth(req: NextRequest) { - const authToken = req.headers.get("Authorization") ?? ""; - - // check if it is openai api key or user token - const { accessCode, apiKey } = parseApiKey(authToken); - - const hashedCode = md5.hash(accessCode ?? "").trim(); - - const serverConfig = getServerSideConfig(); - console.log("[Auth] allowed hashed codes: ", [...serverConfig.codes]); - console.log("[Auth] got access code:", accessCode); - console.log("[Auth] hashed access code:", hashedCode); - console.log("[User IP] ", getIP(req)); - console.log("[Time] ", new Date().toLocaleString()); - - if (serverConfig.needCode && !serverConfig.codes.has(hashedCode) && !apiKey) { - return { - error: true, - msg: !accessCode ? "empty access code" : "wrong access code", - }; - } - - if (serverConfig.hideUserApiKey && !!apiKey) { - return { - error: true, - msg: "you are not allowed to access openai with your own api key", - }; - } - - // if user does not provide an api key, inject system api key - if (!apiKey) { - const serverApiKey = serverConfig.googleApiKey; - - if (serverApiKey) { - console.log("[Auth] use system api key"); - req.headers.set( - "Authorization", - `${serverConfig.isAzure ? "" : "Bearer "}${serverApiKey}`, - ); - } else { - console.log("[Auth] admin did not provide an api key"); - } - } else { - console.log("[Auth] use user api key"); - } - - return { - error: false, - }; -} diff --git a/app/api/deepseek.ts b/app/api/deepseek.ts new file mode 100644 index 000000000..a9879eced --- /dev/null +++ b/app/api/deepseek.ts @@ -0,0 +1,128 @@ +import { getServerSideConfig } from "@/app/config/server"; +import { + DEEPSEEK_BASE_URL, + ApiPath, + ModelProvider, + ServiceProvider, +} from "@/app/constant"; +import { prettyObject } from "@/app/utils/format"; +import { NextRequest, NextResponse } from "next/server"; +import { auth } from "@/app/api/auth"; +import { isModelNotavailableInServer } from "@/app/utils/model"; + +const serverConfig = getServerSideConfig(); + +export async function handle( + req: NextRequest, + { params }: { params: { path: string[] } }, +) { + console.log("[DeepSeek Route] params ", params); + + if (req.method === "OPTIONS") { + return NextResponse.json({ body: "OK" }, { status: 200 }); + } + + const authResult = auth(req, ModelProvider.DeepSeek); + if (authResult.error) { + return NextResponse.json(authResult, { + status: 401, + }); + } + + try { + const response = await request(req); + return response; + } catch (e) { + console.error("[DeepSeek] ", e); + return NextResponse.json(prettyObject(e)); + } +} + +async function request(req: NextRequest) { + const controller = new AbortController(); + + // alibaba use base url or just remove the path + let path = `${req.nextUrl.pathname}`.replaceAll(ApiPath.DeepSeek, ""); + + let baseUrl = serverConfig.deepseekUrl || DEEPSEEK_BASE_URL; + + if (!baseUrl.startsWith("http")) { + baseUrl = `https://${baseUrl}`; + } + + if (baseUrl.endsWith("/")) { + baseUrl = baseUrl.slice(0, -1); + } + + console.log("[Proxy] ", path); + console.log("[Base Url]", baseUrl); + + const timeoutId = setTimeout( + () => { + controller.abort(); + }, + 10 * 60 * 1000, + ); + + const fetchUrl = `${baseUrl}${path}`; + const fetchOptions: RequestInit = { + headers: { + "Content-Type": "application/json", + Authorization: req.headers.get("Authorization") ?? "", + }, + method: req.method, + body: req.body, + redirect: "manual", + // @ts-ignore + duplex: "half", + signal: controller.signal, + }; + + // #1815 try to refuse some request to some models + if (serverConfig.customModels && req.body) { + try { + const clonedBody = await req.text(); + fetchOptions.body = clonedBody; + + const jsonBody = JSON.parse(clonedBody) as { model?: string }; + + // not undefined and is false + if ( + isModelNotavailableInServer( + serverConfig.customModels, + jsonBody?.model as string, + ServiceProvider.DeepSeek as string, + ) + ) { + return NextResponse.json( + { + error: true, + message: `you are not allowed to use ${jsonBody?.model} model`, + }, + { + status: 403, + }, + ); + } + } catch (e) { + console.error(`[DeepSeek] filter`, e); + } + } + try { + const res = await fetch(fetchUrl, fetchOptions); + + // to prevent browser prompt for credentials + const newHeaders = new Headers(res.headers); + newHeaders.delete("www-authenticate"); + // to disable nginx buffering + newHeaders.set("X-Accel-Buffering", "no"); + + return new Response(res.body, { + status: res.status, + statusText: res.statusText, + headers: newHeaders, + }); + } finally { + clearTimeout(timeoutId); + } +} diff --git a/app/api/siliconflow.ts b/app/api/siliconflow.ts new file mode 100644 index 000000000..e298a21d4 --- /dev/null +++ b/app/api/siliconflow.ts @@ -0,0 +1,128 @@ +import { getServerSideConfig } from "@/app/config/server"; +import { + SILICONFLOW_BASE_URL, + ApiPath, + ModelProvider, + ServiceProvider, +} from "@/app/constant"; +import { prettyObject } from "@/app/utils/format"; +import { NextRequest, NextResponse } from "next/server"; +import { auth } from "@/app/api/auth"; +import { isModelNotavailableInServer } from "@/app/utils/model"; + +const serverConfig = getServerSideConfig(); + +export async function handle( + req: NextRequest, + { params }: { params: { path: string[] } }, +) { + console.log("[SiliconFlow Route] params ", params); + + if (req.method === "OPTIONS") { + return NextResponse.json({ body: "OK" }, { status: 200 }); + } + + const authResult = auth(req, ModelProvider.SiliconFlow); + if (authResult.error) { + return NextResponse.json(authResult, { + status: 401, + }); + } + + try { + const response = await request(req); + return response; + } catch (e) { + console.error("[SiliconFlow] ", e); + return NextResponse.json(prettyObject(e)); + } +} + +async function request(req: NextRequest) { + const controller = new AbortController(); + + // alibaba use base url or just remove the path + let path = `${req.nextUrl.pathname}`.replaceAll(ApiPath.SiliconFlow, ""); + + let baseUrl = serverConfig.siliconFlowUrl || SILICONFLOW_BASE_URL; + + if (!baseUrl.startsWith("http")) { + baseUrl = `https://${baseUrl}`; + } + + if (baseUrl.endsWith("/")) { + baseUrl = baseUrl.slice(0, -1); + } + + console.log("[Proxy] ", path); + console.log("[Base Url]", baseUrl); + + const timeoutId = setTimeout( + () => { + controller.abort(); + }, + 10 * 60 * 1000, + ); + + const fetchUrl = `${baseUrl}${path}`; + const fetchOptions: RequestInit = { + headers: { + "Content-Type": "application/json", + Authorization: req.headers.get("Authorization") ?? "", + }, + method: req.method, + body: req.body, + redirect: "manual", + // @ts-ignore + duplex: "half", + signal: controller.signal, + }; + + // #1815 try to refuse some request to some models + if (serverConfig.customModels && req.body) { + try { + const clonedBody = await req.text(); + fetchOptions.body = clonedBody; + + const jsonBody = JSON.parse(clonedBody) as { model?: string }; + + // not undefined and is false + if ( + isModelNotavailableInServer( + serverConfig.customModels, + jsonBody?.model as string, + ServiceProvider.SiliconFlow as string, + ) + ) { + return NextResponse.json( + { + error: true, + message: `you are not allowed to use ${jsonBody?.model} model`, + }, + { + status: 403, + }, + ); + } + } catch (e) { + console.error(`[SiliconFlow] filter`, e); + } + } + try { + const res = await fetch(fetchUrl, fetchOptions); + + // to prevent browser prompt for credentials + const newHeaders = new Headers(res.headers); + newHeaders.delete("www-authenticate"); + // to disable nginx buffering + newHeaders.set("X-Accel-Buffering", "no"); + + return new Response(res.body, { + status: res.status, + statusText: res.statusText, + headers: newHeaders, + }); + } finally { + clearTimeout(timeoutId); + } +} diff --git a/app/client/api.ts b/app/client/api.ts index ceb086711..4937d179a 100644 --- a/app/client/api.ts +++ b/app/client/api.ts @@ -21,8 +21,10 @@ import { QwenApi } from "./platforms/alibaba"; import { HunyuanApi } from "./platforms/tencent"; import { MoonshotApi } from "./platforms/moonshot"; import { SparkApi } from "./platforms/iflytek"; +import { DeepSeekApi } from "./platforms/deepseek"; import { XAIApi } from "./platforms/xai"; import { ChatGLMApi } from "./platforms/glm"; +import { SiliconflowApi } from "./platforms/siliconflow"; export const ROLES = ["system", "user", "assistant"] as const; export type MessageRole = (typeof ROLES)[number]; @@ -204,12 +206,18 @@ export class ClientApi { case ModelProvider.Iflytek: this.llm = new SparkApi(); break; + case ModelProvider.DeepSeek: + this.llm = new DeepSeekApi(); + break; case ModelProvider.XAI: this.llm = new XAIApi(); break; case ModelProvider.ChatGLM: this.llm = new ChatGLMApi(); break; + case ModelProvider.SiliconFlow: + this.llm = new SiliconflowApi(); + break; default: this.llm = new ChatGPTApi(); } @@ -298,8 +306,11 @@ export function getHeaders(ignoreHeaders: boolean = false) { const isAlibaba = modelConfig.providerName === ServiceProvider.Alibaba; const isMoonshot = modelConfig.providerName === ServiceProvider.Moonshot; const isIflytek = modelConfig.providerName === ServiceProvider.Iflytek; + const isDeepSeek = modelConfig.providerName === ServiceProvider.DeepSeek; const isXAI = modelConfig.providerName === ServiceProvider.XAI; const isChatGLM = modelConfig.providerName === ServiceProvider.ChatGLM; + const isSiliconFlow = + modelConfig.providerName === ServiceProvider.SiliconFlow; const isEnabledAccessControl = accessStore.enabledAccessControl(); const apiKey = isGoogle ? accessStore.googleApiKey @@ -315,16 +326,20 @@ export function getHeaders(ignoreHeaders: boolean = false) { ? accessStore.moonshotApiKey : isXAI ? accessStore.xaiApiKey - : isChatGLM - ? accessStore.chatglmApiKey - : isIflytek - ? accessStore.iflytekApiKey && - accessStore.iflytekApiSecret - ? accessStore.iflytekApiKey + - ":" + - accessStore.iflytekApiSecret - : "" - : accessStore.openaiApiKey; + : isDeepSeek + ? accessStore.deepseekApiKey + : isChatGLM + ? accessStore.chatglmApiKey + : isSiliconFlow + ? accessStore.siliconflowApiKey + : isIflytek + ? accessStore.iflytekApiKey && + accessStore.iflytekApiSecret + ? accessStore.iflytekApiKey + + ":" + + accessStore.iflytekApiSecret + : "" + : accessStore.openaiApiKey; if (accessStore.isUseOpenAIEndpointForAllModels || ignoreHeaders) { return { isGoogle: false, @@ -335,8 +350,10 @@ export function getHeaders(ignoreHeaders: boolean = false) { isAlibaba: false, isMoonshot: false, isIflytek: false, + isDeepSeek: false, isXAI: false, isChatGLM: false, + isSiliconFlow: false, apiKey: accessStore.openaiApiKey, isEnabledAccessControl, }; @@ -350,8 +367,10 @@ export function getHeaders(ignoreHeaders: boolean = false) { isAlibaba, isMoonshot, isIflytek, + isDeepSeek, isXAI, isChatGLM, + isSiliconFlow, apiKey, isEnabledAccessControl, }; @@ -418,10 +437,14 @@ export function getClientApi(provider: ServiceProvider): ClientApi { return new ClientApi(ModelProvider.Moonshot); case ServiceProvider.Iflytek: return new ClientApi(ModelProvider.Iflytek); + case ServiceProvider.DeepSeek: + return new ClientApi(ModelProvider.DeepSeek); case ServiceProvider.XAI: return new ClientApi(ModelProvider.XAI); case ServiceProvider.ChatGLM: return new ClientApi(ModelProvider.ChatGLM); + case ServiceProvider.SiliconFlow: + return new ClientApi(ModelProvider.SiliconFlow); default: return new ClientApi(ModelProvider.GPT); } diff --git a/app/client/platforms/deepseek.ts b/app/client/platforms/deepseek.ts new file mode 100644 index 000000000..324bfb5c7 --- /dev/null +++ b/app/client/platforms/deepseek.ts @@ -0,0 +1,254 @@ +"use client"; +// azure and openai, using same models. so using same LLMApi. +import { + ApiPath, + DEEPSEEK_BASE_URL, + DeepSeek, + REQUEST_TIMEOUT_MS, +} from "@/app/constant"; +import { + useAccessStore, + useAppConfig, + useChatStore, + ChatMessageTool, + usePluginStore, +} from "@/app/store"; +import { streamWithThink } from "@/app/utils/chat"; +import { + AgentChatOptions, + ChatOptions, + CreateRAGStoreOptions, + getHeaders, + LLMApi, + LLMModel, + SpeechOptions, + TranscriptionOptions, +} from "../api"; +import { getClientConfig } from "@/app/config/client"; +import { + getMessageTextContent, + getMessageTextContentWithoutThinking, +} from "@/app/utils"; +import { RequestPayload } from "./openai"; +import { fetch } from "@/app/utils/stream"; + +export class DeepSeekApi implements LLMApi { + transcription(options: TranscriptionOptions): Promise { + throw new Error("Method not implemented."); + } + toolAgentChat(options: AgentChatOptions): Promise { + throw new Error("Method not implemented."); + } + createRAGStore(options: CreateRAGStoreOptions): Promise { + throw new Error("Method not implemented."); + } + private disableListModels = true; + + path(path: string): string { + const accessStore = useAccessStore.getState(); + + let baseUrl = ""; + + if (accessStore.useCustomConfig) { + baseUrl = accessStore.deepseekUrl; + } + + if (baseUrl.length === 0) { + const isApp = !!getClientConfig()?.isApp; + const apiPath = ApiPath.DeepSeek; + baseUrl = isApp ? DEEPSEEK_BASE_URL : apiPath; + } + + if (baseUrl.endsWith("/")) { + baseUrl = baseUrl.slice(0, baseUrl.length - 1); + } + if (!baseUrl.startsWith("http") && !baseUrl.startsWith(ApiPath.DeepSeek)) { + baseUrl = "https://" + baseUrl; + } + + console.log("[Proxy Endpoint] ", baseUrl, path); + + return [baseUrl, path].join("/"); + } + + extractMessage(res: any) { + return res.choices?.at(0)?.message?.content ?? ""; + } + + speech(options: SpeechOptions): Promise { + throw new Error("Method not implemented."); + } + + async chat(options: ChatOptions) { + const messages: ChatOptions["messages"] = []; + for (const v of options.messages) { + if (v.role === "assistant") { + const content = getMessageTextContentWithoutThinking(v); + messages.push({ role: v.role, content }); + } else { + const content = getMessageTextContent(v); + messages.push({ role: v.role, content }); + } + } + + const modelConfig = { + ...useAppConfig.getState().modelConfig, + ...useChatStore.getState().currentSession().mask.modelConfig, + ...{ + model: options.config.model, + providerName: options.config.providerName, + }, + }; + + const requestPayload: RequestPayload = { + messages, + stream: options.config.stream, + model: modelConfig.model, + temperature: modelConfig.temperature, + presence_penalty: modelConfig.presence_penalty, + frequency_penalty: modelConfig.frequency_penalty, + top_p: modelConfig.top_p, + // max_tokens: Math.max(modelConfig.max_tokens, 1024), + // Please do not ask me why not send max_tokens, no reason, this param is just shit, I dont want to explain anymore. + }; + + console.log("[Request] openai payload: ", requestPayload); + + const shouldStream = !!options.config.stream; + const controller = new AbortController(); + options.onController?.(controller); + + try { + const chatPath = this.path(DeepSeek.ChatPath); + const chatPayload = { + method: "POST", + body: JSON.stringify(requestPayload), + signal: controller.signal, + headers: getHeaders(), + }; + + // console.log(chatPayload); + + // make a fetch request + const requestTimeoutId = setTimeout( + () => controller.abort(), + REQUEST_TIMEOUT_MS, + ); + + if (shouldStream) { + // const [tools, funcs] = usePluginStore + // .getState() + // .getAsTools( + // useChatStore.getState().currentSession().mask?.plugin || [], + // ); + const tools = null; + const funcs: Record = {}; + return streamWithThink( + chatPath, + requestPayload, + getHeaders(), + tools as any, + funcs, + controller, + // parseSSE + (text: string, runTools: ChatMessageTool[]) => { + // console.log("parseSSE", text, runTools); + const json = JSON.parse(text); + const choices = json.choices as Array<{ + delta: { + content: string | null; + tool_calls: ChatMessageTool[]; + reasoning_content: string | null; + }; + }>; + const tool_calls = choices[0]?.delta?.tool_calls; + if (tool_calls?.length > 0) { + const index = tool_calls[0]?.index; + const id = tool_calls[0]?.id; + const args = tool_calls[0]?.function?.arguments; + if (id) { + runTools.push({ + id, + type: tool_calls[0]?.type, + function: { + name: tool_calls[0]?.function?.name as string, + arguments: args, + }, + }); + } else { + // @ts-ignore + runTools[index]["function"]["arguments"] += args; + } + } + const reasoning = choices[0]?.delta?.reasoning_content; + const content = choices[0]?.delta?.content; + + // Skip if both content and reasoning_content are empty or null + if ( + (!reasoning || reasoning.trim().length === 0) && + (!content || content.trim().length === 0) + ) { + return { + isThinking: false, + content: "", + }; + } + + if (reasoning && reasoning.trim().length > 0) { + return { + isThinking: true, + content: reasoning, + }; + } else if (content && content.trim().length > 0) { + return { + isThinking: false, + content: content, + }; + } + + return { + isThinking: false, + content: "", + }; + }, + // processToolMessage, include tool_calls message and tool call results + ( + requestPayload: RequestPayload, + toolCallMessage: any, + toolCallResult: any[], + ) => { + // @ts-ignore + requestPayload?.messages?.splice( + // @ts-ignore + requestPayload?.messages?.length, + 0, + toolCallMessage, + ...toolCallResult, + ); + }, + options, + ); + } else { + const res = await fetch(chatPath, chatPayload); + clearTimeout(requestTimeoutId); + + const resJson = await res.json(); + const message = this.extractMessage(resJson); + options.onFinish(message, res); + } + } catch (e) { + console.log("[Request] failed to make a chat request", e); + options.onError?.(e as Error); + } + } + async usage() { + return { + used: 0, + total: 0, + }; + } + + async models(): Promise { + return []; + } +} diff --git a/app/client/platforms/siliconflow.ts b/app/client/platforms/siliconflow.ts new file mode 100644 index 000000000..6f71b51f5 --- /dev/null +++ b/app/client/platforms/siliconflow.ts @@ -0,0 +1,257 @@ +"use client"; +// azure and openai, using same models. so using same LLMApi. +import { + ApiPath, + SILICONFLOW_BASE_URL, + SiliconFlow, + REQUEST_TIMEOUT_MS, +} from "@/app/constant"; +import { + useAccessStore, + useAppConfig, + useChatStore, + ChatMessageTool, + usePluginStore, +} from "@/app/store"; +import { streamWithThink } from "@/app/utils/chat"; +import { + AgentChatOptions, + ChatOptions, + CreateRAGStoreOptions, + getHeaders, + LLMApi, + LLMModel, + SpeechOptions, + TranscriptionOptions, +} from "../api"; +import { getClientConfig } from "@/app/config/client"; +import { + getMessageTextContent, + getMessageTextContentWithoutThinking, +} from "@/app/utils"; +import { RequestPayload } from "./openai"; +import { fetch } from "@/app/utils/stream"; + +export class SiliconflowApi implements LLMApi { + transcription(options: TranscriptionOptions): Promise { + throw new Error("Method not implemented."); + } + toolAgentChat(options: AgentChatOptions): Promise { + throw new Error("Method not implemented."); + } + createRAGStore(options: CreateRAGStoreOptions): Promise { + throw new Error("Method not implemented."); + } + private disableListModels = true; + + path(path: string): string { + const accessStore = useAccessStore.getState(); + + let baseUrl = ""; + + if (accessStore.useCustomConfig) { + baseUrl = accessStore.siliconflowUrl; + } + + if (baseUrl.length === 0) { + const isApp = !!getClientConfig()?.isApp; + const apiPath = ApiPath.SiliconFlow; + baseUrl = isApp ? SILICONFLOW_BASE_URL : apiPath; + } + + if (baseUrl.endsWith("/")) { + baseUrl = baseUrl.slice(0, baseUrl.length - 1); + } + if ( + !baseUrl.startsWith("http") && + !baseUrl.startsWith(ApiPath.SiliconFlow) + ) { + baseUrl = "https://" + baseUrl; + } + + console.log("[Proxy Endpoint] ", baseUrl, path); + + return [baseUrl, path].join("/"); + } + + extractMessage(res: any) { + return res.choices?.at(0)?.message?.content ?? ""; + } + + speech(options: SpeechOptions): Promise { + throw new Error("Method not implemented."); + } + + async chat(options: ChatOptions) { + const messages: ChatOptions["messages"] = []; + for (const v of options.messages) { + if (v.role === "assistant") { + const content = getMessageTextContentWithoutThinking(v); + messages.push({ role: v.role, content }); + } else { + const content = getMessageTextContent(v); + messages.push({ role: v.role, content }); + } + } + + const modelConfig = { + ...useAppConfig.getState().modelConfig, + ...useChatStore.getState().currentSession().mask.modelConfig, + ...{ + model: options.config.model, + providerName: options.config.providerName, + }, + }; + + const requestPayload: RequestPayload = { + messages, + stream: options.config.stream, + model: modelConfig.model, + temperature: modelConfig.temperature, + presence_penalty: modelConfig.presence_penalty, + frequency_penalty: modelConfig.frequency_penalty, + top_p: modelConfig.top_p, + // max_tokens: Math.max(modelConfig.max_tokens, 1024), + // Please do not ask me why not send max_tokens, no reason, this param is just shit, I dont want to explain anymore. + }; + + console.log("[Request] openai payload: ", requestPayload); + + const shouldStream = !!options.config.stream; + const controller = new AbortController(); + options.onController?.(controller); + + try { + const chatPath = this.path(SiliconFlow.ChatPath); + const chatPayload = { + method: "POST", + body: JSON.stringify(requestPayload), + signal: controller.signal, + headers: getHeaders(), + }; + + // console.log(chatPayload); + + // make a fetch request + const requestTimeoutId = setTimeout( + () => controller.abort(), + REQUEST_TIMEOUT_MS, + ); + + if (shouldStream) { + // const [tools, funcs] = usePluginStore + // .getState() + // .getAsTools( + // useChatStore.getState().currentSession().mask?.plugin || [], + // ); + const tools = null; + const funcs: Record = {}; + return streamWithThink( + chatPath, + requestPayload, + getHeaders(), + tools as any, + funcs, + controller, + // parseSSE + (text: string, runTools: ChatMessageTool[]) => { + // console.log("parseSSE", text, runTools); + const json = JSON.parse(text); + const choices = json.choices as Array<{ + delta: { + content: string | null; + tool_calls: ChatMessageTool[]; + reasoning_content: string | null; + }; + }>; + const tool_calls = choices[0]?.delta?.tool_calls; + if (tool_calls?.length > 0) { + const index = tool_calls[0]?.index; + const id = tool_calls[0]?.id; + const args = tool_calls[0]?.function?.arguments; + if (id) { + runTools.push({ + id, + type: tool_calls[0]?.type, + function: { + name: tool_calls[0]?.function?.name as string, + arguments: args, + }, + }); + } else { + // @ts-ignore + runTools[index]["function"]["arguments"] += args; + } + } + const reasoning = choices[0]?.delta?.reasoning_content; + const content = choices[0]?.delta?.content; + + // Skip if both content and reasoning_content are empty or null + if ( + (!reasoning || reasoning.trim().length === 0) && + (!content || content.trim().length === 0) + ) { + return { + isThinking: false, + content: "", + }; + } + + if (reasoning && reasoning.trim().length > 0) { + return { + isThinking: true, + content: reasoning, + }; + } else if (content && content.trim().length > 0) { + return { + isThinking: false, + content: content, + }; + } + + return { + isThinking: false, + content: "", + }; + }, + // processToolMessage, include tool_calls message and tool call results + ( + requestPayload: RequestPayload, + toolCallMessage: any, + toolCallResult: any[], + ) => { + // @ts-ignore + requestPayload?.messages?.splice( + // @ts-ignore + requestPayload?.messages?.length, + 0, + toolCallMessage, + ...toolCallResult, + ); + }, + options, + ); + } else { + const res = await fetch(chatPath, chatPayload); + clearTimeout(requestTimeoutId); + + const resJson = await res.json(); + const message = this.extractMessage(resJson); + options.onFinish(message, res); + } + } catch (e) { + console.log("[Request] failed to make a chat request", e); + options.onError?.(e as Error); + } + } + async usage() { + return { + used: 0, + total: 0, + }; + } + + async models(): Promise { + return []; + } +} diff --git a/app/components/settings.tsx b/app/components/settings.tsx index 9b3f3afb6..21449a9f3 100644 --- a/app/components/settings.tsx +++ b/app/components/settings.tsx @@ -71,6 +71,8 @@ import { Stability, Iflytek, ChatGLM, + SiliconFlow, + DeepSeek, } from "../constant"; import { Prompt, SearchService, usePromptStore } from "../store/prompt"; import { ErrorBoundary } from "./error"; @@ -1172,6 +1174,47 @@ export function Settings() { ); + const deepseekConfigComponent = accessStore.provider === + ServiceProvider.DeepSeek && ( + <> + + + accessStore.update( + (access) => (access.deepseekUrl = e.currentTarget.value), + ) + } + > + + + { + accessStore.update( + (access) => (access.deepseekApiKey = e.currentTarget.value), + ); + }} + /> + + + ); + const XAIConfigComponent = accessStore.provider === ServiceProvider.XAI && ( <> ); + const siliconflowConfigComponent = accessStore.provider === + ServiceProvider.SiliconFlow && ( + <> + + + accessStore.update( + (access) => (access.siliconflowUrl = e.currentTarget.value), + ) + } + > + + + { + accessStore.update( + (access) => (access.siliconflowApiKey = e.currentTarget.value), + ); + }} + /> + + + ); + const stabilityConfigComponent = accessStore.provider === ServiceProvider.Stability && ( <> @@ -1711,10 +1795,12 @@ export function Settings() { {alibabaConfigComponent} {tencentConfigComponent} {moonshotConfigComponent} + {deepseekConfigComponent} {stabilityConfigComponent} {lflytekConfigComponent} {XAIConfigComponent} {chatglmConfigComponent} + {siliconflowConfigComponent} )} diff --git a/app/config/server.ts b/app/config/server.ts index 83ba21799..a3358ff33 100644 --- a/app/config/server.ts +++ b/app/config/server.ts @@ -22,9 +22,14 @@ declare global { DISABLE_FAST_LINK?: string; // disallow parse settings from url or not CUSTOM_MODELS?: string; // to control custom models DEFAULT_MODEL?: string; // to control default model in every new chat window + VISION_MODELS?: string; // to control vision models + + // stability only + STABILITY_URL?: string; + STABILITY_API_KEY?: string; // azure only - AZURE_URL?: string; // https://{azure-url}/openai/deployments + AZURE_URL?: string; // https://{azure-url}/openai/deployments/{deploy-name} AZURE_API_KEY?: string; AZURE_API_VERSION?: string; @@ -53,8 +58,39 @@ declare global { ALIBABA_URL?: string; ALIBABA_API_KEY?: string; + // tencent only + TENCENT_URL?: string; + TENCENT_SECRET_KEY?: string; + TENCENT_SECRET_ID?: string; + + // moonshot only + MOONSHOT_URL?: string; + MOONSHOT_API_KEY?: string; + + // iflytek only + IFLYTEK_URL?: string; + IFLYTEK_API_KEY?: string; + IFLYTEK_API_SECRET?: string; + + DEEPSEEK_URL?: string; + DEEPSEEK_API_KEY?: string; + + // xai only + XAI_URL?: string; + XAI_API_KEY?: string; + + // chatglm only + CHATGLM_URL?: string; + CHATGLM_API_KEY?: string; + + // siliconflow only + SILICONFLOW_URL?: string; + SILICONFLOW_API_KEY?: string; + // custom template for preprocessing user input DEFAULT_INPUT_TEMPLATE?: string; + + ENABLE_MCP?: string; // enable mcp functionality } } } @@ -128,8 +164,10 @@ export const getServerSideConfig = () => { const isAlibaba = !!process.env.ALIBABA_API_KEY; const isMoonshot = !!process.env.MOONSHOT_API_KEY; const isIflytek = !!process.env.IFLYTEK_API_KEY; + const isDeepSeek = !!process.env.DEEPSEEK_API_KEY; const isXAI = !!process.env.XAI_API_KEY; const isChatGLM = !!process.env.CHATGLM_API_KEY; + const isSiliconFlow = !!process.env.SILICONFLOW_API_KEY; // const apiKeyEnvVar = process.env.OPENAI_API_KEY ?? ""; // const apiKeys = apiKeyEnvVar.split(",").map((v) => v.trim()); // const randomIndex = Math.floor(Math.random() * apiKeys.length); @@ -192,6 +230,10 @@ export const getServerSideConfig = () => { iflytekApiKey: process.env.IFLYTEK_API_KEY, iflytekApiSecret: process.env.IFLYTEK_API_SECRET, + isDeepSeek, + deepseekUrl: process.env.DEEPSEEK_URL, + deepseekApiKey: getApiKey(process.env.DEEPSEEK_API_KEY), + isXAI, xaiUrl: process.env.XAI_URL, xaiApiKey: getApiKey(process.env.XAI_API_KEY), @@ -205,6 +247,10 @@ export const getServerSideConfig = () => { cloudflareKVApiKey: getApiKey(process.env.CLOUDFLARE_KV_API_KEY), cloudflareKVTTL: process.env.CLOUDFLARE_KV_TTL, + isSiliconFlow, + siliconFlowUrl: process.env.SILICONFLOW_URL, + siliconFlowApiKey: getApiKey(process.env.SILICONFLOW_API_KEY), + gtmId: process.env.GTM_ID, gaId: process.env.GA_ID || DEFAULT_GA_ID, diff --git a/app/constant.ts b/app/constant.ts index 8628db66e..a3b51684b 100644 --- a/app/constant.ts +++ b/app/constant.ts @@ -28,10 +28,14 @@ export const TENCENT_BASE_URL = "https://hunyuan.tencentcloudapi.com"; export const MOONSHOT_BASE_URL = "https://api.moonshot.cn"; export const IFLYTEK_BASE_URL = "https://spark-api-open.xf-yun.com"; +export const DEEPSEEK_BASE_URL = "https://api.deepseek.com"; + export const XAI_BASE_URL = "https://api.x.ai"; export const CHATGLM_BASE_URL = "https://open.bigmodel.cn"; +export const SILICONFLOW_BASE_URL = "https://api.siliconflow.cn"; + export const CACHE_URL_PREFIX = "/api/cache"; export const UPLOAD_URL = `${CACHE_URL_PREFIX}/upload`; @@ -62,6 +66,8 @@ export enum ApiPath { Artifacts = "/api/artifacts", XAI = "/api/xai", ChatGLM = "/api/chatglm", + DeepSeek = "/api/deepseek", + SiliconFlow = "/api/siliconflow", } export enum SlotID { @@ -116,6 +122,8 @@ export enum ServiceProvider { Iflytek = "Iflytek", XAI = "XAI", ChatGLM = "ChatGLM", + DeepSeek = "DeepSeek", + SiliconFlow = "SiliconFlow", } // Google API safety settings, see https://ai.google.dev/gemini-api/docs/safety-settings @@ -140,6 +148,8 @@ export enum ModelProvider { Iflytek = "Iflytek", XAI = "XAI", ChatGLM = "ChatGLM", + DeepSeek = "DeepSeek", + SiliconFlow = "SiliconFlow", } export const Stability = { @@ -222,6 +232,11 @@ export const Iflytek = { ChatPath: "v1/chat/completions", }; +export const DeepSeek = { + ExampleEndpoint: DEEPSEEK_BASE_URL, + ChatPath: "chat/completions", +}; + export const XAI = { ExampleEndpoint: XAI_BASE_URL, ChatPath: "v1/chat/completions", @@ -232,6 +247,11 @@ export const ChatGLM = { ChatPath: "/api/paas/v4/chat/completions", }; +export const SiliconFlow = { + ExampleEndpoint: SILICONFLOW_BASE_URL, + ChatPath: "v1/chat/completions", +}; + export const DEFAULT_INPUT_TEMPLATE = `{{input}}`; // input / time / model / lang // export const DEFAULT_SYSTEM_TEMPLATE = ` // You are ChatGPT, a large language model trained by {{ServiceProvider}}. @@ -252,6 +272,7 @@ Latex block: $$e=mc^2$$ export const SUMMARIZE_MODEL = "gpt-4o-mini"; export const GEMINI_SUMMARIZE_MODEL = "gemini-pro"; +export const DEEPSEEK_SUMMARIZE_MODEL = "deepseek-chat"; export const KnowledgeCutOffDate: Record = { default: "2021-09", @@ -261,17 +282,25 @@ export const KnowledgeCutOffDate: Record = { "gpt-4o": "2023-10", "gpt-4o-2024-05-13": "2023-10", "gpt-4o-2024-08-06": "2023-10", + "gpt-4o-2024-11-20": "2023-10", "chatgpt-4o-latest": "2023-10", "gpt-4o-mini": "2023-10", "gpt-4o-mini-2024-07-18": "2023-10", "gpt-4-vision-preview": "2023-04", + "o1-mini-2024-09-12": "2023-10", "o1-mini": "2023-10", + "o1-preview-2024-09-12": "2023-10", "o1-preview": "2023-10", + "o1-2024-12-17": "2023-10", + o1: "2023-10", + "o3-mini-2025-01-31": "2023-10", + "o3-mini": "2023-10", // After improvements, // it's now easier to add "KnowledgeCutOffDate" instead of stupid hardcoding it, as was done previously. "gemini-pro": "2023-12", "gemini-pro-vision": "2023-12", - "gemini-exp-1114": "2024-11", + "deepseek-chat": "2024-07", + "deepseek-coder": "2024-07", }; export const DEFAULT_TTS_ENGINE = "OpenAI-TTS"; @@ -288,10 +317,29 @@ export const DEFAULT_TTS_VOICES = [ "shimmer", ]; +export const VISION_MODEL_REGEXES = [ + /vision/, + /gpt-4o/, + /claude-3/, + /gemini-1\.5/, + /gemini-exp/, + /gemini-2\.0/, + /learnlm/, + /qwen-vl/, + /qwen2-vl/, + /gpt-4-turbo(?!.*preview)/, // Matches "gpt-4-turbo" but not "gpt-4-turbo-preview" + /^dall-e-3$/, // Matches exactly "dall-e-3" + /glm-4v/, +]; + +export const EXCLUDE_VISION_MODEL_REGEXES = [/claude-3-5-haiku-20241022/]; + export const DEFAULT_STT_ENGINE = "WebAPI"; export const DEFAULT_STT_ENGINES = ["WebAPI", "OpenAI Whisper"]; export const FIREFOX_DEFAULT_STT_ENGINE = "OpenAI Whisper"; const openaiModels = [ + // As of July 2024, gpt-4o-mini should be used in place of gpt-3.5-turbo, + // as it is cheaper, more capable, multimodal, and just as fast. gpt-3.5-turbo is still available for use in the API. "gpt-3.5-turbo", "gpt-3.5-turbo-1106", "gpt-3.5-turbo-0125", @@ -304,6 +352,7 @@ const openaiModels = [ "gpt-4o", "gpt-4o-2024-05-13", "gpt-4o-2024-08-06", + "gpt-4o-2024-11-20", "chatgpt-4o-latest", "gpt-4o-mini", "gpt-4o-mini-2024-07-18", @@ -313,14 +362,29 @@ const openaiModels = [ "dall-e-3", "o1-mini", "o1-preview", + "o3-mini", ]; const googleModels = [ - "gemini-1.0-pro", + "gemini-1.0-pro", // Deprecated on 2/15/2025 "gemini-1.5-pro-latest", + "gemini-1.5-pro", + "gemini-1.5-pro-002", + "gemini-1.5-pro-exp-0827", "gemini-1.5-flash-latest", - "gemini-pro-vision", + "gemini-1.5-flash-8b-latest", + "gemini-1.5-flash", + "gemini-1.5-flash-8b", + "gemini-1.5-flash-002", + "gemini-1.5-flash-exp-0827", + "learnlm-1.5-pro-experimental", "gemini-exp-1114", + "gemini-exp-1121", + "gemini-exp-1206", + "gemini-2.0-flash-exp", + "gemini-2.0-flash-thinking-exp", + "gemini-2.0-flash-thinking-exp-1219", + "gemini-2.0-flash-thinking-exp-01-21", ]; const anthropicModels = [ @@ -331,10 +395,11 @@ const anthropicModels = [ "claude-3-opus-20240229", "claude-3-opus-latest", "claude-3-haiku-20240307", + "claude-3-5-haiku-20241022", + "claude-3-5-haiku-latest", "claude-3-5-sonnet-20240620", "claude-3-5-sonnet-20241022", "claude-3-5-sonnet-latest", - "claude-3-5-haiku-latest", ]; const baiduModels = [ @@ -390,6 +455,8 @@ const iflytekModels = [ "4.0Ultra", ]; +const deepseekModels = ["deepseek-chat", "deepseek-coder", "deepseek-reasoner"]; + const xAIModes = ["grok-beta"]; const chatglmModels = [ @@ -401,6 +468,30 @@ const chatglmModels = [ "glm-4-long", "glm-4-flashx", "glm-4-flash", + "glm-4v-plus", + "glm-4v", + "glm-4v-flash", // free + "cogview-3-plus", + "cogview-3", + "cogview-3-flash", // free + // 目前无法适配轮询任务 + // "cogvideox", + // "cogvideox-flash", // free +]; + +const siliconflowModels = [ + "Qwen/Qwen2.5-7B-Instruct", + "Qwen/Qwen2.5-72B-Instruct", + "deepseek-ai/DeepSeek-R1", + "deepseek-ai/DeepSeek-R1-Distill-Llama-70B", + "deepseek-ai/DeepSeek-R1-Distill-Llama-8B", + "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B", + "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B", + "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", + "deepseek-ai/DeepSeek-V3", + "meta-llama/Llama-3.3-70B-Instruct", + "THUDM/glm-4-9b-chat", ]; let seq = 1000; // 内置的模型序号生成器从1000开始 @@ -537,6 +628,28 @@ export const DEFAULT_MODELS = [ sorted: 12, }, })), + ...deepseekModels.map((name) => ({ + name, + available: true, + sorted: seq++, + provider: { + id: "deepseek", + providerName: "DeepSeek", + providerType: "deepseek", + sorted: 13, + }, + })), + ...siliconflowModels.map((name) => ({ + name, + available: true, + sorted: seq++, + provider: { + id: "siliconflow", + providerName: "SiliconFlow", + providerType: "siliconflow", + sorted: 14, + }, + })), ] as const; export const CHAT_PAGE_SIZE = 15; diff --git a/app/locales/cn.ts b/app/locales/cn.ts index 41998de46..4b9230dae 100644 --- a/app/locales/cn.ts +++ b/app/locales/cn.ts @@ -448,6 +448,17 @@ const cn = { SubTitle: "样例：", }, }, + DeepSeek: { + ApiKey: { + Title: "接口密钥", + SubTitle: "使用自定义DeepSeek API Key", + Placeholder: "DeepSeek API Key", + }, + Endpoint: { + Title: "接口地址", + SubTitle: "样例：", + }, + }, XAI: { ApiKey: { Title: "接口密钥", @@ -470,6 +481,17 @@ const cn = { SubTitle: "样例：", }, }, + SiliconFlow: { + ApiKey: { + Title: "接口密钥", + SubTitle: "使用自定义硅基流动 API Key", + Placeholder: "硅基流动 API Key", + }, + Endpoint: { + Title: "接口地址", + SubTitle: "样例：", + }, + }, Stability: { ApiKey: { Title: "接口密钥", diff --git a/app/locales/en.ts b/app/locales/en.ts index 4f3b987a8..0b8c25d20 100644 --- a/app/locales/en.ts +++ b/app/locales/en.ts @@ -432,6 +432,17 @@ const en: LocaleType = { SubTitle: "Example: ", }, }, + DeepSeek: { + ApiKey: { + Title: "DeepSeek API Key", + SubTitle: "Use a custom DeepSeek API Key", + Placeholder: "DeepSeek API Key", + }, + Endpoint: { + Title: "Endpoint Address", + SubTitle: "Example: ", + }, + }, XAI: { ApiKey: { Title: "XAI API Key", @@ -454,6 +465,17 @@ const en: LocaleType = { SubTitle: "Example: ", }, }, + SiliconFlow: { + ApiKey: { + Title: "SiliconFlow API Key", + SubTitle: "Use a custom SiliconFlow API Key", + Placeholder: "SiliconFlow API Key", + }, + Endpoint: { + Title: "Endpoint Address", + SubTitle: "Example: ", + }, + }, Stability: { ApiKey: { Title: "Stability API Key", diff --git a/app/store/access.ts b/app/store/access.ts index 20a8ab1e0..b962f95f3 100644 --- a/app/store/access.ts +++ b/app/store/access.ts @@ -15,6 +15,8 @@ import { IFLYTEK_BASE_URL, XAI_BASE_URL, CHATGLM_BASE_URL, + DEEPSEEK_BASE_URL, + SILICONFLOW_BASE_URL, } from "../constant"; import { getHeaders } from "../client/api"; import { getClientConfig } from "../config/client"; @@ -47,10 +49,16 @@ const DEFAULT_STABILITY_URL = isApp ? STABILITY_BASE_URL : ApiPath.Stability; const DEFAULT_IFLYTEK_URL = isApp ? IFLYTEK_BASE_URL : ApiPath.Iflytek; +const DEFAULT_DEEPSEEK_URL = isApp ? DEEPSEEK_BASE_URL : ApiPath.DeepSeek; + const DEFAULT_XAI_URL = isApp ? XAI_BASE_URL : ApiPath.XAI; const DEFAULT_CHATGLM_URL = isApp ? CHATGLM_BASE_URL : ApiPath.ChatGLM; +const DEFAULT_SILICONFLOW_URL = isApp + ? SILICONFLOW_BASE_URL + : ApiPath.SiliconFlow; + const DEFAULT_ACCESS_STATE = { accessCode: "", useCustomConfig: false, @@ -108,6 +116,10 @@ const DEFAULT_ACCESS_STATE = { iflytekApiKey: "", iflytekApiSecret: "", + // deepseek + deepseekUrl: DEFAULT_DEEPSEEK_URL, + deepseekApiKey: "", + // xai xaiUrl: DEFAULT_XAI_URL, xaiApiKey: "", @@ -116,6 +128,10 @@ const DEFAULT_ACCESS_STATE = { chatglmUrl: DEFAULT_CHATGLM_URL, chatglmApiKey: "", + // siliconflow + siliconflowUrl: DEFAULT_SILICONFLOW_URL, + siliconflowApiKey: "", + // server config needCode: true, hideUserApiKey: false, @@ -123,8 +139,9 @@ const DEFAULT_ACCESS_STATE = { disableGPT4: false, disableFastLink: false, customModels: "", - isEnableRAG: false, defaultModel: "", + visionModels: "", + isEnableRAG: false, // tts config edgeTTSVoiceName: "zh-CN-YunxiNeural", @@ -212,15 +229,18 @@ export const useAccessStore = createPersistStore( isValidIflytek() { return ensure(get(), ["iflytekApiKey"]); }, - + isValidDeepSeek() { + return ensure(get(), ["deepseekApiKey"]); + }, isValidXAI() { return ensure(get(), ["xaiApiKey"]); }, - isValidChatGLM() { return ensure(get(), ["chatglmApiKey"]); }, - + isValidSiliconFlow() { + return ensure(get(), ["siliconflowApiKey"]); + }, isAuthorized() { this.fetch(); @@ -236,8 +256,10 @@ export const useAccessStore = createPersistStore( this.isValidTencent() || this.isValidMoonshot() || this.isValidIflytek() || + this.isValidDeepSeek() || this.isValidXAI() || this.isValidChatGLM() || + this.isValidSiliconFlow() || !this.enabledAccessControl() || (this.enabledAccessControl() && ensure(get(), ["accessCode"])) ); diff --git a/app/utils.ts b/app/utils.ts index 72775be19..80ab75053 100644 --- a/app/utils.ts +++ b/app/utils.ts @@ -239,6 +239,28 @@ export function getMessageTextContent(message: RequestMessage) { return ""; } +export function getMessageTextContentWithoutThinking(message: RequestMessage) { + let content = ""; + + if (typeof message.content === "string") { + content = message.content; + } else { + for (const c of message.content) { + if (c.type === "text") { + content = c.text ?? ""; + break; + } + } + } + + // Filter out thinking lines (starting with "> ") + return content + .split("\n") + .filter((line) => !line.startsWith("> ") && line.trim() !== "") + .join("\n") + .trim(); +} + export function getMessageImages(message: RequestMessage): string[] { if (typeof message.content === "string") { return []; diff --git a/app/utils/chat.ts b/app/utils/chat.ts index 067552cc2..b090a8551 100644 --- a/app/utils/chat.ts +++ b/app/utils/chat.ts @@ -10,6 +10,7 @@ import { fetchEventSource, } from "@fortaine/fetch-event-source"; import { prettyObject } from "./format"; +import { fetch as tauriFetch } from "./stream"; export function compressImage(file: Blob, maxSize: number): Promise { return new Promise((resolve, reject) => { @@ -352,3 +353,262 @@ export function stream( console.debug("[ChatAPI] start"); chatApi(chatPath, headers, requestPayload, tools); // call fetchEventSource } + +export function streamWithThink( + chatPath: string, + requestPayload: any, + headers: any, + tools: any[], + funcs: Record, + controller: AbortController, + parseSSE: ( + text: string, + runTools: any[], + ) => { + isThinking: boolean; + content: string | undefined; + }, + processToolMessage: ( + requestPayload: any, + toolCallMessage: any, + toolCallResult: any[], + ) => void, + options: any, +) { + let responseText = ""; + let remainText = ""; + let finished = false; + let running = false; + let runTools: any[] = []; + let responseRes: Response; + let isInThinkingMode = false; + let lastIsThinking = false; + + // animate response to make it looks smooth + function animateResponseText() { + if (finished || controller.signal.aborted) { + responseText += remainText; + console.log("[Response Animation] finished"); + if (responseText?.length === 0) { + options.onError?.(new Error("empty response from server")); + } + return; + } + + if (remainText.length > 0) { + const fetchCount = Math.max(1, Math.round(remainText.length / 60)); + const fetchText = remainText.slice(0, fetchCount); + responseText += fetchText; + remainText = remainText.slice(fetchCount); + options.onUpdate?.(responseText, fetchText); + } + + requestAnimationFrame(animateResponseText); + } + + // start animaion + animateResponseText(); + + const finish = () => { + if (!finished) { + if (!running && runTools.length > 0) { + const toolCallMessage = { + role: "assistant", + tool_calls: [...runTools], + }; + running = true; + runTools.splice(0, runTools.length); // empty runTools + return Promise.all( + toolCallMessage.tool_calls.map((tool) => { + options?.onBeforeTool?.(tool); + return Promise.resolve( + // @ts-ignore + funcs[tool.function.name]( + // @ts-ignore + tool?.function?.arguments + ? JSON.parse(tool?.function?.arguments) + : {}, + ), + ) + .then((res) => { + let content = res.data || res?.statusText; + // hotfix #5614 + content = + typeof content === "string" + ? content + : JSON.stringify(content); + if (res.status >= 300) { + return Promise.reject(content); + } + return content; + }) + .then((content) => { + options?.onAfterTool?.({ + ...tool, + content, + isError: false, + }); + return content; + }) + .catch((e) => { + options?.onAfterTool?.({ + ...tool, + isError: true, + errorMsg: e.toString(), + }); + return e.toString(); + }) + .then((content) => ({ + name: tool.function.name, + role: "tool", + content, + tool_call_id: tool.id, + })); + }), + ).then((toolCallResult) => { + processToolMessage(requestPayload, toolCallMessage, toolCallResult); + setTimeout(() => { + // call again + console.debug("[ChatAPI] restart"); + running = false; + chatApi(chatPath, headers, requestPayload, tools); // call fetchEventSource + }, 60); + }); + return; + } + if (running) { + return; + } + console.debug("[ChatAPI] end"); + finished = true; + options.onFinish(responseText + remainText, responseRes); + } + }; + + controller.signal.onabort = finish; + + function chatApi( + chatPath: string, + headers: any, + requestPayload: any, + tools: any, + ) { + const chatPayload = { + method: "POST", + body: JSON.stringify({ + ...requestPayload, + tools: tools && tools.length ? tools : undefined, + }), + signal: controller.signal, + headers, + }; + const requestTimeoutId = setTimeout( + () => controller.abort(), + REQUEST_TIMEOUT_MS, + ); + fetchEventSource(chatPath, { + fetch: tauriFetch as any, + ...chatPayload, + async onopen(res) { + clearTimeout(requestTimeoutId); + const contentType = res.headers.get("content-type"); + console.log("[Request] response content type: ", contentType); + responseRes = res; + + if (contentType?.startsWith("text/plain")) { + responseText = await res.clone().text(); + return finish(); + } + + if ( + !res.ok || + !res.headers + .get("content-type") + ?.startsWith(EventStreamContentType) || + res.status !== 200 + ) { + const responseTexts = [responseText]; + let extraInfo = await res.clone().text(); + try { + const resJson = await res.clone().json(); + extraInfo = prettyObject(resJson); + } catch {} + + if (res.status === 401) { + responseTexts.push(Locale.Error.Unauthorized); + } + + if (extraInfo) { + responseTexts.push(extraInfo); + } + + responseText = responseTexts.join("\n\n"); + + return finish(); + } + }, + onmessage(msg) { + if (msg.data === "[DONE]" || finished) { + return finish(); + } + const text = msg.data; + // Skip empty messages + if (!text || text.trim().length === 0) { + return; + } + try { + const chunk = parseSSE(text, runTools); + // Skip if content is empty + if (!chunk?.content || chunk.content.trim().length === 0) { + return; + } + // Check if thinking mode changed + const isThinkingChanged = lastIsThinking !== chunk.isThinking; + lastIsThinking = chunk.isThinking; + + if (chunk.isThinking) { + // If in thinking mode + if (!isInThinkingMode || isThinkingChanged) { + // If this is a new thinking block or mode changed, add prefix + isInThinkingMode = true; + if (remainText.length > 0) { + remainText += "\n"; + } + remainText += "> " + chunk.content; + } else { + // Handle newlines in thinking content + if (chunk.content.includes("\n\n")) { + const lines = chunk.content.split("\n\n"); + remainText += lines.join("\n\n> "); + } else { + remainText += chunk.content; + } + } + } else { + // If in normal mode + if (isInThinkingMode || isThinkingChanged) { + // If switching from thinking mode to normal mode + isInThinkingMode = false; + remainText += "\n\n" + chunk.content; + } else { + remainText += chunk.content; + } + } + } catch (e) { + console.error("[Request] parse error", text, msg, e); + // Don't throw error for parse failures, just log them + } + }, + onclose() { + finish(); + }, + onerror(e) { + options?.onError?.(e); + throw e; + }, + openWhenHidden: true, + }); + } + console.debug("[ChatAPI] start"); + chatApi(chatPath, headers, requestPayload, tools); // call fetchEventSource +} diff --git a/app/utils/model.ts b/app/utils/model.ts index a1b7df1b6..f460babcd 100644 --- a/app/utils/model.ts +++ b/app/utils/model.ts @@ -1,4 +1,4 @@ -import { DEFAULT_MODELS } from "../constant"; +import { DEFAULT_MODELS, ServiceProvider } from "../constant"; import { LLMModel } from "../client/api"; const CustomSeq = { @@ -202,3 +202,57 @@ export function isModelAvailableInServer( const modelTable = collectModelTable(DEFAULT_MODELS, customModels); return modelTable[fullName]?.available === false; } + +/** + * Check if the model name is a GPT-4 related model + * + * @param modelName The name of the model to check + * @returns True if the model is a GPT-4 related model (excluding gpt-4o-mini) + */ +export function isGPT4Model(modelName: string): boolean { + return ( + (modelName.startsWith("gpt-4") || + modelName.startsWith("chatgpt-4o") || + modelName.startsWith("o1")) && + !modelName.startsWith("gpt-4o-mini") + ); +} + +/** + * Checks if a model is not available on any of the specified providers in the server. + * + * @param {string} customModels - A string of custom models, comma-separated. + * @param {string} modelName - The name of the model to check. + * @param {string|string[]} providerNames - A string or array of provider names to check against. + * + * @returns {boolean} True if the model is not available on any of the specified providers, false otherwise. + */ +export function isModelNotavailableInServer( + customModels: string, + modelName: string, + providerNames: string | string[], +): boolean { + // Check DISABLE_GPT4 environment variable + if ( + process.env.DISABLE_GPT4 === "1" && + isGPT4Model(modelName.toLowerCase()) + ) { + return true; + } + + const modelTable = collectModelTable(DEFAULT_MODELS, customModels); + + const providerNamesArray = Array.isArray(providerNames) + ? providerNames + : [providerNames]; + for (const providerName of providerNamesArray) { + // if model provider is bytedance, use model config name to check if not avaliable + if (providerName === ServiceProvider.ByteDance) { + return !Object.values(modelTable).filter((v) => v.name === modelName)?.[0] + ?.available; + } + const fullName = `${modelName}@${providerName.toLowerCase()}`; + if (modelTable?.[fullName]?.available === true) return false; + } + return true; +}