Compare commits

..

21 Commits

Author SHA1 Message Date
RiverRay
c9ef6d58ed Update README.md 2025-04-19 15:50:17 +08:00
RiverRay
11b37c15bd Merge pull request #6450 from stephen-zeng/main
Some checks failed
Run Tests / test (push) Has been cancelled
Add gpt-4.1 family & gpt-4.5-preview support
2025-04-17 08:29:19 +08:00
QwQwQ
1d0038f17d add gpt-4.5-preview support 2025-04-16 22:10:47 +08:00
QwQwQ
619fa519c0 add gpt-4.1 family support 2025-04-16 22:02:35 +08:00
RiverRay
48469bd8ca Merge pull request #6392 from ChatGPTNextWeb/Leizhenpeng-patch-6
Some checks failed
Run Tests / test (push) Has been cancelled
Update README.md
2025-03-20 17:52:02 +08:00
RiverRay
5a5e887f2b Update README.md 2025-03-20 17:51:47 +08:00
RiverRay
b6f5d75656 Merge pull request #6344 from vangie/fix/jest-setup-esm
Some checks failed
Run Tests / test (push) Has been cancelled
test: fix unit test failures
2025-03-14 20:04:56 +08:00
Vangie Du
0d41a17ef6 test: fix unit test failures 2025-03-07 14:49:17 +08:00
RiverRay
f7cde17919 Merge pull request #6292 from Little-LittleProgrammer/feature/alibaba-omni-support
Some checks failed
Run Tests / test (push) Has been cancelled
feat(alibaba): Added alibaba vision model and omni model support
2025-03-01 10:25:16 +08:00
RiverRay
570cbb34b6 Merge pull request #6310 from agi-dude/patch-1
Remove duplicate links
2025-03-01 10:24:38 +08:00
RiverRay
7aa9ae0a3e Merge pull request #6311 from ChatGPTNextWeb/6305-bugthe-first-message-except-the-system-message-of-deepseek-reasoner-must-be-a-user-message-but-an-assistant-message-detected
Some checks are pending
Run Tests / test (push) Waiting to run
fix: enforce that the first message (excluding system messages) is a …
2025-02-28 19:48:09 +08:00
Kadxy
2d4180f5be fix: update request payload to use filtered messages in Deepseek API 2025-02-28 13:59:30 +08:00
Kadxy
9f0182b55e fix: enforce that the first message (excluding system messages) is a user message in the Deepseek API 2025-02-28 13:54:58 +08:00
Mr. AGI
ad6666eeaf Update README.md 2025-02-28 10:47:52 +05:00
EvanWu
a2c4e468a0 fix(app/utils/chat.ts): fix type error 2025-02-26 19:58:32 +08:00
RiverRay
2167076652 Merge pull request #6293 from hyiip/main
Some checks failed
Run Tests / test (push) Has been cancelled
claude 3.7 support
2025-02-26 18:41:28 +08:00
RiverRay
e123076250 Merge pull request #6295 from rexkyng/patch-1
Fix: Improve Mistral icon detection and remove redundant code.
2025-02-26 18:39:59 +08:00
Rex Ng
ebcb4db245 Fix: Improve Mistral icon detection and remove redundant code.
- Added "codestral" to the list of acceptable names for the Mistral icon, ensuring proper detection.
- Removed duplicate `toLowerCase()` calls.
2025-02-25 14:30:18 +08:00
EvanWu
0a25a1a8cb refacto(app/utils/chat.ts)r: optimize function preProcessImageContentBase 2025-02-25 09:22:47 +08:00
hyiip
f3154b20a5 claude 3.7 support 2025-02-25 03:55:24 +08:00
EvanWu
b709ee3983 feat(alibaba): Added alibaba vision model and omni model support 2025-02-24 20:18:07 +08:00
27 changed files with 1277 additions and 4433 deletions

View File

@@ -76,12 +76,6 @@ ANTHROPIC_URL=
### (optional)
WHITE_WEBDAV_ENDPOINTS=
### bedrock (optional)
AWS_REGION=
AWS_ACCESS_KEY=AKIA
AWS_SECRET_KEY=
### siliconflow Api key (optional)
SILICONFLOW_API_KEY=

View File

@@ -7,7 +7,7 @@
<h1 align="center">NextChat (ChatGPT Next Web)</h1>
<h1 align="center">NextChat</h1>
English / [简体中文](./README_CN.md)
@@ -22,7 +22,6 @@ English / [简体中文](./README_CN.md)
[![MacOS][MacOS-image]][download-url]
[![Linux][Linux-image]][download-url]
[NextChatAI](https://nextchat.dev/chat?utm_source=readme) / [Web App Demo](https://app.nextchat.dev) / [Desktop App](https://github.com/Yidadaa/ChatGPT-Next-Web/releases)
[NextChatAI](https://nextchat.club?utm_source=readme) / [Web App Demo](https://app.nextchat.dev) / [Desktop App](https://github.com/Yidadaa/ChatGPT-Next-Web/releases) / [Discord](https://discord.gg/YCkeafCafC) / [Enterprise Edition](#enterprise-edition) / [Twitter](https://twitter.com/NextChatDev)
@@ -41,31 +40,12 @@ English / [简体中文](./README_CN.md)
</div>
## 👋 Hey, NextChat is going to develop a native app!
## 🥳 Cheer for NextChat iOS Version Online!
> [ 👉 Click Here Install Now](https://apps.apple.com/us/app/nextchat-ai/id6743085599)
> This week we are going to start working on iOS and Android APP, and we want to find some reliable friends to do it together!
![Github iOS Image](https://github.com/user-attachments/assets/e0aa334f-4c13-4dc9-8310-e3b09fa4b9f3)
✨ Several key points:
- Starting from 0, you are a veteran
- Completely open source, not hidden
- Native development, pursuing the ultimate experience
Will you come and do something together? 😎
https://github.com/ChatGPTNextWeb/NextChat/issues/6269
#Seeking for talents is thirsty #lack of people
## 🥳 Cheer for DeepSeek, China's AI star!
> Purpose-Built UI for DeepSeek Reasoner Model
<img src="https://github.com/user-attachments/assets/f3952210-3af1-4dc0-9b81-40eaa4847d9a"/>
## 🫣 NextChat Support MCP !
> Before build, please set env ENABLE_MCP=true

View File

@@ -1,7 +1,6 @@
import { ApiPath } from "@/app/constant";
import { NextRequest } from "next/server";
import { handle as openaiHandler } from "../../openai";
import { handle as bedrockHandler } from "../../bedrock";
import { handle as azureHandler } from "../../azure";
import { handle as googleHandler } from "../../google";
import { handle as anthropicHandler } from "../../anthropic";
@@ -24,15 +23,12 @@ async function handle(
const apiPath = `/api/${params.provider}`;
console.log(`[${params.provider} Route] params `, params);
switch (apiPath) {
case ApiPath.Bedrock:
return bedrockHandler(req, { params });
case ApiPath.Azure:
return azureHandler(req, { params });
case ApiPath.Google:
return googleHandler(req, { params });
case ApiPath.Anthropic:
return anthropicHandler(req, { params });
case ApiPath.Baidu:
return baiduHandler(req, { params });
case ApiPath.ByteDance:

View File

@@ -52,6 +52,7 @@ export function auth(req: NextRequest, modelProvider: ModelProvider) {
msg: "you are not allowed to access with your own api key",
};
}
// if user does not provide an api key, inject system api key
if (!apiKey) {
const serverConfig = getServerSideConfig();
@@ -100,14 +101,6 @@ export function auth(req: NextRequest, modelProvider: ModelProvider) {
case ModelProvider.ChatGLM:
systemApiKey = serverConfig.chatglmApiKey;
break;
case ModelProvider.Bedrock:
systemApiKey =
serverConfig.awsRegion +
":" +
serverConfig.awsAccessKey +
":" +
serverConfig.awsSecretKey;
break;
case ModelProvider.SiliconFlow:
systemApiKey = serverConfig.siliconFlowApiKey;
break;

View File

@@ -1,177 +0,0 @@
import { NextRequest, NextResponse } from "next/server";
import { auth } from "./auth";
import {
sign,
decrypt,
getBedrockEndpoint,
BedrockCredentials,
} from "../utils/aws";
import { getServerSideConfig } from "../config/server";
import { ModelProvider } from "../constant";
import { prettyObject } from "../utils/format";
const ALLOWED_PATH = new Set(["chat", "models"]);
async function getBedrockCredentials(
req: NextRequest,
): Promise<BedrockCredentials> {
// Get AWS credentials from server config first
const config = getServerSideConfig();
let awsRegion = config.awsRegion;
let awsAccessKey = config.awsAccessKey;
let awsSecretKey = config.awsSecretKey;
// If server-side credentials are not available, parse from Authorization header
if (!awsRegion || !awsAccessKey || !awsSecretKey) {
const authHeader = req.headers.get("Authorization");
if (!authHeader || !authHeader.startsWith("Bearer ")) {
throw new Error("Missing or invalid Authorization header");
}
const [_, credentials] = authHeader.split("Bearer ");
const [encryptedRegion, encryptedAccessKey, encryptedSecretKey] =
credentials.split(":");
if (!encryptedRegion || !encryptedAccessKey || !encryptedSecretKey) {
throw new Error("Invalid Authorization header format");
}
const encryptionKey = req.headers.get("XEncryptionKey") || "";
// Decrypt the credentials
[awsRegion, awsAccessKey, awsSecretKey] = await Promise.all([
decrypt(encryptedRegion, encryptionKey),
decrypt(encryptedAccessKey, encryptionKey),
decrypt(encryptedSecretKey, encryptionKey),
]);
if (!awsRegion || !awsAccessKey || !awsSecretKey) {
throw new Error(
"Failed to decrypt AWS credentials. Please ensure ENCRYPTION_KEY is set correctly.",
);
}
}
return {
region: awsRegion,
accessKeyId: awsAccessKey,
secretAccessKey: awsSecretKey,
};
}
async function requestBedrock(req: NextRequest) {
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), 10 * 60 * 1000);
try {
// Get credentials and model info
const credentials = await getBedrockCredentials(req);
const modelId = req.headers.get("XModelID");
const shouldStream = req.headers.get("ShouldStream") !== "false";
if (!modelId) {
throw new Error("Missing model ID");
}
// Parse and validate request body
const bodyText = await req.clone().text();
if (!bodyText) {
throw new Error("Request body is empty");
}
let bodyJson;
try {
bodyJson = JSON.parse(bodyText);
} catch (e) {
throw new Error(`Invalid JSON in request body: ${e}`);
}
console.log("[Bedrock Request] Initiating request");
// Get endpoint and prepare request
const endpoint = getBedrockEndpoint(
credentials.region,
modelId,
shouldStream,
);
const requestBody: any = {
...bodyJson,
};
// Sign request
const headers = await sign({
method: "POST",
url: endpoint,
region: credentials.region,
accessKeyId: credentials.accessKeyId,
secretAccessKey: credentials.secretAccessKey,
body: JSON.stringify(requestBody),
service: "bedrock",
isStreaming: shouldStream,
});
// Make request to AWS Bedrock
// console.log(
// "[Bedrock Request] Final Body:",
// JSON.stringify(requestBody, null, 2),
// );
const res = await fetch(endpoint, {
method: "POST",
headers,
body: JSON.stringify(requestBody),
redirect: "manual",
// @ts-ignore
duplex: "half",
signal: controller.signal,
});
if (!res.ok) {
const error = await res.text();
console.error("[Bedrock Error] Request failed with status:", res.status);
try {
const errorJson = JSON.parse(error);
throw new Error(errorJson.message || error);
} catch {
throw new Error(
`Bedrock request failed with status ${res.status}: ${
error || "No error message"
}`,
);
}
}
if (!res.body) {
console.error("[Bedrock Error] Empty response body");
throw new Error(
"Empty response from Bedrock. Please check AWS credentials and permissions.",
);
}
return res;
} catch (e) {
console.error("[Bedrock Request Error]:", e);
throw e;
} finally {
clearTimeout(timeoutId);
}
}
export async function handle(
req: NextRequest,
{ params }: { params: { path: string[] } },
) {
const subpath = params.path.join("/");
if (!ALLOWED_PATH.has(subpath)) {
return NextResponse.json(
{ error: true, msg: "you are not allowed to request " + subpath },
{ status: 403 },
);
}
const authResult = auth(req, ModelProvider.Bedrock);
if (authResult.error) {
return NextResponse.json(authResult, {
status: 401,
});
}
try {
return await requestBedrock(req);
} catch (e) {
console.error("Handler error:", e);
return NextResponse.json(prettyObject(e));
}
}

View File

@@ -23,10 +23,8 @@ import { SparkApi } from "./platforms/iflytek";
import { DeepSeekApi } from "./platforms/deepseek";
import { XAIApi } from "./platforms/xai";
import { ChatGLMApi } from "./platforms/glm";
import { BedrockApi } from "./platforms/bedrock";
import { SiliconflowApi } from "./platforms/siliconflow";
export const ROLES = ["system", "user", "assistant"] as const;
export type MessageRole = (typeof ROLES)[number];
@@ -42,6 +40,11 @@ export interface MultimodalContent {
};
}
export interface MultimodalContentForAlibaba {
text?: string;
image?: string;
}
export interface RequestMessage {
role: MessageRole;
content: string | MultimodalContent[];
@@ -134,9 +137,6 @@ export class ClientApi {
constructor(provider: ModelProvider = ModelProvider.GPT) {
switch (provider) {
case ModelProvider.Bedrock:
this.llm = new BedrockApi();
break;
case ModelProvider.GeminiPro:
this.llm = new GeminiProApi();
break;
@@ -252,7 +252,6 @@ export function getHeaders(ignoreHeaders: boolean = false) {
function getConfig() {
const modelConfig = chatStore.currentSession().mask.modelConfig;
const isBedrock = modelConfig.providerName === ServiceProvider.Bedrock;
const isGoogle = modelConfig.providerName === ServiceProvider.Google;
const isAzure = modelConfig.providerName === ServiceProvider.Azure;
const isAnthropic = modelConfig.providerName === ServiceProvider.Anthropic;
@@ -293,7 +292,6 @@ export function getHeaders(ignoreHeaders: boolean = false) {
: ""
: accessStore.openaiApiKey;
return {
isBedrock,
isGoogle,
isAzure,
isAnthropic,
@@ -322,7 +320,6 @@ export function getHeaders(ignoreHeaders: boolean = false) {
}
const {
isBedrock,
isGoogle,
isAzure,
isAnthropic,
@@ -343,23 +340,17 @@ export function getHeaders(ignoreHeaders: boolean = false) {
const authHeader = getAuthHeader();
if (isBedrock) {
if (apiKey) {
headers[authHeader] = getBearerToken(apiKey);
}
} else {
const bearerToken = getBearerToken(
apiKey,
isAzure || isAnthropic || isGoogle,
);
const bearerToken = getBearerToken(
apiKey,
isAzure || isAnthropic || isGoogle,
);
if (bearerToken) {
headers[authHeader] = bearerToken;
} else if (isEnabledAccessControl && validString(accessStore.accessCode)) {
headers["Authorization"] = getBearerToken(
ACCESS_CODE_PREFIX + accessStore.accessCode,
);
}
if (bearerToken) {
headers[authHeader] = bearerToken;
} else if (isEnabledAccessControl && validString(accessStore.accessCode)) {
headers["Authorization"] = getBearerToken(
ACCESS_CODE_PREFIX + accessStore.accessCode,
);
}
return headers;
@@ -367,8 +358,6 @@ export function getHeaders(ignoreHeaders: boolean = false) {
export function getClientApi(provider: ServiceProvider): ClientApi {
switch (provider) {
case ServiceProvider.Bedrock:
return new ClientApi(ModelProvider.Bedrock);
case ServiceProvider.Google:
return new ClientApi(ModelProvider.GeminiPro);
case ServiceProvider.Anthropic:

View File

@@ -7,7 +7,10 @@ import {
ChatMessageTool,
usePluginStore,
} from "@/app/store";
import { streamWithThink } from "@/app/utils/chat";
import {
preProcessImageContentForAlibabaDashScope,
streamWithThink,
} from "@/app/utils/chat";
import {
ChatOptions,
getHeaders,
@@ -15,12 +18,14 @@ import {
LLMModel,
SpeechOptions,
MultimodalContent,
MultimodalContentForAlibaba,
} from "../api";
import { getClientConfig } from "@/app/config/client";
import {
getMessageTextContent,
getMessageTextContentWithoutThinking,
getTimeoutMSByModel,
isVisionModel,
} from "@/app/utils";
import { fetch } from "@/app/utils/stream";
@@ -89,14 +94,6 @@ export class QwenApi implements LLMApi {
}
async chat(options: ChatOptions) {
const messages = options.messages.map((v) => ({
role: v.role,
content:
v.role === "assistant"
? getMessageTextContentWithoutThinking(v)
: getMessageTextContent(v),
}));
const modelConfig = {
...useAppConfig.getState().modelConfig,
...useChatStore.getState().currentSession().mask.modelConfig,
@@ -105,6 +102,21 @@ export class QwenApi implements LLMApi {
},
};
const visionModel = isVisionModel(options.config.model);
const messages: ChatOptions["messages"] = [];
for (const v of options.messages) {
const content = (
visionModel
? await preProcessImageContentForAlibabaDashScope(v.content)
: v.role === "assistant"
? getMessageTextContentWithoutThinking(v)
: getMessageTextContent(v)
) as any;
messages.push({ role: v.role, content });
}
const shouldStream = !!options.config.stream;
const requestPayload: RequestPayload = {
model: modelConfig.model,
@@ -129,7 +141,7 @@ export class QwenApi implements LLMApi {
"X-DashScope-SSE": shouldStream ? "enable" : "disable",
};
const chatPath = this.path(Alibaba.ChatPath);
const chatPath = this.path(Alibaba.ChatPath(modelConfig.model));
const chatPayload = {
method: "POST",
body: JSON.stringify(requestPayload),
@@ -162,7 +174,7 @@ export class QwenApi implements LLMApi {
const json = JSON.parse(text);
const choices = json.output.choices as Array<{
message: {
content: string | null;
content: string | null | MultimodalContentForAlibaba[];
tool_calls: ChatMessageTool[];
reasoning_content: string | null;
};
@@ -212,7 +224,9 @@ export class QwenApi implements LLMApi {
} else if (content && content.length > 0) {
return {
isThinking: false,
content: content,
content: Array.isArray(content)
? content.map((item) => item.text).join(",")
: content,
};
}

View File

@@ -1,859 +0,0 @@
"use client";
import { ChatOptions, getHeaders, LLMApi, SpeechOptions } from "../api";
import {
useAppConfig,
usePluginStore,
useChatStore,
useAccessStore,
ChatMessageTool,
} from "@/app/store";
import { preProcessImageContent } from "@/app/utils/chat";
import { getMessageTextContent, isVisionModel } from "@/app/utils";
import { ApiPath, BEDROCK_BASE_URL, REQUEST_TIMEOUT_MS } from "@/app/constant";
import { getClientConfig } from "@/app/config/client";
import {
extractMessage,
processMessage,
processChunks,
parseEventData,
sign,
} from "@/app/utils/aws";
import { prettyObject } from "@/app/utils/format";
import Locale from "@/app/locales";
import { encrypt } from "@/app/utils/aws";
const ClaudeMapper = {
assistant: "assistant",
user: "user",
system: "user",
} as const;
const MistralMapper = {
system: "system",
user: "user",
assistant: "assistant",
} as const;
type MistralRole = keyof typeof MistralMapper;
interface Tool {
function?: {
name?: string;
description?: string;
parameters?: any;
};
}
const isApp = !!getClientConfig()?.isApp;
// const isApp = true;
async function getBedrockHeaders(
modelId: string,
chatPath: string,
finalRequestBody: any,
shouldStream: boolean,
): Promise<Record<string, string>> {
const accessStore = useAccessStore.getState();
const bedrockHeaders = isApp
? await sign({
method: "POST",
url: chatPath,
region: accessStore.awsRegion,
accessKeyId: accessStore.awsAccessKey,
secretAccessKey: accessStore.awsSecretKey,
body: finalRequestBody,
service: "bedrock",
headers: {},
isStreaming: shouldStream,
})
: getHeaders();
if (!isApp) {
const { awsRegion, awsAccessKey, awsSecretKey, encryptionKey } =
accessStore;
const bedrockHeadersConfig = {
XModelID: modelId,
XEncryptionKey: encryptionKey,
ShouldStream: String(shouldStream),
Authorization: await createAuthHeader(
awsRegion,
awsAccessKey,
awsSecretKey,
encryptionKey,
),
};
Object.assign(bedrockHeaders, bedrockHeadersConfig);
}
return bedrockHeaders;
}
// Helper function to create Authorization header
async function createAuthHeader(
region: string,
accessKey: string,
secretKey: string,
encryptionKey: string,
): Promise<string> {
const encryptedValues = await Promise.all([
encrypt(region, encryptionKey),
encrypt(accessKey, encryptionKey),
encrypt(secretKey, encryptionKey),
]);
return `Bearer ${encryptedValues.join(":")}`;
}
export class BedrockApi implements LLMApi {
speech(options: SpeechOptions): Promise<ArrayBuffer> {
throw new Error("Speech not implemented for Bedrock.");
}
formatRequestBody(messages: ChatOptions["messages"], modelConfig: any) {
const model = modelConfig.model;
const visionModel = isVisionModel(modelConfig.model);
// Get tools if available
const [tools] = usePluginStore
.getState()
.getAsTools(useChatStore.getState().currentSession().mask?.plugin || []);
const toolsArray = (tools as Tool[]) || [];
// Handle Nova models
if (model.includes("amazon.nova")) {
// Extract system message if present
const systemMessage = messages.find((m) => m.role === "system");
const conversationMessages = messages.filter((m) => m.role !== "system");
const requestBody: any = {
schemaVersion: "messages-v1",
messages: conversationMessages.map((message) => {
const content = Array.isArray(message.content)
? message.content
: [{ text: getMessageTextContent(message) }];
return {
role: message.role,
content: content.map((item: any) => {
// Handle text content
if (item.text || typeof item === "string") {
return { text: item.text || item };
}
// Handle image content
if (item.image_url?.url) {
const { url = "" } = item.image_url;
const colonIndex = url.indexOf(":");
const semicolonIndex = url.indexOf(";");
const comma = url.indexOf(",");
// Extract format from mime type
const mimeType = url.slice(colonIndex + 1, semicolonIndex);
const format = mimeType.split("/")[1];
const data = url.slice(comma + 1);
return {
image: {
format,
source: {
bytes: data,
},
},
};
}
return item;
}),
};
}),
inferenceConfig: {
temperature: modelConfig.temperature || 0.7,
top_p: modelConfig.top_p || 0.9,
top_k: modelConfig.top_k || 50,
max_new_tokens: modelConfig.max_tokens || 1000,
stopSequences: modelConfig.stop || [],
},
};
// Add system message if present
if (systemMessage) {
requestBody.system = [
{
text: getMessageTextContent(systemMessage),
},
];
}
// Add tools if available - exact Nova format
if (toolsArray.length > 0) {
requestBody.toolConfig = {
tools: toolsArray.map((tool) => ({
toolSpec: {
name: tool?.function?.name || "",
description: tool?.function?.description || "",
inputSchema: {
json: {
type: "object",
properties: tool?.function?.parameters?.properties || {},
required: tool?.function?.parameters?.required || [],
},
},
},
})),
toolChoice: { auto: {} },
};
}
return requestBody;
}
// Handle Titan models
if (model.startsWith("amazon.titan")) {
const inputText = messages
.map((message) => {
return `${message.role}: ${getMessageTextContent(message)}`;
})
.join("\n\n");
return {
inputText,
textGenerationConfig: {
maxTokenCount: modelConfig.max_tokens,
temperature: modelConfig.temperature,
stopSequences: [],
},
};
}
// Handle LLaMA models
if (model.includes("meta.llama")) {
let prompt = "<|begin_of_text|>";
// Extract system message if present
const systemMessage = messages.find((m) => m.role === "system");
if (systemMessage) {
prompt += `<|start_header_id|>system<|end_header_id|>\n${getMessageTextContent(
systemMessage,
)}<|eot_id|>`;
}
// Format the conversation
const conversationMessages = messages.filter((m) => m.role !== "system");
for (const message of conversationMessages) {
const role = message.role === "assistant" ? "assistant" : "user";
const content = getMessageTextContent(message);
prompt += `<|start_header_id|>${role}<|end_header_id|>\n${content}<|eot_id|>`;
}
// Add the final assistant header to prompt completion
prompt += "<|start_header_id|>assistant<|end_header_id|>";
return {
prompt,
max_gen_len: modelConfig.max_tokens || 512,
temperature: modelConfig.temperature || 0.7,
top_p: modelConfig.top_p || 0.9,
};
}
// Handle Mistral models
if (model.includes("mistral.mistral")) {
const formattedMessages = messages.map((message) => ({
role: MistralMapper[message.role as MistralRole] || "user",
content: getMessageTextContent(message),
}));
const requestBody: any = {
messages: formattedMessages,
max_tokens: modelConfig.max_tokens || 4096,
temperature: modelConfig.temperature || 0.7,
top_p: modelConfig.top_p || 0.9,
};
// Add tools if available
if (toolsArray.length > 0) {
requestBody.tool_choice = "auto";
requestBody.tools = toolsArray.map((tool) => ({
type: "function",
function: {
name: tool?.function?.name,
description: tool?.function?.description,
parameters: tool?.function?.parameters,
},
}));
}
return requestBody;
}
// Handle Claude models
const keys = ["system", "user"];
// roles must alternate between "user" and "assistant" in claude, so add a fake assistant message between two user messages
for (let i = 0; i < messages.length - 1; i++) {
const message = messages[i];
const nextMessage = messages[i + 1];
if (keys.includes(message.role) && keys.includes(nextMessage.role)) {
messages[i] = [
message,
{
role: "assistant",
content: ";",
},
] as any;
}
}
const prompt = messages
.flat()
.filter((v) => {
if (!v.content) return false;
if (typeof v.content === "string" && !v.content.trim()) return false;
return true;
})
.map((v) => {
const { role, content } = v;
const insideRole = ClaudeMapper[role] ?? "user";
if (!visionModel || typeof content === "string") {
return {
role: insideRole,
content: getMessageTextContent(v),
};
}
return {
role: insideRole,
content: content
.filter((v) => v.image_url || v.text)
.map(({ type, text, image_url }) => {
if (type === "text") {
return {
type,
text: text!,
};
}
const { url = "" } = image_url || {};
const colonIndex = url.indexOf(":");
const semicolonIndex = url.indexOf(";");
const comma = url.indexOf(",");
const mimeType = url.slice(colonIndex + 1, semicolonIndex);
const encodeType = url.slice(semicolonIndex + 1, comma);
const data = url.slice(comma + 1);
return {
type: "image" as const,
source: {
type: encodeType,
media_type: mimeType,
data,
},
};
}),
};
});
if (prompt[0]?.role === "assistant") {
prompt.unshift({
role: "user",
content: ";",
});
}
const requestBody: any = {
anthropic_version: useAccessStore.getState().bedrockAnthropicVersion,
max_tokens: modelConfig.max_tokens,
messages: prompt,
temperature: modelConfig.temperature,
top_p: modelConfig.top_p || 0.9,
top_k: modelConfig.top_k || 5,
};
// Add tools if available for Claude models
if (toolsArray.length > 0 && model.includes("anthropic.claude")) {
requestBody.tools = toolsArray.map((tool) => ({
name: tool?.function?.name || "",
description: tool?.function?.description || "",
input_schema: tool?.function?.parameters || {},
}));
}
return requestBody;
}
async chat(options: ChatOptions) {
const accessStore = useAccessStore.getState();
const shouldStream = !!options.config.stream;
const modelConfig = {
...useAppConfig.getState().modelConfig,
...useChatStore.getState().currentSession().mask.modelConfig,
...{
model: options.config.model,
},
};
// try get base64image from local cache image_url
const messages: ChatOptions["messages"] = [];
for (const v of options.messages) {
const content = await preProcessImageContent(v.content);
messages.push({ role: v.role, content });
}
const controller = new AbortController();
options.onController?.(controller);
let finalRequestBody = this.formatRequestBody(messages, modelConfig);
try {
const bedrockAPIPath = `${BEDROCK_BASE_URL}/model/${
modelConfig.model
}/invoke${shouldStream ? "-with-response-stream" : ""}`;
const chatPath = isApp ? bedrockAPIPath : ApiPath.Bedrock + "/chat";
if (process.env.NODE_ENV !== "production") {
console.debug("[Bedrock Client] Request:", {
path: chatPath,
model: modelConfig.model,
messages: messages.length,
stream: shouldStream,
});
}
if (shouldStream) {
const [tools, funcs] = usePluginStore
.getState()
.getAsTools(
useChatStore.getState().currentSession().mask?.plugin || [],
);
return bedrockStream(
modelConfig.model,
chatPath,
finalRequestBody,
funcs,
controller,
// processToolMessage, include tool_calls message and tool call results
(
requestPayload: any[],
toolCallMessage: any,
toolCallResult: any[],
) => {
const modelId = modelConfig.model;
const isMistral = modelId.includes("mistral.mistral");
const isClaude = modelId.includes("anthropic.claude");
const isNova = modelId.includes("amazon.nova");
if (isClaude) {
// Format for Claude
// @ts-ignore
requestPayload?.messages?.splice(
// @ts-ignore
requestPayload?.messages?.length,
0,
{
role: "assistant",
content: toolCallMessage.tool_calls.map(
(tool: ChatMessageTool) => ({
type: "tool_use",
id: tool.id,
name: tool?.function?.name,
input: tool?.function?.arguments
? JSON.parse(tool?.function?.arguments)
: {},
}),
),
},
// @ts-ignore
...toolCallResult.map((result) => ({
role: "user",
content: [
{
type: "tool_result",
tool_use_id: result.tool_call_id,
content: result.content,
},
],
})),
);
} else if (isMistral) {
// Format for Mistral
// @ts-ignore
requestPayload?.messages?.splice(
// @ts-ignore
requestPayload?.messages?.length,
0,
{
role: "assistant",
content: "",
// @ts-ignore
tool_calls: toolCallMessage.tool_calls.map(
(tool: ChatMessageTool) => ({
id: tool.id,
function: {
name: tool?.function?.name,
arguments: tool?.function?.arguments || "{}",
},
}),
),
},
...toolCallResult.map((result) => ({
role: "tool",
tool_call_id: result.tool_call_id,
content: result.content,
})),
);
} else if (isNova) {
// Format for Nova - Updated format
// @ts-ignore
requestPayload?.messages?.splice(
// @ts-ignore
requestPayload?.messages?.length,
0,
{
role: "assistant",
content: [
{
toolUse: {
toolUseId: toolCallMessage.tool_calls[0].id,
name: toolCallMessage.tool_calls[0]?.function?.name,
input:
typeof toolCallMessage.tool_calls[0]?.function
?.arguments === "string"
? JSON.parse(
toolCallMessage.tool_calls[0]?.function
?.arguments,
)
: toolCallMessage.tool_calls[0]?.function
?.arguments || {},
},
},
],
},
{
role: "user",
content: [
{
toolResult: {
toolUseId: toolCallResult[0].tool_call_id,
content: [
{
json: {
content: toolCallResult[0].content,
},
},
],
},
},
],
},
);
} else {
console.warn(
`[Bedrock Client] Unhandled model type for tool calls: ${modelId}`,
);
}
},
options,
);
} else {
try {
controller.signal.onabort = () =>
options.onFinish("", new Response(null, { status: 400 }));
const newHeaders = await getBedrockHeaders(
modelConfig.model,
chatPath,
JSON.stringify(finalRequestBody),
shouldStream,
);
const res = await fetch(chatPath, {
method: "POST",
headers: newHeaders,
body: JSON.stringify(finalRequestBody),
});
const contentType = res.headers.get("content-type");
console.log(
"[Bedrock Not Stream Request] response content type: ",
contentType,
);
const resJson = await res.json();
const message = extractMessage(resJson);
options.onFinish(message, res);
} catch (e) {
const error =
e instanceof Error ? e : new Error("Unknown error occurred");
console.error("[Bedrock Client] Chat failed:", error.message);
options.onError?.(error);
}
}
} catch (e) {
console.error("[Bedrock Client] Chat error:", e);
options.onError?.(e as Error);
}
}
async usage() {
return { used: 0, total: 0 };
}
async models() {
return [];
}
}
function bedrockStream(
modelId: string,
chatPath: string,
requestPayload: any,
funcs: Record<string, Function>,
controller: AbortController,
processToolMessage: (
requestPayload: any,
toolCallMessage: any,
toolCallResult: any[],
) => void,
options: any,
) {
let responseText = "";
let remainText = "";
let finished = false;
let running = false;
let runTools: any[] = [];
let responseRes: Response;
let index = -1;
let chunks: Uint8Array[] = [];
let pendingChunk: Uint8Array | null = null;
function animateResponseText() {
if (finished || controller.signal.aborted) {
responseText += remainText;
console.log("[Response Animation] finished");
if (responseText?.length === 0) {
options.onError?.(new Error("empty response from server"));
}
return;
}
if (remainText.length > 0) {
const fetchCount = Math.max(1, Math.round(remainText.length / 60));
const fetchText = remainText.slice(0, fetchCount);
responseText += fetchText;
remainText = remainText.slice(fetchCount);
options.onUpdate?.(responseText, fetchText);
}
requestAnimationFrame(animateResponseText);
}
animateResponseText();
const finish = () => {
if (!finished) {
if (!running && runTools.length > 0) {
const toolCallMessage = {
role: "assistant",
tool_calls: [...runTools],
};
running = true;
runTools.splice(0, runTools.length);
return Promise.all(
toolCallMessage.tool_calls.map((tool) => {
options?.onBeforeTool?.(tool);
const funcName = tool?.function?.name || tool?.name;
if (!funcName || !funcs[funcName]) {
console.error(`Function ${funcName} not found in funcs:`, funcs);
return Promise.reject(`Function ${funcName} not found`);
}
return Promise.resolve(
funcs[funcName](
tool?.function?.arguments
? JSON.parse(tool?.function?.arguments)
: {},
),
)
.then((res) => {
let content = res.data || res?.statusText;
content =
typeof content === "string"
? content
: JSON.stringify(content);
if (res.status >= 300) {
return Promise.reject(content);
}
return content;
})
.then((content) => {
options?.onAfterTool?.({
...tool,
content,
isError: false,
});
return content;
})
.catch((e) => {
options?.onAfterTool?.({
...tool,
isError: true,
errorMsg: e.toString(),
});
return e.toString();
})
.then((content) => ({
name: funcName,
role: "tool",
content,
tool_call_id: tool.id,
}));
}),
).then((toolCallResult) => {
processToolMessage(requestPayload, toolCallMessage, toolCallResult);
setTimeout(() => {
console.debug("[BedrockAPI for toolCallResult] restart");
running = false;
bedrockChatApi(modelId, chatPath, requestPayload, true);
}, 60);
});
}
if (running) {
return;
}
console.debug("[BedrockAPI] end");
finished = true;
options.onFinish(responseText + remainText, responseRes);
}
};
controller.signal.onabort = finish;
async function bedrockChatApi(
modelId: string,
chatPath: string,
requestPayload: any,
shouldStream: boolean,
) {
const requestTimeoutId = setTimeout(
() => controller.abort(),
REQUEST_TIMEOUT_MS,
);
const newHeaders = await getBedrockHeaders(
modelId,
chatPath,
JSON.stringify(requestPayload),
shouldStream,
);
try {
const res = await fetch(chatPath, {
method: "POST",
headers: newHeaders,
body: JSON.stringify(requestPayload),
redirect: "manual",
// @ts-ignore
duplex: "half",
signal: controller.signal,
});
clearTimeout(requestTimeoutId);
responseRes = res;
const contentType = res.headers.get("content-type");
// console.log(
// "[Bedrock Stream Request] response content type: ",
// contentType,
// );
if (contentType?.startsWith("text/plain")) {
responseText = await res.text();
return finish();
}
if (
!res.ok ||
res.status !== 200 ||
!contentType?.startsWith("application/vnd.amazon.eventstream")
) {
const responseTexts = [responseText];
let extraInfo = await res.text();
try {
const resJson = await res.clone().json();
extraInfo = prettyObject(resJson);
} catch {}
if (res.status === 401) {
responseTexts.push(Locale.Error.Unauthorized);
}
if (extraInfo) {
responseTexts.push(extraInfo);
}
responseText = responseTexts.join("\n\n");
return finish();
}
const reader = res.body?.getReader();
if (!reader) {
throw new Error("No response body reader available");
}
try {
while (true) {
const { done, value } = await reader.read();
if (done) {
if (pendingChunk) {
try {
const parsed = parseEventData(pendingChunk);
if (parsed) {
const result = processMessage(
parsed,
remainText,
runTools,
index,
);
remainText = result.remainText;
index = result.index;
}
} catch (e) {
console.error("[Final Chunk Process Error]:", e);
}
}
break;
}
chunks.push(value);
const result = processChunks(
chunks,
pendingChunk,
remainText,
runTools,
index,
);
chunks = result.chunks;
pendingChunk = result.pendingChunk;
remainText = result.remainText;
index = result.index;
}
} catch (err) {
console.error(
"[Bedrock Stream]:",
err instanceof Error ? err.message : "Stream processing failed",
);
throw new Error("Failed to process stream response");
} finally {
reader.releaseLock();
finish();
}
} catch (e) {
if (e instanceof Error && e.name === "AbortError") {
console.log("[Bedrock Client] Aborted by user");
return;
}
console.error(
"[Bedrock Request] Failed:",
e instanceof Error ? e.message : "Request failed",
);
options.onError?.(e);
throw new Error("Request processing failed");
}
}
console.debug("[BedrockAPI] start");
bedrockChatApi(modelId, chatPath, requestPayload, true);
}

View File

@@ -75,6 +75,25 @@ export class DeepSeekApi implements LLMApi {
}
}
// 检测并修复消息顺序确保除system外的第一个消息是user
const filteredMessages: ChatOptions["messages"] = [];
let hasFoundFirstUser = false;
for (const msg of messages) {
if (msg.role === "system") {
// Keep all system messages
filteredMessages.push(msg);
} else if (msg.role === "user") {
// User message directly added
filteredMessages.push(msg);
hasFoundFirstUser = true;
} else if (hasFoundFirstUser) {
// After finding the first user message, all subsequent non-system messages are retained.
filteredMessages.push(msg);
}
// If hasFoundFirstUser is false and it is not a system message, it will be skipped.
}
const modelConfig = {
...useAppConfig.getState().modelConfig,
...useChatStore.getState().currentSession().mask.modelConfig,
@@ -85,7 +104,7 @@ export class DeepSeekApi implements LLMApi {
};
const requestPayload: RequestPayload = {
messages,
messages: filteredMessages,
stream: options.config.stream,
model: modelConfig.model,
temperature: modelConfig.temperature,

View File

@@ -750,4 +750,4 @@
transform: translateX(0);
}
}
}
}

View File

@@ -66,11 +66,11 @@ export function Avatar(props: { model?: ModelType; avatar?: string }) {
LlmIcon = BotIconGemma;
} else if (modelName.startsWith("claude")) {
LlmIcon = BotIconClaude;
} else if (modelName.toLowerCase().includes("llama")) {
} else if (modelName.includes("llama")) {
LlmIcon = BotIconMeta;
} else if (modelName.startsWith("mixtral")) {
} else if (modelName.startsWith("mixtral") || modelName.startsWith("codestral")) {
LlmIcon = BotIconMistral;
} else if (modelName.toLowerCase().includes("deepseek")) {
} else if (modelName.includes("deepseek")) {
LlmIcon = BotIconDeepseek;
} else if (modelName.startsWith("moonshot")) {
LlmIcon = BotIconMoonshot;
@@ -85,7 +85,7 @@ export function Avatar(props: { model?: ModelType; avatar?: string }) {
} else if (modelName.startsWith("doubao") || modelName.startsWith("ep-")) {
LlmIcon = BotIconDoubao;
} else if (
modelName.toLowerCase().includes("glm") ||
modelName.includes("glm") ||
modelName.startsWith("cogview-") ||
modelName.startsWith("cogvideox-")
) {

View File

@@ -1,4 +1,5 @@
import { useState, useEffect, useMemo } from "react";
import styles from "./settings.module.scss";
import ResetIcon from "../icons/reload.svg";
@@ -966,89 +967,7 @@ export function Settings() {
</ListItem>
</>
);
const bedrockConfigComponent = accessStore.provider ===
ServiceProvider.Bedrock && (
<>
<ListItem
title={Locale.Settings.Access.Bedrock.Region.Title}
subTitle={Locale.Settings.Access.Bedrock.Region.SubTitle}
>
<input
aria-label={Locale.Settings.Access.Bedrock.Region.Title}
type="text"
value={accessStore.awsRegion}
placeholder="us-west-2"
onChange={(e) =>
accessStore.update((access) => {
const region = e.currentTarget.value;
access.awsRegion = region;
})
}
/>
</ListItem>
<ListItem
title={Locale.Settings.Access.Bedrock.AccessKey.Title}
subTitle={Locale.Settings.Access.Bedrock.AccessKey.SubTitle}
>
<PasswordInput
aria-label={Locale.Settings.Access.Bedrock.AccessKey.Title}
value={accessStore.awsAccessKey}
type="text"
placeholder={Locale.Settings.Access.Bedrock.AccessKey.Placeholder}
onChange={(e) => {
accessStore.update((access) => {
const accessKey = e.currentTarget.value;
access.awsAccessKey = accessKey;
});
}}
maskWhenShow={true}
/>
</ListItem>
<ListItem
title={Locale.Settings.Access.Bedrock.SecretKey.Title}
subTitle={Locale.Settings.Access.Bedrock.SecretKey.SubTitle}
>
<PasswordInput
aria-label={Locale.Settings.Access.Bedrock.SecretKey.Title}
value={accessStore.awsSecretKey}
type="text"
placeholder={Locale.Settings.Access.Bedrock.SecretKey.Placeholder}
onChange={(e) => {
accessStore.update((access) => {
const secretKey = e.currentTarget.value;
access.awsSecretKey = secretKey;
});
}}
maskWhenShow={true}
/>
</ListItem>
<ListItem
title={Locale.Settings.Access.Bedrock.EncryptionKey.Title}
subTitle={Locale.Settings.Access.Bedrock.EncryptionKey.SubTitle}
>
<PasswordInput
aria-label={Locale.Settings.Access.Bedrock.EncryptionKey.Title}
value={accessStore.encryptionKey}
type="text"
placeholder={Locale.Settings.Access.Bedrock.EncryptionKey.Placeholder}
onChange={(e) => {
accessStore.update(
(access) => (access.encryptionKey = e.currentTarget.value),
);
}}
onBlur={(e) => {
const value = e.currentTarget.value;
if (!value || value.length < 8) {
showToast(Locale.Settings.Access.Bedrock.EncryptionKey.Invalid);
accessStore.update((access) => (access.encryptionKey = ""));
return;
}
}}
maskWhenShow={true}
/>
</ListItem>
</>
);
const baiduConfigComponent = accessStore.provider ===
ServiceProvider.Baidu && (
<>
@@ -1889,7 +1808,6 @@ export function Settings() {
</ListItem>
{openAIConfigComponent}
{bedrockConfigComponent}
{azureConfigComponent}
{googleConfigComponent}
{anthropicConfigComponent}

View File

@@ -11,7 +11,6 @@ import MaxIcon from "../icons/max.svg";
import MinIcon from "../icons/min.svg";
import Locale from "../locales";
import { maskSensitiveValue } from "../utils/aws";
import { createRoot } from "react-dom/client";
import React, {
@@ -271,25 +270,13 @@ export function Input(props: InputProps) {
}
export function PasswordInput(
props: HTMLProps<HTMLInputElement> & {
aria?: string;
maskWhenShow?: boolean;
},
props: HTMLProps<HTMLInputElement> & { aria?: string },
) {
const [visible, setVisible] = useState(false);
const [isEditing, setIsEditing] = useState(false);
const { maskWhenShow, onChange, value, ...inputProps } = props;
function changeVisibility() {
setVisible(!visible);
}
// Get display value - use masked value only when showing and maskWhenShow is true and not editing
const displayValue =
maskWhenShow && visible && value && !isEditing
? maskSensitiveValue(value as string)
: value;
return (
<div className={"password-input-container"}>
<IconButton
@@ -299,11 +286,7 @@ export function PasswordInput(
className={"password-eye"}
/>
<input
{...inputProps}
value={displayValue}
onChange={onChange}
onFocus={() => setIsEditing(true)}
onBlur={() => setIsEditing(false)}
{...props}
type={visible ? "text" : "password"}
className={"password-input"}
/>
@@ -569,7 +552,6 @@ export function Selector<T>(props: {
</div>
);
}
export function FullScreen(props: any) {
const { children, right = 10, top = 10, ...rest } = props;
const ref = useRef<HTMLDivElement>();

View File

@@ -13,12 +13,6 @@ declare global {
BASE_URL?: string;
OPENAI_ORG_ID?: string; // openai only
// bedrock only
AWS_REGION?: string;
AWS_ACCESS_KEY?: string;
AWS_SECRET_KEY?: string;
ENCRYPTION_KEY?: string;
VERCEL?: string;
BUILD_MODE?: "standalone" | "export";
BUILD_APP?: string; // is building desktop app
@@ -154,10 +148,7 @@ export const getServerSideConfig = () => {
}
const isStability = !!process.env.STABILITY_API_KEY;
const isBedrock =
!!process.env.AWS_REGION &&
!!process.env.AWS_ACCESS_KEY &&
!!process.env.AWS_SECRET_KEY;
const isAzure = !!process.env.AZURE_URL;
const isGoogle = !!process.env.GOOGLE_API_KEY;
const isAnthropic = !!process.env.ANTHROPIC_API_KEY;
@@ -189,12 +180,6 @@ export const getServerSideConfig = () => {
apiKey: getApiKey(process.env.OPENAI_API_KEY),
openaiOrgId: process.env.OPENAI_ORG_ID,
isBedrock,
awsRegion: process.env.AWS_REGION,
awsAccessKey: process.env.AWS_ACCESS_KEY,
awsSecretKey: process.env.AWS_SECRET_KEY,
encryptionKey: process.env.ENCRYPTION_KEY,
isStability,
stabilityUrl: process.env.STABILITY_URL,
stabilityApiKey: getApiKey(process.env.STABILITY_API_KEY),

View File

@@ -56,7 +56,6 @@ export enum Path {
export enum ApiPath {
Cors = "",
Bedrock = "/api/bedrock",
Azure = "/api/azure",
OpenAI = "/api/openai",
Anthropic = "/api/anthropic",
@@ -130,7 +129,6 @@ export enum ServiceProvider {
XAI = "XAI",
ChatGLM = "ChatGLM",
DeepSeek = "DeepSeek",
Bedrock = "Bedrock",
SiliconFlow = "SiliconFlow",
}
@@ -157,7 +155,6 @@ export enum ModelProvider {
XAI = "XAI",
ChatGLM = "ChatGLM",
DeepSeek = "DeepSeek",
Bedrock = "Bedrock",
SiliconFlow = "SiliconFlow",
}
@@ -224,7 +221,12 @@ export const ByteDance = {
export const Alibaba = {
ExampleEndpoint: ALIBABA_BASE_URL,
ChatPath: "v1/services/aigc/text-generation/generation",
ChatPath: (modelName: string) => {
if (modelName.includes("vl") || modelName.includes("omni")) {
return "v1/services/aigc/multimodal-generation/generation";
}
return `v1/services/aigc/text-generation/generation`;
},
};
export const Tencent = {
@@ -258,15 +260,6 @@ export const ChatGLM = {
VideoPath: "api/paas/v4/videos/generations",
};
export const Bedrock = {
ChatPath: "model", // Simplified path since we'll append the full path in bedrock.ts
ApiVersion: "2023-11-01",
getEndpoint: (region: string = "us-west-2") =>
`https://bedrock-runtime.${region}.amazonaws.com`,
};
// Get the region from access store for BEDROCK_BASE_URL
export const BEDROCK_BASE_URL = Bedrock.getEndpoint();
export const SiliconFlow = {
ExampleEndpoint: SILICONFLOW_BASE_URL,
ChatPath: "v1/chat/completions",
@@ -424,6 +417,14 @@ export const KnowledgeCutOffDate: Record<string, string> = {
"gpt-4-turbo": "2023-12",
"gpt-4-turbo-2024-04-09": "2023-12",
"gpt-4-turbo-preview": "2023-12",
"gpt-4.1": "2024-06",
"gpt-4.1-2025-04-14": "2024-06",
"gpt-4.1-mini": "2024-06",
"gpt-4.1-mini-2025-04-14": "2024-06",
"gpt-4.1-nano": "2024-06",
"gpt-4.1-nano-2025-04-14": "2024-06",
"gpt-4.5-preview": "2023-10",
"gpt-4.5-preview-2025-02-27": "2023-10",
"gpt-4o": "2023-10",
"gpt-4o-2024-05-13": "2023-10",
"gpt-4o-2024-08-06": "2023-10",
@@ -465,6 +466,7 @@ export const DEFAULT_TTS_VOICES = [
export const VISION_MODEL_REGEXES = [
/vision/,
/gpt-4o/,
/gpt-4\.1/,
/claude-3/,
/gemini-1\.5/,
/gemini-exp/,
@@ -475,8 +477,6 @@ export const VISION_MODEL_REGEXES = [
/gpt-4-turbo(?!.*preview)/, // Matches "gpt-4-turbo" but not "gpt-4-turbo-preview"
/^dall-e-3$/, // Matches exactly "dall-e-3"
/glm-4v/,
/nova-lite/,
/nova-pro/,
/vl/i,
];
@@ -494,6 +494,14 @@ const openaiModels = [
"gpt-4-32k-0613",
"gpt-4-turbo",
"gpt-4-turbo-preview",
"gpt-4.1",
"gpt-4.1-2025-04-14",
"gpt-4.1-mini",
"gpt-4.1-mini-2025-04-14",
"gpt-4.1-nano",
"gpt-4.1-nano-2025-04-14",
"gpt-4.5-preview",
"gpt-4.5-preview-2025-02-27",
"gpt-4o",
"gpt-4o-2024-05-13",
"gpt-4o-2024-08-06",
@@ -510,28 +518,6 @@ const openaiModels = [
"o3-mini",
];
const bedrockModels = [
// Amazon nova Models
"us.amazon.nova-micro-v1:0",
"us.amazon.nova-lite-v1:0",
"us.amazon.nova-pro-v1:0",
// Claude Models
"anthropic.claude-3-haiku-20240307-v1:0",
"anthropic.claude-3-5-haiku-20241022-v1:0",
"anthropic.claude-3-sonnet-20240229-v1:0",
"anthropic.claude-3-5-sonnet-20241022-v2:0",
"anthropic.claude-3-opus-20240229-v1:0",
// Meta Llama Models
"us.meta.llama3-1-8b-instruct-v1:0",
"us.meta.llama3-1-70b-instruct-v1:0",
"us.meta.llama3-2-11b-instruct-v1:0",
"us.meta.llama3-2-90b-instruct-v1:0",
"us.meta.llama3-3-70b-instruct-v1:0",
// Mistral Models
"mistral.mistral-large-2402-v1:0",
"mistral.mistral-large-2407-v1:0",
];
const googleModels = [
"gemini-1.0-pro", // Deprecated on 2/15/2025
"gemini-1.5-pro-latest",
@@ -571,6 +557,8 @@ const anthropicModels = [
"claude-3-5-sonnet-20240620",
"claude-3-5-sonnet-20241022",
"claude-3-5-sonnet-latest",
"claude-3-7-sonnet-20250219",
"claude-3-7-sonnet-latest",
];
const baiduModels = [
@@ -604,6 +592,9 @@ const alibabaModes = [
"qwen-max-0403",
"qwen-max-0107",
"qwen-max-longcontext",
"qwen-omni-turbo",
"qwen-vl-plus",
"qwen-vl-max",
];
const tencentModels = [
@@ -799,7 +790,6 @@ export const DEFAULT_MODELS = [
sorted: 11,
},
})),
...chatglmModels.map((name) => ({
name,
available: true,
@@ -833,17 +823,6 @@ export const DEFAULT_MODELS = [
sorted: 14,
},
})),
...bedrockModels.map((name) => ({
name,
available: true,
sorted: seq++,
provider: {
id: "bedrock",
providerName: "Bedrock",
providerType: "bedrock",
sorted: 15,
},
})),
] as const;
export const CHAT_PAGE_SIZE = 15;

View File

@@ -343,32 +343,6 @@ const cn = {
SubTitle: "除默认地址外,必须包含 http(s)://",
},
},
Bedrock: {
Region: {
Title: "AWS 区域",
SubTitle: "Bedrock 服务所在的 AWS 区域",
Placeholder: "us-west-2",
Invalid: "无效的 AWS 区域格式。示例us-west-2",
},
AccessKey: {
Title: "AWS 访问密钥 ID",
SubTitle: "用于 Bedrock 服务的 AWS 访问密钥 ID",
Placeholder: "AKIA...",
Invalid: "无效的 AWS Access Key 格式。必须为20个字符。",
},
SecretKey: {
Title: "AWS 私有访问密钥",
SubTitle: "用于 Bedrock 服务的 AWS 私有访问密钥",
Placeholder: "****",
Invalid: "无效的 AWS Secret Key 格式。必须为40个字符。",
},
EncryptionKey: {
Title: "加密密钥",
SubTitle: "用于配置数据的加密密钥",
Placeholder: "输入加密密钥",
Invalid: "无效的加密密钥。必须至少包含8个字符",
},
},
Azure: {
ApiKey: {
Title: "接口密钥",

View File

@@ -347,33 +347,6 @@ const en: LocaleType = {
SubTitle: "Must start with http(s):// or use /api/openai as default",
},
},
Bedrock: {
Region: {
Title: "AWS Region",
SubTitle: "The AWS region where Bedrock service is located",
Placeholder: "us-west-2",
Invalid: "Invalid AWS region format. Example: us-west-2",
},
AccessKey: {
Title: "AWS Access Key ID",
SubTitle: "Your AWS access key ID for Bedrock service",
Placeholder: "AKIA...",
Invalid: "Invalid AWS access key format. Must be 20 characters long.",
},
SecretKey: {
Title: "AWS Secret Access Key",
SubTitle: "Your AWS secret access key for Bedrock service",
Placeholder: "****",
Invalid: "Invalid AWS secret key format. Must be 40 characters long.",
},
EncryptionKey: {
Title: "Encryption Key",
SubTitle: "Your encryption key for configuration data",
Placeholder: "Enter encryption key",
Invalid:
"Invalid encryption key format. Must no less than 8 characters long!",
},
},
Azure: {
ApiKey: {
Title: "Azure Api Key",

View File

@@ -16,7 +16,6 @@ import {
DEEPSEEK_BASE_URL,
XAI_BASE_URL,
CHATGLM_BASE_URL,
BEDROCK_BASE_URL,
SILICONFLOW_BASE_URL,
} from "../constant";
import { getHeaders } from "../client/api";
@@ -25,26 +24,36 @@ import { createPersistStore } from "../utils/store";
import { ensure } from "../utils/clone";
import { DEFAULT_CONFIG } from "./config";
import { getModelProvider } from "../utils/model";
import { encrypt, decrypt } from "../utils/aws";
let fetchState = 0; // 0 not fetch, 1 fetching, 2 done
const isApp = getClientConfig()?.buildMode === "export";
const DEFAULT_OPENAI_URL = isApp ? OPENAI_BASE_URL : ApiPath.OpenAI;
const DEFAULT_GOOGLE_URL = isApp ? GEMINI_BASE_URL : ApiPath.Google;
const DEFAULT_ANTHROPIC_URL = isApp ? ANTHROPIC_BASE_URL : ApiPath.Anthropic;
const DEFAULT_BAIDU_URL = isApp ? BAIDU_BASE_URL : ApiPath.Baidu;
const DEFAULT_BYTEDANCE_URL = isApp ? BYTEDANCE_BASE_URL : ApiPath.ByteDance;
const DEFAULT_ALIBABA_URL = isApp ? ALIBABA_BASE_URL : ApiPath.Alibaba;
const DEFAULT_TENCENT_URL = isApp ? TENCENT_BASE_URL : ApiPath.Tencent;
const DEFAULT_MOONSHOT_URL = isApp ? MOONSHOT_BASE_URL : ApiPath.Moonshot;
const DEFAULT_STABILITY_URL = isApp ? STABILITY_BASE_URL : ApiPath.Stability;
const DEFAULT_IFLYTEK_URL = isApp ? IFLYTEK_BASE_URL : ApiPath.Iflytek;
const DEFAULT_DEEPSEEK_URL = isApp ? DEEPSEEK_BASE_URL : ApiPath.DeepSeek;
const DEFAULT_XAI_URL = isApp ? XAI_BASE_URL : ApiPath.XAI;
const DEFAULT_CHATGLM_URL = isApp ? CHATGLM_BASE_URL : ApiPath.ChatGLM;
const DEFAULT_BEDROCK_URL = isApp ? BEDROCK_BASE_URL : ApiPath.Bedrock;
const DEFAULT_SILICONFLOW_URL = isApp
? SILICONFLOW_BASE_URL
@@ -119,19 +128,10 @@ const DEFAULT_ACCESS_STATE = {
chatglmUrl: DEFAULT_CHATGLM_URL,
chatglmApiKey: "",
// aws bedrock
bedrockUrl: DEFAULT_BEDROCK_URL,
awsRegion: "",
awsAccessKey: "",
awsSecretKey: "",
encryptionKey: "",
bedrockAnthropicVersion: "bedrock-2023-05-31",
// siliconflow
siliconflowUrl: DEFAULT_SILICONFLOW_URL,
siliconflowApiKey: "",
// server config
needCode: true,
hideUserApiKey: false,
@@ -148,9 +148,11 @@ const DEFAULT_ACCESS_STATE = {
export const useAccessStore = createPersistStore(
{ ...DEFAULT_ACCESS_STATE },
(set, get) => ({
enabledAccessControl() {
this.fetch();
return get().needCode;
},
getVisionModels() {
@@ -159,6 +161,7 @@ export const useAccessStore = createPersistStore(
},
edgeVoiceName() {
this.fetch();
return get().edgeTTSVoiceName;
},
@@ -197,7 +200,6 @@ export const useAccessStore = createPersistStore(
isValidMoonshot() {
return ensure(get(), ["moonshotApiKey"]);
},
isValidIflytek() {
return ensure(get(), ["iflytekApiKey"]);
},
@@ -213,19 +215,8 @@ export const useAccessStore = createPersistStore(
return ensure(get(), ["chatglmApiKey"]);
},
isValidBedrock() {
return ensure(get(), [
"awsRegion",
"awsAccessKey",
"awsSecretKey",
"encryptionKey",
]);
},
isValidSiliconFlow() {
return ensure(get(), ["siliconflowApiKey"]);
},
isAuthorized() {
@@ -246,13 +237,11 @@ export const useAccessStore = createPersistStore(
this.isValidDeepSeek() ||
this.isValidXAI() ||
this.isValidChatGLM() ||
this.isValidBedrock() ||
this.isValidSiliconFlow() ||
!this.enabledAccessControl() ||
(this.enabledAccessControl() && ensure(get(), ["accessCode"]))
);
},
fetch() {
if (fetchState > 0 || getClientConfig()?.buildMode === "export") return;
fetchState = 1;
@@ -271,6 +260,7 @@ export const useAccessStore = createPersistStore(
DEFAULT_CONFIG.modelConfig.model = model;
DEFAULT_CONFIG.modelConfig.providerName = providerName as any;
}
return res;
})
.then((res: DangerConfig) => {
@@ -284,43 +274,6 @@ export const useAccessStore = createPersistStore(
fetchState = 2;
});
},
// Override the set method to encrypt AWS credentials before storage
set: (partial: { [key: string]: any }) => {
if (partial.awsAccessKey) {
partial.awsAccessKey = encrypt(
partial.awsAccessKey,
partial.encryptionKey,
);
}
if (partial.awsSecretKey) {
partial.awsSecretKey = encrypt(
partial.awsSecretKey,
partial.encryptionKey,
);
}
if (partial.awsRegion) {
partial.awsRegion = encrypt(partial.awsRegion, partial.encryptionKey);
}
set(partial);
},
// Add getter to decrypt AWS credentials when needed
get: () => {
const state = get();
return {
...state,
awsRegion: state.awsRegion
? decrypt(state.awsRegion, state.encryptionKey)
: "",
awsAccessKey: state.awsAccessKey
? decrypt(state.awsAccessKey, state.encryptionKey)
: "",
awsSecretKey: state.awsSecretKey
? decrypt(state.awsSecretKey, state.encryptionKey)
: "",
};
},
}),
{
name: StoreKey.Access,

View File

@@ -344,13 +344,6 @@ export function showPlugins(provider: ServiceProvider, model: string) {
if (provider == ServiceProvider.Anthropic && !model.includes("claude-2")) {
return true;
}
if (
(provider == ServiceProvider.Bedrock && model.includes("claude-3")) ||
model.includes("mistral-large") ||
model.includes("amazon.nova")
) {
return true;
}
if (provider == ServiceProvider.Google && !model.includes("vision")) {
return true;
}

View File

@@ -1,696 +0,0 @@
// Types and Interfaces
export interface BedrockCredentials {
region: string;
accessKeyId: string;
secretAccessKey: string;
}
// Type definitions for better type safety
type ParsedEvent = Record<string, any>;
type EventResult = ParsedEvent[];
// Using a dot as separator since it's not used in Base64
const SEPARATOR = "~";
// Unified crypto utilities for both frontend and backend
async function generateKey(
password: string,
salt: Uint8Array,
): Promise<CryptoKey> {
const enc = new TextEncoder();
const keyMaterial = await crypto.subtle.importKey(
"raw",
enc.encode(password),
{ name: "PBKDF2" },
false,
["deriveBits", "deriveKey"],
);
return crypto.subtle.deriveKey(
{
name: "PBKDF2",
salt,
iterations: 100000,
hash: "SHA-256",
},
keyMaterial,
{ name: "AES-GCM", length: 256 },
false,
["encrypt", "decrypt"],
);
}
function arrayBufferToBase64(buffer: ArrayBuffer | Uint8Array): string {
const bytes = buffer instanceof Uint8Array ? buffer : new Uint8Array(buffer);
return btoa(String.fromCharCode(...bytes));
}
function base64ToArrayBuffer(base64: string): Uint8Array {
const binaryString = atob(base64);
const bytes = new Uint8Array(binaryString.length);
for (let i = 0; i < binaryString.length; i++) {
bytes[i] = binaryString.charCodeAt(i);
}
return bytes;
}
export async function encrypt(
data: string,
encryptionKey: string,
): Promise<string> {
if (!data) return "";
if (!encryptionKey) {
throw new Error("Encryption key is required for AWS credential encryption");
}
try {
const enc = new TextEncoder();
const salt = crypto.getRandomValues(new Uint8Array(16));
const iv = crypto.getRandomValues(new Uint8Array(12));
const key = await generateKey(encryptionKey, salt);
const encrypted = await crypto.subtle.encrypt(
{
name: "AES-GCM",
iv,
},
key,
enc.encode(data),
);
// Convert to base64 strings
const encryptedBase64 = arrayBufferToBase64(encrypted);
const saltBase64 = arrayBufferToBase64(salt);
const ivBase64 = arrayBufferToBase64(iv);
return [saltBase64, ivBase64, encryptedBase64].join(SEPARATOR);
} catch (error) {
// console.error("[Encryption Error]:", error);
throw new Error("Failed to encrypt AWS credentials");
}
}
export async function decrypt(
encryptedData: string,
encryptionKey: string,
): Promise<string> {
if (!encryptedData) return "";
if (!encryptionKey) {
throw new Error("Encryption key is required for AWS credential decryption");
}
try {
const [saltBase64, ivBase64, cipherBase64] = encryptedData.split(SEPARATOR);
// Convert base64 strings back to Uint8Arrays
const salt = base64ToArrayBuffer(saltBase64);
const iv = base64ToArrayBuffer(ivBase64);
const cipherData = base64ToArrayBuffer(cipherBase64);
const key = await generateKey(encryptionKey, salt);
const decrypted = await crypto.subtle.decrypt(
{
name: "AES-GCM",
iv,
},
key,
cipherData,
);
const dec = new TextDecoder();
return dec.decode(decrypted);
} catch (error) {
throw new Error("Failed to decrypt AWS credentials");
}
}
export function maskSensitiveValue(value: string): string {
if (!value) return "";
if (value.length <= 6) return value;
const masked = "*".repeat(value.length - 6);
return value.slice(0, 3) + masked + value.slice(-3);
}
// AWS Signing
export interface SignParams {
method: string;
url: string;
region: string;
accessKeyId: string;
secretAccessKey: string;
body: string | object;
service: string;
headers?: Record<string, string>;
isStreaming?: boolean;
}
async function createHmac(
key: ArrayBuffer | Uint8Array,
data: string,
): Promise<ArrayBuffer> {
const encoder = new TextEncoder();
const keyData = key instanceof Uint8Array ? key : new Uint8Array(key);
const keyObject = await crypto.subtle.importKey(
"raw",
keyData,
{ name: "HMAC", hash: "SHA-256" },
false,
["sign"],
);
return crypto.subtle.sign("HMAC", keyObject, encoder.encode(data));
}
async function getSigningKey(
secretKey: string,
dateStamp: string,
region: string,
service: string,
): Promise<ArrayBuffer> {
const encoder = new TextEncoder();
const kDate = await createHmac(encoder.encode("AWS4" + secretKey), dateStamp);
const kRegion = await createHmac(kDate, region);
const kService = await createHmac(kRegion, service);
const kSigning = await createHmac(kService, "aws4_request");
return kSigning;
}
function normalizeHeaderValue(value: string): string {
return value.replace(/\s+/g, " ").trim();
}
function encodeRFC3986(str: string): string {
return encodeURIComponent(str)
.replace(
/[!'()*]/g,
(c) => "%" + c.charCodeAt(0).toString(16).toUpperCase(),
)
.replace(/[-_.~]/g, (c) => c);
}
function getCanonicalUri(path: string): string {
if (!path || path === "/") return "/";
return (
"/" +
path
.split("/")
.map((segment) => {
if (!segment) return "";
if (segment === "invoke-with-response-stream") return segment;
if (segment.includes("model/")) {
return segment
.split(/(model\/)/)
.map((part) => {
if (part === "model/") return part;
return part
.split(/([.:])/g)
.map((subpart, i) =>
i % 2 === 1 ? subpart : encodeRFC3986(subpart),
)
.join("");
})
.join("");
}
return encodeRFC3986(segment);
})
.join("/")
);
}
export async function sign({
method,
url,
region,
accessKeyId,
secretAccessKey,
body,
service,
headers: customHeaders = {},
isStreaming = true,
}: SignParams): Promise<Record<string, string>> {
try {
const endpoint = new URL(url);
const canonicalUri = getCanonicalUri(endpoint.pathname.slice(1));
const canonicalQueryString = endpoint.search.slice(1);
const now = new Date();
const amzDate = now.toISOString().replace(/[:-]|\.\d{3}/g, "");
const dateStamp = amzDate.slice(0, 8);
const bodyString = typeof body === "string" ? body : JSON.stringify(body);
const encoder = new TextEncoder();
const payloadBuffer = await crypto.subtle.digest(
"SHA-256",
encoder.encode(bodyString),
);
const payloadHash = Array.from(new Uint8Array(payloadBuffer))
.map((b) => b.toString(16).padStart(2, "0"))
.join("");
const headers: Record<string, string> = {
accept: isStreaming
? "application/vnd.amazon.eventstream"
: "application/json",
"content-type": "application/json",
host: endpoint.host,
"x-amz-content-sha256": payloadHash,
"x-amz-date": amzDate,
...customHeaders,
};
// Add x-amzn-bedrock-accept header for streaming requests
if (isStreaming) {
headers["x-amzn-bedrock-accept"] = "*/*";
}
const sortedHeaderKeys = Object.keys(headers).sort((a, b) =>
a.toLowerCase().localeCompare(b.toLowerCase()),
);
const canonicalHeaders = sortedHeaderKeys
.map(
(key) => `${key.toLowerCase()}:${normalizeHeaderValue(headers[key])}\n`,
)
.join("");
const signedHeaders = sortedHeaderKeys
.map((key) => key.toLowerCase())
.join(";");
const canonicalRequest = [
method.toUpperCase(),
canonicalUri,
canonicalQueryString,
canonicalHeaders,
signedHeaders,
payloadHash,
].join("\n");
const algorithm = "AWS4-HMAC-SHA256";
const credentialScope = `${dateStamp}/${region}/${service}/aws4_request`;
const canonicalRequestHash = Array.from(
new Uint8Array(
await crypto.subtle.digest("SHA-256", encoder.encode(canonicalRequest)),
),
)
.map((b) => b.toString(16).padStart(2, "0"))
.join("");
const stringToSign = [
algorithm,
amzDate,
credentialScope,
canonicalRequestHash,
].join("\n");
const signingKey = await getSigningKey(
secretAccessKey,
dateStamp,
region,
service,
);
const signature = Array.from(
new Uint8Array(await createHmac(signingKey, stringToSign)),
)
.map((b) => b.toString(16).padStart(2, "0"))
.join("");
const authorization = [
`${algorithm} Credential=${accessKeyId}/${credentialScope}`,
`SignedHeaders=${signedHeaders}`,
`Signature=${signature}`,
].join(", ");
return {
...headers,
Authorization: authorization,
};
} catch (error) {
console.error("[AWS Signing Error]: Failed to sign request");
throw new Error("Failed to sign AWS request");
}
}
// Bedrock utilities
function decodeBase64(base64String: string): string {
try {
const bytes = Buffer.from(base64String, "base64");
const decoder = new TextDecoder("utf-8");
return decoder.decode(bytes);
} catch (e) {
console.error("[Base64 Decode Error]:", e);
return "";
}
}
export function parseEventData(chunk: Uint8Array): EventResult {
const decoder = new TextDecoder("utf-8");
const text = decoder.decode(chunk);
const results: EventResult = [];
try {
// First try to parse as regular JSON
const parsed = JSON.parse(text);
if (parsed.bytes) {
const decoded = decodeBase64(parsed.bytes);
try {
const decodedJson = JSON.parse(decoded);
results.push(decodedJson);
} catch (e) {
results.push({ output: decoded });
}
return results;
}
if (typeof parsed.body === "string") {
try {
const parsedBody = JSON.parse(parsed.body);
results.push(parsedBody);
} catch (e) {
results.push({ output: parsed.body });
}
return results;
}
results.push(parsed.body || parsed);
return results;
} catch (e) {
// If regular JSON parse fails, try to extract event content
const eventRegex = /:event-type[^\{]+(\{[^\}]+\})/g;
let match;
while ((match = eventRegex.exec(text)) !== null) {
try {
const eventData = match[1];
const parsed = JSON.parse(eventData);
if (parsed.bytes) {
const decoded = decodeBase64(parsed.bytes);
try {
const decodedJson = JSON.parse(decoded);
if (decodedJson.choices?.[0]?.message?.content) {
results.push({ output: decodedJson.choices[0].message.content });
} else {
results.push(decodedJson);
}
} catch (e) {
results.push({ output: decoded });
}
} else {
results.push(parsed);
}
} catch (e) {
console.debug("[Event Parse Warning]:", e);
}
}
// If no events were found, try to extract clean text
if (results.length === 0) {
// Remove event metadata markers and clean the text
const cleanText = text
.replace(/\{KG[^:]+:event-type[^}]+\}/g, "") // Remove event markers
.replace(/[\x00-\x1F\x7F-\x9F\uFEFF]/g, "") // Remove control characters
.trim();
if (cleanText) {
results.push({ output: cleanText });
}
}
}
return results;
}
export function processMessage(
data: ParsedEvent,
remainText: string,
runTools: any[],
index: number,
): { remainText: string; index: number } {
if (!data) return { remainText, index };
try {
// Handle Nova's tool calls with exact schema match
// console.log("processMessage data=========================",data);
if (data.contentBlockStart?.start?.toolUse) {
const toolUse = data.contentBlockStart.start.toolUse;
index += 1;
runTools.push({
id: toolUse.toolUseId,
type: "function",
function: {
name: toolUse.name || "", // Ensure name is always present
arguments: "{}", // Initialize empty arguments
},
});
return { remainText, index };
}
// Handle Nova's tool input in contentBlockDelta
if (data.contentBlockDelta?.delta?.toolUse?.input) {
if (runTools[index]) {
runTools[index].function.arguments =
data.contentBlockDelta.delta.toolUse.input;
}
return { remainText, index };
}
// Handle Nova's text content
if (data.output?.message?.content?.[0]?.text) {
remainText += data.output.message.content[0].text;
return { remainText, index };
}
// Handle Nova's messageStart event
if (data.messageStart) {
return { remainText, index };
}
// Handle Nova's text delta
if (data.contentBlockDelta?.delta?.text) {
remainText += data.contentBlockDelta.delta.text;
return { remainText, index };
}
// Handle Nova's contentBlockStop event
if (data.contentBlockStop) {
return { remainText, index };
}
// Handle Nova's messageStop event
if (data.messageStop) {
return { remainText, index };
}
// Handle message_start event (for other models)
if (data.type === "message_start") {
return { remainText, index };
}
// Handle content_block_start event (for other models)
if (data.type === "content_block_start") {
if (data.content_block?.type === "tool_use") {
index += 1;
runTools.push({
id: data.content_block.id,
type: "function",
function: {
name: data.content_block.name || "", // Ensure name is always present
arguments: "",
},
});
}
return { remainText, index };
}
// Handle content_block_delta event (for other models)
if (data.type === "content_block_delta") {
if (data.delta?.type === "input_json_delta" && runTools[index]) {
runTools[index].function.arguments += data.delta.partial_json;
} else if (data.delta?.type === "text_delta") {
const newText = data.delta.text || "";
remainText += newText;
}
return { remainText, index };
}
// Handle tool calls for other models
if (data.choices?.[0]?.message?.tool_calls) {
for (const toolCall of data.choices[0].message.tool_calls) {
index += 1;
runTools.push({
id: toolCall.id || `tool-${Date.now()}`,
type: "function",
function: {
name: toolCall.function?.name || "", // Ensure name is always present
arguments: toolCall.function?.arguments || "",
},
});
}
return { remainText, index };
}
// Handle various response formats
let newText = "";
if (data.delta?.text) {
newText = data.delta.text;
} else if (data.choices?.[0]?.message?.content) {
newText = data.choices[0].message.content;
} else if (data.content?.[0]?.text) {
newText = data.content[0].text;
} else if (data.generation) {
newText = data.generation;
} else if (data.outputText) {
newText = data.outputText;
} else if (data.response) {
newText = data.response;
} else if (data.output) {
newText = data.output;
}
// Only append if we have new text
if (newText) {
remainText += newText;
}
} catch (e) {
console.warn("Failed to process Bedrock message:");
}
return { remainText, index };
}
export function processChunks(
chunks: Uint8Array[],
pendingChunk: Uint8Array | null,
remainText: string,
runTools: any[],
index: number,
): {
chunks: Uint8Array[];
pendingChunk: Uint8Array | null;
remainText: string;
index: number;
} {
let currentText = remainText;
let currentIndex = index;
while (chunks.length > 0) {
const chunk = chunks[0];
try {
// If there's a pending chunk, try to merge it with the current chunk
let chunkToProcess = chunk;
if (pendingChunk) {
const mergedChunk = new Uint8Array(pendingChunk.length + chunk.length);
mergedChunk.set(pendingChunk);
mergedChunk.set(chunk, pendingChunk.length);
chunkToProcess = mergedChunk;
pendingChunk = null;
}
// Try to process the chunk
const parsedEvents = parseEventData(chunkToProcess);
if (parsedEvents.length > 0) {
// Process each event in the chunk
for (const parsed of parsedEvents) {
const result = processMessage(
parsed,
currentText,
runTools,
currentIndex,
);
currentText = result.remainText;
currentIndex = result.index;
}
chunks.shift(); // Remove processed chunk
} else {
// If parsing fails, it might be an incomplete chunk
pendingChunk = chunkToProcess;
chunks.shift();
}
} catch (e) {
// console.error("[Chunk Process Error]:", e);
// chunks.shift(); // Remove error chunk
// pendingChunk = null; // Reset pending chunk on error
console.warn("Failed to process chunk, attempting recovery");
// Attempt to recover by processing the next chunk
if (chunks.length > 1) {
chunks.shift();
pendingChunk = null;
} else {
// If this is the last chunk, throw to prevent data loss
throw new Error("Failed to process final chunk");
}
}
}
return {
chunks,
pendingChunk,
remainText: currentText,
index: currentIndex,
};
}
export function getBedrockEndpoint(
region: string,
modelId: string,
shouldStream: boolean,
): string {
if (!region || !modelId) {
throw new Error("Region and model ID are required for Bedrock endpoint");
}
const baseEndpoint = `https://bedrock-runtime.${region}.amazonaws.com`;
const endpoint =
shouldStream === false
? `${baseEndpoint}/model/${modelId}/invoke`
: `${baseEndpoint}/model/${modelId}/invoke-with-response-stream`;
return endpoint;
}
export function extractMessage(res: any, modelId: string = ""): string {
if (!res) {
throw new Error("Empty response received");
}
let message = "";
// Handle Nova model response format
if (modelId.toLowerCase().includes("nova")) {
if (res.output?.message?.content?.[0]?.text) {
message = res.output.message.content[0].text;
} else {
message = res.output || "";
}
}
// Handle Mistral model response format
else if (modelId.toLowerCase().includes("mistral")) {
if (res.choices?.[0]?.message?.content) {
message = res.choices[0].message.content;
} else {
message = res.output || "";
}
}
// Handle Llama model response format
else if (modelId.toLowerCase().includes("llama")) {
message = res?.generation || "";
}
// Handle Titan model response format
else if (modelId.toLowerCase().includes("titan")) {
message = res?.outputText || "";
}
// Handle Claude and other models
else if (res.content?.[0]?.text) {
message = res.content[0].text;
}
// Handle other response formats
else {
message = res.output || res.response || res.message || "";
}
return message;
}

View File

@@ -3,7 +3,7 @@ import {
UPLOAD_URL,
REQUEST_TIMEOUT_MS,
} from "@/app/constant";
import { RequestMessage } from "@/app/client/api";
import { MultimodalContent, RequestMessage } from "@/app/client/api";
import Locale from "@/app/locales";
import {
EventStreamContentType,
@@ -70,8 +70,9 @@ export function compressImage(file: Blob, maxSize: number): Promise<string> {
});
}
export async function preProcessImageContent(
export async function preProcessImageContentBase(
content: RequestMessage["content"],
transformImageUrl: (url: string) => Promise<{ [key: string]: any }>,
) {
if (typeof content === "string") {
return content;
@@ -81,7 +82,7 @@ export async function preProcessImageContent(
if (part?.type == "image_url" && part?.image_url?.url) {
try {
const url = await cacheImageToBase64Image(part?.image_url?.url);
result.push({ type: part.type, image_url: { url } });
result.push(await transformImageUrl(url));
} catch (error) {
console.error("Error processing image URL:", error);
}
@@ -92,6 +93,23 @@ export async function preProcessImageContent(
return result;
}
export async function preProcessImageContent(
content: RequestMessage["content"],
) {
return preProcessImageContentBase(content, async (url) => ({
type: "image_url",
image_url: { url },
})) as Promise<MultimodalContent[] | string>;
}
export async function preProcessImageContentForAlibabaDashScope(
content: RequestMessage["content"],
) {
return preProcessImageContentBase(content, async (url) => ({
image: url,
}));
}
const imageCaches: Record<string, string> = {};
export function cacheImageToBase64Image(imageUrl: string) {
if (imageUrl.includes(CACHE_URL_PREFIX)) {

View File

@@ -1,258 +0,0 @@
# Understanding Bedrock Response Format
The AWS Bedrock streaming response format consists of multiple Server-Sent Events (SSE) chunks. Each chunk follows this structure:
```
:event-type chunk
:content-type application/json
:message-type event
{"bytes":"base64_encoded_data","p":"signature"}
```
## Model-Specific Response Formats
### Claude 3 Format
When using Claude 3 models (e.g., claude-3-haiku-20240307), the decoded messages include:
1. **message_start**
```json
{
"type": "message_start",
"message": {
"id": "msg_bdrk_01A6sahWac4XVTR9sX3rgvsZ",
"type": "message",
"role": "assistant",
"model": "claude-3-haiku-20240307",
"content": [],
"stop_reason": null,
"stop_sequence": null,
"usage": {
"input_tokens": 8,
"output_tokens": 1
}
}
}
```
2. **content_block_start**
```json
{
"type": "content_block_start",
"index": 0,
"content_block": {
"type": "text",
"text": ""
}
}
```
3. **content_block_delta**
```json
{
"type": "content_block_delta",
"index": 0,
"delta": {
"type": "text_delta",
"text": "Hello"
}
}
```
### Mistral Format
When using Mistral models (e.g., mistral-large-2407), the decoded messages have a different structure:
```json
{
"id": "b0098812-0ad9-42da-9f17-a5e2f554eb6b",
"object": "chat.completion.chunk",
"created": 1732582566,
"model": "mistral-large-2407",
"choices": [{
"index": 0,
"logprobs": null,
"context_logits": null,
"generation_logits": null,
"message": {
"role": null,
"content": "Hello",
"tool_calls": null,
"index": null,
"tool_call_id": null
},
"stop_reason": null
}],
"usage": null,
"p": null
}
```
### Llama Format
When using Llama models (3.1 or 3.2), the decoded messages use a simpler structure focused on generation tokens:
```json
{
"generation": "Hello",
"prompt_token_count": null,
"generation_token_count": 2,
"stop_reason": null
}
```
Each chunk contains:
- generation: The generated text piece
- prompt_token_count: Token count of the input (only present in first chunk)
- generation_token_count: Running count of generated tokens
- stop_reason: Indicates completion (null until final chunk)
First chunk example (includes prompt_token_count):
```json
{
"generation": "\n\n",
"prompt_token_count": 10,
"generation_token_count": 1,
"stop_reason": null
}
```
### Titan Text Format
When using Amazon's Titan models (text or TG1), the response comes as a single chunk with complete text and metrics:
```json
{
"outputText": "\nBot: Hello! How can I help you today?",
"index": 0,
"totalOutputTextTokenCount": 13,
"completionReason": "FINISH",
"inputTextTokenCount": 3,
"amazon-bedrock-invocationMetrics": {
"inputTokenCount": 3,
"outputTokenCount": 13,
"invocationLatency": 833,
"firstByteLatency": 833
}
}
```
Both Titan text and Titan TG1 use the same response format, with only minor differences in token counts and latency values. For example, here's a TG1 response:
```json
{
"outputText": "\nBot: Hello! How can I help you?",
"index": 0,
"totalOutputTextTokenCount": 12,
"completionReason": "FINISH",
"inputTextTokenCount": 3,
"amazon-bedrock-invocationMetrics": {
"inputTokenCount": 3,
"outputTokenCount": 12,
"invocationLatency": 845,
"firstByteLatency": 845
}
}
```
Key fields:
- outputText: The complete generated response
- totalOutputTextTokenCount: Total tokens in the response
- completionReason: Reason for completion (e.g., "FINISH")
- inputTextTokenCount: Number of input tokens
- amazon-bedrock-invocationMetrics: Detailed performance metrics
## Model-Specific Completion Metrics
### Mistral
```json
{
"usage": {
"prompt_tokens": 5,
"total_tokens": 29,
"completion_tokens": 24
},
"amazon-bedrock-invocationMetrics": {
"inputTokenCount": 5,
"outputTokenCount": 24,
"invocationLatency": 719,
"firstByteLatency": 148
}
}
```
### Claude 3
Included in the message_delta with stop_reason.
### Llama
Included in the final chunk with stop_reason "stop":
```json
{
"amazon-bedrock-invocationMetrics": {
"inputTokenCount": 10,
"outputTokenCount": 11,
"invocationLatency": 873,
"firstByteLatency": 550
}
}
```
### Titan
Both Titan text and TG1 include metrics in the single response chunk:
```json
{
"amazon-bedrock-invocationMetrics": {
"inputTokenCount": 3,
"outputTokenCount": 12,
"invocationLatency": 845,
"firstByteLatency": 845
}
}
```
## How the Response is Processed
1. The raw response is first split into chunks based on SSE format
2. For each chunk:
- The base64 encoded data is decoded
- The JSON is parsed to extract the message content
- Based on the model type and message type, different processing is applied:
### Claude 3 Processing
- message_start: Initializes a new message with model info and usage stats
- content_block_start: Starts a new content block (text, tool use, etc.)
- content_block_delta: Adds incremental content to the current block
- message_delta: Updates message metadata
### Mistral Processing
- Each chunk contains a complete message object with choices array
- The content is streamed through the message.content field
- Final chunk includes token usage and invocation metrics
### Llama Processing
- Each chunk contains a generation field with the text piece
- First chunk includes prompt_token_count
- Tracks generation progress through generation_token_count
- Simple streaming format focused on text generation
- Final chunk includes complete metrics
### Titan Processing
- Single chunk response with complete text
- No streaming - returns full response at once
- Includes comprehensive metrics in the same chunk
## Handling in Code
The response is processed by the `transformBedrockStream` function in `app/utils/aws.ts`, which:
1. Reads the stream chunks
2. Parses each chunk using `parseEventData`
3. Handles model-specific formats:
- For Claude: Processes message_start, content_block_start, content_block_delta
- For Mistral: Extracts content from choices[0].message.content
- For Llama: Uses the generation field directly
- For Titan: Uses the outputText field from the single response
4. Transforms the parsed data into a consistent format for the client
5. Yields the transformed data as SSE events
This allows for real-time streaming of the model's response while maintaining a consistent format for the client application, regardless of which model is being used.

View File

@@ -15,6 +15,8 @@ const config: Config = {
moduleNameMapper: {
"^@/(.*)$": "<rootDir>/$1",
},
extensionsToTreatAsEsm: [".ts", ".tsx"],
injectGlobals: true,
};
// createJestConfig is exported this way to ensure that next/jest can load the Next.js config which is async

View File

@@ -1,24 +1,22 @@
// Learn more: https://github.com/testing-library/jest-dom
import "@testing-library/jest-dom";
import { jest } from "@jest/globals";
global.fetch = jest.fn(() =>
Promise.resolve({
ok: true,
status: 200,
json: () => Promise.resolve({}),
json: () => Promise.resolve([]),
headers: new Headers(),
redirected: false,
statusText: "OK",
type: "basic",
url: "",
clone: function () {
return this;
},
body: null,
bodyUsed: false,
arrayBuffer: () => Promise.resolve(new ArrayBuffer(0)),
blob: () => Promise.resolve(new Blob()),
formData: () => Promise.resolve(new FormData()),
text: () => Promise.resolve(""),
}),
} as Response),
);

View File

@@ -17,8 +17,8 @@
"prompts": "node ./scripts/fetch-prompts.mjs",
"prepare": "husky install",
"proxy-dev": "sh ./scripts/init-proxy.sh && proxychains -f ./scripts/proxychains.conf yarn dev",
"test": "jest --watch",
"test:ci": "jest --ci"
"test": "node --no-warnings --experimental-vm-modules $(yarn bin jest) --watch",
"test:ci": "node --no-warnings --experimental-vm-modules $(yarn bin jest) --ci"
},
"dependencies": {
"@fortaine/fetch-event-source": "^3.0.6",
@@ -94,4 +94,4 @@
"lint-staged/yaml": "^2.2.2"
},
"packageManager": "yarn@1.22.19"
}
}

View File

@@ -1,3 +1,4 @@
import { jest } from "@jest/globals";
import { isVisionModel } from "../app/utils";
describe("isVisionModel", () => {
@@ -50,7 +51,7 @@ describe("isVisionModel", () => {
test("should identify models from VISION_MODELS env var", () => {
process.env.VISION_MODELS = "custom-vision-model,another-vision-model";
expect(isVisionModel("custom-vision-model")).toBe(true);
expect(isVisionModel("another-vision-model")).toBe(true);
expect(isVisionModel("unrelated-model")).toBe(false);
@@ -64,4 +65,4 @@ describe("isVisionModel", () => {
expect(isVisionModel("unrelated-model")).toBe(false);
expect(isVisionModel("gpt-4-vision")).toBe(true);
});
});
});

3177
yarn.lock

File diff suppressed because it is too large Load Diff