更新Bedrock

2025-10-10 03:56:37 +08:00 · 2024-06-13 17:31:42 +08:00 · 2024-06-13 17:31:42 +08:00 · f021826220
commit f021826220
parent b6735bffe4
13 changed files with 454 additions and 11 deletions
--- a/.env.template
+++ b/.env.template
@ -54,10 +54,18 @@ ANTHROPIC_API_KEY=
 ### anthropic claude Api version. (optional)
 ANTHROPIC_API_VERSION=

-
-
 ### anthropic claude Api url (optional)
 ANTHROPIC_URL=

+# AWS Bedrock API Key.(optional)
+AWS_API_KEY=
+
+# AWS Bedrock API url (optional)
+AWS_URL=
+
+# AWS Bedrock API version (optional)
+AWS_API_VERSION=
+
+
 ### (optional)
 WHITE_WEBDEV_ENDPOINTS=
--- a/app/api/auth.ts
+++ b/app/api/auth.ts
@ -73,6 +73,9 @@ export function auth(req: NextRequest, modelProvider: ModelProvider) {
      case ModelProvider.Claude:
        systemApiKey = serverConfig.anthropicApiKey;
        break;
+      case ModelProvider.Bedrock:
+        systemApiKey = serverConfig.awsApiKey;
+        break;
      case ModelProvider.GPT:
      default:
        if (serverConfig.isAzure) {
--- a/app/client/api.ts
+++ b/app/client/api.ts
@ -9,6 +9,7 @@ import { ChatMessage, ModelType, useAccessStore, useChatStore } from "../store";
 import { ChatGPTApi } from "./platforms/openai";
 import { GeminiProApi } from "./platforms/google";
 import { ClaudeApi } from "./platforms/anthropic";
+import { BedrockApi } from "./platforms/aws";
 export const ROLES = ["system", "user", "assistant"] as const;
 export type MessageRole = (typeof ROLES)[number];

@ -70,7 +71,7 @@ export abstract class LLMApi {
  abstract models(): Promise<LLMModel[]>;
 }

-type ProviderName = "openai" | "azure" | "claude" | "palm";
+type ProviderName = "aws" | "openai" | "azure" | "claude" | "palm";

 interface Model {
  name: string;
@ -102,6 +103,9 @@ export class ClientApi {
      case ModelProvider.Claude:
        this.llm = new ClaudeApi();
        break;
+      case ModelProvider.Bedrock:
+        this.llm = new BedrockApi();
+        break;
      default:
        this.llm = new ChatGPTApi();
    }
@ -162,11 +166,15 @@ export function getHeaders() {
  const modelConfig = useChatStore.getState().currentSession().mask.modelConfig;
  const isGoogle = modelConfig.model.startsWith("gemini");
  const isAzure = accessStore.provider === ServiceProvider.Azure;
+  const isAWS = accessStore.provider === ServiceProvider.AWS;
  const authHeader = isAzure ? "api-key" : "Authorization";
+
  const apiKey = isGoogle
    ? accessStore.googleApiKey
    : isAzure
    ? accessStore.azureApiKey
+    : isAWS
+    ? accessStore.awsApiKey
    : accessStore.openaiApiKey;
  const clientConfig = getClientConfig();
  const makeBearer = (s: string) => `${isAzure ? "" : "Bearer "}${s.trim()}`;
--- a/app/client/platforms/aws.ts
+++ b/app/client/platforms/aws.ts
@ -0,0 +1,330 @@
+"use client";
+import { DEFAULT_MODELS, AWS, REQUEST_TIMEOUT_MS } from "@/app/constant";
+import { useAccessStore, useAppConfig, useChatStore } from "@/app/store";
+
+import {
+  ChatOptions,
+  getHeaders,
+  LLMApi,
+  LLMModel,
+  LLMUsage,
+  MultimodalContent,
+} from "../api";
+import Locale from "../../locales";
+import {
+  EventStreamContentType,
+  fetchEventSource,
+} from "@fortaine/fetch-event-source";
+import { prettyObject } from "@/app/utils/format";
+import { getClientConfig } from "@/app/config/client";
+import {
+  getMessageTextContent,
+  getMessageImages,
+  isVisionModel,
+} from "@/app/utils";
+
+export interface OpenAIListModelResponse {
+  object: string;
+  data: Array<{
+    id: string;
+    object: string;
+    root: string;
+  }>;
+}
+
+export class BedrockApi implements LLMApi {
+  private disableListModels = true;
+
+  path(path: string): string {
+    const accessStore = useAccessStore.getState();
+
+    if (!accessStore.awsUrl) {
+      throw Error("Please set your access url.");
+    }
+    let baseUrl = accessStore.awsUrl;
+    if (baseUrl.endsWith("/")) {
+      baseUrl = baseUrl.slice(0, baseUrl.length - 1);
+    }
+    console.log("[Proxy Endpoint] ", baseUrl, path);
+    return [baseUrl, path].join("/");
+  }
+
+  extractMessage(res: any) {
+    return res.choices?.at(0)?.message?.content ?? "";
+  }
+
+  async chat(options: ChatOptions) {
+    const visionModel = isVisionModel(options.config.model);
+    const messages = options.messages.map((v) => ({
+      role: v.role,
+      content: visionModel ? v.content : getMessageTextContent(v),
+    }));
+
+    const modelConfig = {
+      ...useAppConfig.getState().modelConfig,
+      ...useChatStore.getState().currentSession().mask.modelConfig,
+      ...{
+        model: options.config.model,
+      },
+    };
+
+    const requestPayload = {
+      messages,
+      stream: options.config.stream,
+      model: modelConfig.model,
+      temperature: modelConfig.temperature,
+      presence_penalty: modelConfig.presence_penalty,
+      frequency_penalty: modelConfig.frequency_penalty,
+      top_p: modelConfig.top_p,
+      max_tokens: modelConfig.max_tokens,
+    };
+
+    // add max_tokens to vision model
+    if (visionModel) {
+      Object.defineProperty(requestPayload, "max_tokens", {
+        enumerable: true,
+        configurable: true,
+        writable: true,
+        value: modelConfig.max_tokens,
+      });
+    }
+
+    console.log("[Request] aws bedrock payload: ", requestPayload);
+
+    const shouldStream = !!options.config.stream;
+    const controller = new AbortController();
+    options.onController?.(controller);
+
+    try {
+      const chatPath = this.path(AWS.ChatPath);
+      const chatPayload = {
+        method: "POST",
+        body: JSON.stringify(requestPayload),
+        signal: controller.signal,
+        headers: getHeaders(),
+      };
+
+      // make a fetch request
+      const requestTimeoutId = setTimeout(
+        () => controller.abort(),
+        REQUEST_TIMEOUT_MS,
+      );
+
+      if (shouldStream) {
+        let responseText = "";
+        let remainText = "";
+        let finished = false;
+
+        // animate response to make it looks smooth
+        function animateResponseText() {
+          if (finished || controller.signal.aborted) {
+            responseText += remainText;
+            console.log("[Response Animation] finished");
+            return;
+          }
+
+          if (remainText.length > 0) {
+            const fetchCount = Math.max(1, Math.round(remainText.length / 60));
+            const fetchText = remainText.slice(0, fetchCount);
+            responseText += fetchText;
+            remainText = remainText.slice(fetchCount);
+            options.onUpdate?.(responseText, fetchText);
+          }
+
+          requestAnimationFrame(animateResponseText);
+        }
+
+        // start animaion
+        animateResponseText();
+
+        const finish = () => {
+          if (!finished) {
+            finished = true;
+            options.onFinish(responseText + remainText);
+          }
+        };
+
+        controller.signal.onabort = finish;
+
+        fetchEventSource(chatPath, {
+          ...chatPayload,
+          async onopen(res) {
+            clearTimeout(requestTimeoutId);
+            const contentType = res.headers.get("content-type");
+            console.log("[AWS] request response content type: ", contentType);
+
+            if (contentType?.startsWith("text/plain")) {
+              responseText = await res.clone().text();
+              return finish();
+            }
+
+            if (
+              !res.ok ||
+              !res.headers
+                .get("content-type")
+                ?.startsWith(EventStreamContentType) ||
+              res.status !== 200
+            ) {
+              const responseTexts = [responseText];
+              let extraInfo = await res.clone().text();
+              try {
+                const resJson = await res.clone().json();
+                extraInfo = prettyObject(resJson);
+              } catch {}
+
+              if (res.status === 401) {
+                responseTexts.push(Locale.Error.Unauthorized);
+              }
+
+              if (extraInfo) {
+                responseTexts.push(extraInfo);
+              }
+
+              responseText = responseTexts.join("\n\n");
+
+              return finish();
+            }
+          },
+          onmessage(msg) {
+            if (msg.data === "[DONE]" || finished) {
+              return finish();
+            }
+            const text = msg.data;
+            try {
+              const json = JSON.parse(text) as {
+                choices: Array<{
+                  delta: {
+                    content: string;
+                  };
+                }>;
+              };
+              const delta = json.choices[0]?.delta?.content;
+              if (delta) {
+                remainText += delta;
+              }
+            } catch (e) {
+              console.error("[Request] parse error", text);
+            }
+          },
+          onclose() {
+            finish();
+          },
+          onerror(e) {
+            options.onError?.(e);
+            throw e;
+          },
+          openWhenHidden: true,
+        });
+      } else {
+        const res = await fetch(chatPath, chatPayload);
+        clearTimeout(requestTimeoutId);
+
+        const resJson = await res.json();
+        const message = this.extractMessage(resJson);
+        options.onFinish(message);
+      }
+    } catch (e) {
+      console.log("[Request] failed to make a chat request", e);
+      options.onError?.(e as Error);
+    }
+  }
+
+  async usage() {
+    const formatDate = (d: Date) =>
+      `${d.getFullYear()}-${(d.getMonth() + 1).toString().padStart(2, "0")}-${d
+        .getDate()
+        .toString()
+        .padStart(2, "0")}`;
+    const ONE_DAY = 1 * 24 * 60 * 60 * 1000;
+    const now = new Date();
+    const startOfMonth = new Date(now.getFullYear(), now.getMonth(), 1);
+    const startDate = formatDate(startOfMonth);
+    const endDate = formatDate(new Date(Date.now() + ONE_DAY));
+
+    const [used, subs] = await Promise.all([
+      fetch(
+        this.path(
+          `${AWS.UsagePath}?start_date=${startDate}&end_date=${endDate}`,
+        ),
+        {
+          method: "GET",
+          headers: getHeaders(),
+        },
+      ),
+      fetch(this.path(AWS.SubsPath), {
+        method: "GET",
+        headers: getHeaders(),
+      }),
+    ]);
+
+    if (used.status === 401) {
+      throw new Error(Locale.Error.Unauthorized);
+    }
+
+    if (!used.ok || !subs.ok) {
+      throw new Error("Failed to query usage from openai");
+    }
+
+    const response = (await used.json()) as {
+      total_usage?: number;
+      error?: {
+        type: string;
+        message: string;
+      };
+    };
+
+    const total = (await subs.json()) as {
+      hard_limit_usd?: number;
+    };
+
+    if (response.error && response.error.type) {
+      throw Error(response.error.message);
+    }
+
+    if (response.total_usage) {
+      response.total_usage = Math.round(response.total_usage) / 100;
+    }
+
+    if (total.hard_limit_usd) {
+      total.hard_limit_usd = Math.round(total.hard_limit_usd * 100) / 100;
+    }
+
+    return {
+      used: response.total_usage,
+      total: total.hard_limit_usd,
+    } as LLMUsage;
+  }
+
+  async models(): Promise<LLMModel[]> {
+    if (this.disableListModels) {
+      return DEFAULT_MODELS.slice();
+    }
+
+    const res = await fetch(this.path(AWS.ListModelPath), {
+      method: "GET",
+      headers: {
+        ...getHeaders(),
+      },
+    });
+
+    const resJson = (await res.json()) as OpenAIListModelResponse;
+    const chatModels = resJson.data;
+    console.log("[Models]", chatModels);
+
+    if (!chatModels) {
+      return [];
+    }
+
+    return chatModels.map((m) => ({
+      name: m.id,
+      displayName: m.id,
+      available: true,
+      provider: {
+        id: "bedrock",
+        providerName: "Bedrock",
+        providerType: "bedrock",
+      },
+    }));
+  }
+}
+export { AWS };
--- a/app/components/exporter.tsx
+++ b/app/components/exporter.tsx
@ -317,7 +317,8 @@ export function PreviewActions(props: {
    if (config.modelConfig.model.startsWith("gemini")) {
      api = new ClientApi(ModelProvider.GeminiPro);
    } else if (identifyDefaultClaudeModel(config.modelConfig.model)) {
-      api = new ClientApi(ModelProvider.Claude);
+      //api = new ClientApi(ModelProvider.Claude);
+      api = new ClientApi(ModelProvider.Bedrock);
    } else {
      api = new ClientApi(ModelProvider.GPT);
    }
--- a/app/components/home.tsx
+++ b/app/components/home.tsx
@ -175,7 +175,8 @@ export function useLoadData() {
  if (config.modelConfig.model.startsWith("gemini")) {
    api = new ClientApi(ModelProvider.GeminiPro);
  } else if (identifyDefaultClaudeModel(config.modelConfig.model)) {
-    api = new ClientApi(ModelProvider.Claude);
+    //api = new ClientApi(ModelProvider.Claude);
+    api = new ClientApi(ModelProvider.Bedrock);
  } else {
    api = new ClientApi(ModelProvider.GPT);
  }
--- a/app/components/settings.tsx
+++ b/app/components/settings.tsx
@ -1187,6 +1187,46 @@ export function Settings() {
                      </ListItem>
                    </>
                  )}
+                  {accessStore.provider === ServiceProvider.AWS && (
+                    <>
+                      <ListItem
+                        title={Locale.Settings.Access.AWS.Endpoint.Title}
+                        subTitle={Locale.Settings.Access.AWS.Endpoint.SubTitle}
+                      >
+                        <input
+                          type="text"
+                          value={accessStore.awsUrl}
+                          placeholder={
+                            Locale.Settings.Access.AWS.Endpoint.SubTitle
+                          }
+                          onChange={(e) =>
+                            accessStore.update(
+                              (access) =>
+                                (access.awsUrl = e.currentTarget.value),
+                            )
+                          }
+                        ></input>
+                      </ListItem>
+                      <ListItem
+                        title={Locale.Settings.Access.AWS.ApiKey.Title}
+                        subTitle={Locale.Settings.Access.AWS.ApiKey.SubTitle}
+                      >
+                        <PasswordInput
+                          value={accessStore.awsApiKey}
+                          type="text"
+                          placeholder={
+                            Locale.Settings.Access.AWS.ApiKey.Placeholder
+                          }
+                          onChange={(e) => {
+                            accessStore.update(
+                              (access) =>
+                                (access.awsApiKey = e.currentTarget.value),
+                            );
+                          }}
+                        />
+                      </ListItem>
+                    </>
+                  )}
                </>
              )}
            </>
--- a/app/components/sidebar.tsx
+++ b/app/components/sidebar.tsx
@ -216,11 +216,11 @@ export function SideBar(props: { className?: string }) {
              <IconButton icon={<SettingsIcon />} shadow />
            </Link>
          </div>
-          <div className={styles["sidebar-action"]}>
+          {/* <div className={styles["sidebar-action"]}>
            <a href={REPO_URL} target="_blank" rel="noopener noreferrer">
              <IconButton icon={<GithubIcon />} shadow />
            </a>
-          </div>
+          </div> */}
        </div>
        <div>
          <IconButton
--- a/app/config/server.ts
+++ b/app/config/server.ts
@ -92,6 +92,7 @@ export const getServerSideConfig = () => {
  const isAzure = !!process.env.AZURE_URL;
  const isGoogle = !!process.env.GOOGLE_API_KEY;
  const isAnthropic = !!process.env.ANTHROPIC_API_KEY;
+  const isAWS = !!process.env.AWS_API_KEY;

  // const apiKeyEnvVar = process.env.OPENAI_API_KEY ?? "";
  // const apiKeys = apiKeyEnvVar.split(",").map((v) => v.trim());
@ -124,6 +125,10 @@ export const getServerSideConfig = () => {
    anthropicApiVersion: process.env.ANTHROPIC_API_VERSION,
    anthropicUrl: process.env.ANTHROPIC_URL,

+    isAWS,
+    awsApiKey: getApiKey(process.env.AWS_API_KEY),
+    awsUrl: process.env.AWS_URL,
+
    gtmId: process.env.GTM_ID,

    needCode: ACCESS_CODES.size > 0,
--- a/app/constant.ts
+++ b/app/constant.ts
@ -1,4 +1,4 @@
-export const OWNER = "Yidadaa";
+export const OWNER = "Ryder Tsui";
 export const REPO = "ChatGPT-Next-Web";
 export const REPO_URL = `https://github.com/${OWNER}/${REPO}`;
 export const ISSUE_URL = `https://github.com/${OWNER}/${REPO}/issues`;
@ -70,12 +70,14 @@ export enum ServiceProvider {
  Azure = "Azure",
  Google = "Google",
  Anthropic = "Anthropic",
+  AWS = "AWS",
 }

 export enum ModelProvider {
  GPT = "GPT",
  GeminiPro = "GeminiPro",
  Claude = "Claude",
+  Bedrock = "Bedrock",
 }

 export const Anthropic = {
@ -101,6 +103,12 @@ export const Google = {
  ChatPath: (modelName: string) => `v1beta/models/${modelName}:generateContent`,
 };

+export const AWS = {
+  ExampleEndpoint: "http://localhost:8866",
+  ChatPath: "v1/chat/completions",
+  ListModelPath: "v1/models",
+};
+
 export const DEFAULT_INPUT_TEMPLATE = `{{input}}`; // input / time / model / lang
 // export const DEFAULT_SYSTEM_TEMPLATE = `
 // You are ChatGPT, a large language model trained by {{ServiceProvider}}.
@ -168,6 +176,14 @@ const anthropicModels = [
  "claude-3-haiku-20240307",
 ];

+const bedrockModels = [
+  "llama3-70b",
+  "llama3-8b",
+  "claude-3-haiku",
+  "claude-3-sonnet",
+  "claude-3-opus",
+];
+
 export const DEFAULT_MODELS = [
  ...openaiModels.map((name) => ({
    name,
@ -196,6 +212,15 @@ export const DEFAULT_MODELS = [
      providerType: "anthropic",
    },
  })),
+  ...bedrockModels.map((name) => ({
+    name,
+    available: true,
+    provider: {
+      id: "AWS",
+      providerName: "Bedrock",
+      providerType: "bedrock",
+    },
+  })),
 ] as const;

 export const CHAT_PAGE_SIZE = 15;
--- a/app/locales/cn.ts
+++ b/app/locales/cn.ts
@ -347,6 +347,21 @@ const cn = {
          SubTitle: "选择一个特定的 API 版本",
        },
      },
+      AWS: {
+        Endpoint: {
+          Title: "接口地址",
+          SubTitle: "样例：http://localhost:8866",
+        },
+        ApiKey: {
+          Title: "API Key",
+          SubTitle: "API Key",
+          Placeholder: "输入您的 AWS API 密钥",
+        },
+        ApiVerion: {
+          Title: "接口版本 (aws api version)",
+          SubTitle: "选择指定的部分版本",
+        },
+      },
      CustomModel: {
        Title: "自定义模型名",
        SubTitle: "增加自定义模型可选项，使用英文逗号隔开",
--- a/app/store/access.ts
+++ b/app/store/access.ts
@ -42,6 +42,11 @@ const DEFAULT_ACCESS_STATE = {
  anthropicApiVersion: "2023-06-01",
  anthropicUrl: "",

+  //AWS
+  awsUrl: "",
+  awsApiKey: "",
+  awsApiVersion: "",
+
  // server config
  needCode: true,
  hideUserApiKey: false,
--- a/app/store/chat.ts
+++ b/app/store/chat.ts
@ -366,8 +366,9 @@ export const useChatStore = createPersistStore(
        var api: ClientApi;
        if (modelConfig.model.startsWith("gemini")) {
          api = new ClientApi(ModelProvider.GeminiPro);
-        } else if (identifyDefaultClaudeModel(modelConfig.model)) {
-          api = new ClientApi(ModelProvider.Claude);
+        } else if (modelConfig.model.startsWith("claude")) {
+          api = new ClientApi(ModelProvider.Bedrock);
+          //api = new ClientApi(ModelProvider.Claude);
        } else {
          api = new ClientApi(ModelProvider.GPT);
        }
@ -551,7 +552,8 @@ export const useChatStore = createPersistStore(
        if (modelConfig.model.startsWith("gemini")) {
          api = new ClientApi(ModelProvider.GeminiPro);
        } else if (identifyDefaultClaudeModel(modelConfig.model)) {
-          api = new ClientApi(ModelProvider.Claude);
+          //api = new ClientApi(ModelProvider.Claude);
+          api = new ClientApi(ModelProvider.Bedrock);
        } else {
          api = new ClientApi(ModelProvider.GPT);
        }