Merge 69fcb92a3b into c30ddfbb07

Fix: Encourage markdown inclusion in model responses for o1/o3

README.md

@@ -7,7 +7,7 @@
 
 
 
-<h1 align="center">NextChat (ChatGPT Next Web)</h1>
+<h1 align="center">NextChat</h1>
 
 English / [简体中文](./README_CN.md)
 
@@ -22,8 +22,7 @@ English / [简体中文](./README_CN.md)
 [![MacOS][MacOS-image]][download-url]
 [![Linux][Linux-image]][download-url]
 
-[NextChatAI](https://nextchat.dev/chat?utm_source=readme) / [Web App Demo](https://app.nextchat.dev) / [Desktop App](https://github.com/Yidadaa/ChatGPT-Next-Web/releases) 
-[NextChatAI](https://nextchat.club?utm_source=readme) / [Web App Demo](https://app.nextchat.dev) / [Desktop App](https://github.com/Yidadaa/ChatGPT-Next-Web/releases) / [Discord](https://discord.gg/YCkeafCafC) / [Enterprise Edition](#enterprise-edition) / [Twitter](https://twitter.com/NextChatDev)
+[NextChatAI](https://nextchat.club?utm_source=readme) / [iOS APP](https://apps.apple.com/us/app/nextchat-ai/id6743085599) / [Web App Demo](https://app.nextchat.dev) / [Desktop App](https://github.com/Yidadaa/ChatGPT-Next-Web/releases) / [Enterprise Edition](#enterprise-edition) 
 
 
 [saas-url]: https://nextchat.club?utm_source=readme
@@ -41,31 +40,14 @@ English / [简体中文](./README_CN.md)
 
 </div>
 
-## 👋 Hey, NextChat is going to develop a native app!
+## 🥳 Cheer for NextChat iOS Version Online!
+> [👉 Click Here to Install Now](https://apps.apple.com/us/app/nextchat-ai/id6743085599)
 
-> This week we are going to start working on iOS and Android APP, and we want to find some reliable friends to do it together!
+> [❤️ Source Code Coming Soon](https://github.com/ChatGPTNextWeb/NextChat-iOS)
 
+![Github iOS Image](https://github.com/user-attachments/assets/e0aa334f-4c13-4dc9-8310-e3b09fa4b9f3)
 
-✨ Several key points:
-
-- Starting from 0, you are a veteran
-- Completely open source, not hidden
-- Native development, pursuing the ultimate experience
-
-Will you come and do something together? 😎
-
-https://github.com/ChatGPTNextWeb/NextChat/issues/6269
-
-#Seeking for talents is thirsty #lack of people
-
-
-## 🥳 Cheer for DeepSeek, China's AI star!
- > Purpose-Built UI for DeepSeek Reasoner Model
  
-<img src="https://github.com/user-attachments/assets/f3952210-3af1-4dc0-9b81-40eaa4847d9a"/>
-
-
-
 ## 🫣 NextChat Support MCP  ! 
 > Before build, please set env ENABLE_MCP=true
 
@@ -327,11 +309,14 @@ If you want to disable parse settings from url, set this to 1.
 
 To control custom models, use `+` to add a custom model, use `-` to hide a model, use `name=displayName` to customize model name, separated by comma.
 
-User `-all` to disable all default models, `+all` to enable all default models.
+Use `-all` to disable all default models, `+all` to enable all default models.
+Use `-*provider` to disable specified models. 
+Current valid providers: `openai,azure,google,anthropic,baidu,bytedance,alibaba,tencent,moonshot,iflytek,xai,chatglm` and more to come.
 
 For Azure: use `modelName@Azure=deploymentName` to customize model name and deployment name.
 > Example: `+gpt-3.5-turbo@Azure=gpt35` will show option `gpt35(Azure)` in model list.
 > If you only can use Azure model, `-all,+gpt-3.5-turbo@Azure=gpt35` will `gpt35(Azure)` the only option in model list.
+> If you don't want to use Azure model, using `-*azure` will prevent Azure models from appearing in the model list.
 
 For ByteDance: use `modelName@bytedance=deploymentName` to customize model name and deployment name.
 > Example: `+Doubao-lite-4k@bytedance=ep-xxxxx-xxx` will show option `Doubao-lite-4k(ByteDance)` in model list.

app/client/api.ts

@@ -40,6 +40,11 @@ export interface MultimodalContent {
   };
 }
 
+export interface MultimodalContentForAlibaba {
+  text?: string;
+  image?: string;
+}
+
 export interface RequestMessage {
   role: MessageRole;
   content: string | MultimodalContent[];

app/client/platforms/alibaba.ts

@@ -7,7 +7,10 @@ import {
   ChatMessageTool,
   usePluginStore,
 } from "@/app/store";
-import { streamWithThink } from "@/app/utils/chat";
+import {
+  preProcessImageContentForAlibabaDashScope,
+  streamWithThink,
+} from "@/app/utils/chat";
 import {
   ChatOptions,
   getHeaders,
@@ -15,12 +18,14 @@ import {
   LLMModel,
   SpeechOptions,
   MultimodalContent,
+  MultimodalContentForAlibaba,
 } from "../api";
 import { getClientConfig } from "@/app/config/client";
 import {
   getMessageTextContent,
   getMessageTextContentWithoutThinking,
   getTimeoutMSByModel,
+  isVisionModel,
 } from "@/app/utils";
 import { fetch } from "@/app/utils/stream";
 
@@ -89,14 +94,6 @@ export class QwenApi implements LLMApi {
   }
 
   async chat(options: ChatOptions) {
-    const messages = options.messages.map((v) => ({
-      role: v.role,
-      content:
-        v.role === "assistant"
-          ? getMessageTextContentWithoutThinking(v)
-          : getMessageTextContent(v),
-    }));
-
     const modelConfig = {
       ...useAppConfig.getState().modelConfig,
       ...useChatStore.getState().currentSession().mask.modelConfig,
@@ -105,6 +102,21 @@ export class QwenApi implements LLMApi {
       },
     };
 
+    const visionModel = isVisionModel(options.config.model);
+
+    const messages: ChatOptions["messages"] = [];
+    for (const v of options.messages) {
+      const content = (
+        visionModel
+          ? await preProcessImageContentForAlibabaDashScope(v.content)
+          : v.role === "assistant"
+          ? getMessageTextContentWithoutThinking(v)
+          : getMessageTextContent(v)
+      ) as any;
+
+      messages.push({ role: v.role, content });
+    }
+
     const shouldStream = !!options.config.stream;
     const requestPayload: RequestPayload = {
       model: modelConfig.model,
@@ -129,7 +141,7 @@ export class QwenApi implements LLMApi {
         "X-DashScope-SSE": shouldStream ? "enable" : "disable",
       };
 
-      const chatPath = this.path(Alibaba.ChatPath);
+      const chatPath = this.path(Alibaba.ChatPath(modelConfig.model));
       const chatPayload = {
         method: "POST",
         body: JSON.stringify(requestPayload),
@@ -162,7 +174,7 @@ export class QwenApi implements LLMApi {
             const json = JSON.parse(text);
             const choices = json.output.choices as Array<{
               message: {
-                content: string | null;
+                content: string | null | MultimodalContentForAlibaba[];
                 tool_calls: ChatMessageTool[];
                 reasoning_content: string | null;
               };
@@ -212,7 +224,9 @@ export class QwenApi implements LLMApi {
             } else if (content && content.length > 0) {
               return {
                 isThinking: false,
-                content: content,
+                content: Array.isArray(content)
+                  ? content.map((item) => item.text).join(",")
+                  : content,
               };
             }
 

app/client/platforms/openai.ts

@@ -56,7 +56,7 @@ export interface OpenAIListModelResponse {
 
 export interface RequestPayload {
   messages: {
-    role: "system" | "user" | "assistant";
+    role: "developer" | "system" | "user" | "assistant";
     content: string | MultimodalContent[];
   }[];
   stream?: boolean;
@@ -198,7 +198,8 @@ export class ChatGPTApi implements LLMApi {
     const isDalle3 = _isDalle3(options.config.model);
     const isO1OrO3 =
       options.config.model.startsWith("o1") ||
-      options.config.model.startsWith("o3");
+      options.config.model.startsWith("o3") ||
+      options.config.model.startsWith("o4-mini");
     if (isDalle3) {
       const prompt = getMessageTextContent(
         options.messages.slice(-1)?.pop() as any,
@@ -237,13 +238,21 @@ export class ChatGPTApi implements LLMApi {
         // Please do not ask me why not send max_tokens, no reason, this param is just shit, I dont want to explain anymore.
       };
 
-      // O1 使用 max_completion_tokens 控制token数 (https://platform.openai.com/docs/guides/reasoning#controlling-costs)
       if (isO1OrO3) {
+        // by default the o1/o3 models will not attempt to produce output that includes markdown formatting
+        // manually add "Formatting re-enabled" developer message to encourage markdown inclusion in model responses
+        // (https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/reasoning?tabs=python-secure#markdown-output)
+        requestPayload["messages"].unshift({
+          role: "developer",
+          content: "Formatting re-enabled",
+        });
+
+        // o1/o3 uses max_completion_tokens to control the number of tokens (https://platform.openai.com/docs/guides/reasoning#controlling-costs)
         requestPayload["max_completion_tokens"] = modelConfig.max_tokens;
       }
 
       // add max_tokens to vision model
-      if (visionModel) {
+      if (visionModel && !isO1OrO3) {
         requestPayload["max_tokens"] = Math.max(modelConfig.max_tokens, 4000);
       }
     }

app/constant.ts

@@ -221,7 +221,12 @@ export const ByteDance = {
 
 export const Alibaba = {
   ExampleEndpoint: ALIBABA_BASE_URL,
-  ChatPath: "v1/services/aigc/text-generation/generation",
+  ChatPath: (modelName: string) => {
+    if (modelName.includes("vl") || modelName.includes("omni")) {
+      return "v1/services/aigc/multimodal-generation/generation";
+    }
+    return `v1/services/aigc/text-generation/generation`;
+  },
 };
 
 export const Tencent = {
@@ -412,6 +417,14 @@ export const KnowledgeCutOffDate: Record<string, string> = {
   "gpt-4-turbo": "2023-12",
   "gpt-4-turbo-2024-04-09": "2023-12",
   "gpt-4-turbo-preview": "2023-12",
+  "gpt-4.1": "2024-06",
+  "gpt-4.1-2025-04-14": "2024-06",
+  "gpt-4.1-mini": "2024-06",
+  "gpt-4.1-mini-2025-04-14": "2024-06",
+  "gpt-4.1-nano": "2024-06",
+  "gpt-4.1-nano-2025-04-14": "2024-06",
+  "gpt-4.5-preview": "2023-10",
+  "gpt-4.5-preview-2025-02-27": "2023-10",
   "gpt-4o": "2023-10",
   "gpt-4o-2024-05-13": "2023-10",
   "gpt-4o-2024-08-06": "2023-10",
@@ -453,6 +466,7 @@ export const DEFAULT_TTS_VOICES = [
 export const VISION_MODEL_REGEXES = [
   /vision/,
   /gpt-4o/,
+  /gpt-4\.1/,
   /claude-3/,
   /gemini-1\.5/,
   /gemini-exp/,
@@ -464,6 +478,8 @@ export const VISION_MODEL_REGEXES = [
   /^dall-e-3$/, // Matches exactly "dall-e-3"
   /glm-4v/,
   /vl/i,
+  /o3/,
+  /o4-mini/,
 ];
 
 export const EXCLUDE_VISION_MODEL_REGEXES = [/claude-3-5-haiku-20241022/];
@@ -480,6 +496,14 @@ const openaiModels = [
   "gpt-4-32k-0613",
   "gpt-4-turbo",
   "gpt-4-turbo-preview",
+  "gpt-4.1",
+  "gpt-4.1-2025-04-14",
+  "gpt-4.1-mini",
+  "gpt-4.1-mini-2025-04-14",
+  "gpt-4.1-nano",
+  "gpt-4.1-nano-2025-04-14",
+  "gpt-4.5-preview",
+  "gpt-4.5-preview-2025-02-27",
   "gpt-4o",
   "gpt-4o-2024-05-13",
   "gpt-4o-2024-08-06",
@@ -494,6 +518,8 @@ const openaiModels = [
   "o1-mini",
   "o1-preview",
   "o3-mini",
+  "o3",
+  "o4-mini",
 ];
 
 const googleModels = [
@@ -520,6 +546,7 @@ const googleModels = [
   "gemini-2.0-flash-thinking-exp-01-21",
   "gemini-2.0-pro-exp",
   "gemini-2.0-pro-exp-02-05",
+  "gemini-2.5-pro-preview-06-05",
 ];
 
 const anthropicModels = [
@@ -570,6 +597,9 @@ const alibabaModes = [
   "qwen-max-0403",
   "qwen-max-0107",
   "qwen-max-longcontext",
+  "qwen-omni-turbo",
+  "qwen-vl-plus",
+  "qwen-vl-max",
 ];
 
 const tencentModels = [
@@ -603,6 +633,18 @@ const xAIModes = [
   "grok-2-vision-1212",
   "grok-2-vision",
   "grok-2-vision-latest",
+  "grok-3-mini-fast-beta",
+  "grok-3-mini-fast",
+  "grok-3-mini-fast-latest",
+  "grok-3-mini-beta",
+  "grok-3-mini",
+  "grok-3-mini-latest",
+  "grok-3-fast-beta",
+  "grok-3-fast",
+  "grok-3-fast-latest",
+  "grok-3-beta",
+  "grok-3",
+  "grok-3-latest",
 ];
 
 const chatglmModels = [

app/utils/chat.ts

@@ -3,7 +3,7 @@ import {
   UPLOAD_URL,
   REQUEST_TIMEOUT_MS,
 } from "@/app/constant";
-import { RequestMessage } from "@/app/client/api";
+import { MultimodalContent, RequestMessage } from "@/app/client/api";
 import Locale from "@/app/locales";
 import {
   EventStreamContentType,
@@ -70,8 +70,9 @@ export function compressImage(file: Blob, maxSize: number): Promise<string> {
   });
 }
 
-export async function preProcessImageContent(
+export async function preProcessImageContentBase(
   content: RequestMessage["content"],
+  transformImageUrl: (url: string) => Promise<{ [key: string]: any }>,
 ) {
   if (typeof content === "string") {
     return content;
@@ -81,7 +82,7 @@ export async function preProcessImageContent(
     if (part?.type == "image_url" && part?.image_url?.url) {
       try {
         const url = await cacheImageToBase64Image(part?.image_url?.url);
-        result.push({ type: part.type, image_url: { url } });
+        result.push(await transformImageUrl(url));
       } catch (error) {
         console.error("Error processing image URL:", error);
       }
@@ -92,6 +93,23 @@ export async function preProcessImageContent(
   return result;
 }
 
+export async function preProcessImageContent(
+  content: RequestMessage["content"],
+) {
+  return preProcessImageContentBase(content, async (url) => ({
+    type: "image_url",
+    image_url: { url },
+  })) as Promise<MultimodalContent[] | string>;
+}
+
+export async function preProcessImageContentForAlibabaDashScope(
+  content: RequestMessage["content"],
+) {
+  return preProcessImageContentBase(content, async (url) => ({
+    image: url,
+  }));
+}
+
 const imageCaches: Record<string, string> = {};
 export function cacheImageToBase64Image(imageUrl: string) {
   if (imageUrl.includes(CACHE_URL_PREFIX)) {

app/utils/model.ts

@@ -76,6 +76,7 @@ export function collectModelTable(
   // server custom models
   customModels
     .split(",")
+    .map((v) => v.trim())
     .filter((v) => !!v && v.length > 0)
     .forEach((m) => {
       const available = !m.startsWith("-");
@@ -88,6 +89,13 @@ export function collectModelTable(
         Object.values(modelTable).forEach(
           (model) => (model.available = available),
         );
+      } else if (name.startsWith("*")) {
+        const modelId = name.substring(1).toLowerCase();
+        Object.values(modelTable).forEach((model) => {
+          if (model?.provider?.id === modelId) {
+            model.available = available;
+          }
+        });
       } else {
         // 1. find model by name, and set available value
         const [customModelName, customProviderName] = getModelProvider(name);

jest.config.ts

@@ -15,6 +15,8 @@ const config: Config = {
   moduleNameMapper: {
     "^@/(.*)$": "<rootDir>/$1",
   },
+  extensionsToTreatAsEsm: [".ts", ".tsx"],
+  injectGlobals: true,
 };
 
 // createJestConfig is exported this way to ensure that next/jest can load the Next.js config which is async

jest.setup.ts

@@ -1,24 +1,22 @@
 // Learn more: https://github.com/testing-library/jest-dom
 import "@testing-library/jest-dom";
+import { jest } from "@jest/globals";
 
 global.fetch = jest.fn(() =>
   Promise.resolve({
     ok: true,
     status: 200,
-    json: () => Promise.resolve({}),
+    json: () => Promise.resolve([]),
     headers: new Headers(),
     redirected: false,
     statusText: "OK",
     type: "basic",
     url: "",
-    clone: function () {
-      return this;
-    },
     body: null,
     bodyUsed: false,
     arrayBuffer: () => Promise.resolve(new ArrayBuffer(0)),
     blob: () => Promise.resolve(new Blob()),
     formData: () => Promise.resolve(new FormData()),
     text: () => Promise.resolve(""),
-  }),
+  } as Response),
 );

package.json

@@ -17,8 +17,8 @@
     "prompts": "node ./scripts/fetch-prompts.mjs",
     "prepare": "husky install",
     "proxy-dev": "sh ./scripts/init-proxy.sh && proxychains -f ./scripts/proxychains.conf yarn dev",
-    "test": "jest --watch",
-    "test:ci": "jest --ci"
+    "test": "node --no-warnings --experimental-vm-modules $(yarn bin jest) --watch",
+    "test:ci": "node --no-warnings --experimental-vm-modules $(yarn bin jest) --ci"
   },
   "dependencies": {
     "@fortaine/fetch-event-source": "^3.0.6",

test/model.test.ts

@@ -0,0 +1,142 @@
+import { collectModelTable } from "@/app/utils/model"
+import { LLMModel,LLMModelProvider } from "@/app/client/api";
+
+describe('collectModelTable', () => {
+  const mockModels: readonly LLMModel[] = [
+    {
+      name: 'gpt-3.5-turbo',
+      available: true,
+      provider: {
+        id: 'openai',
+        providerName: 'OpenAI',
+        providerType: 'openai',
+      } as LLMModelProvider,
+      sorted: 1,
+    },
+    {
+      name: 'gpt-4',
+      available: true,
+      provider: {
+        id: 'openai',
+        providerName: 'OpenAI',
+        providerType: 'openai',
+      } as LLMModelProvider,
+      sorted: 1,
+    },
+    {
+      name: 'gpt-3.5-turbo',
+      available: true,
+      provider: {
+        id: 'azure',
+        providerName: 'Azure',
+        providerType: 'azure',
+      } as LLMModelProvider,
+      sorted: 2,
+    },
+    {
+      name: 'gpt-4',
+      available: true,
+      provider: {
+        id: 'azure',
+        providerName: 'Azure',
+        providerType: 'azure',
+      } as LLMModelProvider,
+      sorted: 2,
+    },
+    {
+      name: 'gemini-pro',
+      available: true,
+      provider: {
+        id: 'google',
+        providerName: 'Google',
+        providerType: 'google',
+      } as LLMModelProvider,
+      sorted: 3,
+    },
+    {
+      name: 'claude-3-haiku-20240307',
+      available: true,
+      provider: {
+        id: 'anthropic',
+        providerName: 'Anthropic',
+        providerType: 'anthropic',
+      } as LLMModelProvider,
+      sorted: 4,
+    },
+    {
+      name: 'grok-beta',
+      available: true,
+      provider: {
+        id: 'xai',
+        providerName: 'XAI',
+        providerType: 'xai',
+      } as LLMModelProvider,
+      sorted: 11,
+    },
+  ];
+
+  test('all models shoule be available', () => {
+    const customModels = '';
+    const result = collectModelTable(mockModels, customModels);
+
+    expect(result['gpt-3.5-turbo@openai'].available).toBe(true);
+    expect(result['gpt-4@openai'].available).toBe(true);
+    expect(result['gpt-3.5-turbo@azure'].available).toBe(true);
+    expect(result['gpt-4@azure'].available).toBe(true);
+    expect(result['gemini-pro@google'].available).toBe(true);
+    expect(result['claude-3-haiku-20240307@anthropic'].available).toBe(true);
+    expect(result['grok-beta@xai'].available).toBe(true);
+  });
+  test('should exclude all models when custom is "-all"', () => {
+    const customModels = '-all';
+    const result = collectModelTable(mockModels, customModels);
+
+    expect(result['gpt-3.5-turbo@openai'].available).toBe(false);
+    expect(result['gpt-4@openai'].available).toBe(false);
+    expect(result['gpt-3.5-turbo@azure'].available).toBe(false);
+    expect(result['gpt-4@azure'].available).toBe(false);
+    expect(result['gemini-pro@google'].available).toBe(false);
+    expect(result['claude-3-haiku-20240307@anthropic'].available).toBe(false);
+    expect(result['grok-beta@xai'].available).toBe(false);
+  });
+
+  test('should exclude all Azure models when custom is "-*azure"', () => {
+    const customModels = '-*azure';
+    const result = collectModelTable(mockModels, customModels);
+
+    expect(result['gpt-3.5-turbo@openai'].available).toBe(true);
+    expect(result['gpt-4@openai'].available).toBe(true);
+    expect(result['gpt-3.5-turbo@azure'].available).toBe(false);
+    expect(result['gpt-4@azure'].available).toBe(false);
+    expect(result['gemini-pro@google'].available).toBe(true);
+    expect(result['claude-3-haiku-20240307@anthropic'].available).toBe(true);
+    expect(result['grok-beta@xai'].available).toBe(true);
+  });
+
+  test('should exclude Google and XAI models when custom is "-*google,-*xai"', () => {
+    const customModels = '-*google,-*xai';
+    const result = collectModelTable(mockModels, customModels);
+
+    expect(result['gpt-3.5-turbo@openai'].available).toBe(true);
+    expect(result['gpt-4@openai'].available).toBe(true);
+    expect(result['gpt-3.5-turbo@azure'].available).toBe(true);
+    expect(result['gpt-4@azure'].available).toBe(true);
+    expect(result['gemini-pro@google'].available).toBe(false);
+    expect(result['claude-3-haiku-20240307@anthropic'].available).toBe(true);
+    expect(result['grok-beta@xai'].available).toBe(false);
+  });
+
+  test('All models except OpenAI should be excluded, and additional models should be added when customized as "-all, +*openai,gpt-4o@azure"', () => {
+    const customModels = '-all,+*openai,gpt-4o@azure';
+    const result = collectModelTable(mockModels, customModels);
+
+    expect(result['gpt-3.5-turbo@openai'].available).toBe(true);
+    expect(result['gpt-4@openai'].available).toBe(true);
+    expect(result['gpt-3.5-turbo@azure'].available).toBe(false);
+    expect(result['gpt-4@azure'].available).toBe(false);
+    expect(result['gemini-pro@google'].available).toBe(false);
+    expect(result['claude-3-haiku-20240307@anthropic'].available).toBe(false);
+    expect(result['grok-beta@xai'].available).toBe(false);
+    expect(result['gpt-4o@azure'].available).toBe(true);
+  });
+});

test/vision-model-checker.test.ts

@@ -1,3 +1,4 @@
+import { jest } from "@jest/globals";
 import { isVisionModel } from "../app/utils";
 
 describe("isVisionModel", () => {
@@ -50,7 +51,7 @@ describe("isVisionModel", () => {
 
   test("should identify models from VISION_MODELS env var", () => {
     process.env.VISION_MODELS = "custom-vision-model,another-vision-model";
-    
+
     expect(isVisionModel("custom-vision-model")).toBe(true);
     expect(isVisionModel("another-vision-model")).toBe(true);
     expect(isVisionModel("unrelated-model")).toBe(false);
@@ -64,4 +65,4 @@ describe("isVisionModel", () => {
     expect(isVisionModel("unrelated-model")).toBe(false);
     expect(isVisionModel("gpt-4-vision")).toBe(true);
   });
-});
+});