Merge branch 'main' into main

app/api/config/route.ts

@@ -13,6 +13,7 @@ const DANGER_CONFIG = {
   hideBalanceQuery: serverConfig.hideBalanceQuery,
   disableFastLink: serverConfig.disableFastLink,
   customModels: serverConfig.customModels,
+  isEnableRAG: serverConfig.isEnableRAG,
 };
 
 declare global {

app/api/file/[...path]/route.ts

@@ -2,6 +2,7 @@ import { getServerSideConfig } from "@/app/config/server";
 import LocalFileStorage from "@/app/utils/local_file_storage";
 import S3FileStorage from "@/app/utils/s3_file_storage";
 import { NextRequest, NextResponse } from "next/server";
+import mime from "mime";
 
 async function handle(
   req: NextRequest,
@@ -13,19 +14,27 @@ async function handle(
 
   try {
     const serverConfig = getServerSideConfig();
+    const fileName = params.path[0];
+    const contentType = mime.getType(fileName);
+
     if (serverConfig.isStoreFileToLocal) {
-      var fileBuffer = await LocalFileStorage.get(params.path[0]);
+      var fileBuffer = await LocalFileStorage.get(fileName);
       return new Response(fileBuffer, {
         headers: {
-          "Content-Type": "image/png",
+          "Content-Type": contentType ?? "application/octet-stream",
         },
       });
     } else {
-      var file = await S3FileStorage.get(params.path[0]);
-      return new Response(file?.transformToWebStream(), {
-        headers: {
-          "Content-Type": "image/png",
-        },
+      var file = await S3FileStorage.get(fileName);
+      if (file) {
+        return new Response(file?.transformToWebStream(), {
+          headers: {
+            "Content-Type": contentType ?? "application/octet-stream",
+          },
+        });
+      }
+      return new Response("not found", {
+        status: 404,
       });
     }
   } catch (e) {

app/api/file/upload/route.ts

@@ -4,6 +4,7 @@ import { auth } from "@/app/api/auth";
 import LocalFileStorage from "@/app/utils/local_file_storage";
 import { getServerSideConfig } from "@/app/config/server";
 import S3FileStorage from "@/app/utils/s3_file_storage";
+import path from "path";
 
 async function handle(req: NextRequest) {
   if (req.method === "OPTIONS") {
@@ -19,20 +20,14 @@ async function handle(req: NextRequest) {
 
   try {
     const formData = await req.formData();
-    const image = formData.get("file") as File;
+    const file = formData.get("file") as File;
+    const fileData = await file.arrayBuffer();
+    const originalFileName = file?.name;
 
-    const imageReader = image.stream().getReader();
-    const imageData: number[] = [];
-
-    while (true) {
-      const { done, value } = await imageReader.read();
-      if (done) break;
-      imageData.push(...value);
-    }
-
-    const buffer = Buffer.from(imageData);
-
-    var fileName = `${Date.now()}.png`;
+    if (!fileData) throw new Error("Get file buffer error");
+    const buffer = Buffer.from(fileData);
+    const fileType = path.extname(originalFileName).slice(1);
+    var fileName = `${Date.now()}.${fileType}`;
     var filePath = "";
     const serverConfig = getServerSideConfig();
     if (serverConfig.isStoreFileToLocal) {

app/api/langchain-tools/nodejs_tools.ts

@@ -10,16 +10,15 @@ import { WolframAlphaTool } from "@/app/api/langchain-tools/wolframalpha";
 import { BilibiliVideoInfoTool } from "./bilibili_vid_info";
 import { BilibiliVideoSearchTool } from "./bilibili_vid_search";
 import { BilibiliMusicRecognitionTool } from "./bilibili_music_recognition";
+import { RAGSearch } from "./rag_search";
 
 export class NodeJSTool {
   private apiKey: string | undefined;
-
   private baseUrl: string;
-
   private model: BaseLanguageModel;
-
   private embeddings: Embeddings;
-
+  private sessionId: string;
+  private ragEmbeddings: Embeddings;
   private callback?: (data: string) => Promise<void>;
 
   constructor(
@@ -27,12 +26,16 @@ export class NodeJSTool {
     baseUrl: string,
     model: BaseLanguageModel,
     embeddings: Embeddings,
+    sessionId: string,
+    ragEmbeddings: Embeddings,
     callback?: (data: string) => Promise<void>,
   ) {
     this.apiKey = apiKey;
     this.baseUrl = baseUrl;
     this.model = model;
     this.embeddings = embeddings;
+    this.sessionId = sessionId;
+    this.ragEmbeddings = ragEmbeddings;
     this.callback = callback;
   }
 
@@ -66,6 +69,9 @@ export class NodeJSTool {
       bilibiliVideoSearchTool,
       bilibiliMusicRecognitionTool,
     ];
+    if (!!process.env.ENABLE_RAG) {
+      tools.push(new RAGSearch(this.sessionId, this.model, this.ragEmbeddings));
+    }
     return tools;
   }
 }

app/api/langchain-tools/rag_search.ts

@@ -0,0 +1,79 @@
+import { Tool } from "@langchain/core/tools";
+import { CallbackManagerForToolRun } from "@langchain/core/callbacks/manager";
+import { BaseLanguageModel } from "langchain/dist/base_language";
+import { formatDocumentsAsString } from "langchain/util/document";
+import { Embeddings } from "langchain/dist/embeddings/base.js";
+import { RunnableSequence } from "@langchain/core/runnables";
+import { StringOutputParser } from "@langchain/core/output_parsers";
+import { Pinecone } from "@pinecone-database/pinecone";
+import { PineconeStore } from "@langchain/pinecone";
+import { getServerSideConfig } from "@/app/config/server";
+import { QdrantVectorStore } from "@langchain/community/vectorstores/qdrant";
+
+export class RAGSearch extends Tool {
+  static lc_name() {
+    return "RAGSearch";
+  }
+
+  get lc_namespace() {
+    return [...super.lc_namespace, "ragsearch"];
+  }
+
+  private sessionId: string;
+  private model: BaseLanguageModel;
+  private embeddings: Embeddings;
+
+  constructor(
+    sessionId: string,
+    model: BaseLanguageModel,
+    embeddings: Embeddings,
+  ) {
+    super();
+    this.sessionId = sessionId;
+    this.model = model;
+    this.embeddings = embeddings;
+  }
+
+  /** @ignore */
+  async _call(inputs: string, runManager?: CallbackManagerForToolRun) {
+    const serverConfig = getServerSideConfig();
+    if (!serverConfig.isEnableRAG)
+      throw new Error("env ENABLE_RAG not configured");
+    // const pinecone = new Pinecone();
+    // const pineconeIndex = pinecone.Index(serverConfig.pineconeIndex!);
+    // const vectorStore = await PineconeStore.fromExistingIndex(this.embeddings, {
+    //   pineconeIndex,
+    // });
+    const vectorStore = await QdrantVectorStore.fromExistingCollection(
+      this.embeddings,
+      {
+        url: process.env.QDRANT_URL,
+        apiKey: process.env.QDRANT_API_KEY,
+        collectionName: this.sessionId,
+      },
+    );
+
+    let context;
+    const returnCunt = serverConfig.ragReturnCount
+      ? parseInt(serverConfig.ragReturnCount, 10)
+      : 4;
+    console.log("[rag-search]", { inputs, returnCunt });
+    // const results = await vectorStore.similaritySearch(inputs, returnCunt, {
+    //   sessionId: this.sessionId,
+    // });
+    const results = await vectorStore.similaritySearch(inputs, returnCunt);
+    context = formatDocumentsAsString(results);
+    console.log("[rag-search]", { context });
+    return context;
+    // const input = `Text:${context}\n\nQuestion:${inputs}\n\nI need you to answer the question based on the text.`;
+
+    // console.log("[rag-search]", input);
+
+    // const chain = RunnableSequence.from([this.model, new StringOutputParser()]);
+    // return chain.invoke(input, runManager?.getChild());
+  }
+
+  name = "rag-search";
+
+  description = `It is used to query documents entered by the user.The input content is the keywords extracted from the user's question, and multiple keywords are separated by spaces and passed in.`;
+}

app/api/langchain/rag/search/route.ts

@@ -0,0 +1,120 @@
+import { NextRequest, NextResponse } from "next/server";
+import { auth } from "@/app/api/auth";
+import { ACCESS_CODE_PREFIX, ModelProvider } from "@/app/constant";
+import { OpenAIEmbeddings } from "@langchain/openai";
+import { Pinecone } from "@pinecone-database/pinecone";
+import { PineconeStore } from "@langchain/pinecone";
+import { QdrantVectorStore } from "@langchain/community/vectorstores/qdrant";
+import { getServerSideConfig } from "@/app/config/server";
+
+interface RequestBody {
+  sessionId: string;
+  query: string;
+  baseUrl?: string;
+}
+
+async function handle(req: NextRequest) {
+  if (req.method === "OPTIONS") {
+    return NextResponse.json({ body: "OK" }, { status: 200 });
+  }
+  try {
+    const authResult = auth(req, ModelProvider.GPT);
+    if (authResult.error) {
+      return NextResponse.json(authResult, {
+        status: 401,
+      });
+    }
+
+    const reqBody: RequestBody = await req.json();
+    const authToken = req.headers.get("Authorization") ?? "";
+    const token = authToken.trim().replaceAll("Bearer ", "").trim();
+    const serverConfig = getServerSideConfig();
+    // const pinecone = new Pinecone();
+    // const pineconeIndex = pinecone.Index(serverConfig.pineconeIndex!);
+    const apiKey = getOpenAIApiKey(token);
+    const baseUrl = getOpenAIBaseUrl(reqBody.baseUrl);
+    const embeddings = new OpenAIEmbeddings(
+      {
+        modelName: serverConfig.ragEmbeddingModel ?? "text-embedding-3-large",
+        openAIApiKey: apiKey,
+      },
+      { basePath: baseUrl },
+    );
+    // const vectorStore = await PineconeStore.fromExistingIndex(embeddings, {
+    //   pineconeIndex,
+    // });
+    // const results = await vectorStore.similaritySearch(reqBody.query, 4, {
+    //   sessionId: reqBody.sessionId,
+    // });
+    const vectorStore = await QdrantVectorStore.fromExistingCollection(
+      embeddings,
+      {
+        url: process.env.QDRANT_URL,
+        apiKey: process.env.QDRANT_API_KEY,
+        collectionName: reqBody.sessionId,
+      },
+    );
+    const returnCunt = serverConfig.ragReturnCount
+      ? parseInt(serverConfig.ragReturnCount, 10)
+      : 4;
+    const response = await vectorStore.similaritySearch(
+      reqBody.query,
+      returnCunt,
+    );
+    return NextResponse.json(response, {
+      status: 200,
+    });
+  } catch (e) {
+    console.error(e);
+    return new Response(JSON.stringify({ error: (e as any).message }), {
+      status: 500,
+      headers: { "Content-Type": "application/json" },
+    });
+  }
+}
+
+function getOpenAIApiKey(token: string) {
+  const serverConfig = getServerSideConfig();
+  const isApiKey = !token.startsWith(ACCESS_CODE_PREFIX);
+
+  let apiKey = serverConfig.apiKey;
+  if (isApiKey && token) {
+    apiKey = token;
+  }
+  return apiKey;
+}
+
+function getOpenAIBaseUrl(reqBaseUrl: string | undefined) {
+  const serverConfig = getServerSideConfig();
+  let baseUrl = "https://api.openai.com/v1";
+  if (serverConfig.baseUrl) baseUrl = serverConfig.baseUrl;
+  if (reqBaseUrl?.startsWith("http://") || reqBaseUrl?.startsWith("https://"))
+    baseUrl = reqBaseUrl;
+  if (!baseUrl.endsWith("/v1"))
+    baseUrl = baseUrl.endsWith("/") ? `${baseUrl}v1` : `${baseUrl}/v1`;
+  console.log("[baseUrl]", baseUrl);
+  return baseUrl;
+}
+
+export const POST = handle;
+
+export const runtime = "nodejs";
+export const preferredRegion = [
+  "arn1",
+  "bom1",
+  "cdg1",
+  "cle1",
+  "cpt1",
+  "dub1",
+  "fra1",
+  "gru1",
+  "hnd1",
+  "iad1",
+  "icn1",
+  "kix1",
+  "lhr1",
+  "pdx1",
+  "sfo1",
+  "sin1",
+  "syd1",
+];

app/api/langchain/rag/store/route.ts

@@ -0,0 +1,221 @@
+import { NextRequest, NextResponse } from "next/server";
+import { auth } from "@/app/api/auth";
+import { ACCESS_CODE_PREFIX, ModelProvider } from "@/app/constant";
+import { OpenAI, OpenAIEmbeddings } from "@langchain/openai";
+import { PDFLoader } from "langchain/document_loaders/fs/pdf";
+import { TextLoader } from "langchain/document_loaders/fs/text";
+import { CSVLoader } from "langchain/document_loaders/fs/csv";
+import { DocxLoader } from "langchain/document_loaders/fs/docx";
+import { EPubLoader } from "langchain/document_loaders/fs/epub";
+import { JSONLoader } from "langchain/document_loaders/fs/json";
+import { JSONLinesLoader } from "langchain/document_loaders/fs/json";
+import { OpenAIWhisperAudio } from "langchain/document_loaders/fs/openai_whisper_audio";
+// import { PPTXLoader } from "langchain/document_loaders/fs/pptx";
+import { SRTLoader } from "langchain/document_loaders/fs/srt";
+import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
+import { Pinecone } from "@pinecone-database/pinecone";
+import { PineconeStore } from "@langchain/pinecone";
+import { getServerSideConfig } from "@/app/config/server";
+import { FileInfo } from "@/app/client/platforms/utils";
+import mime from "mime";
+import LocalFileStorage from "@/app/utils/local_file_storage";
+import S3FileStorage from "@/app/utils/s3_file_storage";
+import { QdrantVectorStore } from "@langchain/community/vectorstores/qdrant";
+
+interface RequestBody {
+  sessionId: string;
+  fileInfos: FileInfo[];
+  baseUrl?: string;
+}
+
+function getLoader(
+  fileName: string,
+  fileBlob: Blob,
+  openaiApiKey: string,
+  openaiBaseUrl: string,
+) {
+  const extension = fileName.split(".").pop();
+  switch (extension) {
+    case "txt":
+    case "md":
+      return new TextLoader(fileBlob);
+    case "pdf":
+      return new PDFLoader(fileBlob);
+    case "docx":
+      return new DocxLoader(fileBlob);
+    case "csv":
+      return new CSVLoader(fileBlob);
+    case "json":
+      return new JSONLoader(fileBlob);
+    // case 'pptx':
+    //   return new PPTXLoader(fileBlob);
+    case "srt":
+      return new SRTLoader(fileBlob);
+    case "mp3":
+      return new OpenAIWhisperAudio(fileBlob, {
+        clientOptions: {
+          apiKey: openaiApiKey,
+          baseURL: openaiBaseUrl,
+        },
+      });
+    default:
+      throw new Error(`Unsupported file type: ${extension}`);
+  }
+}
+
+async function handle(req: NextRequest) {
+  if (req.method === "OPTIONS") {
+    return NextResponse.json({ body: "OK" }, { status: 200 });
+  }
+  try {
+    const authResult = auth(req, ModelProvider.GPT);
+    if (authResult.error) {
+      return NextResponse.json(authResult, {
+        status: 401,
+      });
+    }
+
+    const reqBody: RequestBody = await req.json();
+    const authToken = req.headers.get("Authorization") ?? "";
+    const token = authToken.trim().replaceAll("Bearer ", "").trim();
+    const apiKey = getOpenAIApiKey(token);
+    const baseUrl = getOpenAIBaseUrl(reqBody.baseUrl);
+    const serverConfig = getServerSideConfig();
+    // const pinecone = new Pinecone();
+    // const pineconeIndex = pinecone.Index(serverConfig.pineconeIndex!);
+    const embeddings = new OpenAIEmbeddings(
+      {
+        modelName: serverConfig.ragEmbeddingModel,
+        openAIApiKey: apiKey,
+      },
+      { basePath: baseUrl },
+    );
+    // https://js.langchain.com/docs/integrations/vectorstores/pinecone
+    // https://js.langchain.com/docs/integrations/vectorstores/qdrant
+    // process files
+    for (let i = 0; i < reqBody.fileInfos.length; i++) {
+      const fileInfo = reqBody.fileInfos[i];
+      const contentType = mime.getType(fileInfo.fileName);
+      // get file buffer
+      var fileBuffer: Buffer | undefined;
+      if (serverConfig.isStoreFileToLocal) {
+        fileBuffer = await LocalFileStorage.get(fileInfo.fileName);
+      } else {
+        var file = await S3FileStorage.get(fileInfo.fileName);
+        var fileByteArray = await file?.transformToByteArray();
+        if (fileByteArray) fileBuffer = Buffer.from(fileByteArray);
+      }
+      if (!fileBuffer || !contentType) {
+        console.error(`get ${fileInfo.fileName} buffer fail`);
+        continue;
+      }
+      // load file to docs
+      const fileBlob = bufferToBlob(fileBuffer, contentType);
+      const loader = getLoader(fileInfo.fileName, fileBlob, apiKey, baseUrl);
+      const docs = await loader.load();
+      // modify doc meta
+      docs.forEach((doc) => {
+        doc.metadata = {
+          ...doc.metadata,
+          sessionId: reqBody.sessionId,
+          sourceFileName: fileInfo.originalFilename,
+          fileName: fileInfo.fileName,
+        };
+      });
+      // split
+      const chunkSize = serverConfig.ragChunkSize
+        ? parseInt(serverConfig.ragChunkSize, 10)
+        : 2000;
+      const chunkOverlap = serverConfig.ragChunkOverlap
+        ? parseInt(serverConfig.ragChunkOverlap, 10)
+        : 200;
+      const textSplitter = new RecursiveCharacterTextSplitter({
+        chunkSize: chunkSize,
+        chunkOverlap: chunkOverlap,
+      });
+      const splits = await textSplitter.splitDocuments(docs);
+      const vectorStore = await QdrantVectorStore.fromDocuments(
+        splits,
+        embeddings,
+        {
+          url: process.env.QDRANT_URL,
+          apiKey: process.env.QDRANT_API_KEY,
+          collectionName: reqBody.sessionId,
+        },
+      );
+      // await PineconeStore.fromDocuments(splits, embeddings, {
+      //   pineconeIndex,
+      //   maxConcurrency: 5,
+      // });
+      // const vectorStore = await PineconeStore.fromExistingIndex(embeddings, {
+      //   pineconeIndex,
+      // });
+    }
+    return NextResponse.json(
+      {
+        sessionId: reqBody.sessionId,
+      },
+      {
+        status: 200,
+      },
+    );
+  } catch (e) {
+    console.error(e);
+    return new Response(JSON.stringify({ error: (e as any).message }), {
+      status: 500,
+      headers: { "Content-Type": "application/json" },
+    });
+  }
+}
+
+function bufferToBlob(buffer: Buffer, mimeType?: string): Blob {
+  const arrayBuffer: ArrayBuffer = buffer.buffer.slice(
+    buffer.byteOffset,
+    buffer.byteOffset + buffer.byteLength,
+  );
+  return new Blob([arrayBuffer], { type: mimeType || "" });
+}
+function getOpenAIApiKey(token: string) {
+  const serverConfig = getServerSideConfig();
+  const isApiKey = !token.startsWith(ACCESS_CODE_PREFIX);
+
+  let apiKey = serverConfig.apiKey;
+  if (isApiKey && token) {
+    apiKey = token;
+  }
+  return apiKey;
+}
+function getOpenAIBaseUrl(reqBaseUrl: string | undefined) {
+  const serverConfig = getServerSideConfig();
+  let baseUrl = "https://api.openai.com/v1";
+  if (serverConfig.baseUrl) baseUrl = serverConfig.baseUrl;
+  if (reqBaseUrl?.startsWith("http://") || reqBaseUrl?.startsWith("https://"))
+    baseUrl = reqBaseUrl;
+  if (!baseUrl.endsWith("/v1"))
+    baseUrl = baseUrl.endsWith("/") ? `${baseUrl}v1` : `${baseUrl}/v1`;
+  console.log("[baseUrl]", baseUrl);
+  return baseUrl;
+}
+
+export const POST = handle;
+
+export const runtime = "nodejs";
+export const preferredRegion = [
+  "arn1",
+  "bom1",
+  "cdg1",
+  "cle1",
+  "cpt1",
+  "dub1",
+  "fra1",
+  "gru1",
+  "hnd1",
+  "iad1",
+  "icn1",
+  "kix1",
+  "lhr1",
+  "pdx1",
+  "sfo1",
+  "sin1",
+  "syd1",
+];

app/api/langchain/tool/agent/agentapi.ts

@@ -44,6 +44,7 @@ export interface RequestMessage {
 }
 
 export interface RequestBody {
+  chatSessionId: string;
   messages: RequestMessage[];
   isAzure: boolean;
   azureApiVersion?: string;

app/api/langchain/tool/agent/nodejs/route.ts

@@ -44,6 +44,13 @@ async function handle(req: NextRequest) {
       },
       { basePath: baseUrl },
     );
+    const ragEmbeddings = new OpenAIEmbeddings(
+      {
+        modelName: process.env.RAG_EMBEDDING_MODEL ?? "text-embedding-3-large",
+        openAIApiKey: apiKey,
+      },
+      { basePath: baseUrl },
+    );
 
     var dalleCallback = async (data: string) => {
       var response = new ResponseBody();
@@ -62,6 +69,8 @@ async function handle(req: NextRequest) {
       baseUrl,
       model,
       embeddings,
+      reqBody.chatSessionId,
+      ragEmbeddings,
       dalleCallback,
     );
     var nodejsTools = await nodejsTool.getCustomTools();