feat: 增加dall-e插件

app/api/langchain-tools/dalle_image_generator.ts

@@ -0,0 +1,65 @@
+import * as fs from "fs";
+import * as path from "path";
+import axios from "axios";
+
+import { Tool } from "langchain/tools";
+import OpenAI from "openai";
+
+export class DallEAPIWrapper extends Tool {
+  name = "dalle_image_generator";
+  n = 1;
+  size: "256x256" | "512x512" | "1024x1024" | null = "1024x1024";
+  apiKey: string;
+  baseURL?: string;
+
+  constructor(apiKey?: string | undefined, baseURL?: string | undefined) {
+    super();
+    if (!apiKey) {
+      throw new Error("OpenAI API key not set.");
+    }
+    this.apiKey = apiKey;
+    this.baseURL = baseURL;
+  }
+
+  async saveImageFromUrl(url: string) {
+    const response = await axios.get(url, { responseType: "arraybuffer" });
+
+    const uploadsDir = "public/uploads";
+    console.log("[fileUpload]", { uploadsDir });
+    if (!fs.existsSync(uploadsDir)) {
+      fs.mkdirSync(uploadsDir, { recursive: true });
+    }
+
+    const filename = `${Date.now()}.png`;
+    const filePath = path.join(uploadsDir, filename);
+
+    fs.writeFileSync(filePath, Buffer.from(response.data, "binary"));
+
+    return `uploads/${filename}`;
+  }
+
+  /** @ignore */
+  async _call(prompt: string) {
+    const openai = new OpenAI({
+      apiKey: this.apiKey,
+      baseURL: this.baseURL,
+    });
+    const response = await openai.images.generate({
+      prompt: prompt,
+      n: this.n,
+      size: this.size,
+    });
+
+    let image_url = response.data[0].url;
+    console.log(image_url);
+    if (!image_url) return "No image was generated";
+    let filePath = await this.saveImageFromUrl(image_url);
+    console.log(filePath);
+    return filePath;
+  }
+
+  description = `openai's dall-e image generator.
+    input must be a english prompt.
+    output will be the image link url.
+    use markdown to display images.`;
+}

app/api/langchain/tool/agent/route.ts

@@ -18,6 +18,7 @@ import { DuckDuckGo } from "@/app/api/langchain-tools/duckduckgo_search";
 import { WebBrowser } from "langchain/tools/webbrowser";
 import { Calculator } from "langchain/tools/calculator";
 import { DynamicTool, Tool } from "langchain/tools";
+import { DallEAPIWrapper } from "@/app/api/langchain-tools/dalle_image_generator";
 
 const serverConfig = getServerSideConfig();
 
@@ -214,9 +215,11 @@ async function handle(req: NextRequest) {
     ];
     const webBrowserTool = new WebBrowser({ model, embeddings });
     const calculatorTool = new Calculator();
+    const dallEAPITool = new DallEAPIWrapper(apiKey, baseUrl);
     if (useTools.includes("web-search")) tools.push(searchTool);
     if (useTools.includes(webBrowserTool.name)) tools.push(webBrowserTool);
     if (useTools.includes(calculatorTool.name)) tools.push(calculatorTool);
+    if (useTools.includes(dallEAPITool.name)) tools.push(dallEAPITool);
 
     useTools.forEach((toolName) => {
       if (toolName) {

app/plugins/cn.ts

@@ -38,4 +38,13 @@ export const CN_PLUGINS: BuiltinPlugin[] = [
     createdAt: 1694235989000,
     enable: false,
   },
+  {
+    name: "DALL·E",
+    toolName: "dalle_image_generator",
+    lang: "cn",
+    description: "DALL·E 可以根据自然语言的描述创建逼真的图像和艺术。",
+    builtin: true,
+    createdAt: 1694703673000,
+    enable: false,
+  },
 ];

app/plugins/en.ts

@@ -40,4 +40,14 @@ export const EN_PLUGINS: BuiltinPlugin[] = [
     createdAt: 1694235989000,
     enable: false,
   },
+  {
+    name: "DALL·E",
+    toolName: "dalle_image_generator",
+    lang: "en",
+    description:
+      "DALL·E 2 is an AI system that can create realistic images and art from a description in natural language.",
+    builtin: true,
+    createdAt: 1694703673000,
+    enable: false,
+  },
 ];