diff --git a/Dockerfile b/Dockerfile index 957063c46..45892c8f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,6 +21,9 @@ RUN # 清理遗留的缓存 RUN yarn cache clean RUN yarn install +# 避免下面那个报错 +RUN mkdir -p "/app/node_modules/tiktoken" + FROM base AS builder ENV OPENAI_API_KEY="" @@ -48,6 +51,9 @@ COPY --from=builder /app/.next/standalone ./ COPY --from=builder /app/.next/static ./.next/static COPY --from=builder /app/.next/server ./.next/server +# 一个插件一直有问题。 +COPY --from=deps /app/node_modules/tiktoken ./node_modules/tiktoken + RUN rm -f .env EXPOSE 3000 diff --git a/app/api/logs/[...path]/route.ts b/app/api/logs/[...path]/route.ts index b2cc5b584..f9a885247 100644 --- a/app/api/logs/[...path]/route.ts +++ b/app/api/logs/[...path]/route.ts @@ -1,7 +1,27 @@ import { NextRequest, NextResponse } from "next/server"; import prisma from "@/lib/prisma"; import { insertUser } from "@/lib/auth"; -import { getTokenLength } from "@/app/utils/token"; +// import { getTokenLength } from "@/app/utils/token"; +// import { Tiktoken } from "tiktoken/lite" +// import cl100k_base from "tiktoken/encoders/cl100k_base.json" +import "tiktoken"; +import { get_encoding } from "tiktoken"; + +function getTokenLength(input: string): number { + // const { Tiktoken } = require("tiktoken/lite"); + // const cl100k_base = require("tiktoken/encoders/cl100k_base.json"); + // const encoding = new Tiktoken( + // cl100k_base.bpe_ranks, + // cl100k_base.special_tokens, + // cl100k_base.pat_str, + // ); + const encoding = get_encoding("cl100k_base"); + + const tokenLength = encoding.encode(input).length; + // console.log('[TOKEN],=========', input, tokenLength) + + return tokenLength; +} async function handle( req: NextRequest, diff --git a/next.config.mjs b/next.config.mjs index 3fb779f62..dc1a5bca7 100644 --- a/next.config.mjs +++ b/next.config.mjs @@ -11,8 +11,8 @@ console.log("[Next] build with chunk: ", disableChunk); // 为了修复tiktoken的插件问题 import CopyPlugin from "copy-webpack-plugin"; - const nextConfig = { + // transpilePackages: ['tiktoken'], webpack(config) { config.module.rules.push({ test: /\.svg$/, @@ -24,17 +24,22 @@ const nextConfig = { new webpack.optimize.LimitChunkCountPlugin({ maxChunks: 1 }), ) } - config.plugins.push( - new CopyPlugin({ - patterns: [ - { - from: ".//node_modules/tiktoken/tiktoken_bg.wasm", - to: "tiktoken_bg.wasm", - toType: "file", - } - ] - }) - ) + // turn off static file serving of WASM files + // we need to let Webpack handle WASM import + // config.module.rules + // .find((i) => "oneOf" in i) + // .oneOf.find((i) => i.type === "asset/resource") + // .exclude.push(/\.wasm$/); + // config.plugins.push( + // new CopyPlugin({ + // patterns: [ + // { + // from: ".//node_modules/tiktoken/, + // to: "", + // } + // ] + // }) + // ) config.optimization.minimize = true config.optimization.splitChunks = {