From b064e16278efdd5d97f2c5e6d4ff6f250bd2aa0c Mon Sep 17 00:00:00 2001 From: sijinhui Date: Fri, 22 Dec 2023 23:18:19 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96token=E8=AE=A1=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/api/logs/[...path]/route.ts | 24 ++++---- app/app/(admin)/admin/usage-analysis.tsx | 73 ------------------------ app/app/(admin)/admin/usage-by-model.tsx | 8 ++- app/utils/token.ts | 19 ++++++ next.config.mjs | 7 ++- package.json | 3 +- prisma/schema.prisma | 1 + 7 files changed, 45 insertions(+), 90 deletions(-) delete mode 100644 app/app/(admin)/admin/usage-analysis.tsx diff --git a/app/api/logs/[...path]/route.ts b/app/api/logs/[...path]/route.ts index 9fccab3d7..accf77036 100644 --- a/app/api/logs/[...path]/route.ts +++ b/app/api/logs/[...path]/route.ts @@ -1,17 +1,7 @@ import { NextRequest, NextResponse } from "next/server"; import prisma from "@/lib/prisma"; import { insertUser } from "@/lib/auth"; - -// function cleanObjectKeys(input: { [key: string]: string }): { -// [key: string]: string; -// } { -// const cleanedObj: { [key: string]: string } = {}; -// Object.keys(input).forEach((key) => { -// cleanedObj[key] = input[key].trim(); -// }); -// console.log('========', input, cleanedObj) -// return cleanedObj; -// } +import { getTokenLength } from "@/app/utils/token"; async function handle( req: NextRequest, @@ -23,12 +13,22 @@ async function handle( await insertUser({ name: request_data?.userName }); } // console.log("===========4", request_data); + if (request_data?.logEntry) { + const regex = /\[(.*)]/g; + const matchResponse = request_data.logEntry.match(regex); + if (matchResponse.length > 0) { + request_data.logToken = getTokenLength(matchResponse[0]); + } + // console.log('=======', request_data.logEntry, '=====', matchResponse); + } + await prisma.logEntry.create({ data: request_data, }); } catch (e) { + console.log("[LOG]", e); + return NextResponse.json({ status: 0 }); - // console.log("[LOG]", e); } return NextResponse.json({ status: 1 }); diff --git a/app/app/(admin)/admin/usage-analysis.tsx b/app/app/(admin)/admin/usage-analysis.tsx deleted file mode 100644 index 2349f74de..000000000 --- a/app/app/(admin)/admin/usage-analysis.tsx +++ /dev/null @@ -1,73 +0,0 @@ -// "use client"; -import { BarList, Bold, Card, Flex, Text, Title } from "@tremor/react"; -import prisma from "@/lib/prisma"; -import { estimateTokenLength } from "@/app/utils/token"; - -function HandleLogData(todayLog: [{ userName: string; logEntry: string }]) { - const data1 = todayLog.map((log) => { - return { - name: log.userName ?? "unknown", - value: estimateTokenLength(log.logEntry ?? ""), - }; - }); - - type Accumulator = { - [key: string]: number; - }; - const data2 = data1.reduce((acc, item) => { - if (acc[item.name]) { - acc[item.name] += item.value; - } else { - acc[item.name] = item.value; - } - return acc; - }, {}); - const data3 = Object.entries(data2) - .map(([name, value]) => ({ - name, - value, - })) - .sort((a, b) => { - return b.value - a.value; - }); - return data3; -} - -export default async function UsageAnalysis() { - // 今天日期的开始和结束 - const startDate = new Date(); - startDate.setHours(0, 0, 0, 0); - - const endDate = new Date(); - endDate.setHours(23, 59, 59, 999); - const todayLog = await prisma.logEntry.findMany({ - where: { - createdAt: { - gte: startDate, // gte 表示 '大于等于' - lte: endDate, // lte 表示 '小于等于' - }, - }, - include: { - user: true, - }, - }); - - // @ts-ignore - const log_data = HandleLogData(todayLog); - - // @ts-ignore - return ( - - Website Analytics - - - Source - - - Visits - - - - - ); -} diff --git a/app/app/(admin)/admin/usage-by-model.tsx b/app/app/(admin)/admin/usage-by-model.tsx index fbc17e74c..5aa8d23c2 100644 --- a/app/app/(admin)/admin/usage-by-model.tsx +++ b/app/app/(admin)/admin/usage-by-model.tsx @@ -2,17 +2,19 @@ import * as echarts from "echarts"; import { EChartsOption } from "echarts"; import dynamic from "next/dynamic"; import prisma from "@/lib/prisma"; -import { estimateTokenLength } from "@/app/utils/token"; +// import { getTokenLength } from "@/app/utils/token"; const UsageByModelChart = dynamic(() => import("./usage-by-model-chart"), { ssr: false, }); -function HandleLogData(todayLog: [{ userName: string; logEntry: string }]) { +function HandleLogData( + todayLog: [{ userName: string; logEntry: string; logToken: number }], +) { const data1 = todayLog.map((log) => { return { name: log.userName ?? "unknown", - value: estimateTokenLength(log.logEntry ?? ""), + value: log.logToken, }; }); diff --git a/app/utils/token.ts b/app/utils/token.ts index ec8139b20..1e803408c 100644 --- a/app/utils/token.ts +++ b/app/utils/token.ts @@ -20,3 +20,22 @@ export function estimateTokenLength(input: string): number { return tokenLength; } + +// import { get_encoding } from "tiktoken"; + +export function getTokenLength(input: string): number { + // const { get_encoding } = require( "tiktoken" ); + // const encoding = get_encoding("cl100k_base"); + + const { Tiktoken } = require("tiktoken/lite"); + const cl100k_base = require("tiktoken/encoders/cl100k_base.json"); + const encoding = new Tiktoken( + cl100k_base.bpe_ranks, + cl100k_base.special_tokens, + cl100k_base.pat_str, + ); + const tokenLength = encoding.encode(input).length; + // console.log('[TOKEN],=========', input, tokenLength) + + return tokenLength; +} diff --git a/next.config.mjs b/next.config.mjs index b6b788aad..68d9a78ca 100644 --- a/next.config.mjs +++ b/next.config.mjs @@ -28,11 +28,16 @@ const nextConfig = { } // console.log('=======', config.optimization) - config.resolve.fallback = { child_process: false, }; + // tiktoken + config.experiments = { + asyncWebAssembly: true, + layers: true, + }; + return config; }, output: mode, diff --git a/package.json b/package.json index b344344db..c21149ba5 100644 --- a/package.json +++ b/package.json @@ -17,9 +17,9 @@ }, "dependencies": { "@fortaine/fetch-event-source": "^3.0.6", + "@hello-pangea/dnd": "^16.5.0", "@next-auth/prisma-adapter": "^1.0.7", "@prisma/client": "^5.7.0", - "@hello-pangea/dnd": "^16.5.0", "@svgr/webpack": "^8.1.0", "@tremor/react": "^3.12.1", "@vercel/analytics": "^1.1.1", @@ -46,6 +46,7 @@ "sonner": "^1.2.0", "spark-md5": "^3.0.2", "tailwind-merge": "^2.0.0", + "tiktoken": "^1.0.11", "use-debounce": "^10.0.0", "zustand": "^4.3.8" }, diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 8778db45b..78b2fae42 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -71,6 +71,7 @@ model LogEntry { userName String? @db.VarChar(50) createdAt DateTime @default(now()) logEntry String? @db.Text + logToken Int? @default(0) user User? @relation(fields: [userName], references: [name], onDelete: NoAction) }