xinyin025/ChatGPT-Next-Web
1
0
Fork 0
mirror of https://github.com/ChatGPTNextWeb/ChatGPT-Next-Web.git synced 2025-10-03 00:26:40 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

优化token计算

Browse Source
This commit is contained in:
sijinhui 2024-01-05 13:19:59 +08:00
parent 167b86642b
commit af2800eb2d
1 changed files with 10 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
app/api/logs/[...path]/route.ts
View File

@ -10,6 +10,7 @@ import { addHours, subMinutes } from "date-fns";
function getTokenLength(input: string): number { function getTokenLength(input: string): number {
const encoding = get_encoding("cl100k_base"); const encoding = get_encoding("cl100k_base");
// console.log('tokens: ', input, encoding.countTokens())
return encoding.encode(input).length; return encoding.encode(input).length;
} }
@ -25,10 +26,15 @@ async function handle(
// console.log("===========4", request_data); // console.log("===========4", request_data);
try { try {
if (request_data?.logEntry) { if (request_data?.logEntry) {
const regex = /\[(.*)]/g; // const regex = /\[(.*)]/g;
const matchResponse = request_data.logEntry.match(regex); // const matchResponse = request_data.logEntry.match(regex);
if (matchResponse.length > 0) { const regex_message = /(?<="content":")(.*?)(?="}[,\]])/g;
request_data.logToken = getTokenLength(matchResponse[0]); const matchAllMessage = request_data.logEntry.match(regex_message);
console.log(matchAllMessage, "=====");
if (matchAllMessage.length > 0) {
request_data.logToken =
getTokenLength(matchAllMessage.join(" ")) +
matchAllMessage.length * 3;
} }
} }
} catch (e) { } catch (e) {