xinyin025/ChatGPT-Next-Web
1
0
Fork 0
mirror of https://github.com/ChatGPTNextWeb/ChatGPT-Next-Web.git synced 2025-11-16 22:13:47 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

用户页面增加使用token显示

Browse Source
This commit is contained in:
sijinhui
2024-03-01 15:45:12 +08:00
parent 3a5f0b2d0e
commit ae03aa7a23
6 changed files with 95 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
app/utils/token.ts
View File

@@ -23,19 +23,19 @@ export function estimateTokenLength(input: string): number {
// import { get_encoding } from "tiktoken";
export function getTokenLength(input: string): number {
// const { get_encoding } = require( "tiktoken" );
// const encoding = get_encoding("cl100k_base");
const { Tiktoken } = require("tiktoken/lite");
const cl100k_base = require("tiktoken/encoders/cl100k_base.json");
const encoding = new Tiktoken(
cl100k_base.bpe_ranks,
cl100k_base.special_tokens,
cl100k_base.pat_str,
);
const tokenLength = encoding.encode(input).length;
// console.log('[TOKEN],=========', input, tokenLength)
return tokenLength;
}
// export function getTokenLength(input: string): number {
// // const { get_encoding } = require( "tiktoken" );
// // const encoding = get_encoding("cl100k_base");
//
// const { Tiktoken } = require("tiktoken/lite");
// const cl100k_base = require("tiktoken/encoders/cl100k_base.json");
// const encoding = new Tiktoken(
// cl100k_base.bpe_ranks,
// cl100k_base.special_tokens,
// cl100k_base.pat_str,
// );
// const tokenLength = encoding.encode(input).length;
// // console.log('[TOKEN],=========', input, tokenLength)
//
// return tokenLength;
// }