xinyin025/ChatGPT-Next-Web
1
0
Fork 0
mirror of https://github.com/ChatGPTNextWeb/ChatGPT-Next-Web.git synced 2025-09-27 13:46:37 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

docs: add documentation for countTokens function

Browse Source
This commit is contained in:
dakai 2025-02-07 01:56:31 +08:00
parent 40947a5fa7
commit f6ebb19ab3
1 changed files with 6 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
app/utils.ts
View File

@ -40,6 +40,12 @@ export const readFileContent = async (file: UploadFile): Promise<string> => {
} }
}; };
/**
* Estimates the token count of a text file using character-based weights.
* Note: This is a rough estimation as standard tokenizers cannot be used due to environment constraints.
* @param file - The file to analyze
* @returns Estimated token count in thousands (K)
*/
export const countTokens = async (file: UploadFile) => { export const countTokens = async (file: UploadFile) => {
const text = await readFileContent(file); const text = await readFileContent(file);
let totalTokens = 0; let totalTokens = 0;