xinyin025/ChatGPT-Next-Web
1
0
Fork 0
mirror of https://github.com/ChatGPTNextWeb/ChatGPT-Next-Web.git synced 2025-11-16 22:13:47 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

refactor: refactor token counting logic in countTokens function

Browse Source
This commit is contained in:
dakai
2024-10-06 10:29:01 +08:00
parent 4acc742456
commit 586c4a493d
4 changed files with 69 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
app/store/chat.ts
View File

@@ -19,7 +19,7 @@ import {
StoreKey,
} from "../constant";
import Locale, { getLang } from "../locales";
import { isDalle3, safeLocalStorage } from "../utils";
import { isDalle3, safeLocalStorage, readFileContent } from "../utils";
import { prettyObject } from "../utils/format";
import { createPersistStore } from "../utils/store";
import { estimateTokenLength } from "../utils/token";
@@ -154,23 +154,6 @@ function fillTemplateWith(input: string, modelConfig: ModelConfig) {
return output;
}
const readFileContent = async (file: UploadFile): Promise<string> => {
try {
const response = await fetch(file.url);
if (!response.ok) {
throw new Error(
`Failed to fetch content from ${file.url}: ${response.statusText}`,
);
}
const content = await response.text();
const result = file.name + "\n" + content;
return result;
} catch (error) {
console.error("Error reading file content:", error);
return "";
}
};
const DEFAULT_CHAT_STATE = {
sessions: [createEmptySession()],
currentSessionIndex: 0,