xinyin025/99AI
1
1
Fork 0
mirror of https://github.com/vastxie/99AI.git synced 2025-11-13 12:13:43 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

v3.7.0

Browse Source
This commit is contained in:
vastxie
2024-07-31 14:12:56 +08:00
parent dd0e1dafd5
commit c831009379
366 changed files with 1881 additions and 1540 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
dist/common/utils/getTokenCount.js vendored
View File

@@ -1,7 +1,7 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.getTokenCount = void 0;
const tiktoken_1 = require("@dqbd/tiktoken");
const gpt_tokenizer_1 = require("gpt-tokenizer");
const getTokenCount = async (input) => {
let text = '';
if (Array.isArray(input)) {
@@ -25,7 +25,6 @@ const getTokenCount = async (input) => {
text = String(input);
}
text = text.replace(/<\|endoftext\|>/g, '');
const tokenizer = (0, tiktoken_1.get_encoding)('cl100k_base');
return tokenizer.encode(text).length;
return (0, gpt_tokenizer_1.encode)(text).length;
};
exports.getTokenCount = getTokenCount;