xinyin025/new-api
1
0
Fork 0
mirror of https://github.com/linux-do/new-api.git synced 2025-11-14 09:53:42 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

refactor: use tiktoken-go to calculate token number

Browse Source
This commit is contained in:
JustSong
2023-04-28 18:36:17 +08:00
parent aea6c859e7
commit b08cd7e104
3 changed files with 21 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@@ -25,6 +25,7 @@ require (
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dlclark/regexp2 v1.8.1 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
@@ -44,6 +45,7 @@ require (
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pelletier/go-toml/v2 v2.0.7 // indirect
github.com/pkoukk/tiktoken-go v0.1.1 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.2.11 // indirect
golang.org/x/arch v0.3.0 // indirect