From ae03aa7a23aeb871c6b5068f29f079482e9ea821 Mon Sep 17 00:00:00 2001 From: sijinhui Date: Fri, 1 Mar 2024 15:45:12 +0800 Subject: [PATCH] =?UTF-8?q?=E7=94=A8=E6=88=B7=E9=A1=B5=E9=9D=A2=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E4=BD=BF=E7=94=A8token=E6=98=BE=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/api/logs/[...path]/route.ts | 15 ++++---- app/components/chat.tsx | 63 ++++++++++++++++++++++++++++++-- app/utils/token.ts | 32 ++++++++-------- lib/auth_list.ts | 3 +- lib/utils.ts | 14 ++++--- public/grinning-face.webp | Bin 0 -> 4282 bytes 6 files changed, 95 insertions(+), 32 deletions(-) create mode 100644 public/grinning-face.webp diff --git a/app/api/logs/[...path]/route.ts b/app/api/logs/[...path]/route.ts index ad6bc33f3..5083853dd 100644 --- a/app/api/logs/[...path]/route.ts +++ b/app/api/logs/[...path]/route.ts @@ -4,15 +4,16 @@ import { insertUser } from "@/lib/auth"; // import { getTokenLength } from "@/app/utils/token"; // import { Tiktoken } from "tiktoken/lite" // import cl100k_base from "tiktoken/encoders/cl100k_base.json" -import "tiktoken"; -import { get_encoding } from "tiktoken"; +// import "tiktoken"; +// import { get_encoding } from "tiktoken"; import { addHours, subMinutes } from "date-fns"; +import { getTokenLength } from "@/lib/utils"; -function getTokenLength(input: string): number { - const encoding = get_encoding("cl100k_base"); - // console.log('tokens: ', input, encoding.countTokens()) - return encoding.encode(input).length; -} +// function getTokenLength(input: string): number { +// const encoding = get_encoding("cl100k_base"); +// // console.log('tokens: ', input, encoding.countTokens()) +// return encoding.encode(input).length; +// } async function handle( req: NextRequest, diff --git a/app/components/chat.tsx b/app/components/chat.tsx index 51a4b54af..acd85f8ce 100644 --- a/app/components/chat.tsx +++ b/app/components/chat.tsx @@ -50,6 +50,7 @@ import { useAppConfig, DEFAULT_TOPIC, ModelType, + ChatSession, } from "../store"; import { @@ -101,6 +102,7 @@ import { Button } from "emoji-picker-react/src/components/atoms/Button"; import Image from "next/image"; import { useAllModels } from "../utils/hooks"; import { MultimodalContent } from "../client/api"; +import { getTokenLength } from "@/lib/utils"; const Markdown = dynamic(async () => (await import("./markdown")).Markdown, { loading: () => , @@ -350,9 +352,10 @@ function ChatAction(props: { icon: 16, }); const allModels = useAllModels().map((item) => item.displayName); - const customModelClassName = allModels.includes(props.text) - ? "chat-input-action-long-weight" - : ""; + let customModelClassName = ""; + if (props.text.includes("使用") || allModels.includes(props.text)) { + customModelClassName = "chat-input-action-long-weight"; + } function updateWidth() { if (!iconRef.current || !textRef.current) return; @@ -466,6 +469,7 @@ export function ChatActions(props: { ); const [showModelSelector, setShowModelSelector] = useState(false); const [showUploadImage, setShowUploadImage] = useState(false); + const current_day_token = localStorage.getItem("current_day_token") ?? ""; useEffect(() => { const show = isVisionModel(currentModel); @@ -602,6 +606,22 @@ export function ChatActions(props: { }} /> )} + + false} + text={"使用 " + current_day_token} + icon={ + 😀 + } + /> ); } @@ -1643,8 +1663,45 @@ function _Chat() { ); } +function getCurrentDayToken(sessions: ChatSession[]): number { + try { + const currentTime = new Date(); + const startOfTheDayInTimeZone = new Date( + currentTime.getFullYear(), + currentTime.getMonth(), + currentTime.getDate(), + 0, + 0, + 0, + ); + const current_day_message = sessions + .reduce((acc, item) => { + // @ts-ignore + return acc.concat(item.messages); + }, []) + .filter((item1) => { + // @ts-ignore + const dateToCheck = new Date(item1.date); + return startOfTheDayInTimeZone < dateToCheck; + }); + // @ts-ignore + const all_current_day_content = current_day_message + .map((item) => item.content) + .join(" "); + // 获取会话之后，再整合content， + return getTokenLength(all_current_day_content); + } catch (e) { + return 0; + } +} + export function Chat() { const chatStore = useChatStore(); const sessionIndex = chatStore.currentSessionIndex; + // 这里计先计算一下当天总token数。 + localStorage.setItem( + "current_day_token", + String(getCurrentDayToken(chatStore.sessions)), + ); return <_Chat key={sessionIndex}>; } diff --git a/app/utils/token.ts b/app/utils/token.ts index 1e803408c..6ff0f5df2 100644 --- a/app/utils/token.ts +++ b/app/utils/token.ts @@ -23,19 +23,19 @@ export function estimateTokenLength(input: string): number { // import { get_encoding } from "tiktoken"; -export function getTokenLength(input: string): number { - // const { get_encoding } = require( "tiktoken" ); - // const encoding = get_encoding("cl100k_base"); - - const { Tiktoken } = require("tiktoken/lite"); - const cl100k_base = require("tiktoken/encoders/cl100k_base.json"); - const encoding = new Tiktoken( - cl100k_base.bpe_ranks, - cl100k_base.special_tokens, - cl100k_base.pat_str, - ); - const tokenLength = encoding.encode(input).length; - // console.log('[TOKEN],=========', input, tokenLength) - - return tokenLength; -} +// export function getTokenLength(input: string): number { +// // const { get_encoding } = require( "tiktoken" ); +// // const encoding = get_encoding("cl100k_base"); +// +// const { Tiktoken } = require("tiktoken/lite"); +// const cl100k_base = require("tiktoken/encoders/cl100k_base.json"); +// const encoding = new Tiktoken( +// cl100k_base.bpe_ranks, +// cl100k_base.special_tokens, +// cl100k_base.pat_str, +// ); +// const tokenLength = encoding.encode(input).length; +// // console.log('[TOKEN],=========', input, tokenLength) +// +// return tokenLength; +// } diff --git a/lib/auth_list.ts b/lib/auth_list.ts index 27d7616cc..c28ea6df2 100644 --- a/lib/auth_list.ts +++ b/lib/auth_list.ts @@ -4,7 +4,8 @@ export const DENY_LIST: string[] = [ ] export const ADMIN_LIST: string[] = [ "司金辉", "sijinhui", "sijinhui@qq.com", - "yuchuan", "于川" + "yuchuan", "于川", + "jujujujuju", ] diff --git a/lib/utils.ts b/lib/utils.ts index f43cf2645..c2c0e3fad 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -1,9 +1,13 @@ -// import { clsx, type ClassValue } from "clsx"; -import { twMerge } from "tailwind-merge"; -// export function cn(...inputs: ClassValue[]) { -// return twMerge(clsx(inputs)); -// } +import {get_encoding} from "tiktoken"; + + +export function getTokenLength(input: string): number { + const encoding = get_encoding("cl100k_base"); + // console.log('tokens: ', input, encoding.countTokens()) + return encoding.encode(input).length; +} + export async function fetcher( input: RequestInfo, init?: RequestInit, diff --git a/public/grinning-face.webp b/public/grinning-face.webp new file mode 100644 index 0000000000000000000000000000000000000000..6e350846abf620e19de5d2feed95aa0bfc98652b GIT binary patch literal 4282 zcmbW42{e@7|Htn$t7WV+)+|$&v5hUuAO>UK_oc`(GclIIFr<`{R47YjjUtg$N?B5f zl!}O=y+}f-w1^PDnQ!M*|Nl9^-~WGp_dL%%ulu?0d+z7?d_K=}p6g+6W78lEfUmWs zn~xjGj}HK#&AGMyx$G$vH#-bx2rza|3}y_Bql{)Iusm!n@qYdr@#ua41H1qQbN~pU z#>YB)+IVr*_SO`9JV(UkKkfSnfViUvL_0S;{*V3tMIzK#RssN!8%MpKMvtd*IGw{$ zNeQuB`z(iXp^;pMA-K%qILKj1E{FYMjej)1*qqC>XeN!L;rbj)i>7h8o5TAP*>n!W zoj9D4$e<^2_#uZCqu9|54li&R7fla|2LQp1j$#6x8qQ&E;4myN4+@7j0KkI@`-?;W z;sp9uj-LQfVq%k7jIi(oyb4tnuVZ9H!rRf4qUZ?;M7I!XWC)9fr^H0ZhA@)>`1Q=( zRX~&*TRbPo20BIt21H#=PWXSf|2X+u^`C*8+h2;W?*HTr67TzC`?L0sEv6Cx(i|r? zxqob-#Q-$#1wiuYADiL{0EE&3Xd3wIdBnN(5}uF{YpkV}l$4~&pi?!u0sYD32uR2a32hVXJ8aefcIb)d7&bwYQbLFfrI22Dd>phXxQhJoQ=vM^au_**{EkAQ#8Cv4A<7QrkBUHT zM`feRQ1z(ms6o^?Y7UJ?OQF@#rf64m2s#0sfi6a$MR%Zw&=cqd9)2Eq9$g+Ao{c%*L$8W^%&d=cA#b3mKf&VuD82=B9C`JuK!31E~m_wKv%yrBY%vS*c z0c8PmfsFzQ0*3@@1#Squ6j%_%3Tg`43(^Er1xp2+1cwAa3h@dl30Vkj7TP9MAaqG+ zKxjsoS6Eq?A{-*TQ@B+4itr=hIT5Ugj)<#Bv`Dtd8Ie08Q=({5Wl?KUnrOP{Nzogk z<5)OW0ZYMBv3syp*qhi1912Ik+2bN`S-5k!e%!2>sFa5c3u%P3 znzXxgl61NBP3f66xHTqgsB1FUG^}|l1Cvpc@sQarb5f>HW=>X4)>bxFwpjLt?2Mea zoVi?t+!48UxoLTh{)l{s{5!lD-U1(mFT!`@XV=QCwOh+xTd}r(?b15JI`4I9 z>+09NP~cM_DQrkwNkYKb(p$=IzzooeNY3bVWh#- zsML5w#1JiriNreMn5MX7Gq-@e{Bczd~QL523qaVfw#tFs^#-B`xCXps*Or|yvHqbXzZ5TIQXBuK!VLEDt zHw!j9VKz!$OAaAdl3$xEnA6Ow%_l8XEg~$=T70zBwq#j0TFz69DLW|Ft>9L+R+(1) z)}q$l)+N?2ZPwX@+0@z0+UnbGx4mwMv~#q}vwLJOYfrVWwV!n`blBn0?TB&obS!av z?WE?!a%y#kIXgHXc7E=no4uk@Ne0O-srQjdgI(C z>rF>DO$6u%WCV-^5(2jd-q|d@IcjrTkWf%a(4}Bxuy630;N=k4kjjv`Q2Wqhp|ey9 zwUGLrW==aoo2Hx6^XZdYOt<83nF=!vI~+D0P7W^!pJ7lKC5+Dzb`i%T<|ADrYa&;p ze52~4d7^`(uP{ZKk<9KGxtOgngR$zd8L^{r#&JjEW?7D`>UbzVFuo-L$N8D+W2>ol6;abZ57$d+}f9{mV6+2a+}SzQ`-^SsoT3!)~4)E8QWp8qhcqtGjwNH zszPdd>clSFU8i^R?T*}iFHJiwKkeHd-#xAAvgy0h$M@Rqt;-P1U}ZeoXR`13e#Cyp z{{92&4-_9*$)si8&C<>)$Xd>(X5T%ibFk>(>Y*)%`g06&%5zb<%-oSYa^C5Dk^F7> zgO#ks}HCE+E*rIw}j$7GHjI5uBK zD;p{|FF$`=_ITFu#S@GZk1K2|nktnm3r?a=CZ2p-BTDm%``ezNZX7sej z>F!#?+S)TRXL9Obb?mz7vjJ!CpR+pGa-Mj;qF%f{>jJpIzVPm1@Wqh^$A+#;q)X>7 zD_uU;h;7Vlf|`<>W}CyCM_c?`2Cq0=>1s7;ZM>>^wdUH|Yo%>sZ8`0H?dk2S9ostQ zuE$=V=?v=}?+WUAcEj(+!*2KP{+o_BZ{M=Hb)(0kr{lKi?Q3_8?_9Y{y4&1q(A(6f z-`99g|6XIiL4VVL;XuoMqx-Fc8wT5l$U~hEtRCEYX#cQx*kyQd#CzoNqku=Fk7+|ZD?5|Q^ zYrg4y>zZ?$d-*;3`_g>og4Dw4AEX~Wi@uAKOG(S<<)WXeKU-HESDvp%udZ4+x!Fx~ zegdG_O_9$I7hNKHNhOA6Uq%u!u>Rcmb7F0^L&l=V<7X7o4nDIvH1#f2Kjf9$(t~fe z#^r8ZUJj3U${$z$LcFAVv(#+!m1HN=Q+%c^?GA6Gr|6BxzShzDUFvW;JH72R3Ql|s zI@;jN&%4j}rc8RJmU8{w;>%y2XyFrA9DTdI?_t|>(~YW3#x?25S}L1us4m5ZL)nB0 zMwP#ohEK`B=LexK$2?xe8q7V+CdIk59gpj>zqMf{Efu*c5=2Om{le~_7kOE;EOUJl zL++V1x-$@x(m%c(@yc^Su4-PRt!FGAwq5YY0O!oWIaA?1-~b3?K^df`%{g0{u{H8- z6e}w#G)aQNxq=6xK-qe?OG5R@X`Zi6MTh6TtR*|X&079+K}cI@*eq8rnCwfAXd7z- z+d!(v%7GEzDcOnCrG2x$OIb^0v%E{#)yG5cuZe$;cYk;2*|kk)EA88T;snn&^iJJE ztf%!m-8U3_+VNyTW?^N5l>tQ=M>-(Q<}&t(hf1AOl{hNj9kFXDqS_pQmDR_st8e_+ zMC;v{9{KN$h8jn;dpi=)u1yr{HXXV9PC`C`sB$^ZK6(d zr(x6g%{vd)B`F5YEZDI$_bKjISX{b9L49gF8Ni(Vs$cdlF;87NqHkV@Fnj2muzTMb znz%U+<+>qRth;T_PvPhEC+ic^;c79XZ&-E<8X9sLvD?Ek2*WevOFXa!g^s;Rqh}X= zDtb>%->AA3m_s^tq4JB&)D@d9xeJt(JO$MwSm~ZXeO{G3`3Vrh#T5j zWlwk*?<1Ln!A&-1&mx{aAUL~p`(!R#gnTOOxHVieUG^qFxwA(59zkk{lthH)r;g66 z)H62+dp3N-#Z-)*6gZMI9jiOGgpS&`cWGDn%uwp$g^n3T_`x1`l=SLJ@?g6DB(v?> z_JX4FcB9XPu`T7ez|}YX!h}t^dH#iXvAcnIih#L^4?1J;9iwNl!i`-1UIdAy0n3#Q z^Q`{)MlB@aLjZ5JcxLax>L`Ol<>b`2M4x^7J{4jQJ2q$6Pt+uyj=p1BXPmj5TQYSq zYQFu$H-hNyoYk_R6KP|j881>nvv$<#QdH!7_8hsl>>RurZd+j|V|8eXFw(i>wOxmw Q>+|~#S9?QpIp2_f0XJeIpa1{> literal 0 HcmV?d00001