Merge pull request #8 from sijinhui/dev

优化token计算

.github/workflows/dockerToHub-dev.yml

@@ -12,29 +12,29 @@ on:
 #      - 'package.json'
 
 jobs:
-  build:
+#  build:
-    name: build image to aly
+#    name: build image to aly
-    # runs-on: "103.200"
+#    # runs-on: "103.200"
-    runs-on: thinkpad
+#    runs-on: thinkpad
-    #    runs-on: ubuntu-latest
+#    #    runs-on: ubuntu-latest
-    #    runs-on: self-hosted
+#    #    runs-on: self-hosted
-    steps:
+#    steps:
-      - name: Check out the repo
+#      - name: Check out the repo
-        uses: actions/checkout@v3
+#        uses: actions/checkout@v3
-        with:
+#        with:
-          clean: true
+#          clean: true
-          ref: 'dev'
+#          ref: 'dev'
-      - name: build and deploy to Docker Hub
+#      - name: build and deploy to Docker Hub
-        run: |
+#        run: |
-          echo ${{ secrets.ALY_DOCKER_PASSWORD }} | docker login registry.cn-hangzhou.aliyuncs.com -u ${{ secrets.ALY_DOCKER_USERNAME }} --password-stdin
+#          echo ${{ secrets.ALY_DOCKER_PASSWORD }} | docker login registry.cn-hangzhou.aliyuncs.com -u ${{ secrets.ALY_DOCKER_USERNAME }} --password-stdin
-          echo "${{ secrets.DOCKER_ENV }}" > .env
+#          echo "${{ secrets.DOCKER_ENV }}" > .env
-          docker-compose build
+#          docker-compose build
-          docker-compose push
+#          docker-compose push
-          yes | docker system prune --filter "until=168h"
+#          yes | docker system prune --filter "until=168h"
   deploy:
     name: 部署到dev服务器
-    needs: build
+    #needs: z4
-    runs-on: thinkpad
+    runs-on: z4
     steps:
       - name: Check out the repo
         uses: actions/checkout@v3
@@ -45,7 +45,7 @@ jobs:
         uses: webfactory/ssh-agent@v0.8.0
         with:
           ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
-      - name: Sync repository to ty
+      - name: Sync repository to tx
         run: |
           yes | docker image prune
           rsync -az -e 'ssh -o StrictHostKeyChecking=no' --delete $GITHUB_WORKSPACE/ root@tx.xiaosi.cc:/data/ChatGPT-Next-Web
@@ -61,9 +61,7 @@ jobs:
           script: |
             cd $SERVER_WORKDIR #进入到工作目录
             echo "${{ secrets.DOCKER_ENV }}" > .env
-            echo ${{ secrets.ALY_DOCKER_PASSWORD }} | docker login registry.cn-hangzhou.aliyuncs.com -u ${{ secrets.ALY_DOCKER_USERNAME }} --password-stdin
+            bash ./start.sh
-            docker-compose pull && docker-compose up -d
-            yes | docker image prune
             rm -rf /www/server/nginx/proxy_cache_dir/*
             rm -rf /www/server/nginx/proxy_temp_dir/*
             sleep 2

app/api/logs/[...path]/route.ts

@@ -10,6 +10,7 @@ import { addHours, subMinutes } from "date-fns";
 
 function getTokenLength(input: string): number {
   const encoding = get_encoding("cl100k_base");
+  // console.log('tokens: ', input, encoding.countTokens())
   return encoding.encode(input).length;
 }
 
@@ -25,10 +26,15 @@ async function handle(
     // console.log("===========4", request_data);
     try {
       if (request_data?.logEntry) {
-        const regex = /\[(.*)]/g;
+        // const regex = /\[(.*)]/g;
-        const matchResponse = request_data.logEntry.match(regex);
+        // const matchResponse = request_data.logEntry.match(regex);
-        if (matchResponse.length > 0) {
+        const regex_message = /(?<="content":")(.*?)(?="}[,\]])/g;
-          request_data.logToken = getTokenLength(matchResponse[0]);
+        const matchAllMessage = request_data.logEntry.match(regex_message);
+        console.log(matchAllMessage, "=====");
+        if (matchAllMessage.length > 0) {
+          request_data.logToken =
+            getTokenLength(matchAllMessage.join(" ")) +
+            matchAllMessage.length * 3;
         }
       }
     } catch (e) {

start.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+yarn config set registry 'https://registry.npmmirror.com/'
+yarn config set sharp_binary_host "https://npm.taobao.org/mirrors/sharp"
+yarn config set sharp_libvips_binary_host "https://npm.taobao.org/mirrors/sharp-libvips"
+
+#yarn cache clean
+yarn install
+yarn run build
+
+mkdir -p "./node_modules/tiktoken"
+export OUT_DIR="out"
+
+mkdir -p ${OUT_DIR}
+
+rsync -az --delete ./.next/standalone/ ${OUT_DIR}
+rsync -az --delete ./public/ ${OUT_DIR}/public
+rsync -az --delete ./.next/static/ ${OUT_DIR}/.next/static
+rsync -az --delete ./.next/server/ ${OUT_DIR}/.next/server
+rsync -az --delete ./.next/server/ ${OUT_DIR}/.next/server
+rsync -az --delete "./node_modules/tiktoken/" ${OUT_DIR}/node_modules/tiktoken