From 823ebc0357ae158df6af826236fe4592a9b67139 Mon Sep 17 00:00:00 2001 From: sijinhui Date: Thu, 4 Jan 2024 08:53:01 +0800 Subject: [PATCH 1/5] =?UTF-8?q?=E6=B5=8B=E8=AF=95dev=E9=83=A8=E7=BD=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/dockerToHub-dev.yml | 47 ++++++++++++++------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/.github/workflows/dockerToHub-dev.yml b/.github/workflows/dockerToHub-dev.yml index bdd9662cf..a9303173a 100644 --- a/.github/workflows/dockerToHub-dev.yml +++ b/.github/workflows/dockerToHub-dev.yml @@ -12,28 +12,28 @@ on: # - 'package.json' jobs: - build: - name: build image to aly - # runs-on: "103.200" - runs-on: thinkpad - # runs-on: ubuntu-latest - # runs-on: self-hosted - steps: - - name: Check out the repo - uses: actions/checkout@v3 - with: - clean: true - ref: 'dev' - - name: build and deploy to Docker Hub - run: | - echo ${{ secrets.ALY_DOCKER_PASSWORD }} | docker login registry.cn-hangzhou.aliyuncs.com -u ${{ secrets.ALY_DOCKER_USERNAME }} --password-stdin - echo "${{ secrets.DOCKER_ENV }}" > .env - docker-compose build - docker-compose push - yes | docker system prune --filter "until=168h" +# build: +# name: build image to aly +# # runs-on: "103.200" +# runs-on: thinkpad +# # runs-on: ubuntu-latest +# # runs-on: self-hosted +# steps: +# - name: Check out the repo +# uses: actions/checkout@v3 +# with: +# clean: true +# ref: 'dev' +# - name: build and deploy to Docker Hub +# run: | +# echo ${{ secrets.ALY_DOCKER_PASSWORD }} | docker login registry.cn-hangzhou.aliyuncs.com -u ${{ secrets.ALY_DOCKER_USERNAME }} --password-stdin +# echo "${{ secrets.DOCKER_ENV }}" > .env +# docker-compose build +# docker-compose push +# yes | docker system prune --filter "until=168h" deploy: name: 部署到dev服务器 - needs: build + needs: z4 runs-on: thinkpad steps: - name: Check out the repo @@ -61,9 +61,10 @@ jobs: script: | cd $SERVER_WORKDIR #进入到工作目录 echo "${{ secrets.DOCKER_ENV }}" > .env - echo ${{ secrets.ALY_DOCKER_PASSWORD }} | docker login registry.cn-hangzhou.aliyuncs.com -u ${{ secrets.ALY_DOCKER_USERNAME }} --password-stdin - docker-compose pull && docker-compose up -d - yes | docker image prune + #echo ${{ secrets.ALY_DOCKER_PASSWORD }} | docker login registry.cn-hangzhou.aliyuncs.com -u ${{ secrets.ALY_DOCKER_USERNAME }} --password-stdin + #docker-compose pull && docker-compose up -d + #yes | docker image prune + yarn install && yarn run build rm -rf /www/server/nginx/proxy_cache_dir/* rm -rf /www/server/nginx/proxy_temp_dir/* sleep 2 From 818c17058ac1de51f50a09c6d40076c2db59da30 Mon Sep 17 00:00:00 2001 From: sijinhui Date: Thu, 4 Jan 2024 08:53:45 +0800 Subject: [PATCH 2/5] =?UTF-8?q?=E6=B5=8B=E8=AF=95dev=E9=83=A8=E7=BD=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/dockerToHub-dev.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/dockerToHub-dev.yml b/.github/workflows/dockerToHub-dev.yml index a9303173a..1d6f9fa96 100644 --- a/.github/workflows/dockerToHub-dev.yml +++ b/.github/workflows/dockerToHub-dev.yml @@ -64,6 +64,9 @@ jobs: #echo ${{ secrets.ALY_DOCKER_PASSWORD }} | docker login registry.cn-hangzhou.aliyuncs.com -u ${{ secrets.ALY_DOCKER_USERNAME }} --password-stdin #docker-compose pull && docker-compose up -d #yes | docker image prune + yarn config set registry 'https://registry.npmmirror.com/' + yarn config set sharp_binary_host "https://npm.taobao.org/mirrors/sharp" + yarn config set sharp_libvips_binary_host "https://npm.taobao.org/mirrors/sharp-libvips" yarn install && yarn run build rm -rf /www/server/nginx/proxy_cache_dir/* rm -rf /www/server/nginx/proxy_temp_dir/* From b1198dab55fc240416e904ed621ccbbef04c12ad Mon Sep 17 00:00:00 2001 From: sijinhui Date: Thu, 4 Jan 2024 08:54:49 +0800 Subject: [PATCH 3/5] =?UTF-8?q?=E6=B5=8B=E8=AF=95dev=E9=83=A8=E7=BD=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/dockerToHub-dev.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dockerToHub-dev.yml b/.github/workflows/dockerToHub-dev.yml index 1d6f9fa96..d872d6495 100644 --- a/.github/workflows/dockerToHub-dev.yml +++ b/.github/workflows/dockerToHub-dev.yml @@ -33,8 +33,8 @@ jobs: # yes | docker system prune --filter "until=168h" deploy: name: 部署到dev服务器 - needs: z4 - runs-on: thinkpad + #needs: z4 + runs-on: z4 steps: - name: Check out the repo uses: actions/checkout@v3 From 167b86642b4d9867abc00c80c2f5e8f0f6c5f766 Mon Sep 17 00:00:00 2001 From: sijinhui Date: Thu, 4 Jan 2024 10:56:49 +0800 Subject: [PATCH 4/5] =?UTF-8?q?=E9=85=8D=E7=BD=AE=E6=96=B0=E7=9A=84?= =?UTF-8?q?=E9=83=A8=E7=BD=B2=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/dockerToHub-dev.yml | 10 ++-------- start.sh | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 8 deletions(-) create mode 100755 start.sh diff --git a/.github/workflows/dockerToHub-dev.yml b/.github/workflows/dockerToHub-dev.yml index d872d6495..9980020f0 100644 --- a/.github/workflows/dockerToHub-dev.yml +++ b/.github/workflows/dockerToHub-dev.yml @@ -45,7 +45,7 @@ jobs: uses: webfactory/ssh-agent@v0.8.0 with: ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} - - name: Sync repository to ty + - name: Sync repository to tx run: | yes | docker image prune rsync -az -e 'ssh -o StrictHostKeyChecking=no' --delete $GITHUB_WORKSPACE/ root@tx.xiaosi.cc:/data/ChatGPT-Next-Web @@ -61,13 +61,7 @@ jobs: script: | cd $SERVER_WORKDIR #进入到工作目录 echo "${{ secrets.DOCKER_ENV }}" > .env - #echo ${{ secrets.ALY_DOCKER_PASSWORD }} | docker login registry.cn-hangzhou.aliyuncs.com -u ${{ secrets.ALY_DOCKER_USERNAME }} --password-stdin - #docker-compose pull && docker-compose up -d - #yes | docker image prune - yarn config set registry 'https://registry.npmmirror.com/' - yarn config set sharp_binary_host "https://npm.taobao.org/mirrors/sharp" - yarn config set sharp_libvips_binary_host "https://npm.taobao.org/mirrors/sharp-libvips" - yarn install && yarn run build + bash ./start.sh rm -rf /www/server/nginx/proxy_cache_dir/* rm -rf /www/server/nginx/proxy_temp_dir/* sleep 2 diff --git a/start.sh b/start.sh new file mode 100755 index 000000000..24c223416 --- /dev/null +++ b/start.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +yarn config set registry 'https://registry.npmmirror.com/' +yarn config set sharp_binary_host "https://npm.taobao.org/mirrors/sharp" +yarn config set sharp_libvips_binary_host "https://npm.taobao.org/mirrors/sharp-libvips" + +#yarn cache clean +yarn install +yarn run build + +mkdir -p "./node_modules/tiktoken" +export OUT_DIR="out" + +mkdir -p ${OUT_DIR} + +rsync -az --delete ./.next/standalone/ ${OUT_DIR} +rsync -az --delete ./public/ ${OUT_DIR}/public +rsync -az --delete ./.next/static/ ${OUT_DIR}/.next/static +rsync -az --delete ./.next/server/ ${OUT_DIR}/.next/server +rsync -az --delete ./.next/server/ ${OUT_DIR}/.next/server +rsync -az --delete "./node_modules/tiktoken/" ${OUT_DIR}/node_modules/tiktoken From af2800eb2d95dca0b0b57a50ebd6ed79eed56042 Mon Sep 17 00:00:00 2001 From: sijinhui Date: Fri, 5 Jan 2024 13:19:59 +0800 Subject: [PATCH 5/5] =?UTF-8?q?=E4=BC=98=E5=8C=96token=E8=AE=A1=E7=AE=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/api/logs/[...path]/route.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/app/api/logs/[...path]/route.ts b/app/api/logs/[...path]/route.ts index 16a6a1b55..6a1f266ec 100644 --- a/app/api/logs/[...path]/route.ts +++ b/app/api/logs/[...path]/route.ts @@ -10,6 +10,7 @@ import { addHours, subMinutes } from "date-fns"; function getTokenLength(input: string): number { const encoding = get_encoding("cl100k_base"); + // console.log('tokens: ', input, encoding.countTokens()) return encoding.encode(input).length; } @@ -25,10 +26,15 @@ async function handle( // console.log("===========4", request_data); try { if (request_data?.logEntry) { - const regex = /\[(.*)]/g; - const matchResponse = request_data.logEntry.match(regex); - if (matchResponse.length > 0) { - request_data.logToken = getTokenLength(matchResponse[0]); + // const regex = /\[(.*)]/g; + // const matchResponse = request_data.logEntry.match(regex); + const regex_message = /(?<="content":")(.*?)(?="}[,\]])/g; + const matchAllMessage = request_data.logEntry.match(regex_message); + console.log(matchAllMessage, "====="); + if (matchAllMessage.length > 0) { + request_data.logToken = + getTokenLength(matchAllMessage.join(" ")) + + matchAllMessage.length * 3; } } } catch (e) {