xinyin025/new-api
1
0
Fork 0
mirror of https://github.com/linux-do/new-api.git synced 2025-09-21 17:56:38 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: 新增数据看板

Browse Source
This commit is contained in:
CaIon 2024-01-07 18:31:14 +08:00
parent c09df83f34
commit bf8794d257
16 changed files with 455 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
common/constants.go
View File

@ -24,6 +24,9 @@ var ChatLink = ""
var QuotaPerUnit = 500 * 1000.0 // $0.002 / 1K tokens
var DisplayInCurrencyEnabled = true
var DisplayTokenStatEnabled = true
var DrawingEnabled = true
var DataExportEnabled = true
var DataExportInterval = 5 // unit: minute
// Any options with "Secret", "Token" in its key won't be return by GetOptions

2
controller/misc.go
View File

@ -34,6 +34,8 @@ func GetStatus(c *gin.Context) {
"quota_per_unit": common.QuotaPerUnit,
"display_in_currency": common.DisplayInCurrencyEnabled,
"enable_batch_update": common.BatchUpdateEnabled,
"enable_drawing": common.DrawingEnabled,
"enable_data_export": common.DataExportEnabled,
},
})
return

24
controller/usedata.go Normal file
View File

@ -0,0 +1,24 @@
package controller
import (
"github.com/gin-gonic/gin"
"net/http"
"one-api/model"
)
func GetAllQuotaDates(c *gin.Context) {
dates, err := model.GetAllQuotaDates()
if err != nil {
c.JSON(http.StatusOK, gin.H{
"success": false,
"message": err.Error(),
})
return
}
c.JSON(http.StatusOK, gin.H{
"success": true,
"message": "",
"data": dates,
})
return
}

3
main.go
View File

@ -67,6 +67,9 @@ func main() {
go model.SyncOptions(common.SyncFrequency)
go model.SyncChannelCache(common.SyncFrequency)
}
if common.DataExportEnabled {
go model.UpdateQuotaData(common.DataExportInterval)
}
if os.Getenv("CHANNEL_UPDATE_FREQUENCY") != "" {
frequency, err := strconv.Atoi(os.Getenv("CHANNEL_UPDATE_FREQUENCY"))
if err != nil {

6
model/log.go
View File

@ -59,9 +59,10 @@ func RecordConsumeLog(ctx context.Context, userId int, channelId int, promptToke
if !common.LogConsumeEnabled {
return
}
username := GetUsernameById(userId)
log := &Log{
UserId: userId,
Username: GetUsernameById(userId),
Username: username,
CreatedAt: common.GetTimestamp(),
Type: LogTypeConsume,
Content: content,
@ -77,6 +78,9 @@ func RecordConsumeLog(ctx context.Context, userId int, channelId int, promptToke
if err != nil {
common.LogError(ctx, "failed to record log: "+err.Error())
}
if common.DataExportEnabled {
LogQuotaData(userId, username, modelName, quota, common.GetTimestamp())
}
}
func GetAllLogs(logType int, startTimestamp int64, endTimestamp int64, modelName string, username string, tokenName string, startIdx int, num int, channel int) (logs []*Log, err error) {

4
model/main.go
View File

@ -127,6 +127,10 @@ func InitDB() (err error) {
if err != nil {
return err
}
err = db.AutoMigrate(&QuotaData{})
if err != nil {
return err
}
common.SysLog("database migrated")
err = createRootAccountIfNeed()
return err

11
model/option.go
View File

@ -37,6 +37,8 @@ func InitOptionMap() {
common.OptionMap["LogConsumeEnabled"] = strconv.FormatBool(common.LogConsumeEnabled)
common.OptionMap["DisplayInCurrencyEnabled"] = strconv.FormatBool(common.DisplayInCurrencyEnabled)
common.OptionMap["DisplayTokenStatEnabled"] = strconv.FormatBool(common.DisplayTokenStatEnabled)
common.OptionMap["DrawingEnabled"] = strconv.FormatBool(common.DrawingEnabled)
common.OptionMap["DataExportEnabled"] = strconv.FormatBool(common.DataExportEnabled)
common.OptionMap["ChannelDisableThreshold"] = strconv.FormatFloat(common.ChannelDisableThreshold, 'f', -1, 64)
common.OptionMap["EmailDomainRestrictionEnabled"] = strconv.FormatBool(common.EmailDomainRestrictionEnabled)
common.OptionMap["EmailDomainWhitelist"] = strings.Join(common.EmailDomainWhitelist, ",")
@ -76,6 +78,7 @@ func InitOptionMap() {
common.OptionMap["ChatLink"] = common.ChatLink
common.OptionMap["QuotaPerUnit"] = strconv.FormatFloat(common.QuotaPerUnit, 'f', -1, 64)
common.OptionMap["RetryTimes"] = strconv.Itoa(common.RetryTimes)
common.OptionMap["DataExportInterval"] = strconv.Itoa(common.DataExportInterval)
common.OptionMapRWMutex.Unlock()
loadOptionsFromDatabase()
@ -157,6 +160,12 @@ func updateOptionMap(key string, value string) (err error) {
common.LogConsumeEnabled = boolValue
case "DisplayInCurrencyEnabled":
common.DisplayInCurrencyEnabled = boolValue
case "DisplayTokenStatEnabled":
common.DisplayTokenStatEnabled = boolValue
case "DrawingEnabled":
common.DrawingEnabled = boolValue
case "DataExportEnabled":
common.DataExportEnabled = boolValue
}
}
switch key {
@ -217,6 +226,8 @@ func updateOptionMap(key string, value string) (err error) {
common.PreConsumedQuota, _ = strconv.Atoi(value)
case "RetryTimes":
common.RetryTimes, _ = strconv.Atoi(value)
case "DataExportInterval":
common.DataExportInterval, _ = strconv.Atoi(value)
case "ModelRatio":
err = common.UpdateModelRatioByJSONString(value)
case "GroupRatio":

87
model/usedata.go Normal file
View File

@ -0,0 +1,87 @@
package model
import (
"fmt"
"one-api/common"
"time"
)
// QuotaData 柱状图数据
type QuotaData struct {
Id int `json:"id"`
UserID int `json:"user_id" gorm:"index"`
Username string `json:"username" gorm:"index:index_quota_data_model_user_name,priority:2;default:''"`
ModelName string `json:"model_name" gorm:"index;index:index_quota_data_model_user_name,priority:1;default:''"`
CreatedAt int64 `json:"created_at" gorm:"bigint;index:index_quota_data_created_at,priority:2"`
Count int `json:"count" gorm:"default:0"`
Quota int `json:"quota" gorm:"default:0"`
}
func UpdateQuotaData(frequency int) {
for {
common.SysLog("正在更新数据看板数据...")
SaveQuotaDataCache()
time.Sleep(time.Duration(frequency) * time.Minute)
}
}
var CacheQuotaData = make(map[string]*QuotaData)
func LogQuotaDataCache(userId int, username string, modelName string, quota int, createdAt int64) {
// 只精确到小时
createdAt = createdAt - (createdAt % 3600)
key := fmt.Sprintf("%d-%s-%s-%d", userId, username, modelName, createdAt)
quotaData, ok := CacheQuotaData[key]
if ok {
quotaData.Count += 1
quotaData.Quota += quota
} else {
quotaData = &QuotaData{
UserID: userId,
Username: username,
ModelName: modelName,
CreatedAt: createdAt,
Count: 1,
Quota: quota,
}
}
CacheQuotaData[key] = quotaData
}
func LogQuotaData(userId int, username string, modelName string, quota int, createdAt int64) {
LogQuotaDataCache(userId, username, modelName, quota, createdAt)
}
func SaveQuotaDataCache() {
// 如果缓存中有数据，就保存到数据库中
// 1. 先查询数据库中是否有数据
// 2. 如果有数据，就更新数据
// 3. 如果没有数据，就插入数据
for _, quotaData := range CacheQuotaData {
quotaDataDB := &QuotaData{}
DB.Table("quota_data").Where("user_id = ? and username = ? and model_name = ? and created_at = ?",
quotaData.UserID, quotaData.Username, quotaData.ModelName, quotaData.CreatedAt).First(quotaDataDB)
if quotaDataDB.Id > 0 {
quotaDataDB.Count += quotaData.Count
quotaDataDB.Quota += quotaData.Quota
DB.Table("quota_data").Save(quotaDataDB)
} else {
DB.Table("quota_data").Create(quotaData)
}
}
CacheQuotaData = make(map[string]*QuotaData)
}
func GetQuotaDataByUsername(username string) (quotaData []*QuotaData, err error) {
var quotaDatas []*QuotaData
// 从quota_data表中查询数据
err = DB.Table("quota_data").Where("username = ?", username).Find(&quotaDatas).Error
return quotaDatas, err
}
func GetAllQuotaDates() (quotaData []*QuotaData, err error) {
var quotaDatas []*QuotaData
// 从quota_data表中查询数据
err = DB.Table("quota_data").Find(&quotaDatas).Error
return quotaDatas, err
}

3
router/api-router.go
View File

@ -114,6 +114,9 @@ func SetApiRouter(router *gin.Engine) {
logRoute.GET("/self", middleware.UserAuth(), controller.GetUserLogs)
logRoute.GET("/self/search", middleware.UserAuth(), controller.SearchUserLogs)
dataRoute := apiRouter.Group("/data")
dataRoute.GET("/", middleware.AdminAuth(), controller.GetAllQuotaDates)
logRoute.Use(middleware.CORS())
{
logRoute.GET("/token", controller.GetLogByKey)

8
web/package.json
View File

@ -3,7 +3,10 @@
"version": "0.1.0",
"private": true,
"dependencies": {
"@douyinfe/semi-ui": "^2.45.2",
"@douyinfe/semi-ui": "^2.46.1",
"@visactor/vchart": "~1.7.2",
"@visactor/react-vchart": "~1.7.2",
"@visactor/vchart-semi-theme": "~1.7.2",
"axios": "^0.27.2",
"history": "^5.3.0",
"marked": "^4.1.1",
@ -44,7 +47,8 @@
]
},
"devDependencies": {
"prettier": "^2.7.1"
"prettier": "^2.7.1",
"typescript": "4.4.2"
},
"prettier": {
"singleQuote": true,

12
web/src/App.js
View File

@ -23,10 +23,10 @@ import Log from './pages/Log';
import Chat from './pages/Chat';
import {Layout} from "@douyinfe/semi-ui";
import Midjourney from "./pages/Midjourney";
import Detail from "./pages/Detail";
const Home = lazy(() => import('./pages/Home'));
const About = lazy(() => import('./pages/About'));
function App() {
const [userState, userDispatch] = useContext(UserContext);
const [statusState, statusDispatch] = useContext(StatusContext);
@ -49,6 +49,8 @@ function App() {
localStorage.setItem('footer_html', data.footer_html);
localStorage.setItem('quota_per_unit', data.quota_per_unit);
localStorage.setItem('display_in_currency', data.display_in_currency);
localStorage.setItem('enable_drawing', data.enable_drawing);
localStorage.setItem('enable_data_export', data.enable_data_export);
if (data.chat_link) {
localStorage.setItem('chat_link', data.chat_link);
} else {
@ -228,6 +230,14 @@ function App() {
</PrivateRoute>
}
/>
<Route
path='/detail'
element={
<PrivateRoute>
<Detail />
</PrivateRoute>
}
/>
<Route
path='/midjourney'
element={

6
web/src/helpers/render.js
View File

@ -76,6 +76,12 @@ export function getQuotaPerUnit() {
return quotaPerUnit;
}
export function getQuotaWithUnit(quota, digits = 6) {
let quotaPerUnit = localStorage.getItem('quota_per_unit');
quotaPerUnit = parseFloat(quotaPerUnit);
return (quota / quotaPerUnit).toFixed(digits);
}
export function renderQuota(quota, digits = 2) {
let quotaPerUnit = localStorage.getItem('quota_per_unit');
let displayInCurrency = localStorage.getItem('display_in_currency');

26
web/src/helpers/utils.js
View File

@ -171,6 +171,32 @@ export function timestamp2string(timestamp) {
);
}
export function timestamp2string1(timestamp) {
let date = new Date(timestamp * 1000);
// let year = date.getFullYear().toString();
let month = (date.getMonth() + 1).toString();
let day = date.getDate().toString();
let hour = date.getHours().toString();
if (month.length === 1) {
month = '0' + month;
}
if (day.length === 1) {
day = '0' + day;
}
if (hour.length === 1) {
hour = '0' + hour;
}
return (
// year +
// '-' +
month +
'-' +
day +
' ' +
hour + ":00"
);
}
export function downloadTextAsFile(text, filename) {
let blob = new Blob([text], { type: 'text/plain;charset=utf-8' });
let url = URL.createObjectURL(blob);

8
web/src/index.js
View File

@ -1,7 +1,8 @@
import { initVChartSemiTheme } from '@visactor/vchart-semi-theme';
import VChart from "@visactor/vchart";
import React from 'react';
import ReactDOM from 'react-dom/client';
import {BrowserRouter} from 'react-router-dom';
import {Container} from 'semantic-ui-react';
import App from './App';
import HeaderBar from './components/HeaderBar';
import Footer from './components/Footer';
@ -14,6 +15,11 @@ import {StatusProvider} from './context/Status';
import {Layout} from "@douyinfe/semi-ui";
import SiderBar from "./components/SiderBar";
// initialization
initVChartSemiTheme({
isWatchingThemeSwitch: true,
});
const root = ReactDOM.createRoot(document.getElementById('root'));
const {Sider, Content, Header} = Layout;
root.render(

1
web/src/pages/About/index.js
View File

@ -1,5 +1,4 @@
import React, { useEffect, useState } from 'react';
import { Header, Segment } from 'semantic-ui-react';
import { API, showError } from '../../helpers';
import { marked } from 'marked';
import {Layout} from "@douyinfe/semi-ui";

257
web/src/pages/Detail/index.js Normal file
View File

@ -0,0 +1,257 @@
import React, {useEffect, useState} from 'react';
import {Button, Col, Form, Layout, Row} from "@douyinfe/semi-ui";
import VChart from '@visactor/vchart';
import {useEffectOnce} from "usehooks-ts";
import {API, isAdmin, showError, timestamp2string, timestamp2string1} from "../../helpers";
import {ITEMS_PER_PAGE} from "../../constants";
import {getQuotaWithUnit} from "../../helpers/render";
const Detail = (props) => {
let now = new Date();
const [inputs, setInputs] = useState({
username: '',
token_name: '',
model_name: '',
start_timestamp: timestamp2string(now.getTime() / 1000 - 86400),
end_timestamp: timestamp2string(now.getTime() / 1000 + 3600),
channel: ''
});
const {username, token_name, model_name, start_timestamp, end_timestamp, channel} = inputs;
const isAdminUser = isAdmin();
let modelDataChart = null;
let modelDataPieChart = null;
const [loading, setLoading] = useState(true);
const [quotaData, setQuotaData] = useState([]);
const [quotaDataPie, setQuotaDataPie] = useState([]);
const [quotaDataLine, setQuotaDataLine] = useState([]);
const handleInputChange = (value, name) => {
setInputs((inputs) => ({...inputs, [name]: value}));
};
const spec_line = {
type: 'bar',
data: [
{
id: 'barData',
values: [
]
}
],
xField: 'Time',
yField: 'Usage',
seriesField: 'Model',
stack: true,
legends: {
visible: true
},
title: {
visible: true,
text: '模型消耗分布'
},
bar: {
// The state style of bar
state: {
hover: {
stroke: '#000',
lineWidth: 1
}
}
}
};
const spec_pie = {
type: 'pie',
data: [
{
id: 'id0',
values: [
{ type: 'null', value: '0' },
]
}
],
outerRadius: 0.8,
innerRadius: 0.5,
padAngle: 0.6,
valueField: 'value',
categoryField: 'type',
pie: {
style: {
cornerRadius: 10
},
state: {
hover: {
outerRadius: 0.85,
stroke: '#000',
lineWidth: 1
},
selected: {
outerRadius: 0.85,
stroke: '#000',
lineWidth: 1
}
}
},
title: {
visible: true,
text: '模型调用次数占比'
},
legends: {
visible: true,
orient: 'left'
},
label: {
visible: true
},
tooltip: {
mark: {
content: [
{
key: datum => datum['type'],
value: datum => datum['value']
}
]
}
}
};
const loadQuotaData = async () => {
setLoading(true);
let url = '';
let localStartTimestamp = Date.parse(start_timestamp) / 1000;
let localEndTimestamp = Date.parse(end_timestamp) / 1000;
if (isAdminUser) {
url = `/api/data`;
} else {
url = `/api/data/self`;
}
const res = await API.get(url);
const {success, message, data} = res.data;
if (success) {
setQuotaData(data);
updateChart(data);
} else {
showError(message);
}
setLoading(false);
};
const refresh = async () => {
await loadQuotaData();
};
const updateChart = (data) => {
if (isAdminUser) {
// 将所有用户的数据累加
let pieData = [];
let lineData = [];
for (let i = 0; i < data.length; i++) {
const item = data[i];
const {count, id, model_name, quota, user_id, username} = item;
// 合并model_name
let pieItem = pieData.find(item => item.model_name === model_name);
if (pieItem) {
pieItem.count += count;
} else {
pieData.push({
"type": model_name,
"value": count
});
}
// 合并created_at和model_name 为 lineData, created_at 数据类型是小时的时间戳
// 转换日期格式
let createTime = timestamp2string1(item.created_at);
let lineItem = lineData.find(item => item.Time === item.createTime && item.Model === model_name);
if (lineItem) {
lineItem.Usage += getQuotaWithUnit(quota);
} else {
lineData.push({
"Time": createTime,
"Model": model_name,
"Usage": getQuotaWithUnit(quota)
});
}
}
// sort by count
pieData.sort((a, b) => b.value - a.value);
spec_line.data[0].values = lineData;
spec_pie.data[0].values = pieData;
// console.log('spec_line', spec_line);
console.log('spec_pie', spec_pie);
// modelDataChart.renderAsync();
modelDataPieChart.updateSpec(spec_pie);
modelDataChart.updateSpec(spec_line);
}
}
useEffect(() => {
refresh();
}, []);
useEffectOnce(() => {
// 创建 vchart 实例
if (!modelDataChart) {
modelDataChart = new VChart(spec_line, {dom: 'model_data'});
// 绘制
modelDataChart.renderAsync();
}
if (!modelDataPieChart) {
modelDataPieChart = new VChart(spec_pie, {dom: 'model_pie'});
// 绘制
modelDataPieChart.renderAsync();
}
console.log('render vchart');
})
return (
<>
<Layout>
<Layout.Header>
<h3>数据看板(24H)</h3>
</Layout.Header>
<Layout.Content>
<Form layout='horizontal' style={{marginTop: 10}}>
<>
<Form.DatePicker field="start_timestamp" label='起始时间' style={{width: 272}}
initValue={start_timestamp}
value={start_timestamp} type='dateTime'
name='start_timestamp'
onChange={value => handleInputChange(value, 'start_timestamp')}/>
<Form.DatePicker field="end_timestamp" fluid label='结束时间' style={{width: 272}}
initValue={end_timestamp}
value={end_timestamp} type='dateTime'
name='end_timestamp'
onChange={value => handleInputChange(value, 'end_timestamp')}/>
{/*<Form.Button fluid label='操作' width={2} onClick={refresh}>查询</Form.Button>*/}
{/*{*/}
{/* isAdminUser && <>*/}
{/* <Form.Input field="username" label='用户名称' style={{width: 176}} value={username}*/}
{/* placeholder={'可选值'} name='username'*/}
{/* onChange={value => handleInputChange(value, 'username')}/>*/}
{/* </>*/}
{/*}*/}
{/*<Form.Section>*/}
{/* <Button label='查询' type="primary" htmlType="submit" className="btn-margin-right"*/}
{/* >查询</Button>*/}
{/*</Form.Section>*/}
</>
</Form>
<div style={{height: 500}}>
<div id="model_pie" style={{width: '100%'}}></div>
</div>
<div style={{height: 500}}>
<div id="model_data" style={{width: '100%'}}></div>
</div>
</Layout.Content>
</Layout>
</>
);
};
export default Detail;