From bf8794d2574f6f428401812466e9e0a5db131e6a Mon Sep 17 00:00:00 2001 From: CaIon <1808837298@qq.com> Date: Sun, 7 Jan 2024 18:31:14 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E7=9C=8B=E6=9D=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/constants.go | 3 + controller/misc.go | 2 + controller/usedata.go | 24 ++++ main.go | 3 + model/log.go | 6 +- model/main.go | 4 + model/option.go | 11 ++ model/usedata.go | 87 ++++++++++++ router/api-router.go | 3 + web/package.json | 8 +- web/src/App.js | 12 +- web/src/helpers/render.js | 6 + web/src/helpers/utils.js | 26 ++++ web/src/index.js | 8 +- web/src/pages/About/index.js | 1 - web/src/pages/Detail/index.js | 257 ++++++++++++++++++++++++++++++++++ 16 files changed, 455 insertions(+), 6 deletions(-) create mode 100644 controller/usedata.go create mode 100644 model/usedata.go create mode 100644 web/src/pages/Detail/index.js diff --git a/common/constants.go b/common/constants.go index 50b7091..95e4628 100644 --- a/common/constants.go +++ b/common/constants.go @@ -24,6 +24,9 @@ var ChatLink = "" var QuotaPerUnit = 500 * 1000.0 // $0.002 / 1K tokens var DisplayInCurrencyEnabled = true var DisplayTokenStatEnabled = true +var DrawingEnabled = true +var DataExportEnabled = true +var DataExportInterval = 5 // unit: minute // Any options with "Secret", "Token" in its key won't be return by GetOptions diff --git a/controller/misc.go b/controller/misc.go index de38c36..80cb795 100644 --- a/controller/misc.go +++ b/controller/misc.go @@ -34,6 +34,8 @@ func GetStatus(c *gin.Context) { "quota_per_unit": common.QuotaPerUnit, "display_in_currency": common.DisplayInCurrencyEnabled, "enable_batch_update": common.BatchUpdateEnabled, + "enable_drawing": common.DrawingEnabled, + "enable_data_export": common.DataExportEnabled, }, }) return diff --git a/controller/usedata.go b/controller/usedata.go new file mode 100644 index 0000000..fc0a326 --- /dev/null +++ b/controller/usedata.go @@ -0,0 +1,24 @@ +package controller + +import ( + "github.com/gin-gonic/gin" + "net/http" + "one-api/model" +) + +func GetAllQuotaDates(c *gin.Context) { + dates, err := model.GetAllQuotaDates() + if err != nil { + c.JSON(http.StatusOK, gin.H{ + "success": false, + "message": err.Error(), + }) + return + } + c.JSON(http.StatusOK, gin.H{ + "success": true, + "message": "", + "data": dates, + }) + return +} diff --git a/main.go b/main.go index 3cfb29f..4253abd 100644 --- a/main.go +++ b/main.go @@ -67,6 +67,9 @@ func main() { go model.SyncOptions(common.SyncFrequency) go model.SyncChannelCache(common.SyncFrequency) } + if common.DataExportEnabled { + go model.UpdateQuotaData(common.DataExportInterval) + } if os.Getenv("CHANNEL_UPDATE_FREQUENCY") != "" { frequency, err := strconv.Atoi(os.Getenv("CHANNEL_UPDATE_FREQUENCY")) if err != nil { diff --git a/model/log.go b/model/log.go index bb3d7d8..3df5c2c 100644 --- a/model/log.go +++ b/model/log.go @@ -59,9 +59,10 @@ func RecordConsumeLog(ctx context.Context, userId int, channelId int, promptToke if !common.LogConsumeEnabled { return } + username := GetUsernameById(userId) log := &Log{ UserId: userId, - Username: GetUsernameById(userId), + Username: username, CreatedAt: common.GetTimestamp(), Type: LogTypeConsume, Content: content, @@ -77,6 +78,9 @@ func RecordConsumeLog(ctx context.Context, userId int, channelId int, promptToke if err != nil { common.LogError(ctx, "failed to record log: "+err.Error()) } + if common.DataExportEnabled { + LogQuotaData(userId, username, modelName, quota, common.GetTimestamp()) + } } func GetAllLogs(logType int, startTimestamp int64, endTimestamp int64, modelName string, username string, tokenName string, startIdx int, num int, channel int) (logs []*Log, err error) { diff --git a/model/main.go b/model/main.go index eab668d..ade3cc1 100644 --- a/model/main.go +++ b/model/main.go @@ -127,6 +127,10 @@ func InitDB() (err error) { if err != nil { return err } + err = db.AutoMigrate(&QuotaData{}) + if err != nil { + return err + } common.SysLog("database migrated") err = createRootAccountIfNeed() return err diff --git a/model/option.go b/model/option.go index d98904c..e32cd93 100644 --- a/model/option.go +++ b/model/option.go @@ -37,6 +37,8 @@ func InitOptionMap() { common.OptionMap["LogConsumeEnabled"] = strconv.FormatBool(common.LogConsumeEnabled) common.OptionMap["DisplayInCurrencyEnabled"] = strconv.FormatBool(common.DisplayInCurrencyEnabled) common.OptionMap["DisplayTokenStatEnabled"] = strconv.FormatBool(common.DisplayTokenStatEnabled) + common.OptionMap["DrawingEnabled"] = strconv.FormatBool(common.DrawingEnabled) + common.OptionMap["DataExportEnabled"] = strconv.FormatBool(common.DataExportEnabled) common.OptionMap["ChannelDisableThreshold"] = strconv.FormatFloat(common.ChannelDisableThreshold, 'f', -1, 64) common.OptionMap["EmailDomainRestrictionEnabled"] = strconv.FormatBool(common.EmailDomainRestrictionEnabled) common.OptionMap["EmailDomainWhitelist"] = strings.Join(common.EmailDomainWhitelist, ",") @@ -76,6 +78,7 @@ func InitOptionMap() { common.OptionMap["ChatLink"] = common.ChatLink common.OptionMap["QuotaPerUnit"] = strconv.FormatFloat(common.QuotaPerUnit, 'f', -1, 64) common.OptionMap["RetryTimes"] = strconv.Itoa(common.RetryTimes) + common.OptionMap["DataExportInterval"] = strconv.Itoa(common.DataExportInterval) common.OptionMapRWMutex.Unlock() loadOptionsFromDatabase() @@ -157,6 +160,12 @@ func updateOptionMap(key string, value string) (err error) { common.LogConsumeEnabled = boolValue case "DisplayInCurrencyEnabled": common.DisplayInCurrencyEnabled = boolValue + case "DisplayTokenStatEnabled": + common.DisplayTokenStatEnabled = boolValue + case "DrawingEnabled": + common.DrawingEnabled = boolValue + case "DataExportEnabled": + common.DataExportEnabled = boolValue } } switch key { @@ -217,6 +226,8 @@ func updateOptionMap(key string, value string) (err error) { common.PreConsumedQuota, _ = strconv.Atoi(value) case "RetryTimes": common.RetryTimes, _ = strconv.Atoi(value) + case "DataExportInterval": + common.DataExportInterval, _ = strconv.Atoi(value) case "ModelRatio": err = common.UpdateModelRatioByJSONString(value) case "GroupRatio": diff --git a/model/usedata.go b/model/usedata.go new file mode 100644 index 0000000..987c2b7 --- /dev/null +++ b/model/usedata.go @@ -0,0 +1,87 @@ +package model + +import ( + "fmt" + "one-api/common" + "time" +) + +// QuotaData 柱状图数据 +type QuotaData struct { + Id int `json:"id"` + UserID int `json:"user_id" gorm:"index"` + Username string `json:"username" gorm:"index:index_quota_data_model_user_name,priority:2;default:''"` + ModelName string `json:"model_name" gorm:"index;index:index_quota_data_model_user_name,priority:1;default:''"` + CreatedAt int64 `json:"created_at" gorm:"bigint;index:index_quota_data_created_at,priority:2"` + Count int `json:"count" gorm:"default:0"` + Quota int `json:"quota" gorm:"default:0"` +} + +func UpdateQuotaData(frequency int) { + for { + common.SysLog("正在更新数据看板数据...") + SaveQuotaDataCache() + time.Sleep(time.Duration(frequency) * time.Minute) + } +} + +var CacheQuotaData = make(map[string]*QuotaData) + +func LogQuotaDataCache(userId int, username string, modelName string, quota int, createdAt int64) { + // 只精确到小时 + createdAt = createdAt - (createdAt % 3600) + key := fmt.Sprintf("%d-%s-%s-%d", userId, username, modelName, createdAt) + quotaData, ok := CacheQuotaData[key] + if ok { + quotaData.Count += 1 + quotaData.Quota += quota + } else { + quotaData = &QuotaData{ + UserID: userId, + Username: username, + ModelName: modelName, + CreatedAt: createdAt, + Count: 1, + Quota: quota, + } + } + CacheQuotaData[key] = quotaData +} + +func LogQuotaData(userId int, username string, modelName string, quota int, createdAt int64) { + LogQuotaDataCache(userId, username, modelName, quota, createdAt) +} + +func SaveQuotaDataCache() { + // 如果缓存中有数据，就保存到数据库中 + // 1. 先查询数据库中是否有数据 + // 2. 如果有数据，就更新数据 + // 3. 如果没有数据，就插入数据 + for _, quotaData := range CacheQuotaData { + quotaDataDB := &QuotaData{} + DB.Table("quota_data").Where("user_id = ? and username = ? and model_name = ? and created_at = ?", + quotaData.UserID, quotaData.Username, quotaData.ModelName, quotaData.CreatedAt).First(quotaDataDB) + if quotaDataDB.Id > 0 { + quotaDataDB.Count += quotaData.Count + quotaDataDB.Quota += quotaData.Quota + DB.Table("quota_data").Save(quotaDataDB) + } else { + DB.Table("quota_data").Create(quotaData) + } + } + CacheQuotaData = make(map[string]*QuotaData) +} + +func GetQuotaDataByUsername(username string) (quotaData []*QuotaData, err error) { + var quotaDatas []*QuotaData + // 从quota_data表中查询数据 + err = DB.Table("quota_data").Where("username = ?", username).Find("aDatas).Error + return quotaDatas, err +} + +func GetAllQuotaDates() (quotaData []*QuotaData, err error) { + var quotaDatas []*QuotaData + // 从quota_data表中查询数据 + err = DB.Table("quota_data").Find("aDatas).Error + return quotaDatas, err +} diff --git a/router/api-router.go b/router/api-router.go index b9f4602..f39a5af 100644 --- a/router/api-router.go +++ b/router/api-router.go @@ -114,6 +114,9 @@ func SetApiRouter(router *gin.Engine) { logRoute.GET("/self", middleware.UserAuth(), controller.GetUserLogs) logRoute.GET("/self/search", middleware.UserAuth(), controller.SearchUserLogs) + dataRoute := apiRouter.Group("/data") + dataRoute.GET("/", middleware.AdminAuth(), controller.GetAllQuotaDates) + logRoute.Use(middleware.CORS()) { logRoute.GET("/token", controller.GetLogByKey) diff --git a/web/package.json b/web/package.json index 14fa905..f371120 100644 --- a/web/package.json +++ b/web/package.json @@ -3,7 +3,10 @@ "version": "0.1.0", "private": true, "dependencies": { - "@douyinfe/semi-ui": "^2.45.2", + "@douyinfe/semi-ui": "^2.46.1", + "@visactor/vchart": "~1.7.2", + "@visactor/react-vchart": "~1.7.2", + "@visactor/vchart-semi-theme": "~1.7.2", "axios": "^0.27.2", "history": "^5.3.0", "marked": "^4.1.1", @@ -44,7 +47,8 @@ ] }, "devDependencies": { - "prettier": "^2.7.1" + "prettier": "^2.7.1", + "typescript": "4.4.2" }, "prettier": { "singleQuote": true, diff --git a/web/src/App.js b/web/src/App.js index 1ffeba7..40e2175 100644 --- a/web/src/App.js +++ b/web/src/App.js @@ -23,10 +23,10 @@ import Log from './pages/Log'; import Chat from './pages/Chat'; import {Layout} from "@douyinfe/semi-ui"; import Midjourney from "./pages/Midjourney"; +import Detail from "./pages/Detail"; const Home = lazy(() => import('./pages/Home')); const About = lazy(() => import('./pages/About')); - function App() { const [userState, userDispatch] = useContext(UserContext); const [statusState, statusDispatch] = useContext(StatusContext); @@ -49,6 +49,8 @@ function App() { localStorage.setItem('footer_html', data.footer_html); localStorage.setItem('quota_per_unit', data.quota_per_unit); localStorage.setItem('display_in_currency', data.display_in_currency); + localStorage.setItem('enable_drawing', data.enable_drawing); + localStorage.setItem('enable_data_export', data.enable_data_export); if (data.chat_link) { localStorage.setItem('chat_link', data.chat_link); } else { @@ -228,6 +230,14 @@ function App() { } /> + + + + } + /> { + + let now = new Date(); + const [inputs, setInputs] = useState({ + username: '', + token_name: '', + model_name: '', + start_timestamp: timestamp2string(now.getTime() / 1000 - 86400), + end_timestamp: timestamp2string(now.getTime() / 1000 + 3600), + channel: '' + }); + const {username, token_name, model_name, start_timestamp, end_timestamp, channel} = inputs; + const isAdminUser = isAdmin(); + let modelDataChart = null; + let modelDataPieChart = null; + const [loading, setLoading] = useState(true); + const [quotaData, setQuotaData] = useState([]); + const [quotaDataPie, setQuotaDataPie] = useState([]); + const [quotaDataLine, setQuotaDataLine] = useState([]); + + const handleInputChange = (value, name) => { + setInputs((inputs) => ({...inputs, [name]: value})); + }; + + const spec_line = { + type: 'bar', + data: [ + { + id: 'barData', + values: [ + ] + } + ], + xField: 'Time', + yField: 'Usage', + seriesField: 'Model', + stack: true, + legends: { + visible: true + }, + title: { + visible: true, + text: '模型消耗分布' + }, + bar: { + // The state style of bar + state: { + hover: { + stroke: '#000', + lineWidth: 1 + } + } + } + }; + + const spec_pie = { + type: 'pie', + data: [ + { + id: 'id0', + values: [ + { type: 'null', value: '0' }, + ] + } + ], + outerRadius: 0.8, + innerRadius: 0.5, + padAngle: 0.6, + valueField: 'value', + categoryField: 'type', + pie: { + style: { + cornerRadius: 10 + }, + state: { + hover: { + outerRadius: 0.85, + stroke: '#000', + lineWidth: 1 + }, + selected: { + outerRadius: 0.85, + stroke: '#000', + lineWidth: 1 + } + } + }, + title: { + visible: true, + text: '模型调用次数占比' + }, + legends: { + visible: true, + orient: 'left' + }, + label: { + visible: true + }, + tooltip: { + mark: { + content: [ + { + key: datum => datum['type'], + value: datum => datum['value'] + } + ] + } + } + }; + + const loadQuotaData = async () => { + setLoading(true); + + let url = ''; + let localStartTimestamp = Date.parse(start_timestamp) / 1000; + let localEndTimestamp = Date.parse(end_timestamp) / 1000; + if (isAdminUser) { + url = `/api/data`; + } else { + url = `/api/data/self`; + } + const res = await API.get(url); + const {success, message, data} = res.data; + if (success) { + setQuotaData(data); + updateChart(data); + } else { + showError(message); + } + setLoading(false); + }; + + const refresh = async () => { + await loadQuotaData(); + }; + + const updateChart = (data) => { + if (isAdminUser) { + // 将所有用户的数据累加 + let pieData = []; + let lineData = []; + for (let i = 0; i < data.length; i++) { + const item = data[i]; + const {count, id, model_name, quota, user_id, username} = item; + // 合并model_name + let pieItem = pieData.find(item => item.model_name === model_name); + if (pieItem) { + pieItem.count += count; + } else { + pieData.push({ + "type": model_name, + "value": count + }); + } + // 合并created_at和model_name 为 lineData, created_at 数据类型是小时的时间戳 + // 转换日期格式 + let createTime = timestamp2string1(item.created_at); + let lineItem = lineData.find(item => item.Time === item.createTime && item.Model === model_name); + if (lineItem) { + lineItem.Usage += getQuotaWithUnit(quota); + } else { + lineData.push({ + "Time": createTime, + "Model": model_name, + "Usage": getQuotaWithUnit(quota) + }); + } + + } + // sort by count + pieData.sort((a, b) => b.value - a.value); + spec_line.data[0].values = lineData; + spec_pie.data[0].values = pieData; + // console.log('spec_line', spec_line); + console.log('spec_pie', spec_pie); + // modelDataChart.renderAsync(); + modelDataPieChart.updateSpec(spec_pie); + modelDataChart.updateSpec(spec_line); + } + } + + useEffect(() => { + refresh(); + }, []); + + useEffectOnce(() => { + // 创建 vchart 实例 + if (!modelDataChart) { + modelDataChart = new VChart(spec_line, {dom: 'model_data'}); + // 绘制 + modelDataChart.renderAsync(); + } + + if (!modelDataPieChart) { + modelDataPieChart = new VChart(spec_pie, {dom: 'model_pie'}); + // 绘制 + modelDataPieChart.renderAsync(); + } + + console.log('render vchart'); + }) + + return ( + <> + + +

数据看板(24H)

+ + +
+ <> + handleInputChange(value, 'start_timestamp')}/> + handleInputChange(value, 'end_timestamp')}/> + {/*查询*/} + {/*{*/} + {/* isAdminUser && <>*/} + {/* handleInputChange(value, 'username')}/>*/} + {/* */} + {/*}*/} + {/**/} + {/* */} + {/**/} + + +
+
+
+
+
+
+
+ + + ); +}; + + +export default Detail;