mirror of
https://github.com/MHSanaei/3x-ui.git
synced 2026-07-15 17:16:07 +00:00
perf(scale): speed up traffic, auto-renew, and node bulk ops at 50k-100k clients
Local hot paths: - autoRenewClients: replace the O(clients x expired) inner scan with an email->traffic map lookup (quadratic at scale). - node traffic sync: scope the client_traffics email-membership query to the snapshot's emails instead of plucking the whole table every poll. - add a (expiry_time, reset) index for the per-tick auto-renew filter. - SQLite: add cache_size/mmap_size/temp_store pragmas (env-tunable); keep the single-file DELETE journal and synchronous=FULL defaults. - scale benchmarks now run on SQLite too via XUI_SCALE_TEST=1 (shared setupScaleDB/resetScaleTables helpers), not just Postgres. Node paths: - bulk add/delete/adjust on a node-attached inbound folded one HTTP RPC per client; above nodeBulkPushThreshold (32) mark the node dirty and let one ReconcileNode push converge it instead of O(M) sequential round-trips. Small ops keep the live per-client path. Also hoist nodePushPlan out of the per-email delete loop. - ReconcileNode skips inbounds whose wire payload is unchanged (per-tag fingerprint on Remote), guarded by node-side tag presence so a restarted node is still re-seeded. Tests: auto-renew multi-inbound correctness, node-path dispatch (large ops fold to dirty, small ops push live) via a manager runtime override seam, and reconcile delta-skip.
This commit is contained in:
@@ -21,6 +21,12 @@ import (
|
||||
|
||||
var reportedRemoteTagConflict sync.Map
|
||||
|
||||
// nodeBulkPushThreshold caps how many per-client RPCs a single operation will
|
||||
// stream to a remote node. Above it, the panel marks the node dirty instead and
|
||||
// lets one ReconcileNode push converge the whole inbound — far cheaper than M
|
||||
// sequential round-trips. Small ops stay on the live per-client path.
|
||||
const nodeBulkPushThreshold = 32
|
||||
|
||||
func (s *InboundService) runtimeFor(ib *model.Inbound) (runtime.Runtime, error) {
|
||||
mgr := runtime.GetManager()
|
||||
if mgr == nil {
|
||||
@@ -90,18 +96,31 @@ func (s *InboundService) ReconcileNode(ctx context.Context, rt *runtime.Remote,
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
remoteTagSet := make(map[string]struct{}, len(remoteTags))
|
||||
for _, tag := range remoteTags {
|
||||
remoteTagSet[tag] = struct{}{}
|
||||
}
|
||||
prefix := nodeTagPrefix(&nodeID)
|
||||
desiredTags := make(map[string]struct{}, len(inbounds)*2)
|
||||
for _, ib := range inbounds {
|
||||
desiredTags[ib.Tag] = struct{}{}
|
||||
// existsOnNode: does the node already report this inbound under any of the
|
||||
// tag forms it may be stored as? If so, an unchanged push can be skipped.
|
||||
_, existsOnNode := remoteTagSet[ib.Tag]
|
||||
if prefix != "" {
|
||||
if stripped, found := strings.CutPrefix(ib.Tag, prefix); found {
|
||||
desiredTags[stripped] = struct{}{}
|
||||
if _, ok := remoteTagSet[stripped]; ok {
|
||||
existsOnNode = true
|
||||
}
|
||||
} else {
|
||||
desiredTags[prefix+ib.Tag] = struct{}{}
|
||||
if _, ok := remoteTagSet[prefix+ib.Tag]; ok {
|
||||
existsOnNode = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := rt.UpdateInbound(ctx, ib, ib); err != nil {
|
||||
if _, err := rt.ReconcileInbound(ctx, ib, existsOnNode); err != nil {
|
||||
return fmt.Errorf("reconcile inbound %q: %w", ib.Tag, err)
|
||||
}
|
||||
}
|
||||
@@ -260,15 +279,6 @@ func (s *InboundService) setRemoteTrafficLocked(nodeID int, snap *runtime.Traffi
|
||||
nodeBaselines[baselineRows[i].Email] = nodeTrafficCounter{Up: baselineRows[i].Up, Down: baselineRows[i].Down}
|
||||
}
|
||||
|
||||
var existingEmailsList []string
|
||||
if err := db.Model(xray.ClientTraffic{}).Pluck("email", &existingEmailsList).Error; err != nil {
|
||||
return false, err
|
||||
}
|
||||
existingEmails := make(map[string]struct{}, len(existingEmailsList))
|
||||
for _, e := range existingEmailsList {
|
||||
existingEmails[e] = struct{}{}
|
||||
}
|
||||
|
||||
var defaultUserId int
|
||||
if len(central) > 0 {
|
||||
defaultUserId = central[0].UserId
|
||||
@@ -312,6 +322,26 @@ func (s *InboundService) setRemoteTrafficLocked(nodeID int, snap *runtime.Traffi
|
||||
}
|
||||
}
|
||||
|
||||
// Membership set for the rowExists checks below. Only the snapshot's emails
|
||||
// are ever probed, so scope the lookup to those instead of plucking the whole
|
||||
// client_traffics table (50k+ rows) on every node poll.
|
||||
existingEmails := make(map[string]struct{}, len(snapEmailsAll))
|
||||
if len(snapEmailsAll) > 0 {
|
||||
snapEmailList := make([]string, 0, len(snapEmailsAll))
|
||||
for email := range snapEmailsAll {
|
||||
snapEmailList = append(snapEmailList, email)
|
||||
}
|
||||
for _, batch := range chunkStrings(snapEmailList, sqliteMaxVars) {
|
||||
var found []string
|
||||
if err := db.Model(xray.ClientTraffic{}).Where("email IN ?", batch).Pluck("email", &found).Error; err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, e := range found {
|
||||
existingEmails[e] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tx := db.Begin()
|
||||
committed := false
|
||||
defer func() {
|
||||
|
||||
Reference in New Issue
Block a user