mirror of
https://github.com/MHSanaei/3x-ui.git
synced 2026-07-12 23:56:07 +00:00
perf(clients): scale add/delete and bulk client operations
Follow-up to the SyncInbound bulk rewrite, fixing the remaining O(M*N) and O(M)-round-trip behaviour in the add/delete and bulk paths that made them time out on large inbounds (worst case minutes), especially on PostgreSQL. - compactOrphans: chunk the "email IN (...)" lookup (400/batch) instead of binding every email at once. A single huge IN exceeded PostgreSQL's 65535-parameter limit (and SQLite's) and made the planner pathological, so add/delete failed outright past ~100k clients. - emailsUsedByOtherInbounds: new batched form used by delInboundClients (BulkDetach) and bulkDelInboundClients (BulkDelete), replacing a per-email global JSON scan (O(M*N)) with one scan, and skipped entirely when keepTraffic is set. - BulkCreate: rewritten to validate/dedup in one pass, then group clients by inbound and add them in a single addInboundClient call per inbound (one getAllEmailSubIDs, one settings rewrite, one SyncInbound) instead of running the full single-create pipeline per client. - Bulk delete/adjust: batch DelClientStat/DelClientIPs with IN deletes and wrap the settings Save + SyncInbound in one transaction, so the per-row writes share a single fsync instead of one per row. Measured on PostgreSQL 16 (one inbound, M=2000 affected clients): - create: 8m35s (M=500) -> ~1-5s - detach: 52s -> ~4s (flat in N) - delete: ~16s -> ~1-4s - adjust: ~20s -> ~7-10s add/delete of a single client on a 200k-client inbound stays in seconds. sync_scale_postgres_test.go adds skip-gated benchmarks (XUI_DB_TYPE= postgres) for the single add/delete and the five bulk operations.
This commit is contained in:
@@ -222,13 +222,100 @@ func TestAddDelClientPostgresScale(t *testing.T) {
|
||||
}
|
||||
delDur := time.Since(start)
|
||||
|
||||
var recCount int64
|
||||
var recCount, linkCount int64
|
||||
db.Model(&model.ClientRecord{}).Count(&recCount)
|
||||
if int(recCount) != n {
|
||||
t.Fatalf("record count after add+del = %d, want %d", recCount, n)
|
||||
}
|
||||
db.Model(&model.ClientInbound{}).Where("inbound_id = ?", ib.Id).Count(&linkCount)
|
||||
|
||||
t.Logf("N=%-7d add=%-10v del=%-10v", n, addDur.Round(time.Millisecond), delDur.Round(time.Millisecond))
|
||||
t.Logf("N=%-7d add=%-10v del=%-10v records=%d links=%d", n,
|
||||
addDur.Round(time.Millisecond), delDur.Round(time.Millisecond), recCount, linkCount)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBulkOpsPostgresScale(t *testing.T) {
|
||||
if strings.TrimSpace(os.Getenv("XUI_DB_DSN")) == "" || os.Getenv("XUI_DB_TYPE") != "postgres" {
|
||||
t.Skip("set XUI_DB_TYPE=postgres and XUI_DB_DSN to run the postgres scale benchmark")
|
||||
}
|
||||
if err := database.InitDB(""); err != nil {
|
||||
t.Fatalf("InitDB: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = database.CloseDB() })
|
||||
|
||||
svc := &ClientService{}
|
||||
inboundSvc := &InboundService{}
|
||||
sizes := []int{5000, 20000, 50000, 100000}
|
||||
const m = 2000
|
||||
|
||||
for _, n := range sizes {
|
||||
t.Run(fmt.Sprintf("N=%d", n), func(t *testing.T) {
|
||||
db := database.GetDB()
|
||||
if err := db.Exec("TRUNCATE TABLE inbounds, clients, client_inbounds, client_traffics RESTART IDENTITY CASCADE").Error; err != nil {
|
||||
t.Fatalf("truncate: %v", err)
|
||||
}
|
||||
|
||||
clients := makeScaleClients(n)
|
||||
exp := time.Now().AddDate(1, 0, 0).UnixMilli()
|
||||
for i := range clients {
|
||||
clients[i].ExpiryTime = exp
|
||||
clients[i].TotalGB = 100 << 30
|
||||
}
|
||||
ib := &model.Inbound{Tag: fmt.Sprintf("bulk-%d", n), Enable: true, Port: 40000, Protocol: model.VLESS, Settings: clientsSettings(t, clients)}
|
||||
if err := db.Create(ib).Error; err != nil {
|
||||
t.Fatalf("create inbound: %v", err)
|
||||
}
|
||||
if err := svc.SyncInbound(nil, ib.Id, clients); err != nil {
|
||||
t.Fatalf("seed SyncInbound: %v", err)
|
||||
}
|
||||
ib2 := &model.Inbound{Tag: fmt.Sprintf("bulk2-%d", n), Enable: true, Port: 40001, Protocol: model.VLESS, Settings: `{"clients":[]}`}
|
||||
if err := db.Create(ib2).Error; err != nil {
|
||||
t.Fatalf("create inbound2: %v", err)
|
||||
}
|
||||
|
||||
emailsM := make([]string, m)
|
||||
for i := 0; i < m; i++ {
|
||||
emailsM[i] = clients[i].Email
|
||||
}
|
||||
|
||||
t0 := time.Now()
|
||||
if _, _, err := svc.BulkAdjust(inboundSvc, emailsM, 7, 1<<30); err != nil {
|
||||
t.Fatalf("BulkAdjust: %v", err)
|
||||
}
|
||||
adjustDur := time.Since(t0)
|
||||
|
||||
t0 = time.Now()
|
||||
if _, _, err := svc.BulkAttach(inboundSvc, emailsM, []int{ib2.Id}); err != nil {
|
||||
t.Fatalf("BulkAttach: %v", err)
|
||||
}
|
||||
attachDur := time.Since(t0)
|
||||
|
||||
t0 = time.Now()
|
||||
if _, _, err := svc.BulkDetach(inboundSvc, emailsM, []int{ib2.Id}); err != nil {
|
||||
t.Fatalf("BulkDetach: %v", err)
|
||||
}
|
||||
detachDur := time.Since(t0)
|
||||
|
||||
payloads := make([]ClientCreatePayload, m)
|
||||
for i := 0; i < m; i++ {
|
||||
payloads[i] = ClientCreatePayload{
|
||||
Client: model.Client{ID: uuid.NewString(), Email: fmt.Sprintf("bulknew-%07d@scale", i), SubID: fmt.Sprintf("bnsub-%07d", i), Enable: true},
|
||||
InboundIds: []int{ib.Id},
|
||||
}
|
||||
}
|
||||
t0 = time.Now()
|
||||
if _, _, err := svc.BulkCreate(inboundSvc, payloads); err != nil {
|
||||
t.Fatalf("BulkCreate: %v", err)
|
||||
}
|
||||
createDur := time.Since(t0)
|
||||
|
||||
t0 = time.Now()
|
||||
if _, _, err := svc.BulkDelete(inboundSvc, emailsM, false); err != nil {
|
||||
t.Fatalf("BulkDelete: %v", err)
|
||||
}
|
||||
deleteDur := time.Since(t0)
|
||||
|
||||
t.Logf("N=%-6d M=%d adjust=%-9v attach=%-9v detach=%-9v create=%-9v delete=%-9v", n, m,
|
||||
adjustDur.Round(time.Millisecond), attachDur.Round(time.Millisecond), detachDur.Round(time.Millisecond),
|
||||
createDur.Round(time.Millisecond), deleteDur.Round(time.Millisecond))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user