mirror of
https://github.com/MHSanaei/3x-ui.git
synced 2026-08-31 15:37:14 +00:00
perf(clients): make SyncInbound bulk to fix large-inbound timeouts (#4885)
Every client mutation funnels through SyncInbound, which ran O(n) DB round-trips per call: one SELECT per client, a Save+UpdateColumn per client, and a per-row junction INSERT. Toggling a single client on a large inbound issued thousands of queries and timed out, badly so on PostgreSQL where each round-trip pays TCP latency. SyncInbound now: - loads existing records with a single chunked SELECT ... email IN (...) instead of one query per client - writes only the records that actually changed (skips no-op Saves), so toggling/editing one client writes one row, not all of them - batch-creates new records and batch-inserts the junction rows Merge and sticky-field semantics are unchanged. Measured on PostgreSQL 16: a single-client toggle on a 50k-client inbound drops from ~8m54s to ~0.9s, and seeding 50k clients from ~2m48s to ~1.6s; 200k clients sync in seconds. A skip-gated benchmark (web/service/sync_scale_postgres_test.go, run with XUI_DB_TYPE=postgres) reproduces and verifies the scaling.
This commit is contained in:
MHSanaei
+119
-58
@@ -196,73 +196,134 @@ func (s *ClientService) SyncInbound(tx *gorm.DB, inboundId int, clients []model.
|
||||
return err
|
||||
}
|
||||
|
||||
emails := make([]string, 0, len(clients))
|
||||
seen := make(map[string]struct{}, len(clients))
|
||||
for i := range clients {
|
||||
c := clients[i]
|
||||
email := strings.TrimSpace(c.Email)
|
||||
email := strings.TrimSpace(clients[i].Email)
|
||||
if email == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[email]; ok {
|
||||
continue
|
||||
}
|
||||
seen[email] = struct{}{}
|
||||
emails = append(emails, email)
|
||||
}
|
||||
|
||||
existing := make(map[string]*model.ClientRecord, len(emails))
|
||||
const selectChunk = 400
|
||||
for start := 0; start < len(emails); start += selectChunk {
|
||||
end := min(start+selectChunk, len(emails))
|
||||
var rows []model.ClientRecord
|
||||
if err := tx.Where("email IN ?", emails[start:end]).Find(&rows).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range rows {
|
||||
r := rows[i]
|
||||
existing[r.Email] = &r
|
||||
}
|
||||
}
|
||||
|
||||
idByEmail := make(map[string]int, len(emails))
|
||||
pending := make(map[string]*model.ClientRecord, len(emails))
|
||||
toCreate := make([]*model.ClientRecord, 0, len(emails))
|
||||
for i := range clients {
|
||||
email := strings.TrimSpace(clients[i].Email)
|
||||
if email == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
incoming := c.ToRecord()
|
||||
row := &model.ClientRecord{}
|
||||
err := tx.Where("email = ?", email).First(row).Error
|
||||
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return err
|
||||
}
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
if err := tx.Create(incoming).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
row = incoming
|
||||
} else {
|
||||
if incoming.UUID != "" {
|
||||
row.UUID = incoming.UUID
|
||||
}
|
||||
if incoming.Password != "" {
|
||||
row.Password = incoming.Password
|
||||
}
|
||||
if incoming.Auth != "" {
|
||||
row.Auth = incoming.Auth
|
||||
}
|
||||
row.Flow = incoming.Flow
|
||||
if incoming.Security != "" {
|
||||
row.Security = incoming.Security
|
||||
}
|
||||
if incoming.Reverse != "" {
|
||||
row.Reverse = incoming.Reverse
|
||||
}
|
||||
row.SubID = incoming.SubID
|
||||
row.LimitIP = incoming.LimitIP
|
||||
row.TotalGB = incoming.TotalGB
|
||||
row.ExpiryTime = incoming.ExpiryTime
|
||||
row.Enable = incoming.Enable
|
||||
row.TgID = incoming.TgID
|
||||
if incoming.Group != "" {
|
||||
row.Group = incoming.Group
|
||||
}
|
||||
row.Comment = incoming.Comment
|
||||
row.Reset = incoming.Reset
|
||||
if incoming.CreatedAt > 0 && (row.CreatedAt == 0 || incoming.CreatedAt < row.CreatedAt) {
|
||||
row.CreatedAt = incoming.CreatedAt
|
||||
}
|
||||
preservedUpdatedAt := max(incoming.UpdatedAt, row.UpdatedAt)
|
||||
row.UpdatedAt = preservedUpdatedAt
|
||||
if err := tx.Save(row).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Model(&model.ClientRecord{}).
|
||||
Where("id = ?", row.Id).
|
||||
UpdateColumn("updated_at", preservedUpdatedAt).Error; err != nil {
|
||||
return err
|
||||
incoming := clients[i].ToRecord()
|
||||
row, ok := existing[email]
|
||||
if !ok {
|
||||
if _, dup := pending[email]; !dup {
|
||||
pending[email] = incoming
|
||||
toCreate = append(toCreate, incoming)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
link := model.ClientInbound{
|
||||
ClientId: row.Id,
|
||||
InboundId: inboundId,
|
||||
FlowOverride: c.Flow,
|
||||
before := *row
|
||||
if incoming.UUID != "" {
|
||||
row.UUID = incoming.UUID
|
||||
}
|
||||
if err := tx.Create(&link).Error; err != nil {
|
||||
if incoming.Password != "" {
|
||||
row.Password = incoming.Password
|
||||
}
|
||||
if incoming.Auth != "" {
|
||||
row.Auth = incoming.Auth
|
||||
}
|
||||
row.Flow = incoming.Flow
|
||||
if incoming.Security != "" {
|
||||
row.Security = incoming.Security
|
||||
}
|
||||
if incoming.Reverse != "" {
|
||||
row.Reverse = incoming.Reverse
|
||||
}
|
||||
row.SubID = incoming.SubID
|
||||
row.LimitIP = incoming.LimitIP
|
||||
row.TotalGB = incoming.TotalGB
|
||||
row.ExpiryTime = incoming.ExpiryTime
|
||||
row.Enable = incoming.Enable
|
||||
row.TgID = incoming.TgID
|
||||
if incoming.Group != "" {
|
||||
row.Group = incoming.Group
|
||||
}
|
||||
row.Comment = incoming.Comment
|
||||
row.Reset = incoming.Reset
|
||||
if incoming.CreatedAt > 0 && (row.CreatedAt == 0 || incoming.CreatedAt < row.CreatedAt) {
|
||||
row.CreatedAt = incoming.CreatedAt
|
||||
}
|
||||
preservedUpdatedAt := max(incoming.UpdatedAt, row.UpdatedAt)
|
||||
row.UpdatedAt = preservedUpdatedAt
|
||||
|
||||
idByEmail[email] = row.Id
|
||||
|
||||
if *row == before {
|
||||
continue
|
||||
}
|
||||
if err := tx.Save(row).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Model(&model.ClientRecord{}).
|
||||
Where("id = ?", row.Id).
|
||||
UpdateColumn("updated_at", preservedUpdatedAt).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if len(toCreate) > 0 {
|
||||
if err := tx.CreateInBatches(toCreate, 200).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for _, rec := range toCreate {
|
||||
idByEmail[rec.Email] = rec.Id
|
||||
}
|
||||
}
|
||||
|
||||
links := make([]model.ClientInbound, 0, len(clients))
|
||||
linked := make(map[int]struct{}, len(clients))
|
||||
for i := range clients {
|
||||
email := strings.TrimSpace(clients[i].Email)
|
||||
if email == "" {
|
||||
continue
|
||||
}
|
||||
id, ok := idByEmail[email]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if _, dup := linked[id]; dup {
|
||||
continue
|
||||
}
|
||||
linked[id] = struct{}{}
|
||||
links = append(links, model.ClientInbound{
|
||||
ClientId: id,
|
||||
InboundId: inboundId,
|
||||
FlowOverride: clients[i].Flow,
|
||||
})
|
||||
}
|
||||
if len(links) > 0 {
|
||||
if err := tx.CreateInBatches(links, 200).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user