perf(scale): speed up traffic, auto-renew, and node bulk ops at 50k-100k clients

Local hot paths:
- autoRenewClients: replace the O(clients x expired) inner scan with an
  email->traffic map lookup (quadratic at scale).
- node traffic sync: scope the client_traffics email-membership query to the
  snapshot's emails instead of plucking the whole table every poll.
- add a (expiry_time, reset) index for the per-tick auto-renew filter.
- SQLite: add cache_size/mmap_size/temp_store pragmas (env-tunable); keep the
  single-file DELETE journal and synchronous=FULL defaults.
- scale benchmarks now run on SQLite too via XUI_SCALE_TEST=1 (shared
  setupScaleDB/resetScaleTables helpers), not just Postgres.

Node paths:
- bulk add/delete/adjust on a node-attached inbound folded one HTTP RPC per
  client; above nodeBulkPushThreshold (32) mark the node dirty and let one
  ReconcileNode push converge it instead of O(M) sequential round-trips.
  Small ops keep the live per-client path. Also hoist nodePushPlan out of the
  per-email delete loop.
- ReconcileNode skips inbounds whose wire payload is unchanged (per-tag
  fingerprint on Remote), guarded by node-side tag presence so a restarted
  node is still re-seeded.

Tests: auto-renew multi-inbound correctness, node-path dispatch (large ops
fold to dirty, small ops push live) via a manager runtime override seam, and
reconcile delta-skip.
This commit is contained in:
MHSanaei
2026-06-20 10:35:46 +02:00
parent e079490144
commit 6a032bcb2a
15 changed files with 623 additions and 137 deletions
+34 -8
View File
@@ -6,6 +6,7 @@ import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"log"
"math"
@@ -886,7 +887,10 @@ func InitDB(dbPath string) error {
if err = os.MkdirAll(dir, 0755); err != nil {
return err
}
dsn := dbPath + "?_journal_mode=DELETE&_busy_timeout=10000&_synchronous=FULL&_txlock=immediate"
// Keep journal_mode=DELETE so the DB stays a single file (no -wal/-shm
// sidecars). synchronous defaults to FULL for durability but is tunable.
sync := sqliteSynchronous()
dsn := dbPath + "?_journal_mode=DELETE&_busy_timeout=10000&_synchronous=" + sync + "&_txlock=immediate"
db, err = gorm.Open(sqlite.Open(dsn), c)
if err != nil {
return err
@@ -895,14 +899,21 @@ func InitDB(dbPath string) error {
if err != nil {
return err
}
if _, err := sqlDB.Exec("PRAGMA journal_mode=DELETE"); err != nil {
return err
// Re-assert the DSN pragmas plus scan-friendly ones for large datasets.
// cache_size/mmap_size/temp_store create no extra files, so the single-file
// guarantee holds; they just cut disk I/O on the 50k-row hot paths.
pragmas := []string{
"PRAGMA journal_mode=DELETE",
"PRAGMA busy_timeout=10000",
"PRAGMA synchronous=" + sync,
fmt.Sprintf("PRAGMA cache_size=-%d", envInt("XUI_DB_CACHE_MB", 32)*1024),
fmt.Sprintf("PRAGMA mmap_size=%d", int64(envInt("XUI_DB_MMAP_MB", 256))*1024*1024),
"PRAGMA temp_store=MEMORY",
}
if _, err := sqlDB.Exec("PRAGMA busy_timeout=10000"); err != nil {
return err
}
if _, err := sqlDB.Exec("PRAGMA synchronous=FULL"); err != nil {
return err
for _, p := range pragmas {
if _, err := sqlDB.Exec(p); err != nil {
return err
}
}
}
@@ -939,6 +950,21 @@ func InitDB(dbPath string) error {
return runSeeders(isUsersEmpty)
}
// sqliteSynchronous returns the SQLite synchronous mode, defaulting to FULL.
// Whitelisted because the value is interpolated directly into a PRAGMA string.
func sqliteSynchronous() string {
switch strings.ToUpper(strings.TrimSpace(os.Getenv("XUI_DB_SYNCHRONOUS"))) {
case "OFF":
return "OFF"
case "NORMAL":
return "NORMAL"
case "EXTRA":
return "EXTRA"
default:
return "FULL"
}
}
func envInt(key string, def int) int {
v := strings.TrimSpace(os.Getenv(key))
if v == "" {
+1
View File
@@ -31,6 +31,7 @@ func TestAutoMigrateCreatesHotPathIndexes(t *testing.T) {
}{
{&model.ClientRecord{}, "idx_client_record_group"},
{&xray.ClientTraffic{}, "idx_client_traffics_inbound"},
{&xray.ClientTraffic{}, "idx_client_traffics_renew"},
}
for _, c := range cases {
if !db.Migrator().HasIndex(c.model, c.index) {