fix(node): stop a departed master's frozen traffic from disabling clients (#6113)

client_global_traffics rows are keyed by (master_guid, email) and are only
ever overwritten by a push from that same master. A master that stops
pushing — decommissioned, reinstalled under a fresh GUID, or detached from
the node — therefore leaves its last snapshot behind permanently.

depletedClientsCond's cross-panel EXISTS branch matched any such row, so a
node kept comparing a client's quota against counters frozen weeks earlier.
Once they exceeded the quota the node disabled the client on every traffic
poll, and the node -> master enable merge latched that off on the master too,
where nothing sets it back. The reported symptom is exactly this: a client at
11 GB of a 24 GB quota, enabled on two nodes, disabled on the third, which
still held a 27-day-old row from a previous master reporting 30 GB.

Bound both the enforcement predicate and the display overlay to rows a master
refreshed within globalTrafficFreshWindow. Masters push every 30s, so a live
master is never affected; a master that is merely unreachable for a while
keeps enforcing for a full day before its numbers are set aside.

The one-way enable merge that makes such a disable permanent on the master is
deliberate (12d84c2a, #4917) and is left alone.
This commit is contained in:
Sanaei
2026-07-27 14:21:56 +02:00
parent 5accd8a611
commit f8e9f2f087
4 changed files with 126 additions and 26 deletions
+37 -16
View File
@@ -49,46 +49,67 @@ func (s *InboundService) disableInvalidInbounds(tx *gorm.DB) (bool, int64, error
return needRestart, count, err
}
// globalTrafficFreshWindow bounds how long a pushed client_global_traffics row
// stays authoritative. Masters refresh their rows every nodeGlobalPushInterval
// (30s), so a row older than this belongs to a master that stopped pushing —
// decommissioned, reinstalled under a new GUID, or detached from this node.
// Such a row keeps its last-seen counters forever, and without this bound a
// long-dead master's numbers permanently trip the cross-panel quota check and
// disable clients that are nowhere near their limit (#6113). The window is far
// wider than any real push gap, so a master that is merely unreachable for a
// while keeps enforcing.
const globalTrafficFreshWindow = 24 * time.Hour
func globalTrafficFreshSince() int64 {
return time.Now().Add(-globalTrafficFreshWindow).UnixMilli()
}
// depletedClientsCond matches clients that exhausted their quota or expired.
// Besides the local counters it also trips on the cross-panel usage a master
// pushed into client_global_traffics — that's what lets a node cut a client
// whose combined usage exceeds the quota even though the local share doesn't
// (placeholders: now).
// whose combined usage exceeds the quota even though the local share doesn't.
// Only rows a master refreshed recently count (placeholders: now, freshSince).
const depletedClientsCond = `((total > 0 AND up + down >= total)
OR (expiry_time > 0 AND expiry_time <= ?)
OR (total > 0 AND EXISTS (
SELECT 1 FROM client_global_traffics g
WHERE g.email = client_traffics.email AND g.up + g.down >= client_traffics.total
WHERE g.email = client_traffics.email
AND g.updated_at >= ?
AND g.up + g.down >= client_traffics.total
)))`
// depletedClientsCondLocal is depletedClientsCond without the cross-panel
// client_global_traffics check. The EXISTS branch is a correlated subquery that
// turns every traffic poll into a full client_traffics scan; on a panel no
// master pushes to (the common case) client_global_traffics is empty, so the
// branch can never match and is pure CPU cost (#5392).
// branch can never match and is pure CPU cost (#5392). Placeholders: now.
const depletedClientsCondLocal = `((total > 0 AND up + down >= total)
OR (expiry_time > 0 AND expiry_time <= ?))`
// depletedCond returns the local-only predicate unless this panel actually
// holds global-traffic rows, in which case the cross-panel EXISTS check is
// needed to enforce combined quota. Both variants take the same single
// expiry_time placeholder, so callers pass identical args either way.
func depletedCond(tx *gorm.DB) string {
// depletedCond returns the predicate matching depleted clients together with
// the arguments it binds. The local-only variant is used unless this panel
// holds a global-traffic row a master still refreshes, in which case the
// cross-panel EXISTS check is needed to enforce combined quota.
func depletedCond(tx *gorm.DB) (string, []any) {
now := time.Now().UnixMilli()
freshSince := globalTrafficFreshSince()
var probe int64
if err := tx.Model(&model.ClientGlobalTraffic{}).Limit(1).Count(&probe).Error; err == nil && probe > 0 {
return depletedClientsCond
err := tx.Model(&model.ClientGlobalTraffic{}).
Where("updated_at >= ?", freshSince).
Limit(1).Count(&probe).Error
if err == nil && probe > 0 {
return depletedClientsCond, []any{now, freshSince}
}
return depletedClientsCondLocal
return depletedClientsCondLocal, []any{now}
}
func (s *InboundService) disableInvalidClients(tx *gorm.DB) (bool, int64, error) {
now := time.Now().Unix() * 1000
needRestart := false
cond := depletedCond(tx)
cond, condArgs := depletedCond(tx)
var depletedRows []xray.ClientTraffic
err := tx.Model(xray.ClientTraffic{}).
Where(cond+" AND enable = ?", now, true).
Where(cond+" AND enable = ?", append(condArgs, true)...).
Find(&depletedRows).Error
if err != nil {
return false, 0, err
@@ -185,7 +206,7 @@ func (s *InboundService) disableInvalidClients(tx *gorm.DB) (bool, int64, error)
if len(depletedEmails) > 0 {
if err := tx.Model(&model.ClientRecord{}).
Where("email IN ?", depletedEmails).
Updates(map[string]any{"enable": false, "updated_at": now}).Error; err != nil {
Updates(map[string]any{"enable": false, "updated_at": time.Now().UnixMilli()}).Error; err != nil {
logger.Warning("disableInvalidClients update clients.enable:", err)
}
}