fix(traffic): apply maintenance side effects only after the commit lands (#6200)

* fix(traffic): apply maintenance after durable commit

* fix(traffic): apply all runtime maintenance after commit

---------

Co-authored-by: n0ctal <293235942+n0ctal@users.noreply.github.com>
This commit is contained in:
n0ctal
2026-08-14 22:42:09 +05:00
committed by GitHub
parent b70c5abce8
commit 1396005082
11 changed files with 543 additions and 329 deletions
+273 -224
View File
@@ -22,60 +22,77 @@ import (
)
func (s *InboundService) AddTraffic(inboundTraffics []*xray.Traffic, clientTraffics []*xray.ClientTraffic) (needRestart bool, clientsDisabled bool, err error) {
var disabledNodeIDs []int
err = submitTrafficWrite(func() error {
var inner error
needRestart, clientsDisabled, inner = s.addTrafficLocked(inboundTraffics, clientTraffics)
needRestart, clientsDisabled, disabledNodeIDs, inner = s.addTrafficLocked(inboundTraffics, clientTraffics)
return inner
})
if err == nil && len(disabledNodeIDs) > 0 {
s.restartRemoteNodesOnDisable(disabledNodeIDs)
}
return
}
func (s *InboundService) addTrafficLocked(inboundTraffics []*xray.Traffic, clientTraffics []*xray.ClientTraffic) (bool, bool, error) {
var err error
func (s *InboundService) addTrafficLocked(inboundTraffics []*xray.Traffic, clientTraffics []*xray.ClientTraffic) (bool, bool, []int, error) {
db := database.GetDB()
tx := db.Begin()
defer func() {
if err != nil {
if rbErr := tx.Rollback().Error; rbErr != nil {
logger.Warning("Error rolling back traffic tx:", rbErr)
}
} else if cErr := tx.Commit().Error; cErr != nil {
logger.Warning("Error committing traffic tx:", cErr)
// Commit durable traffic before best-effort lifecycle maintenance so helper
// failures cannot discard usage already reported by Xray.
if err := db.Transaction(func(tx *gorm.DB) error {
if err := s.addInboundTraffic(tx, inboundTraffics); err != nil {
return err
}
}()
err = s.addInboundTraffic(tx, inboundTraffics)
return s.addClientTraffic(tx, clientTraffics)
}); err != nil {
return false, false, nil, err
}
var (
needRestart bool
clientsDisabled bool
disabledNodeIDs []int
disabledClientsCount int64
)
batch := newTrafficMutationBatch()
err := db.Transaction(func(tx *gorm.DB) error {
needRestart0, count, err := s.autoRenewClients(tx, batch)
if err != nil {
return fmt.Errorf("renew clients: %w", err)
}
if count > 0 {
logger.Debugf("%v clients renewed", count)
}
needRestart1, count, nodeIDs, err := s.disableInvalidClients(tx, batch)
if err != nil {
return fmt.Errorf("disable invalid clients: %w", err)
}
if count > 0 {
logger.Debugf("%v clients disabled", count)
disabledClientsCount = count
}
needRestart2, count, err := s.disableInvalidInbounds(tx, batch)
if err != nil {
return fmt.Errorf("disable invalid inbounds: %w", err)
}
if count > 0 {
logger.Debugf("%v inbounds disabled", count)
}
if err := batch.markNodesTx(tx); err != nil {
return err
}
needRestart = needRestart0 || needRestart1 || needRestart2
clientsDisabled = disabledClientsCount > 0
disabledNodeIDs = nodeIDs
return nil
})
if err != nil {
return false, false, err
logger.Warning("traffic lifecycle maintenance failed after traffic commit:", err)
return false, false, nil, nil
}
err = s.addClientTraffic(tx, clientTraffics)
if err != nil {
return false, false, err
}
needRestart0, count, renewErr := s.autoRenewClients(tx)
if renewErr != nil {
logger.Warning("Error in renew clients:", renewErr)
} else if count > 0 {
logger.Debugf("%v clients renewed", count)
}
disabledClientsCount := int64(0)
needRestart1, count, disableClientsErr := s.disableInvalidClients(tx)
if disableClientsErr != nil {
logger.Warning("Error in disabling invalid clients:", disableClientsErr)
} else if count > 0 {
logger.Debugf("%v clients disabled", count)
disabledClientsCount = count
}
needRestart2, count, disableInboundsErr := s.disableInvalidInbounds(tx)
if disableInboundsErr != nil {
logger.Warning("Error in disabling invalid inbounds:", disableInboundsErr)
} else if count > 0 {
logger.Debugf("%v inbounds disabled", count)
}
return needRestart0 || needRestart1 || needRestart2, disabledClientsCount > 0, nil
needRestart = needRestart || s.applyTrafficMutationBatch(batch)
return needRestart, clientsDisabled, disabledNodeIDs, nil
}
func (s *InboundService) addInboundTraffic(tx *gorm.DB, traffics []*xray.Traffic) error {
@@ -304,11 +321,11 @@ func apiUserFromClient(client map[string]any, cipher string) map[string]any {
return user
}
func (s *InboundService) autoRenewClients(tx *gorm.DB) (bool, int64, error) {
func (s *InboundService) autoRenewClients(tx *gorm.DB, mutationBatch *trafficMutationBatch) (bool, int64, error) {
// check for time expired
var traffics []*xray.ClientTraffic
now := time.Now().Unix() * 1000
var err, err1 error
var err error
// Filter to clients that have at least one local inbound. Using
// client_traffics.inbound_id is wrong: it goes stale after an inbound is
@@ -335,9 +352,8 @@ func (s *InboundService) autoRenewClients(tx *gorm.DB) (bool, int64, error) {
var inbounds []*model.Inbound
needRestart := false
var clientsToAdd []struct {
protocol string
tag string
client map[string]any
inbound model.Inbound
client map[string]any
}
// Resolve the inbounds to renew through the client_inbounds link rather than
@@ -408,13 +424,11 @@ func (s *InboundService) autoRenewClients(tx *gorm.DB) (bool, int64, error) {
c["enable"] = true
clientsToAdd = append(clientsToAdd,
struct {
protocol string
tag string
client map[string]any
inbound model.Inbound
client map[string]any
}{
protocol: string(inbounds[inbound_index].Protocol),
tag: inbounds[inbound_index].Tag,
client: apiUserFromClient(c, cipher),
inbound: *inbounds[inbound_index],
client: apiUserFromClient(c, cipher),
})
}
clients[client_index] = any(c)
@@ -452,18 +466,14 @@ func (s *InboundService) autoRenewClients(tx *gorm.DB) (bool, int64, error) {
if err = clearGlobalTraffic(tx, renewEmails...); err != nil {
return false, 0, err
}
if process := currentXrayProcess(); process != nil {
err1 = s.xrayApi.Init(process.GetAPIPort())
if err1 != nil {
return true, int64(len(traffics)), nil
for _, clientToAdd := range clientsToAdd {
if clientToAdd.inbound.NodeID != nil {
mutationBatch.addNode(*clientToAdd.inbound.NodeID)
continue
}
for _, clientToAdd := range clientsToAdd {
err1 = s.xrayApi.AddUser(clientToAdd.protocol, clientToAdd.tag, clientToAdd.client)
if err1 != nil {
needRestart = true
}
}
s.xrayApi.Close()
mutationBatch.localPlans = append(mutationBatch.localPlans, trafficLocalApplyPlan{
action: trafficAddUser, inbound: clientToAdd.inbound, client: clientToAdd.client,
})
}
return needRestart, int64(len(traffics)), nil
}
@@ -577,56 +587,58 @@ func (s *InboundService) ResetClientTrafficByEmail(clientEmail string) error {
}
func (s *InboundService) ResetClientTraffic(id int, clientEmail string) (needRestart bool, err error) {
var resetInbound *model.Inbound
err = submitTrafficWrite(func() error {
var inner error
needRestart, inner = s.resetClientTrafficLocked(id, clientEmail)
needRestart, resetInbound, inner = s.resetClientTrafficLocked(id, clientEmail)
return inner
})
if err == nil {
s.resetMtprotoClientQuota(clientEmail)
if resetInbound != nil && resetInbound.NodeID != nil {
if rt, rterr := s.runtimeFor(resetInbound); rterr == nil {
if e := rt.ResetClientTraffic(context.Background(), resetInbound, clientEmail); e != nil {
logger.Warning("ResetClientTraffic: remote propagation to", rt.Name(), "failed:", e)
}
} else {
logger.Warning("ResetClientTraffic: runtime lookup failed:", rterr)
}
}
}
return
}
func (s *InboundService) resetClientTrafficLocked(id int, clientEmail string) (bool, error) {
func (s *InboundService) resetClientTrafficLocked(id int, clientEmail string) (bool, *model.Inbound, error) {
needRestart := false
var reenablePlan *trafficLocalApplyPlan
var reenableNodeID *int
traffic, err := s.GetClientTrafficByEmail(clientEmail)
if err != nil {
return false, err
return false, nil, err
}
if !traffic.Enable {
inbound, err := s.GetInbound(id)
if err != nil {
return false, err
return false, nil, err
}
clients, err := s.GetClients(inbound)
if err != nil {
return false, err
return false, nil, err
}
for _, client := range clients {
if client.Email == clientEmail && client.Enable {
rt, push, _, perr := s.nodePushPlan(inbound)
if perr != nil {
return false, perr
}
if !push {
if inbound.NodeID == nil {
needRestart = true
}
break
}
cipher := ""
if string(inbound.Protocol) == "shadowsocks" {
var oldSettings map[string]any
err = json.Unmarshal([]byte(inbound.Settings), &oldSettings)
if err != nil {
return false, err
return false, nil, err
}
cipher, _ = oldSettings["method"].(string)
}
err1 := rt.AddUser(context.Background(), inbound, map[string]any{
clientMap := map[string]any{
"email": client.Email,
"id": client.ID,
"auth": client.Auth,
@@ -634,14 +646,11 @@ func (s *InboundService) resetClientTrafficLocked(id int, clientEmail string) (b
"flow": client.Flow,
"password": client.Password,
"cipher": cipher,
})
if err1 == nil {
logger.Debug("Client enabled on", rt.Name(), "due to reset traffic:", clientEmail)
} else if inbound.NodeID != nil {
logger.Warning("Error in enabling client on", rt.Name(), ":", err1)
}
if inbound.NodeID != nil {
reenableNodeID = inbound.NodeID
} else {
logger.Debug("Error in enabling client on", rt.Name(), ":", err1)
needRestart = true
reenablePlan = &trafficLocalApplyPlan{action: trafficAddUser, inbound: *inbound, client: clientMap}
}
break
}
@@ -656,7 +665,7 @@ func (s *InboundService) resetClientTrafficLocked(id int, clientEmail string) (b
now := time.Now().UnixMilli()
inbound, err := s.GetInbound(id)
if err != nil {
return false, err
return false, nil, err
}
if err := db.Transaction(func(tx *gorm.DB) error {
if err := adjustGroupBaselinesForRemovedTraffic(tx, []string{clientEmail}); err != nil {
@@ -676,25 +685,30 @@ func (s *InboundService) resetClientTrafficLocked(id int, clientEmail string) (b
Update("last_traffic_reset_time", now).Error; err != nil {
return err
}
if reenableNodeID != nil {
return (&NodeService{}).MarkNodeDirtyTx(tx, *reenableNodeID)
}
if inbound != nil && inbound.NodeID != nil {
return (&NodeService{}).MarkNodeDirtyTx(tx, *inbound.NodeID)
}
return nil
}); err != nil {
return false, err
return false, nil, err
}
if inbound != nil && inbound.NodeID != nil {
if rt, rterr := s.runtimeFor(inbound); rterr == nil {
if e := rt.ResetClientTraffic(context.Background(), inbound, clientEmail); e != nil {
logger.Warning("ResetClientTraffic: remote propagation to", rt.Name(), "failed:", e)
}
if reenablePlan != nil {
rt, err := s.runtimeFor(&reenablePlan.inbound)
if err != nil {
needRestart = true
} else if err := rt.AddUser(context.Background(), &reenablePlan.inbound, reenablePlan.client); err != nil {
logger.Debug("Error in enabling client on", rt.Name(), ":", err)
needRestart = true
} else {
logger.Warning("ResetClientTraffic: runtime lookup failed:", rterr)
logger.Debug("Client enabled on", rt.Name(), "due to reset traffic:", clientEmail)
}
}
return needRestart, nil
return needRestart, inbound, nil
}
func (s *InboundService) ResetAllTraffics() error {
@@ -740,16 +754,24 @@ func (s *InboundService) propagateResetAllTrafficsToNodes() {
}
func (s *InboundService) ResetInboundTraffic(id int) error {
var inbound *model.Inbound
if err := submitTrafficWrite(func() error {
return database.GetDB().Model(model.Inbound{}).
db := database.GetDB()
if err := db.Model(model.Inbound{}).
Where("id = ?", id).
Updates(map[string]any{"up": 0, "down": 0}).Error
Updates(map[string]any{"up": 0, "down": 0}).Error; err != nil {
return err
}
var err error
inbound, err = s.GetInbound(id)
if err != nil {
return err
}
return nil
}); err != nil {
return err
}
inbound, err := s.GetInbound(id)
if err == nil && inbound != nil && inbound.NodeID != nil {
if inbound != nil && inbound.NodeID != nil {
if rt, rterr := s.runtimeFor(inbound); rterr == nil {
if e := rt.ResetInboundTraffic(context.Background(), inbound); e != nil {
logger.Warning("ResetInboundTraffic: remote propagation to", rt.Name(), "failed:", e)
@@ -763,134 +785,161 @@ func (s *InboundService) ResetInboundTraffic(id int) error {
func (s *InboundService) DelDepletedClients(id int) (err error) {
db := database.GetDB()
tx := db.Begin()
defer func() {
if err == nil {
tx.Commit()
} else {
tx.Rollback()
var deletedInbounds []model.Inbound
err = db.Transaction(func(tx *gorm.DB) error {
// Collect depleted emails globally — a shared-email row owned by one
// inbound depletes every sibling that lists the email.
now := time.Now().Unix() * 1000
depletedClause := "reset = 0 and ((total > 0 and up + down >= total) or (expiry_time > 0 and expiry_time <= ?))"
var depletedRows []xray.ClientTraffic
if err := tx.Model(xray.ClientTraffic{}).
Where(depletedClause, now).
Find(&depletedRows).Error; err != nil {
return err
}
if len(depletedRows) == 0 {
return nil
}
}()
// Collect depleted emails globally — a shared-email row owned by one
// inbound depletes every sibling that lists the email.
now := time.Now().Unix() * 1000
depletedClause := "reset = 0 and ((total > 0 and up + down >= total) or (expiry_time > 0 and expiry_time <= ?))"
var depletedRows []xray.ClientTraffic
err = db.Model(xray.ClientTraffic{}).
Where(depletedClause, now).
Find(&depletedRows).Error
depletedEmails := make(map[string]struct{}, len(depletedRows))
for _, r := range depletedRows {
if r.Email == "" {
continue
}
depletedEmails[strings.ToLower(r.Email)] = struct{}{}
}
if len(depletedEmails) == 0 {
return nil
}
var inbounds []*model.Inbound
inboundQuery := tx.Model(model.Inbound{})
if id >= 0 {
inboundQuery = inboundQuery.Where("id = ?", id)
}
if err := inboundQuery.Find(&inbounds).Error; err != nil {
return err
}
for _, inbound := range inbounds {
var settings map[string]any
if err := json.Unmarshal([]byte(inbound.Settings), &settings); err != nil {
return err
}
rawClients, ok := settings["clients"].([]any)
if !ok {
continue
}
newClients := make([]any, 0, len(rawClients))
removed := 0
for _, client := range rawClients {
c, ok := client.(map[string]any)
if !ok {
newClients = append(newClients, client)
continue
}
email, _ := c["email"].(string)
if _, isDepleted := depletedEmails[strings.ToLower(email)]; isDepleted {
removed++
continue
}
newClients = append(newClients, client)
}
if removed == 0 {
continue
}
if len(newClients) == 0 {
deletedInbounds = append(deletedInbounds, *inbound)
if err := s.clientService.DetachInbound(tx, inbound.Id); err != nil {
return err
}
if err := tx.Where("inbound_id = ?", inbound.Id).Delete(&model.Host{}).Error; err != nil {
return err
}
if err := tx.Delete(model.Inbound{}, inbound.Id).Error; err != nil {
return err
}
if inbound.NodeID != nil {
if err := (&NodeService{}).MarkNodeDirtyTx(tx, *inbound.NodeID); err != nil {
return err
}
}
continue
}
settings["clients"] = newClients
ns, mErr := json.MarshalIndent(settings, "", " ")
if mErr != nil {
return mErr
}
inbound.Settings = string(ns)
if err := tx.Save(inbound).Error; err != nil {
return err
}
survivingClients, gcErr := s.GetClients(inbound)
if gcErr != nil {
return gcErr
}
if err := s.clientService.SyncInbound(tx, inbound.Id, survivingClients); err != nil {
return err
}
if inbound.NodeID != nil {
if err := (&NodeService{}).MarkNodeDirtyTx(tx, *inbound.NodeID); err != nil {
return err
}
}
}
// Drop now-orphaned rows. With id >= 0, a row is safe to drop only when
// no out-of-scope inbound still references the email.
if id < 0 {
return tx.Where(depletedClause, now).Delete(xray.ClientTraffic{}).Error
}
emails := make([]string, 0, len(depletedEmails))
for e := range depletedEmails {
emails = append(emails, e)
}
var stillReferenced []string
emailExpr := database.JSONFieldText("client.value", "email")
stillQuery := fmt.Sprintf(
"SELECT DISTINCT LOWER(%s) %s WHERE LOWER(%s) IN ?",
emailExpr,
database.JSONClientsFromInbound(),
emailExpr,
)
if err := tx.Raw(stillQuery, emails).Scan(&stillReferenced).Error; err != nil {
return err
}
stillSet := make(map[string]struct{}, len(stillReferenced))
for _, e := range stillReferenced {
stillSet[e] = struct{}{}
}
toDelete := make([]string, 0, len(emails))
for _, e := range emails {
if _, kept := stillSet[e]; !kept {
toDelete = append(toDelete, e)
}
}
if len(toDelete) > 0 {
if err := tx.Where("LOWER(email) IN ?", toDelete).Delete(xray.ClientTraffic{}).Error; err != nil {
return err
}
}
return nil
})
if err != nil {
return err
}
if len(depletedRows) == 0 {
return nil
}
depletedEmails := make(map[string]struct{}, len(depletedRows))
for _, r := range depletedRows {
if r.Email == "" {
continue
for i := range deletedInbounds {
inbound := &deletedInbounds[i]
if rt, rtErr := s.runtimeFor(inbound); rtErr != nil {
logger.Warning("DelDepletedClients: runtime lookup failed after commit:", rtErr)
} else if rtErr = rt.DelInbound(context.Background(), inbound); rtErr != nil && !xray.IsMissingHandlerErr(rtErr) {
logger.Warning("DelDepletedClients: runtime cleanup failed after commit:", rtErr)
}
depletedEmails[strings.ToLower(r.Email)] = struct{}{}
}
if len(depletedEmails) == 0 {
return nil
}
var inbounds []*model.Inbound
inboundQuery := db.Model(model.Inbound{})
if id >= 0 {
inboundQuery = inboundQuery.Where("id = ?", id)
}
if err = inboundQuery.Find(&inbounds).Error; err != nil {
return err
}
for _, inbound := range inbounds {
var settings map[string]any
if err = json.Unmarshal([]byte(inbound.Settings), &settings); err != nil {
return err
}
rawClients, ok := settings["clients"].([]any)
if !ok {
continue
}
newClients := make([]any, 0, len(rawClients))
removed := 0
for _, client := range rawClients {
c, ok := client.(map[string]any)
if !ok {
newClients = append(newClients, client)
continue
if inbound.Tag != "" {
if _, syncErr := (&XraySettingService{}).RemoveInboundTagReferences(inbound.Tag); syncErr != nil {
logger.Warning("DelDepletedClients: routing cleanup failed after commit:", syncErr)
}
email, _ := c["email"].(string)
if _, isDepleted := depletedEmails[strings.ToLower(email)]; isDepleted {
removed++
continue
}
newClients = append(newClients, client)
}
if removed == 0 {
continue
}
if len(newClients) == 0 {
_, _ = s.DelInbound(inbound.Id)
continue
}
settings["clients"] = newClients
ns, mErr := json.MarshalIndent(settings, "", " ")
if mErr != nil {
return mErr
}
inbound.Settings = string(ns)
if err = tx.Save(inbound).Error; err != nil {
return err
}
survivingClients, gcErr := s.GetClients(inbound)
if gcErr != nil {
err = gcErr
return err
}
if err = s.clientService.SyncInbound(tx, inbound.Id, survivingClients); err != nil {
return err
}
}
// Drop now-orphaned rows. With id >= 0, a row is safe to drop only when
// no out-of-scope inbound still references the email.
if id < 0 {
err = tx.Where(depletedClause, now).Delete(xray.ClientTraffic{}).Error
return err
}
emails := make([]string, 0, len(depletedEmails))
for e := range depletedEmails {
emails = append(emails, e)
}
var stillReferenced []string
emailExpr := database.JSONFieldText("client.value", "email")
stillQuery := fmt.Sprintf(
"SELECT DISTINCT LOWER(%s) %s WHERE LOWER(%s) IN ?",
emailExpr,
database.JSONClientsFromInbound(),
emailExpr,
)
if err = tx.Raw(stillQuery, emails).Scan(&stillReferenced).Error; err != nil {
return err
}
stillSet := make(map[string]struct{}, len(stillReferenced))
for _, e := range stillReferenced {
stillSet[e] = struct{}{}
}
toDelete := make([]string, 0, len(emails))
for _, e := range emails {
if _, kept := stillSet[e]; !kept {
toDelete = append(toDelete, e)
}
}
if len(toDelete) > 0 {
if err = tx.Where("LOWER(email) IN ?", toDelete).Delete(xray.ClientTraffic{}).Error; err != nil {
return err
}
}
return nil