fix(node): fan out operations that call every node

An operation that calls every node has to finish inside the panel's 30s
write timeout. Reset all traffic, UpdatePanels and bulk inbound delete
walked the nodes one at a time, up to 10s per hanging node, so 15 hanging
nodes out of 150 kept each request running for 2m41s while the browser
had already been told it failed.

All three now fan out through fanoutInboundResults, bounded by
nodeFanoutConcurrency (32, the heartbeat's bound), and UpdatePanels keeps
its results in request order. Bulk delete still removes the rows one at a
time, since each rewrites shared routing references, and only fans out the
node pushes that delInbound now hands back.
This commit is contained in:
Sanaei
2026-09-15 20:07:18 +02:00
parent a84bbeab2e
commit eb11e8c85a
5 changed files with 210 additions and 15 deletions
+28 -7
View File
@@ -1363,10 +1363,20 @@ func (s *InboundService) AddInbound(inbound *model.Inbound) (*model.Inbound, boo
}
func (s *InboundService) DelInbound(id int) (bool, error) {
needRestart, nodePush, err := s.delInbound(id)
if nodePush != nil {
nodePush()
}
return needRestart, err
}
// delInbound deletes the central row and returns the node push instead of running
// it, so a bulk delete can fan the pushes out once every row is gone.
func (s *InboundService) delInbound(id int) (bool, func(), error) {
db := database.GetDB()
needRestart := false
var postCommitApply func()
var postCommitApply, nodePush func()
var ib model.Inbound
loadErr := db.Model(model.Inbound{}).Where("id = ?", id).First(&ib).Error
if loadErr == nil {
@@ -1377,7 +1387,7 @@ func (s *InboundService) DelInbound(id int) (bool, error) {
if perr != nil {
logger.Warning("DelInbound: node runtime lookup failed, deleting central row anyway:", perr)
} else if push {
postCommitApply = func() {
nodePush = func() {
if err1 := rt.DelInbound(context.Background(), &ib); err1 == nil {
logger.Debug("Inbound deleted on", rt.Name(), ":", ib.Tag)
} else {
@@ -1440,7 +1450,7 @@ func (s *InboundService) DelInbound(id int) (bool, error) {
}
return nil
}); err != nil {
return needRestart, err
return needRestart, nil, err
}
if postCommitApply != nil {
postCommitApply()
@@ -1455,11 +1465,11 @@ func (s *InboundService) DelInbound(id int) (bool, error) {
if !database.IsPostgres() {
var count int64
if err := db.Model(&model.Inbound{}).Count(&count).Error; err != nil {
return needRestart, err
return needRestart, nodePush, err
}
if count == 0 {
if err := db.Exec("DELETE FROM sqlite_sequence WHERE name = ?", "inbounds").Error; err != nil {
return needRestart, err
return needRestart, nodePush, err
}
}
}
@@ -1467,7 +1477,7 @@ func (s *InboundService) DelInbound(id int) (bool, error) {
if mtprotoRoutesThroughXray(&ib) {
needRestart = true
}
return needRestart, nil
return needRestart, nodePush, nil
}
type BulkDelInboundResult struct {
@@ -1487,8 +1497,14 @@ type BulkDelInboundReport struct {
func (s *InboundService) DelInbounds(ids []int) (BulkDelInboundResult, bool, error) {
result := BulkDelInboundResult{}
needRestart := false
var pushIDs []int
var nodePushes []func()
for _, id := range ids {
r, err := s.DelInbound(id)
r, nodePush, err := s.delInbound(id)
if nodePush != nil {
pushIDs = append(pushIDs, id)
nodePushes = append(nodePushes, nodePush)
}
if err != nil {
result.Skipped = append(result.Skipped, BulkDelInboundReport{Id: id, Reason: err.Error()})
continue
@@ -1498,6 +1514,11 @@ func (s *InboundService) DelInbounds(ids []int) (BulkDelInboundResult, bool, err
needRestart = true
}
}
// Rows go one at a time for the shared routing rewrite; only node pushes fan out.
fanoutInboundResults(pushIDs, nodeFanoutConcurrency, func(i int) struct{} {
nodePushes[i]()
return struct{}{}
})
return result, needRestart, nil
}