fix(jobs): isolate per-node background goroutines from panics (#5397)

A panic in a goroutine without a recover takes the whole panel down. The
per-node heartbeat and traffic-sync goroutines run remote network I/O for
each node with no panic isolation, so one misbehaving node could crash the
master.

Add common.GoRecover(name, fn), which runs fn in a goroutine guarded by a
recover that logs the panic with a stack trace instead of crashing, and use
it for the per-node heartbeat, traffic-sync and global-push goroutines. The
deferred WaitGroup/semaphore releases still run during panic unwind, so the
group never stalls. Other background goroutines can adopt the same helper.
This commit is contained in:
n0ctal
2026-06-20 03:38:25 +05:00
committed by GitHub
parent bedbe04bf1
commit f63ed9f510
4 changed files with 67 additions and 7 deletions
+4 -2
View File
@@ -9,6 +9,7 @@ import (
"github.com/mhsanaei/3x-ui/v3/internal/database/model"
"github.com/mhsanaei/3x-ui/v3/internal/eventbus"
"github.com/mhsanaei/3x-ui/v3/internal/logger"
"github.com/mhsanaei/3x-ui/v3/internal/util/common"
"github.com/mhsanaei/3x-ui/v3/internal/web/service"
"github.com/mhsanaei/3x-ui/v3/internal/web/websocket"
)
@@ -50,11 +51,12 @@ func (j *NodeHeartbeatJob) Run() {
}
wg.Add(1)
sem <- struct{}{}
go func(n *model.Node) {
n := n
common.GoRecover("node-heartbeat:"+n.Name, func() {
defer wg.Done()
defer func() { <-sem }()
j.probeOne(n)
}(n)
})
}
wg.Wait()
+7 -5
View File
@@ -8,10 +8,10 @@ import (
"github.com/mhsanaei/3x-ui/v3/internal/database/model"
"github.com/mhsanaei/3x-ui/v3/internal/logger"
"github.com/mhsanaei/3x-ui/v3/internal/util/common"
"github.com/mhsanaei/3x-ui/v3/internal/web/runtime"
"github.com/mhsanaei/3x-ui/v3/internal/web/service"
"github.com/mhsanaei/3x-ui/v3/internal/web/websocket"
"github.com/mhsanaei/3x-ui/v3/internal/xray"
)
const (
@@ -96,11 +96,12 @@ func (j *NodeTrafficSyncJob) Run() {
}
wg.Add(1)
sem <- struct{}{}
go func(n *model.Node) {
n := n
common.GoRecover("node-traffic-sync:"+n.Name, func() {
defer wg.Done()
defer func() { <-sem }()
j.syncOne(mgr, n, doIpSync)
}(n)
})
}
wg.Wait()
@@ -211,7 +212,8 @@ func (j *NodeTrafficSyncJob) maybePushGlobals(mgr *runtime.Manager, nodes []*mod
}
wg.Add(1)
sem <- struct{}{}
go func(n *model.Node, remote *runtime.Remote, traffics []*xray.ClientTraffic) {
n, remote, traffics := n, remote, traffics
common.GoRecover("node-global-push:"+n.Name, func() {
defer wg.Done()
defer func() { <-sem }()
ctx, cancel := context.WithTimeout(context.Background(), nodeTrafficSyncRequestTimeout)
@@ -225,7 +227,7 @@ func (j *NodeTrafficSyncJob) maybePushGlobals(mgr *runtime.Manager, nodes []*mod
logger.Warning("node traffic sync: push globals to", n.Name, "failed:", err)
}
}
}(n, remote, traffics)
})
}
wg.Wait()
}