mirror of
https://github.com/MHSanaei/3x-ui.git
synced 2026-07-25 13:56:10 +00:00
2b1308ca29
Problem: a flaky outbound produces hundreds of false-positive "outbound down" notifications overnight — each fires the moment xray's observatory reports a single failed probe, and the next successful probe fires an "up". applyObservatory forwarded every raw alive:true->false transition straight to EventOutboundDown; xray's observatory has effectively no hysteresis, and nothing on the panel side debounced it (the email/Telegram subscribers are pure formatters). Fix: debounce per outbound. outbound.down now fires only after outboundDownThreshold consecutive FAILED probes (new setting, default 3); outbound.up fires immediately on the first successful probe and only when a down was actually notified. The threshold gates the event itself, so email and Telegram share one knob (exposed next to the outbound.down toggle). The streak counts genuinely new probes (last_try_time advancing), not sampler polls — the sampler runs every 2s but the observatory re-probes per its probeInterval, so counting samples would trip the threshold instantly. outboundDownThreshold=1 reproduces the legacy notify-on-first-failure behaviour. Tuning the observatory's probe interval/timeout is not a workaround: those probes also drive the load balancer's outbound selection, so loosening them to quiet notifications would slow real failover away from a genuinely dead outbound. Notifications don't need observatory-grade latency, so the tolerance belongs at the notification layer, leaving the observatory (and balancer) untouched. Adds TestApplyObservatoryDebounce covering the threshold, probe-vs-sample counting, single-blip suppression and the legacy path. Co-authored-by: Yuriy Khachaturian <y.khachaturian@souzmult.ru> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
123 lines
3.1 KiB
Go
123 lines
3.1 KiB
Go
package service
|
|
|
|
import (
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/mhsanaei/3x-ui/v3/internal/database"
|
|
"github.com/mhsanaei/3x-ui/v3/internal/eventbus"
|
|
)
|
|
|
|
// probe is one observatory sample: whether the outbound is alive and the
|
|
// last_try_time xray reports for it (a new probe advances lastTry).
|
|
type probe struct {
|
|
alive bool
|
|
lastTry int64
|
|
}
|
|
|
|
const testSentinel eventbus.EventType = "test.sentinel"
|
|
|
|
// runObservatory feeds a probe sequence through applyObservatory with the given
|
|
// threshold and returns the outbound.* events it published, in order.
|
|
func runObservatory(t *testing.T, threshold int, seq []probe) []eventbus.EventType {
|
|
t.Helper()
|
|
|
|
ss := SettingService{}
|
|
if err := ss.SetOutboundDownThreshold(threshold); err != nil {
|
|
t.Fatalf("set threshold: %v", err)
|
|
}
|
|
|
|
bus := eventbus.New(256)
|
|
events := make(chan eventbus.Event, 256)
|
|
bus.Subscribe("test", func(e eventbus.Event) { events <- e })
|
|
SetEventBus(bus)
|
|
t.Cleanup(func() {
|
|
SetEventBus(nil)
|
|
bus.Stop()
|
|
})
|
|
|
|
s := &XrayMetricsService{settingService: ss}
|
|
for _, p := range seq {
|
|
s.applyObservatory(time.Unix(p.lastTry, 0), map[string]rawObsEntry{
|
|
"proxy": {Alive: p.alive, Delay: 10, LastTryTime: p.lastTry, OutboundTag: "proxy"},
|
|
})
|
|
}
|
|
|
|
bus.Publish(eventbus.Event{Type: testSentinel, Source: "x"})
|
|
var got []eventbus.EventType
|
|
for {
|
|
select {
|
|
case e := <-events:
|
|
if e.Type == testSentinel {
|
|
return got
|
|
}
|
|
got = append(got, e.Type)
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("timed out waiting for events to drain")
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestApplyObservatoryDebounce(t *testing.T) {
|
|
if err := database.InitDB(filepath.Join(t.TempDir(), "x-ui.db")); err != nil {
|
|
t.Fatalf("init db: %v", err)
|
|
}
|
|
t.Cleanup(func() { _ = database.CloseDB() })
|
|
|
|
tests := []struct {
|
|
name string
|
|
threshold int
|
|
seq []probe
|
|
want []eventbus.EventType
|
|
}{
|
|
{
|
|
name: "notifies only after threshold consecutive failed probes",
|
|
threshold: 3,
|
|
seq: []probe{
|
|
{true, 1},
|
|
{false, 2},
|
|
{false, 3},
|
|
{false, 4},
|
|
{false, 5},
|
|
{true, 6},
|
|
{false, 7},
|
|
{true, 8},
|
|
},
|
|
want: []eventbus.EventType{eventbus.EventOutboundDown, eventbus.EventOutboundUp},
|
|
},
|
|
{
|
|
name: "repeated samples of the same probe do not advance the streak",
|
|
threshold: 3,
|
|
seq: []probe{{false, 2}, {false, 2}, {false, 2}, {false, 2}, {false, 2}},
|
|
want: nil,
|
|
},
|
|
{
|
|
name: "single-probe blip never notifies",
|
|
threshold: 3,
|
|
seq: []probe{{true, 1}, {false, 2}, {true, 3}},
|
|
want: nil,
|
|
},
|
|
{
|
|
name: "threshold 1 keeps the legacy notify-on-first-failure behaviour",
|
|
threshold: 1,
|
|
seq: []probe{{true, 1}, {false, 2}},
|
|
want: []eventbus.EventType{eventbus.EventOutboundDown},
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := runObservatory(t, tt.threshold, tt.seq)
|
|
if len(got) != len(tt.want) {
|
|
t.Fatalf("events = %v, want %v", got, tt.want)
|
|
}
|
|
for i := range got {
|
|
if got[i] != tt.want[i] {
|
|
t.Fatalf("event[%d] = %q, want %q (full: %v)", i, got[i], tt.want[i], got)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|