Files
3x-ui/internal/web/service/xray_metrics_test.go
T
Yuri Khachaturyan 2b1308ca29 feat(notifications): add a consecutive-failure threshold for outbound.down alerts (#5968)
Problem: a flaky outbound produces hundreds of false-positive "outbound down"
notifications overnight — each fires the moment xray's observatory reports a
single failed probe, and the next successful probe fires an "up".

applyObservatory forwarded every raw alive:true->false transition straight to
EventOutboundDown; xray's observatory has effectively no hysteresis, and nothing
on the panel side debounced it (the email/Telegram subscribers are pure
formatters).

Fix: debounce per outbound. outbound.down now fires only after
outboundDownThreshold consecutive FAILED probes (new setting, default 3);
outbound.up fires immediately on the first successful probe and only when a down
was actually notified. The threshold gates the event itself, so email and
Telegram share one knob (exposed next to the outbound.down toggle).

The streak counts genuinely new probes (last_try_time advancing), not sampler
polls — the sampler runs every 2s but the observatory re-probes per its
probeInterval, so counting samples would trip the threshold instantly.
outboundDownThreshold=1 reproduces the legacy notify-on-first-failure behaviour.

Tuning the observatory's probe interval/timeout is not a workaround: those
probes also drive the load balancer's outbound selection, so loosening them to
quiet notifications would slow real failover away from a genuinely dead
outbound. Notifications don't need observatory-grade latency, so the tolerance
belongs at the notification layer, leaving the observatory (and balancer)
untouched.

Adds TestApplyObservatoryDebounce covering the threshold, probe-vs-sample
counting, single-blip suppression and the legacy path.

Co-authored-by: Yuriy Khachaturian <y.khachaturian@souzmult.ru>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-21 15:59:07 +02:00

123 lines
3.1 KiB
Go

package service
import (
"path/filepath"
"testing"
"time"
"github.com/mhsanaei/3x-ui/v3/internal/database"
"github.com/mhsanaei/3x-ui/v3/internal/eventbus"
)
// probe is one observatory sample: whether the outbound is alive and the
// last_try_time xray reports for it (a new probe advances lastTry).
type probe struct {
alive bool
lastTry int64
}
const testSentinel eventbus.EventType = "test.sentinel"
// runObservatory feeds a probe sequence through applyObservatory with the given
// threshold and returns the outbound.* events it published, in order.
func runObservatory(t *testing.T, threshold int, seq []probe) []eventbus.EventType {
t.Helper()
ss := SettingService{}
if err := ss.SetOutboundDownThreshold(threshold); err != nil {
t.Fatalf("set threshold: %v", err)
}
bus := eventbus.New(256)
events := make(chan eventbus.Event, 256)
bus.Subscribe("test", func(e eventbus.Event) { events <- e })
SetEventBus(bus)
t.Cleanup(func() {
SetEventBus(nil)
bus.Stop()
})
s := &XrayMetricsService{settingService: ss}
for _, p := range seq {
s.applyObservatory(time.Unix(p.lastTry, 0), map[string]rawObsEntry{
"proxy": {Alive: p.alive, Delay: 10, LastTryTime: p.lastTry, OutboundTag: "proxy"},
})
}
bus.Publish(eventbus.Event{Type: testSentinel, Source: "x"})
var got []eventbus.EventType
for {
select {
case e := <-events:
if e.Type == testSentinel {
return got
}
got = append(got, e.Type)
case <-time.After(2 * time.Second):
t.Fatal("timed out waiting for events to drain")
}
}
}
func TestApplyObservatoryDebounce(t *testing.T) {
if err := database.InitDB(filepath.Join(t.TempDir(), "x-ui.db")); err != nil {
t.Fatalf("init db: %v", err)
}
t.Cleanup(func() { _ = database.CloseDB() })
tests := []struct {
name string
threshold int
seq []probe
want []eventbus.EventType
}{
{
name: "notifies only after threshold consecutive failed probes",
threshold: 3,
seq: []probe{
{true, 1},
{false, 2},
{false, 3},
{false, 4},
{false, 5},
{true, 6},
{false, 7},
{true, 8},
},
want: []eventbus.EventType{eventbus.EventOutboundDown, eventbus.EventOutboundUp},
},
{
name: "repeated samples of the same probe do not advance the streak",
threshold: 3,
seq: []probe{{false, 2}, {false, 2}, {false, 2}, {false, 2}, {false, 2}},
want: nil,
},
{
name: "single-probe blip never notifies",
threshold: 3,
seq: []probe{{true, 1}, {false, 2}, {true, 3}},
want: nil,
},
{
name: "threshold 1 keeps the legacy notify-on-first-failure behaviour",
threshold: 1,
seq: []probe{{true, 1}, {false, 2}},
want: []eventbus.EventType{eventbus.EventOutboundDown},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := runObservatory(t, tt.threshold, tt.seq)
if len(got) != len(tt.want) {
t.Fatalf("events = %v, want %v", got, tt.want)
}
for i := range got {
if got[i] != tt.want[i] {
t.Fatalf("event[%d] = %q, want %q (full: %v)", i, got[i], tt.want[i], got)
}
}
})
}
}