feat(notifications): add a consecutive-failure threshold for outbound.down alerts (#5968)

Problem: a flaky outbound produces hundreds of false-positive "outbound down"
notifications overnight — each fires the moment xray's observatory reports a
single failed probe, and the next successful probe fires an "up".

applyObservatory forwarded every raw alive:true->false transition straight to
EventOutboundDown; xray's observatory has effectively no hysteresis, and nothing
on the panel side debounced it (the email/Telegram subscribers are pure
formatters).

Fix: debounce per outbound. outbound.down now fires only after
outboundDownThreshold consecutive FAILED probes (new setting, default 3);
outbound.up fires immediately on the first successful probe and only when a down
was actually notified. The threshold gates the event itself, so email and
Telegram share one knob (exposed next to the outbound.down toggle).

The streak counts genuinely new probes (last_try_time advancing), not sampler
polls — the sampler runs every 2s but the observatory re-probes per its
probeInterval, so counting samples would trip the threshold instantly.
outboundDownThreshold=1 reproduces the legacy notify-on-first-failure behaviour.

Tuning the observatory's probe interval/timeout is not a workaround: those
probes also drive the load balancer's outbound selection, so loosening them to
quiet notifications would slow real failover away from a genuinely dead
outbound. Notifications don't need observatory-grade latency, so the tolerance
belongs at the notification layer, leaving the observatory (and balancer)
untouched.

Adds TestApplyObservatoryDebounce covering the threshold, probe-vs-sample
counting, single-blip suppression and the legacy path.

Co-authored-by: Yuriy Khachaturian <y.khachaturian@souzmult.ru>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>

internal/web/service/xray_metrics.go

@@ -46,6 +46,19 @@ type XrayMetricsService struct {
 	state    xrayMetricsState
 	client   *http.Client
 	obsByTag map[string]ObsTagSnapshot
+	health   map[string]outboundHealth
+}
+
+// outboundHealth debounces observatory flapping. Xray flips an outbound's
+// alive flag on a single failed probe, so raw transitions produce a storm of
+// down/up notifications on a flaky link. We instead require failStreak to reach
+// the configured threshold (consecutive FAILED probes, tracked per new probe
+// via lastTry) before publishing outbound.down, and only publish outbound.up
+// once a down has actually been notified.
+type outboundHealth struct {
+	lastTry    int64
+	failStreak int
+	notified   bool
 }
 
 var validObsTag = regexp.MustCompile(`^[a-zA-Z0-9._\-]+$`)
@@ -214,32 +227,57 @@ func (s *XrayMetricsService) applyObservatory(t time.Time, entries map[string]ra
 		xrayMetrics.append(obsHistoryKey(tag), t, float64(e.Delay))
 	}
 
+	threshold := 3
+	if v, err := s.settingService.GetOutboundDownThreshold(); err == nil && v > 0 {
+		threshold = v
+	}
+
 	s.mu.Lock()
-	// Detect transitions and publish events
+	// Debounce observatory flapping into stable down/up notifications.
 	if eventBus != nil {
-		// Check existing tags for state changes
-		for tag, old := range s.obsByTag {
-			cur, exists := next[tag]
-			if !exists {
-				// Tag disappeared from observatory — skip, not a real failure
+		if s.health == nil {
+			s.health = make(map[string]outboundHealth, len(next))
+		}
+		for tag, cur := range next {
+			// React only to a genuinely new probe attempt (lastTry advanced).
+			// The sampler polls far more often than xray probes, so counting
+			// samples instead of probes would trip the threshold instantly.
+			h := s.health[tag]
+			if cur.LastTryTime == 0 || cur.LastTryTime == h.lastTry {
 				continue
 			}
-			if old.Alive && !cur.Alive {
-				errMsg := ""
-				if cur.Delay < 0 {
-					errMsg = "probe failed"
+			h.lastTry = cur.LastTryTime
+			if cur.Alive {
+				if h.notified {
+					eventBus.Publish(eventbus.Event{
+						Type:   eventbus.EventOutboundUp,
+						Source: tag,
+						Data:   &eventbus.OutboundHealthData{Delay: cur.Delay},
+					})
 				}
-				eventBus.Publish(eventbus.Event{
-					Type:   eventbus.EventOutboundDown,
-					Source: tag,
-					Data:   &eventbus.OutboundHealthData{Delay: cur.Delay, Error: errMsg},
-				})
-			} else if !old.Alive && cur.Alive {
-				eventBus.Publish(eventbus.Event{
-					Type:   eventbus.EventOutboundUp,
-					Source: tag,
-					Data:   &eventbus.OutboundHealthData{Delay: cur.Delay},
-				})
+				h.failStreak = 0
+				h.notified = false
+			} else {
+				h.failStreak++
+				if h.failStreak >= threshold && !h.notified {
+					errMsg := ""
+					if cur.Delay < 0 {
+						errMsg = "probe failed"
+					}
+					eventBus.Publish(eventbus.Event{
+						Type:   eventbus.EventOutboundDown,
+						Source: tag,
+						Data:   &eventbus.OutboundHealthData{Delay: cur.Delay, Error: errMsg},
+					})
+					h.notified = true
+				}
+			}
+			s.health[tag] = h
+		}
+		// Forget tags that vanished from the observatory.
+		for tag := range s.health {
+			if _, ok := next[tag]; !ok {
+				delete(s.health, tag)
 			}
 		}
 	}