feat(notifications): add a consecutive-failure threshold for outbound.down alerts (#5968)

Problem: a flaky outbound produces hundreds of false-positive "outbound down"
notifications overnight — each fires the moment xray's observatory reports a
single failed probe, and the next successful probe fires an "up".

applyObservatory forwarded every raw alive:true->false transition straight to
EventOutboundDown; xray's observatory has effectively no hysteresis, and nothing
on the panel side debounced it (the email/Telegram subscribers are pure
formatters).

Fix: debounce per outbound. outbound.down now fires only after
outboundDownThreshold consecutive FAILED probes (new setting, default 3);
outbound.up fires immediately on the first successful probe and only when a down
was actually notified. The threshold gates the event itself, so email and
Telegram share one knob (exposed next to the outbound.down toggle).

The streak counts genuinely new probes (last_try_time advancing), not sampler
polls — the sampler runs every 2s but the observatory re-probes per its
probeInterval, so counting samples would trip the threshold instantly.
outboundDownThreshold=1 reproduces the legacy notify-on-first-failure behaviour.

Tuning the observatory's probe interval/timeout is not a workaround: those
probes also drive the load balancer's outbound selection, so loosening them to
quiet notifications would slow real failover away from a genuinely dead
outbound. Notifications don't need observatory-grade latency, so the tolerance
belongs at the notification layer, leaving the observatory (and balancer)
untouched.

Adds TestApplyObservatoryDebounce covering the threshold, probe-vs-sample
counting, single-blip suppression and the legacy path.

Co-authored-by: Yuriy Khachaturian <y.khachaturian@souzmult.ru>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>

frontend/public/openapi.json

@@ -109,6 +109,11 @@
           "ldapVlessField": {
             "type": "string"
           },
+          "outboundDownThreshold": {
+            "maximum": 100,
+            "minimum": 1,
+            "type": "integer"
+          },
           "pageSize": {
             "maximum": 1000,
             "minimum": 0,
@@ -387,6 +392,7 @@
           "ldapUserAttr",
           "ldapUserFilter",
           "ldapVlessField",
+          "outboundDownThreshold",
           "pageSize",
           "panelOutbound",
           "remarkTemplate",
@@ -570,6 +576,11 @@
           "ldapVlessField": {
             "type": "string"
           },
+          "outboundDownThreshold": {
+            "maximum": 100,
+            "minimum": 1,
+            "type": "integer"
+          },
           "pageSize": {
             "maximum": 1000,
             "minimum": 0,
@@ -855,6 +866,7 @@
           "ldapUserAttr",
           "ldapUserFilter",
           "ldapVlessField",
+          "outboundDownThreshold",
           "pageSize",
           "panelOutbound",
           "remarkTemplate",

frontend/src/components/ui/notifications/EmailNotifications.tsx

@@ -10,7 +10,14 @@ const GROUPS: NotificationGroupConfig[] = [
     icon: <CloudServerOutlined />,
     title: 'eventGroupOutbound',
     events: [
-      { key: 'outbound.down', label: 'eventOutboundDown', settingKey: '' },
+      {
+        key: 'outbound.down',
+        label: 'eventOutboundDown',
+        settingKey: 'outboundDownThreshold',
+        extra: ({ value, onChange, ariaLabel }) => (
+          <InputNumber size="small" min={1} max={100} value={value} onChange={onChange} aria-label={ariaLabel} style={{ width: 80 }} />
+        ),
+      },
       { key: 'outbound.up', label: 'eventOutboundUp', settingKey: '' },
     ],
   },

frontend/src/components/ui/notifications/TelegramNotifications.tsx

@@ -10,7 +10,14 @@ const GROUPS: NotificationGroupConfig[] = [
     icon: <CloudServerOutlined />,
     title: 'eventGroupOutbound',
     events: [
-      { key: 'outbound.down', label: 'eventOutboundDown', settingKey: '' },
+      {
+        key: 'outbound.down',
+        label: 'eventOutboundDown',
+        settingKey: 'outboundDownThreshold',
+        extra: ({ value, onChange, ariaLabel }) => (
+          <InputNumber size="small" min={1} max={100} value={value} onChange={onChange} aria-label={ariaLabel} style={{ width: 80 }} />
+        ),
+      },
       { key: 'outbound.up', label: 'eventOutboundUp', settingKey: '' },
     ],
   },

frontend/src/generated/examples.ts

@@ -26,6 +26,7 @@ export const EXAMPLES: Record<string, unknown> = {
     "ldapUserAttr": "",
     "ldapUserFilter": "",
     "ldapVlessField": "",
+    "outboundDownThreshold": 1,
     "pageSize": 0,
     "panelOutbound": "",
     "remarkTemplate": "",
@@ -136,6 +137,7 @@ export const EXAMPLES: Record<string, unknown> = {
     "ldapUserAttr": "",
     "ldapUserFilter": "",
     "ldapVlessField": "",
+    "outboundDownThreshold": 1,
     "pageSize": 0,
     "panelOutbound": "",
     "remarkTemplate": "",

frontend/src/generated/schemas.ts

@@ -83,6 +83,11 @@ export const SCHEMAS: Record<string, unknown> = {
       "ldapVlessField": {
         "type": "string"
       },
+      "outboundDownThreshold": {
+        "maximum": 100,
+        "minimum": 1,
+        "type": "integer"
+      },
       "pageSize": {
         "maximum": 1000,
         "minimum": 0,
@@ -361,6 +366,7 @@ export const SCHEMAS: Record<string, unknown> = {
       "ldapUserAttr",
       "ldapUserFilter",
       "ldapVlessField",
+      "outboundDownThreshold",
       "pageSize",
       "panelOutbound",
       "remarkTemplate",
@@ -544,6 +550,11 @@ export const SCHEMAS: Record<string, unknown> = {
       "ldapVlessField": {
         "type": "string"
       },
+      "outboundDownThreshold": {
+        "maximum": 100,
+        "minimum": 1,
+        "type": "integer"
+      },
       "pageSize": {
         "maximum": 1000,
         "minimum": 0,
@@ -829,6 +840,7 @@ export const SCHEMAS: Record<string, unknown> = {
       "ldapUserAttr",
       "ldapUserFilter",
       "ldapVlessField",
+      "outboundDownThreshold",
       "pageSize",
       "panelOutbound",
       "remarkTemplate",

frontend/src/generated/types.ts

@@ -32,6 +32,7 @@ export interface AllSetting {
   ldapUserAttr: string;
   ldapUserFilter: string;
   ldapVlessField: string;
+  outboundDownThreshold: number;
   pageSize: number;
   panelOutbound: string;
   remarkTemplate: string;
@@ -143,6 +144,7 @@ export interface AllSettingView {
   ldapUserAttr: string;
   ldapUserFilter: string;
   ldapVlessField: string;
+  outboundDownThreshold: number;
   pageSize: number;
   panelOutbound: string;
   remarkTemplate: string;

frontend/src/generated/zod.ts

@@ -44,6 +44,7 @@ export const AllSettingSchema = z.object({
   ldapUserAttr: z.string(),
   ldapUserFilter: z.string(),
   ldapVlessField: z.string(),
+  outboundDownThreshold: z.number().int().min(1).max(100),
   pageSize: z.number().int().min(0).max(1000),
   panelOutbound: z.string(),
   remarkTemplate: z.string(),
@@ -156,6 +157,7 @@ export const AllSettingViewSchema = z.object({
   ldapUserAttr: z.string(),
   ldapUserFilter: z.string(),
   ldapVlessField: z.string(),
+  outboundDownThreshold: z.number().int().min(1).max(100),
   pageSize: z.number().int().min(0).max(1000),
   panelOutbound: z.string(),
   remarkTemplate: z.string(),

frontend/src/models/setting.ts

@@ -103,6 +103,7 @@ export class AllSetting {
   smtpEnabledEvents = '';
   smtpCpu = 80;
   smtpMemory = 80;
+  outboundDownThreshold = 3;
   hasTgBotToken = false;
   hasTwoFactorToken = false;
   hasLdapPassword = false;
@@ -120,6 +121,8 @@ export class AllSetting {
     }
     const cpu = Math.round(Number(this.tgCpu));
     this.tgCpu = Number.isFinite(cpu) ? Math.min(100, Math.max(0, cpu)) : 80;
+    const threshold = Math.round(Number(this.outboundDownThreshold));
+    this.outboundDownThreshold = Number.isFinite(threshold) ? Math.min(100, Math.max(1, threshold)) : 3;
   }
 
   equals(other: AllSetting): boolean {

frontend/src/schemas/setting.ts

@@ -27,6 +27,7 @@ export const AllSettingSchema = z.object({
   tgRunTime: z.string().optional(),
   tgBotBackup: z.boolean().optional(),
   tgCpu: z.number().int().min(0).max(100).optional(),
+  outboundDownThreshold: z.number().int().min(1).max(100).optional(),
   tgLang: z.string().optional(),
   twoFactorEnable: z.boolean().optional(),
   twoFactorToken: z.string().optional(),