feat(reality): warn when target cert chain is too small for ML-DSA-65 (#6470)

* feat(reality): warn when target cert chain is too small for ML-DSA-65

Expose peer cert-chain DER size from the REALITY scanner and surface a UI
warning when ML-DSA-65 is enabled but the chain is under xray-core's 3500-byte
minimum, so silent fallback failures are easier to catch.

Fixes #5973

* fix(reality): gate scanner ML-DSA tag and sync docs OpenAPI

Only warn on short cert chains in the target scanner when ML-DSA-65 is
enabled. Copy frontend/public/openapi.json to docs/public/openapi.json
and fix oxfmt wrapping in the new test.

---------

Co-authored-by: mrchatam <287639636+mrchatam@users.noreply.github.com>

frontend/public/openapi.json

@@ -3710,6 +3710,11 @@
             "example": "h2",
             "type": "string"
           },
+          "certChainBytes": {
+            "description": "CertChainBytes is the sum of DER lengths of the presented peer chain.\nxray-core ML-DSA-65 REALITY needs >= 3500 bytes (constant lives in xray-core).",
+            "example": 3427,
+            "type": "integer"
+          },
           "certChainValid": {
             "description": "CertChainValid ignores the name: a trusted chain presented for other names\nstill has serverNames the panel can offer instead of the failing SNI.",
             "example": true,
@@ -3792,6 +3797,7 @@
         },
         "required": [
           "alpn",
+          "certChainBytes",
           "certChainValid",
           "certIssuer",
           "certSubject",
@@ -7240,6 +7246,7 @@
                   "success": true,
                   "obj": {
                     "alpn": "h2",
+                    "certChainBytes": 3427,
                     "certChainValid": true,
                     "certIssuer": "Google Trust Services",
                     "certSubject": "cloudflare.com",
@@ -7319,6 +7326,7 @@
                   "obj": [
                     {
                       "alpn": "h2",
+                      "certChainBytes": 3427,
                       "certChainValid": true,
                       "certIssuer": "Google Trust Services",
                       "certSubject": "cloudflare.com",

frontend/src/generated/examples.ts

@@ -916,6 +916,7 @@ export const EXAMPLES: Record<string, unknown> = {
   },
   "RealityScanResult": {
     "alpn": "h2",
+    "certChainBytes": 3427,
     "certChainValid": true,
     "certIssuer": "Google Trust Services",
     "certSubject": "cloudflare.com",

frontend/src/generated/schemas.ts

@@ -3684,6 +3684,11 @@ export const SCHEMAS: Record<string, unknown> = {
         "example": "h2",
         "type": "string"
       },
+      "certChainBytes": {
+        "description": "CertChainBytes is the sum of DER lengths of the presented peer chain.\nxray-core ML-DSA-65 REALITY needs \u003e= 3500 bytes (constant lives in xray-core).",
+        "example": 3427,
+        "type": "integer"
+      },
       "certChainValid": {
         "description": "CertChainValid ignores the name: a trusted chain presented for other names\nstill has serverNames the panel can offer instead of the failing SNI.",
         "example": true,
@@ -3766,6 +3771,7 @@ export const SCHEMAS: Record<string, unknown> = {
     },
     "required": [
       "alpn",
+      "certChainBytes",
       "certChainValid",
       "certIssuer",
       "certSubject",

frontend/src/generated/types.ts

@@ -835,6 +835,7 @@ export interface ProbeResultUI {
 
 export interface RealityScanResult {
   alpn: string;
+  certChainBytes: number;
   certChainValid: boolean;
   certIssuer: string;
   certSubject: string;

frontend/src/generated/zod.ts

@@ -892,6 +892,7 @@ export type ProbeResultUI = z.infer<typeof ProbeResultUISchema>;
 
 export const RealityScanResultSchema = z.object({
   alpn: z.string(),
+  certChainBytes: z.number().int(),
   certChainValid: z.boolean(),
   certIssuer: z.string(),
   certSubject: z.string(),

frontend/src/pages/inbounds/form/security/RealityTargetScannerModal.tsx

@@ -5,11 +5,15 @@ import type { ColumnsType } from 'antd/es/table';
 
 import type { RealityScanResult } from '@/generated/types';
 
+// xray-core ML-DSA-65 REALITY min peer cert-chain size (not defined in this repo).
+export const MLDSA65_MIN_CERT_CHAIN_BYTES = 3500;
+
 interface RealityTargetScannerModalProps {
   open: boolean;
   onClose: () => void;
   scanRealityCandidates: (targets?: string) => Promise<RealityScanResult[]>;
   onPick: (result: RealityScanResult) => void;
+  mldsa65Enabled?: boolean;
 }
 
 export default function RealityTargetScannerModal({
@@ -17,6 +21,7 @@ export default function RealityTargetScannerModal({
   onClose,
   scanRealityCandidates,
   onPick,
+  mldsa65Enabled = false,
 }: RealityTargetScannerModalProps) {
   const { t } = useTranslation();
   const [loading, setLoading] = useState(false);
@@ -126,6 +131,28 @@ export default function RealityTargetScannerModal({
           <Tag>{t('pages.inbounds.form.scanCertInvalid')}</Tag>
         ),
     },
+    {
+      title: t('pages.inbounds.form.scanCertChain'),
+      dataIndex: 'certChainBytes',
+      key: 'certChainBytes',
+      width: 100,
+      render: (bytes: number) => {
+        if (!bytes) return '—';
+        if (mldsa65Enabled && bytes < MLDSA65_MIN_CERT_CHAIN_BYTES) {
+          return (
+            <Tooltip
+              title={t('pages.inbounds.form.scanMldsaCertChainTooSmall', {
+                length: bytes,
+                min: MLDSA65_MIN_CERT_CHAIN_BYTES,
+              })}
+            >
+              <Tag color="warning">{bytes} B</Tag>
+            </Tooltip>
+          );
+        }
+        return `${bytes} B`;
+      },
+    },
     {
       title: t('pages.inbounds.form.scanLatency'),
       dataIndex: 'latencyMs',
@@ -165,7 +192,7 @@ export default function RealityTargetScannerModal({
         </Button>,
       ]}
       title={t('pages.inbounds.form.scanModalTitle')}
-      width={960}
+      width={1080}
     >
       <Space orientation="vertical" size="small" style={{ width: '100%' }}>
         <Typography.Paragraph type="secondary" style={{ marginBottom: 0 }}>

frontend/src/pages/inbounds/form/security/reality.tsx

@@ -1,5 +1,5 @@
 import { useState } from 'react';
-import { useFormContext } from 'react-hook-form';
+import { useFormContext, useWatch } from 'react-hook-form';
 import { useTranslation } from 'react-i18next';
 import {
   Alert,
@@ -25,7 +25,9 @@ import {
   validateRealityTarget,
 } from '@/lib/xray/stream-wire-normalize';
 import type { RealityScanResult } from '@/generated/types';
-import RealityTargetScannerModal from './RealityTargetScannerModal';
+import RealityTargetScannerModal, {
+  MLDSA65_MIN_CERT_CHAIN_BYTES,
+} from './RealityTargetScannerModal';
 
 interface RealityFormProps {
   saving: boolean;
@@ -59,6 +61,18 @@ export default function RealityForm({
   const { t } = useTranslation();
   const { getFieldState, trigger } = useFormContext();
   const [scannerOpen, setScannerOpen] = useState(false);
+  const mldsa65Seed = useWatch({ name: 'streamSettings.realitySettings.mldsa65Seed' });
+  const mldsa65Verify = useWatch({
+    name: 'streamSettings.realitySettings.settings.mldsa65Verify',
+  });
+  const mldsa65Enabled =
+    (typeof mldsa65Seed === 'string' && mldsa65Seed.trim() !== '') ||
+    (typeof mldsa65Verify === 'string' && mldsa65Verify.trim() !== '');
+  const mldsaChainTooSmall =
+    !!scanResult &&
+    mldsa65Enabled &&
+    scanResult.certChainBytes > 0 &&
+    scanResult.certChainBytes < MLDSA65_MIN_CERT_CHAIN_BYTES;
   /*
    * An untrusted certificate (self-signed fronting service on the LAN) is still
    * worth reading, so subject/issuer stay visible and only the verdict is added.
@@ -130,7 +144,11 @@ export default function RealityForm({
       {scanResult && (
         <Form.Item label=" " colon={false}>
           <Alert
-            type={scanResult.feasible && !scanResult.privateTarget ? 'success' : 'warning'}
+            type={
+              scanResult.feasible && !scanResult.privateTarget && !mldsaChainTooSmall
+                ? 'success'
+                : 'warning'
+            }
             showIcon
             title={
               scanResult.feasible
@@ -139,6 +157,14 @@ export default function RealityForm({
             }
             description={
               <>
+                {mldsaChainTooSmall && (
+                  <div style={{ marginBottom: 8 }}>
+                    {t('pages.inbounds.form.scanMldsaCertChainTooSmall', {
+                      length: scanResult.certChainBytes,
+                      min: MLDSA65_MIN_CERT_CHAIN_BYTES,
+                    })}
+                  </div>
+                )}
                 {scanResult.privateTarget && (
                   <div style={{ marginBottom: 8 }}>{t('pages.inbounds.form.scanPrivateNote')}</div>
                 )}
@@ -159,6 +185,9 @@ export default function RealityForm({
                       ? dayjs(scanResult.notAfter).format('YYYY-MM-DD HH:mm')
                       : '—'}
                   </Descriptions.Item>
+                  <Descriptions.Item label={t('pages.inbounds.form.scanCertChain')}>
+                    {scanResult.certChainBytes > 0 ? `${scanResult.certChainBytes} B` : '—'}
+                  </Descriptions.Item>
                   <Descriptions.Item label={t('pages.inbounds.form.scanLatency')}>
                     {scanResult.latencyMs > 0 ? `${scanResult.latencyMs} ms` : '—'}
                   </Descriptions.Item>
@@ -330,6 +359,7 @@ export default function RealityForm({
         onClose={() => setScannerOpen(false)}
         scanRealityCandidates={scanRealityCandidates}
         onPick={(r) => applyRealityScanResult(r, true)}
+        mldsa65Enabled={mldsa65Enabled}
       />
     </>
   );

frontend/src/test/reality-mldsa-cert-chain.test.tsx

@@ -0,0 +1,102 @@
+import { describe, it, expect } from 'vitest';
+import { Form } from 'antd';
+import type { ReactNode } from 'react';
+import { FormProvider, useForm } from 'react-hook-form';
+
+import { RealityForm } from '@/pages/inbounds/form/security';
+import RealityTargetScannerModal from '@/pages/inbounds/form/security/RealityTargetScannerModal';
+import type { InboundFormValues } from '@/schemas/forms/inbound-form';
+import type { RealityScanResult } from '@/generated/types';
+import { renderWithProviders } from './test-utils';
+
+const smallChain: RealityScanResult = {
+  alpn: 'h2',
+  certChainBytes: 3427,
+  certChainValid: true,
+  certIssuer: 'Google Trust Services',
+  certSubject: 'cloudflare.com',
+  certValid: true,
+  curveID: 'X25519',
+  feasible: true,
+  h2: true,
+  host: 'www.cloudflare.com',
+  ip: '104.16.124.96',
+  latencyMs: 180,
+  notAfter: '2026-08-01T00:00:00Z',
+  port: 443,
+  privateTarget: false,
+  reason: '',
+  serverNames: ['www.cloudflare.com'],
+  target: 'www.cloudflare.com:443',
+  tls13: true,
+  tlsVersion: '1.3',
+  x25519: true,
+};
+
+function FormHarness({
+  children,
+  defaultValues,
+}: {
+  children: ReactNode;
+  defaultValues?: Record<string, unknown>;
+}) {
+  const methods = useForm<InboundFormValues>({ defaultValues: defaultValues as never });
+  return (
+    <FormProvider {...methods}>
+      <Form>{children}</Form>
+    </FormProvider>
+  );
+}
+
+const noop = () => {};
+
+function renderRealityForm(
+  scanResult: RealityScanResult | null,
+  defaultValues?: Record<string, unknown>,
+) {
+  return renderWithProviders(
+    <FormHarness defaultValues={defaultValues}>
+      <RealityForm
+        saving={false}
+        scanning={false}
+        scanResult={scanResult}
+        scanRealityTarget={noop}
+        scanRealityCandidates={async () => []}
+        applyRealityScanResult={noop}
+        randomizeShortIds={noop}
+        randomizeSpiderX={noop}
+        genRealityKeypair={noop}
+        clearRealityKeypair={noop}
+        genMldsa65={noop}
+        clearMldsa65={noop}
+      />
+    </FormHarness>,
+  );
+}
+
+describe('ML-DSA-65 cert chain warning', () => {
+  it('warns on the inbound form when ML-DSA-65 is on and the scanned chain is under 3500 bytes', () => {
+    const { getByText } = renderRealityForm(smallChain, {
+      streamSettings: { realitySettings: { mldsa65Seed: 'seed' } },
+    });
+    expect(getByText(/below the 3500-byte minimum required for ML-DSA-65/)).toBeTruthy();
+  });
+
+  it('does not warn on the inbound form when ML-DSA-65 is off', () => {
+    const { queryByText } = renderRealityForm(smallChain);
+    expect(queryByText(/below the 3500-byte minimum required for ML-DSA-65/)).toBeNull();
+  });
+
+  it('tags scanner rows whose cert chain is too small for ML-DSA-65', async () => {
+    const { findByText } = renderWithProviders(
+      <RealityTargetScannerModal
+        open
+        onClose={noop}
+        scanRealityCandidates={async () => [smallChain]}
+        onPick={noop}
+        mldsa65Enabled
+      />,
+    );
+    expect(await findByText('3427 B')).toBeTruthy();
+  });
+});