fix(hysteria2): emit pinSHA256 as hex in subscriptions, not base64

Hysteria2 clients backed by Xray-core hex-decode the pinSHA256 URI param and crash on the base64 value the panel stores for pinnedPeerCertSha256 (xray-core native TLS format). Normalize each pin to bare lowercase hex when building the Hysteria link, accepting base64, bare hex, and colon-separated openssl fingerprints; values that are neither are passed through untouched. Applied in both the backend subscription generator and the frontend link builder. The pcs share-link and JSON-sub paths keep base64 for their consumers. Fixes #4818.

sub/subService.go

@@ -1,7 +1,9 @@
 package sub
 
 import (
+	"crypto/sha256"
 	"encoding/base64"
+	"encoding/hex"
 	"fmt"
 	"maps"
 	"net"
@@ -609,6 +611,9 @@ func (s *SubService) genHysteriaLink(inbound *model.Inbound, email string) strin
 			}
 		}
 		if pins, ok := pinnedSha256List(tlsSettings); ok {
+			for i, p := range pins {
+				pins[i] = hysteriaPinHex(p)
+			}
 			params["pinSHA256"] = strings.Join(pins, ",")
 		}
 	}
@@ -937,6 +942,36 @@ func pinnedSha256List(tlsClientSettings any) ([]string, bool) {
 	return out, true
 }
 
+// hysteriaPinHex normalises a pinnedPeerCertSha256 entry into the 64-character
+// lowercase hex form that Xray-core's Hysteria2 pinSHA256 parser requires.
+//
+// The panel stores pins in several shapes: base64 (xray-core's native TLS
+// format, used by the generate button and the JSON subscription) and hex —
+// either bare or colon-separated as `openssl x509 -fingerprint -sha256` emits
+// it. Hysteria2 clients hex-decode pinSHA256 and crash on a base64 value, so
+// each entry is coerced to bare hex here. Anything that is neither a 32-byte
+// hex nor a 32-byte base64 SHA-256 is returned unchanged so unexpected data is
+// not silently dropped. Mirrors decodeCertPin in web/service/node.go.
+func hysteriaPinHex(pin string) string {
+	pin = strings.TrimSpace(pin)
+	if h := strings.ReplaceAll(pin, ":", ""); len(h) == hex.EncodedLen(sha256.Size) {
+		if _, err := hex.DecodeString(h); err == nil {
+			return strings.ToLower(h)
+		}
+	}
+	for _, enc := range []*base64.Encoding{
+		base64.StdEncoding,
+		base64.RawStdEncoding,
+		base64.URLEncoding,
+		base64.RawURLEncoding,
+	} {
+		if b, err := enc.DecodeString(pin); err == nil && len(b) == sha256.Size {
+			return hex.EncodeToString(b)
+		}
+	}
+	return pin
+}
+
 func applyShareRealityParams(stream map[string]any, params map[string]string) {
 	params["security"] = "reality"
 	realitySetting, _ := stream["realitySettings"].(map[string]any)

sub/subService_test.go

@@ -780,6 +780,40 @@ func TestHasFinalMaskContent(t *testing.T) {
 	}
 }
 
+func TestHysteriaPinHex(t *testing.T) {
+	const hexPin = "c847dd2395d0978c0780b8201c4b289a8b281597d47c275f2d77d3f96d8de9c4"
+
+	cases := []struct {
+		name string
+		in   string
+		want string
+	}{
+		// Std base64 (xray-core's native TLS format / the panel generate button)
+		// must be re-encoded to the hex form Hysteria2 clients expect (#4818).
+		{"std base64", "yEfdI5XQl4wHgLggHEsomosoFZfUfCdfLXfT+W2N6cQ=", hexPin},
+		// A manually pasted hex fingerprint passes through (lowercased).
+		{"hex passthrough", hexPin, hexPin},
+		{"uppercase hex lowercased", strings.ToUpper(hexPin), hexPin},
+		// openssl x509 -fingerprint -sha256 emits colon-separated hex.
+		{"colon hex stripped", "C8:47:DD:23:95:D0:97:8C:07:80:B8:20:1C:4B:28:9A:8B:28:15:97:D4:7C:27:5F:2D:77:D3:F9:6D:8D:E9:C4", hexPin},
+		{"surrounding whitespace trimmed", "  " + hexPin + "  ", hexPin},
+		// URL-safe base64 with the same 32 bytes decodes identically.
+		{"url-safe base64", "yEfdI5XQl4wHgLggHEsomosoFZfUfCdfLXfT-W2N6cQ=", hexPin},
+		// Garbage that is neither valid hex nor a 32-byte base64 is left as-is
+		// rather than silently dropped.
+		{"unrecognized passthrough", "not-a-pin", "not-a-pin"},
+		{"empty", "", ""},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := hysteriaPinHex(tc.in); got != tc.want {
+				t.Fatalf("hysteriaPinHex(%q) = %q, want %q", tc.in, got, tc.want)
+			}
+		})
+	}
+}
+
 func TestHysteriaHopPorts(t *testing.T) {
 	withHop := func(ports any) map[string]any {
 		return map[string]any{