feat(sub): auto-detect subscription format by User-Agent (Updated) (#5826)

* feat(settings): add subscription format controls * feat(sub): auto-detect subscription formats * fix(xray): validate balancer regexes before save * Revert "fix(xray): validate balancer regexes before save" This reverts commit 8a208ce71b. * doc(endpoints): align indent spaces * doc(settings): improve error message formatting in validateSubUserAgentRegex - Use NewErrorf with proper formatting instead of NewError with string concatenation - Add comment explaining the rationale for returning original pattern value - This preserves the intentional design where empty input is stored as empty in the DB and inherited as the runtime default at read time --------- Co-authored-by: Tomilla <5007859+Tomilla@users.noreply.github.com> Co-authored-by: Sanaei <ho3ein.sanaei@gmail.com>
2026-08-11 22:00:59 +00:00 · 2026-07-14 19:01:40 +08:00
parent f2b17397f4
commit 129f50d92a
40 changed files with 1588 additions and 89 deletions
@@ -9,6 +9,7 @@ import (
 	"net/http"
 	"os"
 	"reflect"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -32,6 +33,13 @@ import (
 //go:embed config.json
 var xrayTemplateConfig string

+const (
+	DefaultSubClashUserAgentRegex = `(?i)(clash|mihomo)`
+	DefaultSubJsonUserAgentRegex  = ``
+	DefaultRemarkTemplate         = "{{INBOUND}}-{{EMAIL}}|📊{{TRAFFIC_LEFT}}|⏳{{DAYS_LEFT}}D"
+	maxRegexLength                = 2048
+)
+
 var defaultValueMap = map[string]string{
 	"xrayTemplateConfig": xrayTemplateConfig,
 	"webListen":          "",
@@ -57,7 +65,7 @@ var defaultValueMap = map[string]string{
 	"pageSize":                    "25",
 	"expireDiff":                  "0",
 	"trafficDiff":                 "0",
-	"remarkTemplate":              "{{INBOUND}}-{{EMAIL}}|📊{{TRAFFIC_LEFT}}|⏳{{DAYS_LEFT}}D",
+	"remarkTemplate":              DefaultRemarkTemplate,
 	"timeLocation":                "Local",
 	"tgBotEnable":                 "false",
 	"tgBotToken":                  "",
@@ -73,6 +81,11 @@ var defaultValueMap = map[string]string{
 	"twoFactorToken":              "",
 	"subEnable":                   "true",
 	"subJsonEnable":               "false",
+	"subJsonAutoDetect":           "false",
+	"subJsonAlwaysArray":          "false",
+	"subJsonUserAgentRegex":       "",
+	"subClashAutoDetect":          "false",
+	"subClashUserAgentRegex":      "",
 	"subTitle":                    "",
 	"subSupportUrl":               "",
 	"subProfileUrl":               "",
@@ -707,6 +720,26 @@ func (s *SettingService) GetSubJsonEnable() (bool, error) {
 	return s.getBool("subJsonEnable")
 }

+func (s *SettingService) GetSubJsonAutoDetect() (bool, error) {
+	return s.getBool("subJsonAutoDetect")
+}
+
+func (s *SettingService) GetSubJsonAlwaysArray() (bool, error) {
+	return s.getBool("subJsonAlwaysArray")
+}
+
+func (s *SettingService) GetSubJsonUserAgentRegex() (string, error) {
+	return s.getString("subJsonUserAgentRegex")
+}
+
+func (s *SettingService) GetSubClashAutoDetect() (bool, error) {
+	return s.getBool("subClashAutoDetect")
+}
+
+func (s *SettingService) GetSubClashUserAgentRegex() (string, error) {
+	return s.getString("subClashUserAgentRegex")
+}
+
 func (s *SettingService) GetSubTitle() (string, error) {
 	return s.getString("subTitle")
 }
@@ -1119,6 +1152,9 @@ func (s *SettingService) UpdateAllSetting(allSetting *entity.AllSetting, clears
 	if err := validateSettingsURLs(allSetting); err != nil {
 		return err
 	}
+	if err := validateSubUserAgentRegexes(allSetting); err != nil {
+		return err
+	}
 	if err := allSetting.CheckValid(); err != nil {
 		return err
 	}
@@ -1159,6 +1195,48 @@ func (s *SettingService) UpdateAllSetting(allSetting *entity.AllSetting, clears
 	})
 }

+func validateSubUserAgentRegexes(allSetting *entity.AllSetting) error {
+	jsonPattern, err := validateSubUserAgentRegex("Xray JSON", allSetting.SubJsonUserAgentRegex, DefaultSubJsonUserAgentRegex)
+	if err != nil {
+		return err
+	}
+	clashPattern, err := validateSubUserAgentRegex("Clash/Mihomo", allSetting.SubClashUserAgentRegex, DefaultSubClashUserAgentRegex)
+	if err != nil {
+		return err
+	}
+	allSetting.SubJsonUserAgentRegex = jsonPattern
+	allSetting.SubClashUserAgentRegex = clashPattern
+	return nil
+}
+
+func validateSubUserAgentRegex(name, pattern, defaultPattern string) (string, error) {
+	pattern = strings.TrimSpace(pattern)
+	effectivePattern := pattern
+	if effectivePattern == "" {
+		effectivePattern = defaultPattern
+	}
+	if len(effectivePattern) > maxRegexLength {
+		return "", common.NewErrorf("%s User-Agent regex must not exceed %d characters", name, maxRegexLength)
+	}
+	if _, err := regexp.Compile(effectivePattern); err != nil {
+		return "", common.NewErrorf("%s User-Agent regex is invalid: %v", name, err)
+	}
+	// Return the original pattern (empty string if cleared) so the caller
+	// can distinguish "user explicitly set empty" from "user set a value".
+	// The empty value is stored in the DB and inherited as runtime default.
+	return pattern, nil
+}
+
+func ValidateRegex(pattern string) error {
+	if len(pattern) > maxRegexLength {
+		return common.NewErrorf("Regular expression must not exceed %d characters", maxRegexLength)
+	}
+	if _, err := regexp.Compile(pattern); err != nil {
+		return common.NewError("Regular expression is invalid:", err)
+	}
+	return nil
+}
+
 func (s *SettingService) preserveRedactedSecrets(allSetting *entity.AllSetting, clears SecretClears) error {
 	if !clears.TgBotToken && strings.TrimSpace(allSetting.TgBotToken) == "" {
 		value, err := s.GetTgBotToken()
@@ -0,0 +1,207 @@
+package service
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/mhsanaei/3x-ui/v3/internal/database"
+	"github.com/mhsanaei/3x-ui/v3/internal/database/model"
+)
+
+func TestValidateRegex(t *testing.T) {
+	for _, pattern := range []string{"", `(?i)^jsonclient([ /]|$)`, `(?m)^general-purpose$`} {
+		if err := ValidateRegex(pattern); err != nil {
+			t.Errorf("ValidateRegex(%q) returned %v", pattern, err)
+		}
+	}
+	for _, pattern := range []string{"[", strings.Repeat("a", 2049)} {
+		if err := ValidateRegex(pattern); err == nil {
+			t.Errorf("ValidateRegex(%q) accepted an invalid pattern", pattern)
+		}
+	}
+}
+
+func TestSubscriptionAutoDetectDefaultsWithoutStoredRows(t *testing.T) {
+	setupSettingTestDB(t)
+	keys := []string{"subClashAutoDetect", "subClashUserAgentRegex", "subJsonAutoDetect", "subJsonAlwaysArray", "subJsonUserAgentRegex"}
+	if err := database.GetDB().Where("key IN ?", keys).Delete(&model.Setting{}).Error; err != nil {
+		t.Fatal(err)
+	}
+
+	s := &SettingService{}
+	clashEnabled, err := s.GetSubClashAutoDetect()
+	if err != nil {
+		t.Fatal(err)
+	}
+	jsonEnabled, err := s.GetSubJsonAutoDetect()
+	if err != nil {
+		t.Fatal(err)
+	}
+	jsonAlwaysArray, err := s.GetSubJsonAlwaysArray()
+	if err != nil {
+		t.Fatal(err)
+	}
+	clashRegex, err := s.GetSubClashUserAgentRegex()
+	if err != nil {
+		t.Fatal(err)
+	}
+	jsonRegex, err := s.GetSubJsonUserAgentRegex()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if clashEnabled || jsonEnabled || jsonAlwaysArray {
+		t.Fatalf("missing subscription flags must default off: clashAuto=%v jsonAuto=%v jsonAlwaysArray=%v", clashEnabled, jsonEnabled, jsonAlwaysArray)
+	}
+	if clashRegex != "" {
+		t.Fatalf("missing Clash regex = %q, want empty inherited value", clashRegex)
+	}
+	if jsonRegex != "" {
+		t.Fatalf("missing JSON regex = %q, want empty inherited value", jsonRegex)
+	}
+
+	var count int64
+	if err := database.GetDB().Model(&model.Setting{}).Where("key IN ?", keys).Count(&count).Error; err != nil {
+		t.Fatal(err)
+	}
+	if count != 0 {
+		t.Fatalf("default lookup unexpectedly persisted %d setting rows", count)
+	}
+}
+
+func TestUpdateAllSettingPreservesEmptyUserAgentRegexes(t *testing.T) {
+	setupSettingTestDB(t)
+	s := &SettingService{}
+	settings, err := s.GetAllSetting()
+	if err != nil {
+		t.Fatal(err)
+	}
+	settings.SubJsonUserAgentRegex = "   "
+	settings.SubClashUserAgentRegex = ""
+
+	if err := s.UpdateAllSetting(settings, SecretClears{}); err != nil {
+		t.Fatal(err)
+	}
+
+	for _, key := range []string{"subJsonUserAgentRegex", "subClashUserAgentRegex"} {
+		var stored model.Setting
+		if err := database.GetDB().Where("key = ?", key).First(&stored).Error; err != nil {
+			t.Fatal(err)
+		}
+		if stored.Value != "" {
+			t.Fatalf("%s stored value = %q, want empty inherited value", key, stored.Value)
+		}
+	}
+}
+
+func TestUpdateAllSettingPersistsClashSubscriptionSettings(t *testing.T) {
+	setupSettingTestDB(t)
+	s := &SettingService{}
+
+	settings, err := s.GetAllSetting()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if settings.SubClashAutoDetect {
+		t.Fatal("subClashAutoDetect default = true, want false")
+	}
+	if settings.SubJsonAutoDetect {
+		t.Fatal("subJsonAutoDetect default = true, want false")
+	}
+	if settings.SubJsonAlwaysArray {
+		t.Fatal("subJsonAlwaysArray default = true, want false")
+	}
+	if settings.SubJsonUserAgentRegex != "" {
+		t.Fatalf("subJsonUserAgentRegex = %q, want empty inherited value", settings.SubJsonUserAgentRegex)
+	}
+	if settings.SubClashUserAgentRegex != "" {
+		t.Fatalf("subClashUserAgentRegex = %q, want empty inherited value", settings.SubClashUserAgentRegex)
+	}
+	settings.SubClashAutoDetect = true
+	settings.SubClashUserAgentRegex = `(?i)^custom-clash/`
+	settings.SubJsonAutoDetect = true
+	settings.SubJsonAlwaysArray = true
+	settings.SubJsonUserAgentRegex = `(?i)^custom-json/`
+	settings.SubJsonEnable = true
+	settings.SubJsonPath = "/json-custom/"
+	settings.SubJsonURI = "https://subscriptions.example.com/json-custom/"
+	settings.SubClashEnable = true
+	settings.SubClashPath = "/clash-custom/"
+	settings.SubClashURI = "https://subscriptions.example.com/clash-custom/"
+	settings.SubClashEnableRouting = true
+	settings.SubClashRules = "GEOIP,private,DIRECT"
+
+	if err := s.UpdateAllSetting(settings, SecretClears{}); err != nil {
+		t.Fatal(err)
+	}
+
+	got, err := s.GetAllSetting()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !got.SubClashEnable {
+		t.Fatal("subClashEnable = false, want true")
+	}
+	if !got.SubClashAutoDetect {
+		t.Fatal("subClashAutoDetect = false, want true")
+	}
+	if !got.SubJsonAutoDetect {
+		t.Fatal("subJsonAutoDetect = false, want true")
+	}
+	if !got.SubJsonAlwaysArray {
+		t.Fatal("subJsonAlwaysArray = false, want true")
+	}
+	if !got.SubJsonEnable {
+		t.Fatal("subJsonEnable = false, want true")
+	}
+	if got.SubJsonPath != "/json-custom/" {
+		t.Fatalf("subJsonPath = %q, want %q", got.SubJsonPath, "/json-custom/")
+	}
+	if got.SubJsonURI != "https://subscriptions.example.com/json-custom/" {
+		t.Fatalf("subJsonURI = %q, want %q", got.SubJsonURI, "https://subscriptions.example.com/json-custom/")
+	}
+	if got.SubJsonUserAgentRegex != `(?i)^custom-json/` {
+		t.Fatalf("subJsonUserAgentRegex = %q, want %q", got.SubJsonUserAgentRegex, `(?i)^custom-json/`)
+	}
+	if got.SubClashUserAgentRegex != `(?i)^custom-clash/` {
+		t.Fatalf("subClashUserAgentRegex = %q, want %q", got.SubClashUserAgentRegex, `(?i)^custom-clash/`)
+	}
+	if got.SubClashPath != "/clash-custom/" {
+		t.Fatalf("subClashPath = %q, want %q", got.SubClashPath, "/clash-custom/")
+	}
+	if got.SubClashURI != "https://subscriptions.example.com/clash-custom/" {
+		t.Fatalf("subClashURI = %q, want %q", got.SubClashURI, "https://subscriptions.example.com/clash-custom/")
+	}
+	if !got.SubClashEnableRouting {
+		t.Fatal("subClashEnableRouting = false, want true")
+	}
+	if got.SubClashRules != "GEOIP,private,DIRECT" {
+		t.Fatalf("subClashRules = %q, want %q", got.SubClashRules, "GEOIP,private,DIRECT")
+	}
+}
+
+func TestUpdateAllSettingRejectsInvalidClashUserAgentRegex(t *testing.T) {
+	setupSettingTestDB(t)
+	s := &SettingService{}
+	settings, err := s.GetAllSetting()
+	if err != nil {
+		t.Fatal(err)
+	}
+	settings.SubClashUserAgentRegex = "["
+	if err := s.UpdateAllSetting(settings, SecretClears{}); err == nil {
+		t.Fatal("UpdateAllSetting accepted an invalid Clash/Mihomo User-Agent regex")
+	}
+}
+
+func TestUpdateAllSettingRejectsInvalidJsonUserAgentRegex(t *testing.T) {
+	setupSettingTestDB(t)
+	s := &SettingService{}
+	settings, err := s.GetAllSetting()
+	if err != nil {
+		t.Fatal(err)
+	}
+	settings.SubJsonUserAgentRegex = "["
+	if err := s.UpdateAllSetting(settings, SecretClears{}); err == nil {
+		t.Fatal("UpdateAllSetting accepted an invalid Xray JSON User-Agent regex")
+	}
+}