fix(nodes): apply a rotated master mTLS certificate without restarting the panel (#6194)

* fix(mtls): invalidate pooled clients after credential rotation * fix(mtls): make connection reload read-only --------- Co-authored-by: n0ctal <293235942+n0ctal@users.noreply.github.com>
2026-08-16 00:01:02 +00:00 · 2026-08-15 19:03:42 +05:00
parent 1230559e69
commit 7ecd88b9e3
24 changed files with 747 additions and 101 deletions
@@ -9,6 +9,7 @@ import (
 	"net/http"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"

 	"github.com/mhsanaei/3x-ui/v3/internal/database/model"
@@ -25,6 +26,7 @@ type MasterClientCertProvider func() (tls.Certificate, error)
 var (
 	masterClientCertMu sync.RWMutex
 	masterClientCert   MasterClientCertProvider
+	masterCertEpoch    atomic.Uint64
 )

 // SetMasterClientCertProvider installs the provider used to obtain the master
@@ -45,6 +47,91 @@ func getMasterClientCert() (tls.Certificate, error) {
 	return p()
 }

+// InvalidateMasterClientConnections advances the client-credential generation.
+// Every cached mTLS transport observes the generation before its next request,
+// replaces its TLS transport, and closes the old idle pool. Requests already
+// in flight are not interrupted; no request that starts after invalidation can
+// reuse a connection authenticated with the previous leaf.
+func InvalidateMasterClientConnections() {
+	masterCertEpoch.Add(1)
+}
+
+// ReloadMasterClientConnections validates that the currently configured
+// provider can load the master credential, then invalidates every cached mTLS
+// transport. Operators that rotate the credential outside the process (for
+// example by restoring settings) can call this without restarting the panel.
+func ReloadMasterClientConnections() error {
+	if _, err := getMasterClientCert(); err != nil {
+		return err
+	}
+	InvalidateMasterClientConnections()
+	return nil
+}
+
+type idleClosingRoundTripper interface {
+	http.RoundTripper
+	CloseIdleConnections()
+}
+
+type credentialRotatingTransport struct {
+	mu         sync.Mutex
+	generation uint64
+	current    idleClosingRoundTripper
+	build      func() (idleClosingRoundTripper, error)
+}
+
+func buildStableCredentialTransport(build func() (idleClosingRoundTripper, error)) (idleClosingRoundTripper, uint64, error) {
+	for {
+		before := masterCertEpoch.Load()
+		current, err := build()
+		if err != nil {
+			return nil, 0, err
+		}
+		after := masterCertEpoch.Load()
+		if before == after {
+			return current, after, nil
+		}
+		current.CloseIdleConnections()
+	}
+}
+
+func newCredentialRotatingTransport(build func() (idleClosingRoundTripper, error)) (*credentialRotatingTransport, error) {
+	current, generation, err := buildStableCredentialTransport(build)
+	if err != nil {
+		return nil, err
+	}
+	return &credentialRotatingTransport{
+		generation: generation,
+		current:    current,
+		build:      build,
+	}, nil
+}
+
+func (t *credentialRotatingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	t.mu.Lock()
+	if masterCertEpoch.Load() != t.generation {
+		next, generation, err := buildStableCredentialTransport(t.build)
+		if err != nil {
+			t.mu.Unlock()
+			return nil, err
+		}
+		previous := t.current
+		t.current = next
+		t.generation = generation
+		previous.CloseIdleConnections()
+	}
+	current := t.current
+	t.mu.Unlock()
+	return current.RoundTrip(req)
+}
+
+func (t *credentialRotatingTransport) CloseIdleConnections() {
+	t.mu.Lock()
+	current := t.current
+	t.mu.Unlock()
+	current.CloseIdleConnections()
+}
+
 // defaultNodeHTTPClient reaches nodes trusting the system CA store ("verify"
 // mode or plain http); shared so connections pool across nodes.
 var defaultNodeHTTPClient = &http.Client{
@@ -62,6 +149,30 @@ func HTTPClientForNode(n *model.Node, proxyURL string) (*http.Client, error) {
 		mode = "verify"
 	}
 	if proxyURL != "" {
+		if mode == "mtls" && n.Scheme != "http" {
+			timeout := remoteHTTPTimeout
+			build := func() (idleClosingRoundTripper, error) {
+				client, err := netproxy.NewHTTPClient(proxyURL, remoteHTTPTimeout)
+				if err != nil {
+					return nil, err
+				}
+				transport, ok := client.Transport.(*http.Transport)
+				if !ok {
+					return nil, common.NewError("mtls proxy client transport does not support credential rotation")
+				}
+				tlsCfg, err := tlsConfigForNode(n)
+				if err != nil {
+					return nil, err
+				}
+				transport.TLSClientConfig = tlsCfg
+				return transport, nil
+			}
+			transport, err := newCredentialRotatingTransport(build)
+			if err != nil {
+				return nil, err
+			}
+			return &http.Client{Transport: transport, Timeout: timeout}, nil
+		}
 		client, err := netproxy.NewHTTPClient(proxyURL, remoteHTTPTimeout)
 		if err != nil {
 			return nil, err
@@ -83,6 +194,26 @@ func HTTPClientForNode(n *model.Node, proxyURL string) (*http.Client, error) {
 	if mode == "verify" || n.Scheme == "http" {
 		return defaultNodeHTTPClient, nil
 	}
+	if mode == "mtls" {
+		build := func() (idleClosingRoundTripper, error) {
+			tlsCfg, err := tlsConfigForNode(n)
+			if err != nil {
+				return nil, err
+			}
+			return &http.Transport{
+				MaxIdleConns:        64,
+				MaxIdleConnsPerHost: 4,
+				IdleConnTimeout:     60 * time.Second,
+				DialContext:         netsafe.SSRFGuardedDialContext,
+				TLSClientConfig:     tlsCfg,
+			}, nil
+		}
+		transport, err := newCredentialRotatingTransport(build)
+		if err != nil {
+			return nil, err
+		}
+		return &http.Client{Transport: transport}, nil
+	}
 	tlsCfg, err := tlsConfigForNode(n)
 	if err != nil {
 		return nil, err
@@ -11,12 +11,245 @@ import (
 	"net/url"
 	"strconv"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"testing"

 	"github.com/mhsanaei/3x-ui/v3/internal/database/model"
 	"github.com/mhsanaei/3x-ui/v3/internal/util/crypto"
 )

+type generationProbeTransport struct {
+	id     string
+	closed atomic.Int32
+}
+
+func (t *generationProbeTransport) RoundTrip(*http.Request) (*http.Response, error) {
+	return &http.Response{
+		StatusCode: http.StatusOK,
+		Body:       http.NoBody,
+		Header:     make(http.Header),
+		Request:    &http.Request{},
+	}, nil
+}
+
+func (t *generationProbeTransport) CloseIdleConnections() {
+	t.closed.Add(1)
+}
+
+func TestCredentialRotatingTransportDropsOldPoolBeforeNextRequest(t *testing.T) {
+	var selected atomic.Pointer[generationProbeTransport]
+	oldTransport := &generationProbeTransport{id: "old"}
+	newTransport := &generationProbeTransport{id: "new"}
+	selected.Store(oldTransport)
+
+	rotating, err := newCredentialRotatingTransport(func() (idleClosingRoundTripper, error) {
+		return selected.Load(), nil
+	})
+	if err != nil {
+		t.Fatalf("newCredentialRotatingTransport: %v", err)
+	}
+	rotating.mu.Lock()
+	initial := rotating.current
+	rotating.mu.Unlock()
+	if initial != oldTransport {
+		t.Fatalf("initial transport = %p, want old %p", initial, oldTransport)
+	}
+
+	selected.Store(newTransport)
+	InvalidateMasterClientConnections()
+
+	req := httptest.NewRequest(http.MethodGet, "https://node.example.test/panel/api/server/status", nil)
+	resp, err := rotating.RoundTrip(req)
+	if err != nil {
+		t.Fatalf("RoundTrip after credential rotation: %v", err)
+	}
+	_ = resp.Body.Close()
+
+	rotating.mu.Lock()
+	current := rotating.current
+	rotating.mu.Unlock()
+	if current != newTransport {
+		t.Fatalf("transport after invalidation = %p, want new %p", current, newTransport)
+	}
+	if got := oldTransport.closed.Load(); got != 1 {
+		t.Fatalf("old transport CloseIdleConnections calls = %d, want 1", got)
+	}
+}
+
+func TestReloadMasterClientConnectionsValidatesProviderBeforeInvalidation(t *testing.T) {
+	before := masterCertEpoch.Load()
+	SetMasterClientCertProvider(func() (tls.Certificate, error) {
+		return tls.Certificate{}, context.Canceled
+	})
+	if err := ReloadMasterClientConnections(); err == nil {
+		t.Fatal("reload with an invalid provider unexpectedly succeeded")
+	}
+	if got := masterCertEpoch.Load(); got != before {
+		t.Fatalf("failed reload changed generation from %d to %d", before, got)
+	}
+
+	SetMasterClientCertProvider(func() (tls.Certificate, error) {
+		return masterCertForTest(t), nil
+	})
+	t.Cleanup(func() { SetMasterClientCertProvider(nil) })
+	if err := ReloadMasterClientConnections(); err != nil {
+		t.Fatalf("ReloadMasterClientConnections: %v", err)
+	}
+	if got := masterCertEpoch.Load(); got != before+1 {
+		t.Fatalf("successful reload generation = %d, want %d", got, before+1)
+	}
+}
+
+func TestCredentialRotatingTransportRejectsBuildAcrossInvalidation(t *testing.T) {
+	oldTransport := &generationProbeTransport{id: "old"}
+	newTransport := &generationProbeTransport{id: "new"}
+	var selected atomic.Pointer[generationProbeTransport]
+	selected.Store(oldTransport)
+
+	firstBuildCaptured := make(chan struct{})
+	releaseFirstBuild := make(chan struct{})
+	var once sync.Once
+	build := func() (idleClosingRoundTripper, error) {
+		captured := selected.Load()
+		once.Do(func() {
+			close(firstBuildCaptured)
+			<-releaseFirstBuild
+		})
+		return captured, nil
+	}
+
+	type result struct {
+		transport *credentialRotatingTransport
+		err       error
+	}
+	resultCh := make(chan result, 1)
+	go func() {
+		transport, err := newCredentialRotatingTransport(build)
+		resultCh <- result{transport: transport, err: err}
+	}()
+
+	<-firstBuildCaptured
+	selected.Store(newTransport)
+	InvalidateMasterClientConnections()
+	close(releaseFirstBuild)
+
+	got := <-resultCh
+	if got.err != nil {
+		t.Fatalf("newCredentialRotatingTransport: %v", got.err)
+	}
+	got.transport.mu.Lock()
+	current := got.transport.current
+	got.transport.mu.Unlock()
+	if current != newTransport {
+		t.Fatalf("transport built across invalidation = %p, want new %p", current, newTransport)
+	}
+	if calls := oldTransport.closed.Load(); calls != 1 {
+		t.Fatalf("stale transport CloseIdleConnections calls = %d, want 1", calls)
+	}
+}
+
+func TestHTTPClientForNodeMTLSRebuildsTLSConfigAfterCredentialInvalidation(t *testing.T) {
+	oldCert := masterCertForTest(t)
+	newCert := masterCertForTest(t)
+	selected := oldCert
+	SetMasterClientCertProvider(func() (tls.Certificate, error) { return selected, nil })
+	t.Cleanup(func() { SetMasterClientCertProvider(nil) })
+
+	client, err := HTTPClientForNode(&model.Node{
+		Scheme:        "https",
+		Address:       "node.example.test",
+		Port:          443,
+		TlsVerifyMode: "mtls",
+	}, "")
+	if err != nil {
+		t.Fatalf("HTTPClientForNode: %v", err)
+	}
+	rotating, ok := client.Transport.(*credentialRotatingTransport)
+	if !ok {
+		t.Fatalf("transport = %T, want *credentialRotatingTransport", client.Transport)
+	}
+	leaf := func() []byte {
+		rotating.mu.Lock()
+		defer rotating.mu.Unlock()
+		transport, ok := rotating.current.(*http.Transport)
+		if !ok {
+			t.Fatalf("current transport = %T, want *http.Transport", rotating.current)
+		}
+		return transport.TLSClientConfig.Certificates[0].Certificate[0]
+	}
+	if got := leaf(); string(got) != string(oldCert.Certificate[0]) {
+		t.Fatal("initial TLS config does not contain the old credential")
+	}
+
+	selected = newCert
+	InvalidateMasterClientConnections()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://node.example.test/", nil)
+	if err != nil {
+		t.Fatalf("NewRequestWithContext: %v", err)
+	}
+	if _, err := client.Do(req); err == nil {
+		t.Fatal("canceled request unexpectedly succeeded")
+	}
+	if got := leaf(); string(got) != string(newCert.Certificate[0]) {
+		t.Fatal("TLS config retained the old credential after invalidation")
+	}
+}
+
+func TestHTTPClientForNodeProxyMTLSRebuildKeepsProxyAndNewCredential(t *testing.T) {
+	oldCert := masterCertForTest(t)
+	newCert := masterCertForTest(t)
+	selected := oldCert
+	SetMasterClientCertProvider(func() (tls.Certificate, error) { return selected, nil })
+	t.Cleanup(func() { SetMasterClientCertProvider(nil) })
+
+	const proxyURL = "http://127.0.0.1:18080"
+	client, err := HTTPClientForNode(&model.Node{Scheme: "https", TlsVerifyMode: "mtls"}, proxyURL)
+	if err != nil {
+		t.Fatalf("HTTPClientForNode: %v", err)
+	}
+	rotating, ok := client.Transport.(*credentialRotatingTransport)
+	if !ok {
+		t.Fatalf("transport = %T, want rotating transport", client.Transport)
+	}
+	current := func() *http.Transport {
+		rotating.mu.Lock()
+		defer rotating.mu.Unlock()
+		transport, ok := rotating.current.(*http.Transport)
+		if !ok {
+			t.Fatalf("current transport = %T, want *http.Transport", rotating.current)
+		}
+		return transport
+	}
+	assertProxy := func(transport *http.Transport) {
+		t.Helper()
+		if transport.Proxy == nil {
+			t.Fatalf("proxy function is nil, want %s", proxyURL)
+		}
+		req, _ := http.NewRequest(http.MethodGet, "https://node.example.test/", nil)
+		got, err := transport.Proxy(req)
+		if err != nil || got == nil || got.String() != proxyURL {
+			t.Fatalf("proxy = %v, error = %v, want %s", got, err, proxyURL)
+		}
+	}
+	assertProxy(current())
+
+	selected = newCert
+	InvalidateMasterClientConnections()
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	req, _ := http.NewRequestWithContext(ctx, http.MethodGet, "https://node.example.test/", nil)
+	_, _ = client.Do(req)
+	rebuilt := current()
+	assertProxy(rebuilt)
+	if got := rebuilt.TLSClientConfig.Certificates[0].Certificate[0]; string(got) != string(newCert.Certificate[0]) {
+		t.Fatal("proxy mTLS rebuild retained the old credential")
+	}
+}
+
 // masterCertForTest builds a real CA-signed client certificate for mtls tests.
 func masterCertForTest(t *testing.T) tls.Certificate {
 	t.Helper()
@@ -0,0 +1,162 @@
+package runtime
+
+import (
+	"crypto/sha256"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/hex"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"testing"
+	"time"
+)
+
+type wireObservation struct {
+	pin        string
+	remoteAddr string
+}
+
+func startLeafRecordingServer(t *testing.T) (*httptest.Server, *x509.CertPool, func() []wireObservation) {
+	t.Helper()
+	var mu sync.Mutex
+	var seen []wireObservation
+	srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		observation := wireObservation{remoteAddr: r.RemoteAddr}
+		if r.TLS != nil && len(r.TLS.PeerCertificates) > 0 {
+			sum := sha256.Sum256(r.TLS.PeerCertificates[0].Raw)
+			observation.pin = hex.EncodeToString(sum[:])
+		}
+		mu.Lock()
+		seen = append(seen, observation)
+		mu.Unlock()
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte("ok"))
+	}))
+	srv.TLS = &tls.Config{ClientAuth: tls.RequestClientCert}
+	srv.StartTLS()
+	t.Cleanup(srv.Close)
+	pool := x509.NewCertPool()
+	pool.AddCert(srv.Certificate())
+	return srv, pool, func() []wireObservation {
+		mu.Lock()
+		defer mu.Unlock()
+		result := make([]wireObservation, len(seen))
+		copy(result, seen)
+		return result
+	}
+}
+
+func pinOf(t *testing.T, cert tls.Certificate) string {
+	t.Helper()
+	sum := sha256.Sum256(cert.Certificate[0])
+	return hex.EncodeToString(sum[:])
+}
+
+func rotatingClientForTest(t *testing.T, roots *x509.CertPool) *http.Client {
+	t.Helper()
+	build := func() (idleClosingRoundTripper, error) {
+		cert, err := getMasterClientCert()
+		if err != nil {
+			return nil, err
+		}
+		return &http.Transport{
+			MaxIdleConns:        64,
+			MaxIdleConnsPerHost: 4,
+			IdleConnTimeout:     60 * time.Second,
+			TLSClientConfig: &tls.Config{
+				Certificates: []tls.Certificate{cert},
+				RootCAs:      roots,
+				MinVersion:   tls.VersionTLS12,
+			},
+		}, nil
+	}
+	transport, err := newCredentialRotatingTransport(build)
+	if err != nil {
+		t.Fatalf("newCredentialRotatingTransport: %v", err)
+	}
+	return &http.Client{Transport: transport, Timeout: 10 * time.Second}
+}
+
+func doWireRequest(t *testing.T, client *http.Client, url string) {
+	t.Helper()
+	response, err := client.Get(url)
+	if err != nil {
+		t.Fatalf("request: %v", err)
+	}
+	_, _ = io.Copy(io.Discard, response.Body)
+	_ = response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		t.Fatalf("status=%d want=%d", response.StatusCode, http.StatusOK)
+	}
+}
+
+func TestCredentialRotationPresentsNewLeafOnNextConnection(t *testing.T) {
+	server, roots, observations := startLeafRecordingServer(t)
+	oldCert := masterCertForTest(t)
+	newCert := masterCertForTest(t)
+	oldPin := pinOf(t, oldCert)
+	newPin := pinOf(t, newCert)
+	if oldPin == newPin {
+		t.Fatal("test fixture produced identical leaves")
+	}
+	var providerMu sync.Mutex
+	current := oldCert
+	SetMasterClientCertProvider(func() (tls.Certificate, error) {
+		providerMu.Lock()
+		defer providerMu.Unlock()
+		return current, nil
+	})
+	t.Cleanup(func() { SetMasterClientCertProvider(nil) })
+	client := rotatingClientForTest(t, roots)
+	doWireRequest(t, client, server.URL)
+	doWireRequest(t, client, server.URL)
+	baseline := observations()
+	if len(baseline) != 2 || baseline[0].pin != oldPin || baseline[1].pin != oldPin {
+		t.Fatalf("baseline=%v", baseline)
+	}
+	if baseline[0].remoteAddr != baseline[1].remoteAddr {
+		t.Fatalf("baseline connections differ: %v", baseline)
+	}
+	providerMu.Lock()
+	current = newCert
+	providerMu.Unlock()
+	InvalidateMasterClientConnections()
+	doWireRequest(t, client, server.URL)
+	after := observations()
+	if len(after) != 3 || after[2].pin != newPin {
+		t.Fatalf("rotation observations=%v want new leaf=%s", after, newPin)
+	}
+	if after[2].remoteAddr == baseline[1].remoteAddr {
+		t.Fatalf("rotated request reused stale connection %s", after[2].remoteAddr)
+	}
+}
+
+func TestCredentialRotationControlKeepsOldLeafWithoutInvalidation(t *testing.T) {
+	server, roots, observations := startLeafRecordingServer(t)
+	oldCert := masterCertForTest(t)
+	newCert := masterCertForTest(t)
+	oldPin := pinOf(t, oldCert)
+	var providerMu sync.Mutex
+	current := oldCert
+	SetMasterClientCertProvider(func() (tls.Certificate, error) {
+		providerMu.Lock()
+		defer providerMu.Unlock()
+		return current, nil
+	})
+	t.Cleanup(func() { SetMasterClientCertProvider(nil) })
+	client := rotatingClientForTest(t, roots)
+	doWireRequest(t, client, server.URL)
+	providerMu.Lock()
+	current = newCert
+	providerMu.Unlock()
+	doWireRequest(t, client, server.URL)
+	got := observations()
+	if len(got) != 2 || got[1].pin != oldPin {
+		t.Fatalf("control observations=%v want stale leaf=%s", got, oldPin)
+	}
+	if got[0].remoteAddr != got[1].remoteAddr {
+		t.Fatalf("control did not reuse connection: %v", got)
+	}
+}