perf(clients): apply a multi-inbound client create concurrently

Creating or attaching a client across N inbounds called AddInboundClient
once per inbound, strictly one after another. When those inbounds live on
different nodes each call is a full node round-trip bounded by the 10s
remote timeout, so the request cost the SUM of every node's latency: two
nodes felt instant, three took ~13s and timed out bot callers, which is
how it surfaced as "two out of four account creations fail".

Split the per-inbound preparation from the apply. Preparation stays
ordered and single-threaded because fillProtocolDefaults mints the shared
credentials on the first inbound and every later one reuses them; the
applies then run concurrently, capped at inboundFanoutConcurrency. A
4-node create measured 1.205s -> 0.307s with peak overlap 1 -> 4.

Consequences of no longer aborting at the first failing inbound:

- Every apply error is tagged with its inbound and the failures are
  joined, so all of them reach the caller instead of just the first.
- The fanout goroutines recover their own panics. Off the request
  goroutine gin's Recovery no longer covers them, and an unrecovered
  panic would kill the panel rather than fail one inbound.
- A partly-applied call commits clients on the inbounds that succeeded,
  so the controller and the LDAP job now read needRestart before the
  error check; otherwise Xray was never flagged for the work that landed.
- limitHwid is applied only when every inbound succeeded. Applying it
  after a failure rewrites limit_hwid and trims the registered devices of
  an email that already existed, which is silent data loss on an
  operation the panel reported as failed.

Update the API docs for the new partial-application contract and the
inbound-tagged error strings.

internal/web/service/client_create_fanout_test.go

@@ -1,9 +1,16 @@
 package service
 
 import (
+	"context"
+	"fmt"
+	"strings"
+	"sync/atomic"
 	"testing"
+	"time"
 
+	"github.com/mhsanaei/3x-ui/v3/internal/database"
 	"github.com/mhsanaei/3x-ui/v3/internal/database/model"
+	"github.com/mhsanaei/3x-ui/v3/internal/web/runtime"
 )
 
 func TestCreateAcrossManyInboundsUsesOneEmailSnapshot(t *testing.T) {
@@ -82,3 +89,252 @@ func TestAttachAcrossManyInboundsUsesOneEmailSnapshot(t *testing.T) {
 		t.Fatalf("linked inbounds = %d, want %d", len(linked), len(ids))
 	}
 }
+
+// barrierNodeRuntime holds every AddClient until fanout of them are inside it at
+// once, recording the peak overlap; a sequential caller only ever reaches one.
+type barrierNodeRuntime struct {
+	fakeNodeRuntime
+	fanout   int32
+	inFlight atomic.Int32
+	maxPar   atomic.Int32
+	release  chan struct{}
+	freed    atomic.Bool
+	expired  atomic.Bool
+}
+
+func (b *barrierNodeRuntime) free() {
+	if b.freed.CompareAndSwap(false, true) {
+		close(b.release)
+	}
+}
+
+func (b *barrierNodeRuntime) AddClient(ctx context.Context, ib *model.Inbound, c model.Client) error {
+	n := b.inFlight.Add(1)
+	for {
+		peak := b.maxPar.Load()
+		if n <= peak || b.maxPar.CompareAndSwap(peak, n) {
+			break
+		}
+	}
+	if n == b.fanout {
+		b.free()
+	}
+	select {
+	case <-b.release:
+	case <-time.After(5 * time.Second):
+		// Release everyone on the first timeout so a sequential regression
+		// fails once instead of stalling for fanout x the wait.
+		b.expired.Store(true)
+		b.free()
+	}
+	b.inFlight.Add(-1)
+	return b.fakeNodeRuntime.AddClient(ctx, ib, c)
+}
+
+func fanoutNodeInbounds(t *testing.T, mgr *runtime.Manager, rt runtime.Runtime, n int, basePort int) []int {
+	t.Helper()
+	ids := make([]int, 0, n)
+	for i := range n {
+		node := &model.Node{
+			Name: fmt.Sprintf("%s-%d", t.Name(), i), Address: "127.0.0.1", Port: 2096 + i,
+			ApiToken: "tok", Enable: true, Status: "online",
+		}
+		if err := database.GetDB().Create(node).Error; err != nil {
+			t.Fatalf("create node %d: %v", i, err)
+		}
+		mgr.SetRuntimeOverride(node.Id, rt)
+		ids = append(ids, nodeInbound(t, node.Id, basePort+i, nil).Id)
+	}
+	return ids
+}
+
+// TestCreateAcrossNodesPushesConcurrently pins that a client spanning several
+// node inbounds pushes to them at once, up to inboundFanoutConcurrency at a time.
+func TestCreateAcrossNodesPushesConcurrently(t *testing.T) {
+	setupBulkDB(t)
+	startSerializedWriter(t)
+	mgr := useTestRuntimeManager(t)
+
+	const nodes = inboundFanoutConcurrency + 1
+	bar := &barrierNodeRuntime{fanout: inboundFanoutConcurrency, release: make(chan struct{})}
+	ids := fanoutNodeInbounds(t, mgr, bar, nodes, 40101)
+
+	if _, err := (&ClientService{}).Create(&InboundService{}, &ClientCreatePayload{
+		Client:     model.Client{Email: "fanout@x", ID: "11111111-2222-3333-4444-555555555555", SubID: "sub-fanout", Enable: true},
+		InboundIds: ids,
+	}); err != nil {
+		t.Fatalf("Create across %d node inbounds: %v", nodes, err)
+	}
+
+	if got := bar.addClient.Load(); got != nodes {
+		t.Fatalf("AddClient pushes = %d, want %d", got, nodes)
+	}
+	if got := bar.maxPar.Load(); got < 2 || got != inboundFanoutConcurrency {
+		t.Fatalf("peak node pushes in flight = %d, want overlap at the %d cap (barrier timed out: %v)",
+			got, inboundFanoutConcurrency, bar.expired.Load())
+	}
+}
+
+// TestCreateRecoversPanicInOneInbound pins that a panicking inbound fails only
+// itself: off the request goroutine nothing else would catch it.
+func TestCreateRecoversPanicInOneInbound(t *testing.T) {
+	setupBulkDB(t)
+	startSerializedWriter(t)
+	mgr := useTestRuntimeManager(t)
+
+	node := &model.Node{
+		Name: t.Name(), Address: "127.0.0.1", Port: 2096,
+		ApiToken: "tok", Enable: true, Status: "online",
+	}
+	if err := database.GetDB().Create(node).Error; err != nil {
+		t.Fatalf("create node: %v", err)
+	}
+	mgr.SetRuntimeOverride(node.Id, &panicNodeRuntime{})
+	boom := nodeInbound(t, node.Id, 40201, nil)
+	healthy := mkInbound(t, 40202, model.VLESS, `{"clients":[]}`)
+
+	const uuid = "33333333-4444-5555-6666-777777777777"
+	_, err := (&ClientService{}).Create(&InboundService{}, &ClientCreatePayload{
+		Client:     model.Client{Email: "panic@x", ID: uuid, SubID: "sub-panic", Enable: true},
+		InboundIds: []int{boom.Id, healthy.Id},
+	})
+	if err == nil {
+		t.Fatal("a panicking node runtime produced no error")
+	}
+	if want := fmt.Sprintf("inbound %d: panic:", boom.Id); !strings.Contains(err.Error(), want) {
+		t.Fatalf("error %q does not report %q", err, want)
+	}
+	if !settingsHoldUUID(t, &InboundService{}, healthy.Id, uuid) {
+		t.Fatalf("healthy inbound %d did not get the client", healthy.Id)
+	}
+}
+
+// TestCreateLeavesHwidLimitAloneWhenCreateFails pins that a create the panel
+// reported as failed never rewrites a device cap, so it can never retrim one.
+func TestCreateLeavesHwidLimitAloneWhenCreateFails(t *testing.T) {
+	setupBulkDB(t)
+	startSerializedWriter(t)
+	svc := &ClientService{}
+	inboundSvc := &InboundService{}
+
+	const vipUUID = "44444444-5555-6666-7777-888888888888"
+	seed := mkInbound(t, 41401, model.VLESS, `{"clients":[]}`)
+	if _, err := svc.Create(inboundSvc, &ClientCreatePayload{
+		Client:     model.Client{Email: "vip@x", ID: vipUUID, SubID: "sub-vip", Enable: true},
+		InboundIds: []int{seed.Id},
+		LimitHwid:  3,
+	}); err != nil {
+		t.Fatalf("seed Create: %v", err)
+	}
+
+	broken := mkInbound(t, 41402, model.VLESS, `{"clients":`)
+	if _, err := svc.Create(inboundSvc, &ClientCreatePayload{
+		Client:     model.Client{Email: "vip@x", ID: vipUUID, SubID: "sub-vip", Enable: true},
+		InboundIds: []int{broken.Id},
+		LimitHwid:  1,
+	}); err == nil {
+		t.Fatal("re-adding to an unparsable inbound returned no error")
+	}
+
+	if rec := lookupClientRecord(t, "vip@x"); rec.LimitHwid != 3 {
+		t.Fatalf("limit_hwid = %d, want the untouched 3: a failed create retrimmed a live client", rec.LimitHwid)
+	}
+
+	// Same failure with the seeded inbound alongside it: that one is a dedup
+	// no-op returning no error, which must not read as "an inbound took it".
+	if _, err := svc.Create(inboundSvc, &ClientCreatePayload{
+		Client:     model.Client{Email: "vip@x", ID: vipUUID, SubID: "sub-vip", Enable: true},
+		InboundIds: []int{seed.Id, broken.Id},
+		LimitHwid:  1,
+	}); err == nil {
+		t.Fatal("re-adding over a no-op and an unparsable inbound returned no error")
+	}
+	if rec := lookupClientRecord(t, "vip@x"); rec.LimitHwid != 3 {
+		t.Fatalf("limit_hwid = %d, want the untouched 3: a no-op inbound counted as applied", rec.LimitHwid)
+	}
+
+	// A brand new identity that only partly applies is left uncapped rather than
+	// capped, the deliberate safe side: the operator saw the error and retries.
+	healthy := mkInbound(t, 41403, model.VLESS, `{"clients":[]}`)
+	if _, err := svc.Create(inboundSvc, &ClientCreatePayload{
+		Client:     model.Client{Email: "fresh@x", ID: "55555555-6666-7777-8888-999999999999", SubID: "sub-fresh", Enable: true},
+		InboundIds: []int{healthy.Id, broken.Id},
+		LimitHwid:  5,
+	}); err == nil {
+		t.Fatal("creating over an unparsable inbound returned no error")
+	}
+	if rec := lookupClientRecord(t, "fresh@x"); rec.LimitHwid != 0 {
+		t.Fatalf("limit_hwid = %d, want 0 on a create that failed", rec.LimitHwid)
+	}
+}
+
+func assertNamesFailedInbounds(t *testing.T, err error, broken []*model.Inbound, healthy *model.Inbound) {
+	t.Helper()
+	if err == nil {
+		t.Fatalf("applying %d unparsable inbounds returned no error", len(broken))
+	}
+	for _, ib := range broken {
+		if want := fmt.Sprintf("inbound %d:", ib.Id); !strings.Contains(err.Error(), want) {
+			t.Fatalf("error %q does not name the failing %s", err, want)
+		}
+	}
+	if blamed := fmt.Sprintf("inbound %d:", healthy.Id); strings.Contains(err.Error(), blamed) {
+		t.Fatalf("error %q blames the healthy %s", err, blamed)
+	}
+}
+
+// TestFanoutReportsEveryFailingInbound pins that no inbound aborts the others:
+// each failure names its own inbound, and the healthy ones still get the client.
+func TestFanoutReportsEveryFailingInbound(t *testing.T) {
+	const halfBadUUID = "22222222-3333-4444-5555-666666666666"
+
+	t.Run("create", func(t *testing.T) {
+		setupBulkDB(t)
+		startSerializedWriter(t)
+		svc := &ClientService{}
+		inboundSvc := &InboundService{}
+
+		broken := []*model.Inbound{
+			mkInbound(t, 41201, model.VLESS, `{"clients":`),
+			mkInbound(t, 41202, model.VLESS, `{"clients":`),
+		}
+		healthy := mkInbound(t, 41203, model.VLESS, `{"clients":[]}`)
+
+		_, err := svc.Create(inboundSvc, &ClientCreatePayload{
+			Client:     model.Client{Email: "halfbad@x", ID: halfBadUUID, SubID: "sub-halfbad", Enable: true},
+			InboundIds: []int{broken[0].Id, broken[1].Id, healthy.Id},
+		})
+		assertNamesFailedInbounds(t, err, broken, healthy)
+		if !settingsHoldUUID(t, inboundSvc, healthy.Id, halfBadUUID) {
+			t.Fatalf("healthy inbound %d did not get the client", healthy.Id)
+		}
+	})
+
+	t.Run("attach", func(t *testing.T) {
+		setupBulkDB(t)
+		startSerializedWriter(t)
+		svc := &ClientService{}
+		inboundSvc := &InboundService{}
+
+		seed := mkInbound(t, 41301, model.VLESS, `{"clients":[]}`)
+		if _, err := svc.Create(inboundSvc, &ClientCreatePayload{
+			Client:     model.Client{Email: "halfbad@x", ID: halfBadUUID, SubID: "sub-halfbad", Enable: true},
+			InboundIds: []int{seed.Id},
+		}); err != nil {
+			t.Fatalf("seed Create: %v", err)
+		}
+
+		broken := []*model.Inbound{
+			mkInbound(t, 41302, model.VLESS, `{"clients":`),
+			mkInbound(t, 41303, model.VLESS, `{"clients":`),
+		}
+		healthy := mkInbound(t, 41304, model.VLESS, `{"clients":[]}`)
+
+		rec := lookupClientRecord(t, "halfbad@x")
+		_, err := svc.Attach(inboundSvc, rec.Id, []int{broken[0].Id, broken[1].Id, healthy.Id})
+		assertNamesFailedInbounds(t, err, broken, healthy)
+		if !settingsHoldUUID(t, inboundSvc, healthy.Id, halfBadUUID) {
+			t.Fatalf("healthy inbound %d did not get the client", healthy.Id)
+		}
+	})
+}

internal/web/service/client_crud.go

@@ -6,7 +6,10 @@ import (
 	"errors"
 	"fmt"
 	"net/netip"
+	"runtime/debug"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"time"
 	"unicode"
 
@@ -14,6 +17,7 @@ import (
 
 	"github.com/mhsanaei/3x-ui/v3/internal/database"
 	"github.com/mhsanaei/3x-ui/v3/internal/database/model"
+	"github.com/mhsanaei/3x-ui/v3/internal/logger"
 	"github.com/mhsanaei/3x-ui/v3/internal/util/common"
 	"github.com/mhsanaei/3x-ui/v3/internal/util/random"
 	"github.com/mhsanaei/3x-ui/v3/internal/xray"
@@ -116,6 +120,8 @@ func (s *ClientService) GetClientsByTrafficReset(period string) ([]ClientResetCy
 	return cycles, nil
 }
 
+// Create applies the client to every requested inbound: one failing inbound no
+// longer aborts the others, so the error can name several and needRestart holds.
 func (s *ClientService) Create(inboundSvc *InboundService, payload *ClientCreatePayload) (bool, error) {
 	if payload == nil {
 		return false, common.NewError("empty payload")
@@ -194,14 +200,16 @@ func (s *ClientService) Create(inboundSvc *InboundService, payload *ClientCreate
 		}
 	}
 
-	needRestart := false
+	// Prepared before any inbound is written: fillProtocolDefaults mints the
+	// shared credentials on the first inbound and every later one reuses them.
+	adds := make([]*model.Inbound, 0, len(payload.InboundIds))
 	for _, ibId := range payload.InboundIds {
 		inbound, getErr := inboundSvc.GetInbound(ibId)
 		if getErr != nil {
-			return needRestart, getErr
+			return false, fmt.Errorf("inbound %d: %w", ibId, getErr)
 		}
 		if err := s.fillProtocolDefaults(&client, inbound); err != nil {
-			return needRestart, err
+			return false, fmt.Errorf("inbound %d: %w", ibId, err)
 		}
 		clientForInbound := client
 		if ips, ok := client.AllowedIPsByInbound[ibId]; ok {
@@ -217,23 +225,59 @@ func (s *ClientService) Create(inboundSvc *InboundService, payload *ClientCreate
 		}
 		settingsPayload, mErr := json.Marshal(map[string][]model.Client{"clients": {clientWithInboundFlow(clientForInbound, inbound)}})
 		if mErr != nil {
-			return needRestart, mErr
-		}
-		nr, addErr := s.AddInboundClient(inboundSvc, &model.Inbound{
-			Id:       ibId,
-			Settings: string(settingsPayload),
-		})
-		if addErr != nil {
-			return needRestart, addErr
-		}
-		if nr {
-			needRestart = true
+			return false, fmt.Errorf("inbound %d: %w", ibId, mErr)
 		}
+		adds = append(adds, &model.Inbound{Id: ibId, Settings: string(settingsPayload)})
 	}
-	if err := s.setClientLimitHwidByEmail(nil, client.Email, payload.LimitHwid); err != nil {
-		return needRestart, err
+	needRestart, fanoutErr := s.fanoutInboundClientAdds(inboundSvc, adds)
+	if fanoutErr != nil {
+		// Never on a failed create: this retrims the devices of an email that
+		// already existed, and a create the panel reported as failed must not.
+		return needRestart, fanoutErr
 	}
-	return needRestart, nil
+	return needRestart, s.setClientLimitHwidByEmail(nil, client.Email, payload.LimitHwid)
+}
+
+// inboundFanoutConcurrency caps how many inbounds one create/attach applies at
+// once, so a client spanning many of them can't start an unbounded RPC burst.
+const inboundFanoutConcurrency = 4
+
+// fanoutInboundClientAdds applies one payload per inbound with the node pushes
+// overlapping; unlike the sequential loop, one failure no longer stops the rest.
+func (s *ClientService) fanoutInboundClientAdds(inboundSvc *InboundService, adds []*model.Inbound) (bool, error) {
+	var needRestart atomic.Bool
+	errs := make([]error, len(adds))
+	sem := make(chan struct{}, inboundFanoutConcurrency)
+	var wg sync.WaitGroup
+	for i := range adds {
+		wg.Add(1)
+		sem <- struct{}{}
+		go func() {
+			defer wg.Done()
+			defer func() { <-sem }()
+			// Off the request goroutine gin's Recovery no longer covers this,
+			// so an unrecovered panic here would take the whole panel down.
+			defer func() {
+				if r := recover(); r != nil {
+					// The apply may already have committed, so ask for the
+					// restart the lost return value can no longer report.
+					needRestart.Store(true)
+					errs[i] = fmt.Errorf("inbound %d: panic: %v", adds[i].Id, r)
+					logger.Errorf("panic adding client to inbound %d: %v\n%s", adds[i].Id, r, debug.Stack())
+				}
+			}()
+			nr, err := s.AddInboundClient(inboundSvc, adds[i])
+			if nr {
+				needRestart.Store(true)
+			}
+			if err != nil {
+				errs[i] = fmt.Errorf("inbound %d: %w", adds[i].Id, err)
+			}
+		}()
+	}
+	wg.Wait()
+
+	return needRestart.Load(), errors.Join(errs...)
 }
 
 func (s *ClientService) fillProtocolDefaults(c *model.Client, ib *model.Inbound) error {
@@ -792,6 +836,8 @@ func addressesFitAmneziaWGInbound(addrs []string, ib *model.Inbound) bool {
 	return true
 }
 
+// Attach applies the client to every requested inbound: one failing inbound no
+// longer aborts the others, so the error can name several and needRestart holds.
 func (s *ClientService) Attach(inboundSvc *InboundService, id int, inboundIds []int) (bool, error) {
 	existing, err := s.GetByID(id)
 	if err != nil {
@@ -826,38 +872,29 @@ func (s *ClientService) Attach(inboundSvc *InboundService, id int, inboundIds []
 		clientWire.AllowedIPs = nil
 	}
 
-	needRestart := false
+	adds := make([]*model.Inbound, 0, len(inboundIds))
 	for _, ibId := range inboundIds {
 		if _, attached := have[ibId]; attached {
 			continue
 		}
 		inbound, getErr := inboundSvc.GetInbound(ibId)
 		if getErr != nil {
-			return needRestart, getErr
+			return false, fmt.Errorf("inbound %d: %w", ibId, getErr)
 		}
 		copyClient := *clientWire
 		if !addressesFitAmneziaWGInbound(copyClient.AllowedIPs, inbound) {
 			copyClient.AllowedIPs = nil
 		}
 		if err := s.fillProtocolDefaults(&copyClient, inbound); err != nil {
-			return needRestart, err
+			return false, fmt.Errorf("inbound %d: %w", ibId, err)
 		}
 		settingsPayload, mErr := json.Marshal(map[string][]model.Client{"clients": {clientWithInboundFlow(copyClient, inbound)}})
 		if mErr != nil {
-			return needRestart, mErr
-		}
-		nr, addErr := s.AddInboundClient(inboundSvc, &model.Inbound{
-			Id:       ibId,
-			Settings: string(settingsPayload),
-		})
-		if addErr != nil {
-			return needRestart, addErr
-		}
-		if nr {
-			needRestart = true
+			return false, fmt.Errorf("inbound %d: %w", ibId, mErr)
 		}
+		adds = append(adds, &model.Inbound{Id: ibId, Settings: string(settingsPayload)})
 	}
-	return needRestart, nil
+	return s.fanoutInboundClientAdds(inboundSvc, adds)
 }
 
 func (s *ClientService) CreateOne(inboundSvc *InboundService, inboundId int, client model.Client) (bool, error) {

internal/web/service/node_bulk_dispatch_test.go

@@ -80,15 +80,38 @@ func (f *fakeNodeRuntime) ResetClientTraffic(context.Context, *model.Inbound, st
 func (f *fakeNodeRuntime) ResetInboundTraffic(context.Context, *model.Inbound) error { return nil }
 func (f *fakeNodeRuntime) ResetAllTraffics(context.Context) error                    { return nil }
 
-// setupNodeRuntime wires an online node + a fake runtime override and returns the
-// node id and the fake so a test can drive the service node-dispatch path without
-// a network node.
-func setupNodeRuntime(t *testing.T) (int, *fakeNodeRuntime) {
+// startSerializedWriter runs the single traffic-writer goroutine for the test, so
+// concurrent service writes take the serialized path production uses.
+func startSerializedWriter(t *testing.T) {
+	t.Helper()
+	resetTrafficWriterForTest(t)
+	StartTrafficWriter()
+}
+
+// useTestRuntimeManager swaps in a fresh runtime.Manager for the test and puts
+// the previous one back afterwards, so overrides can't leak between tests.
+func useTestRuntimeManager(t *testing.T) *runtime.Manager {
 	t.Helper()
 	prev := runtime.GetManager()
 	mgr := runtime.NewManager(runtime.LocalDeps{APIPort: func() int { return 0 }, SetNeedRestart: func() {}})
 	runtime.SetManager(mgr)
 	t.Cleanup(func() { runtime.SetManager(prev) })
+	return mgr
+}
+
+// panicNodeRuntime panics on the per-client push, standing in for a bug in the
+// apply path that would otherwise unwind straight out of a fanout goroutine.
+type panicNodeRuntime struct{ fakeNodeRuntime }
+
+func (p *panicNodeRuntime) AddClient(context.Context, *model.Inbound, model.Client) error {
+	panic("boom from node runtime")
+}
+
+// setupNodeRuntime wires an online node + a fake runtime override so a test can
+// drive the service node-dispatch path without a network node.
+func setupNodeRuntime(t *testing.T) (int, *fakeNodeRuntime) {
+	t.Helper()
+	mgr := useTestRuntimeManager(t)
 
 	node := &model.Node{Name: "n1-" + t.Name(), Address: "127.0.0.1", Port: 2096, ApiToken: "tok", Enable: true, Status: "online"}
 	if err := database.GetDB().Create(node).Error; err != nil {