fix(migrate): copy composite-key tables without FindInBatches (#4787)

SQLite to Postgres migration aborted with "copy *model.ClientInbound:
primary key required" on installs whose client_inbounds table exceeds
one read batch (500 rows). gorm's FindInBatches pages between batches
using a single PrioritizedPrimaryField, which composite-key tables
(client_id + inbound_id, no surrogate id) do not have, so it returns
ErrPrimaryKeyRequired once a table holds more than one batch.

Replace FindInBatches in copyTable with explicit LIMIT/OFFSET paging
ordered by the model's primary-key columns. This works for every table
including composite-key ones, keeps memory bounded, and changes no
schema.

Add a Postgres-gated regression test covering a >500-row composite-key
table.
This commit is contained in:
MHSanaei
2026-06-02 04:20:42 +02:00
parent 4f597a08c4
commit c8ad42631c
2 changed files with 94 additions and 12 deletions
+30 -12
View File
@@ -7,6 +7,7 @@ import (
"os"
"path"
"reflect"
"strings"
"time"
"github.com/mhsanaei/3x-ui/v3/database/model"
@@ -103,25 +104,42 @@ func MigrateData(srcPath, dstDSN string) error {
return nil
}
// copyTable streams every row of `mdl` from src to dst in batches.
func copyTable(src, dst *gorm.DB, mdl any) (int, error) {
const batchSize = 500
sliceType := reflect.SliceOf(reflect.PointerTo(reflect.TypeOf(mdl).Elem()))
batchPtr := reflect.New(sliceType)
batchPtr.Elem().Set(reflect.MakeSlice(sliceType, 0, 0))
// Resolve primary-key columns so paging is deterministic across successive
// LIMIT/OFFSET reads. The model set is trusted (not user input).
stmt := &gorm.Statement{DB: src}
if err := stmt.Parse(mdl); err != nil {
return 0, err
}
order := strings.Join(stmt.Schema.PrimaryFieldDBNames, ", ")
total := 0
err := src.Model(mdl).FindInBatches(batchPtr.Interface(), 500, func(tx *gorm.DB, _ int) error {
batch := batchPtr.Elem()
if batch.Len() == 0 {
return nil
for offset := 0; ; offset += batchSize {
batchPtr := reflect.New(sliceType)
q := src.Model(mdl).Limit(batchSize).Offset(offset)
if order != "" {
q = q.Order(order)
}
if err := q.Find(batchPtr.Interface()).Error; err != nil {
return total, err
}
n := batchPtr.Elem().Len()
if n == 0 {
break
}
if err := dst.CreateInBatches(batchPtr.Interface(), 200).Error; err != nil {
return err
return total, err
}
total += batch.Len()
return nil
}).Error
return total, err
total += n
if n < batchSize {
break
}
}
return total, nil
}
// resetPostgresSequences advances each migrated table's id sequence past MAX(id),