mirror of
https://github.com/MHSanaei/3x-ui.git
synced 2026-07-17 01:56:06 +00:00
fix(migrate): copy composite-key tables without FindInBatches (#4787)
SQLite to Postgres migration aborted with "copy *model.ClientInbound: primary key required" on installs whose client_inbounds table exceeds one read batch (500 rows). gorm's FindInBatches pages between batches using a single PrioritizedPrimaryField, which composite-key tables (client_id + inbound_id, no surrogate id) do not have, so it returns ErrPrimaryKeyRequired once a table holds more than one batch. Replace FindInBatches in copyTable with explicit LIMIT/OFFSET paging ordered by the model's primary-key columns. This works for every table including composite-key ones, keeps memory bounded, and changes no schema. Add a Postgres-gated regression test covering a >500-row composite-key table.
This commit is contained in:
+30
-12
@@ -7,6 +7,7 @@ import (
|
||||
"os"
|
||||
"path"
|
||||
"reflect"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mhsanaei/3x-ui/v3/database/model"
|
||||
@@ -103,25 +104,42 @@ func MigrateData(srcPath, dstDSN string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// copyTable streams every row of `mdl` from src to dst in batches.
|
||||
func copyTable(src, dst *gorm.DB, mdl any) (int, error) {
|
||||
const batchSize = 500
|
||||
|
||||
sliceType := reflect.SliceOf(reflect.PointerTo(reflect.TypeOf(mdl).Elem()))
|
||||
batchPtr := reflect.New(sliceType)
|
||||
batchPtr.Elem().Set(reflect.MakeSlice(sliceType, 0, 0))
|
||||
|
||||
// Resolve primary-key columns so paging is deterministic across successive
|
||||
// LIMIT/OFFSET reads. The model set is trusted (not user input).
|
||||
stmt := &gorm.Statement{DB: src}
|
||||
if err := stmt.Parse(mdl); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
order := strings.Join(stmt.Schema.PrimaryFieldDBNames, ", ")
|
||||
|
||||
total := 0
|
||||
err := src.Model(mdl).FindInBatches(batchPtr.Interface(), 500, func(tx *gorm.DB, _ int) error {
|
||||
batch := batchPtr.Elem()
|
||||
if batch.Len() == 0 {
|
||||
return nil
|
||||
for offset := 0; ; offset += batchSize {
|
||||
batchPtr := reflect.New(sliceType)
|
||||
q := src.Model(mdl).Limit(batchSize).Offset(offset)
|
||||
if order != "" {
|
||||
q = q.Order(order)
|
||||
}
|
||||
if err := q.Find(batchPtr.Interface()).Error; err != nil {
|
||||
return total, err
|
||||
}
|
||||
n := batchPtr.Elem().Len()
|
||||
if n == 0 {
|
||||
break
|
||||
}
|
||||
if err := dst.CreateInBatches(batchPtr.Interface(), 200).Error; err != nil {
|
||||
return err
|
||||
return total, err
|
||||
}
|
||||
total += batch.Len()
|
||||
return nil
|
||||
}).Error
|
||||
return total, err
|
||||
total += n
|
||||
if n < batchSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
// resetPostgresSequences advances each migrated table's id sequence past MAX(id),
|
||||
|
||||
Reference in New Issue
Block a user