diff --git a/lightning/pkg/importer/get_pre_info.go b/lightning/pkg/importer/get_pre_info.go index 6a06a459def84..8bcb02d075ee3 100644 --- a/lightning/pkg/importer/get_pre_info.go +++ b/lightning/pkg/importer/get_pre_info.go @@ -617,7 +617,7 @@ func (p *PreImportInfoGetterImpl) sampleDataFromTable( if err != nil { return 0.0, false, errors.Trace(err) } - idAlloc := kv.NewPanickingAllocators(0) + idAlloc := kv.NewPanickingAllocators(tableInfo.SepAutoInc(), 0) tbl, err := tables.TableFromMeta(idAlloc, tableInfo) if err != nil { return 0.0, false, errors.Trace(err) diff --git a/lightning/pkg/importer/meta_manager.go b/lightning/pkg/importer/meta_manager.go index 9d61e072df280..188ecb9987507 100644 --- a/lightning/pkg/importer/meta_manager.go +++ b/lightning/pkg/importer/meta_manager.go @@ -266,6 +266,8 @@ func (m *dbTableMetaMgr) AllocTableRowIDs(ctx context.Context, rawRowIDMax int64 if curStatus == metaStatusInitial { if needAutoID { // maxRowIDMax is the max row_id that other tasks has allocated, we need to rebase the global autoid base first. + // TODO this is not right when AUTO_ID_CACHE=1 and have auto row id, + // the id allocators are separated in this case. if err := common.RebaseGlobalAutoID(ctx, maxRowIDMax, m.tr, m.tr.dbInfo.ID, m.tr.tableInfo.Core); err != nil { return errors.Trace(err) } diff --git a/lightning/pkg/importer/table_import.go b/lightning/pkg/importer/table_import.go index 44ced41f23349..dd6de8667427e 100644 --- a/lightning/pkg/importer/table_import.go +++ b/lightning/pkg/importer/table_import.go @@ -92,7 +92,7 @@ func NewTableImporter( etcdCli *clientv3.Client, logger log.Logger, ) (*TableImporter, error) { - idAlloc := kv.NewPanickingAllocators(cp.AllocBase) + idAlloc := kv.NewPanickingAllocators(tableInfo.Core.SepAutoInc(), cp.AllocBase) tbl, err := tables.TableFromMeta(idAlloc, tableInfo.Core) if err != nil { return nil, errors.Annotatef(err, "failed to tables.TableFromMeta %s", tableName) @@ -953,26 +953,30 @@ func (tr *TableImporter) postProcess( if cp.Status < checkpoints.CheckpointStatusAlteredAutoInc { tblInfo := tr.tableInfo.Core var err error + // TODO why we have to rebase id for tidb backend??? remove it later. if tblInfo.ContainsAutoRandomBits() { ft := &common.GetAutoRandomColumn(tblInfo).FieldType shardFmt := autoid.NewShardIDFormat(ft, tblInfo.AutoRandomBits, tblInfo.AutoRandomRangeBits) maxCap := shardFmt.IncrementalBitsCapacity() err = AlterAutoRandom(ctx, rc.db, tr.tableName, uint64(tr.alloc.Get(autoid.AutoRandomType).Base())+1, maxCap) } else if common.TableHasAutoRowID(tblInfo) || tblInfo.GetAutoIncrementColInfo() != nil { - // only alter auto increment id iff table contains auto-increment column or generated handle. - // ALTER TABLE xxx AUTO_INCREMENT = yyy has a bad naming. - // if a table has implicit _tidb_rowid column & tbl.SepAutoID=false, then it works on _tidb_rowid - // allocator, even if the table has NO auto-increment column. - newBase := uint64(tr.alloc.Get(autoid.RowIDAllocType).Base()) + 1 - err = AlterAutoIncrement(ctx, rc.db, tr.tableName, newBase) - - if err == nil && isLocalBackend(rc.cfg) { + if isLocalBackend(rc.cfg) { // for TiDB version >= 6.5.0, a table might have separate allocators for auto_increment column and _tidb_rowid, // especially when a table has auto_increment non-clustered PK, it will use both allocators. // And in this case, ALTER TABLE xxx AUTO_INCREMENT = xxx only works on the allocator of auto_increment column, // not for allocator of _tidb_rowid. // So we need to rebase IDs for those 2 allocators explicitly. - err = common.RebaseGlobalAutoID(ctx, adjustIDBase(newBase), tr, tr.dbInfo.ID, tr.tableInfo.Core) + err = common.RebaseTableAllocators(ctx, map[autoid.AllocatorType]int64{ + autoid.RowIDAllocType: tr.alloc.Get(autoid.RowIDAllocType).Base(), + autoid.AutoIncrementType: tr.alloc.Get(autoid.AutoIncrementType).Base(), + }, tr, tr.dbInfo.ID, tr.tableInfo.Core) + } else { + // only alter auto increment id iff table contains auto-increment column or generated handle. + // ALTER TABLE xxx AUTO_INCREMENT = yyy has a bad naming. + // if a table has implicit _tidb_rowid column & tbl.SepAutoID=false, then it works on _tidb_rowid + // allocator, even if the table has NO auto-increment column. + newBase := uint64(tr.alloc.Get(autoid.RowIDAllocType).Base()) + 1 + err = AlterAutoIncrement(ctx, rc.db, tr.tableName, newBase) } } saveCpErr := rc.saveStatusCheckpoint(ctx, tr.tableName, checkpoints.WholeTableEngineID, err, checkpoints.CheckpointStatusAlteredAutoInc) diff --git a/lightning/pkg/importer/table_import_test.go b/lightning/pkg/importer/table_import_test.go index 92c5f960f6311..ba28b155a8586 100644 --- a/lightning/pkg/importer/table_import_test.go +++ b/lightning/pkg/importer/table_import_test.go @@ -408,7 +408,7 @@ func (s *tableRestoreSuite) TestRestoreEngineFailed() { mockEngineWriter.EXPECT().IsSynced().Return(true).AnyTimes() mockEngineWriter.EXPECT().Close(gomock.Any()).Return(mockChunkFlushStatus, nil).AnyTimes() - tbl, err := tables.TableFromMeta(kv.NewPanickingAllocators(0), s.tableInfo.Core) + tbl, err := tables.TableFromMeta(kv.NewPanickingAllocators(s.tableInfo.Core.SepAutoInc(), 0), s.tableInfo.Core) require.NoError(s.T(), err) _, indexUUID := backend.MakeUUID("`db`.`table`", -1) _, dataUUID := backend.MakeUUID("`db`.`table`", 0) @@ -1444,7 +1444,7 @@ func (s *tableRestoreSuite) TestEstimate() { controller := gomock.NewController(s.T()) defer controller.Finish() mockEncBuilder := mock.NewMockEncodingBuilder(controller) - idAlloc := kv.NewPanickingAllocators(0) + idAlloc := kv.NewPanickingAllocators(s.tableInfo.Core.SepAutoInc(), 0) tbl, err := tables.TableFromMeta(idAlloc, s.tableInfo.Core) require.NoError(s.T(), err) diff --git a/lightning/tests/lightning_csv/data/auto_incr_id.nonclustered_cache1_shard_autorowid-schema.sql b/lightning/tests/lightning_csv/data/auto_incr_id.nonclustered_cache1_shard_autorowid-schema.sql new file mode 100644 index 0000000000000..4b34758103d03 --- /dev/null +++ b/lightning/tests/lightning_csv/data/auto_incr_id.nonclustered_cache1_shard_autorowid-schema.sql @@ -0,0 +1,9 @@ +/* + 1778961125641936898 is 0001100010110000001000111011011111101011000111011110000000000010 + bigger than the max increment part of sharded auto row id. + */ +CREATE TABLE nonclustered_cache1_shard_autorowid ( + id bigint(20) unsigned NOT NULL AUTO_INCREMENT, + v int, + PRIMARY KEY (id) NONCLUSTERED +) AUTO_ID_CACHE=1 SHARD_ROW_ID_BITS=4; diff --git a/lightning/tests/lightning_csv/data/auto_incr_id.nonclustered_cache1_shard_autorowid.0.csv b/lightning/tests/lightning_csv/data/auto_incr_id.nonclustered_cache1_shard_autorowid.0.csv new file mode 100644 index 0000000000000..75562d4451313 --- /dev/null +++ b/lightning/tests/lightning_csv/data/auto_incr_id.nonclustered_cache1_shard_autorowid.0.csv @@ -0,0 +1,3 @@ +1778961125641936898,1 +1778961125641936899,2 +1778961125641936900,3 diff --git a/lightning/tests/lightning_csv/run.sh b/lightning/tests/lightning_csv/run.sh index 70b5cff66e989..08e1382e08b4f 100755 --- a/lightning/tests/lightning_csv/run.sh +++ b/lightning/tests/lightning_csv/run.sh @@ -60,7 +60,7 @@ function run_with() { run_sql 'SELECT id FROM csv.empty_strings WHERE b <> ""' check_not_contains 'id:' - for table in clustered nonclustered clustered_cache1 nonclustered_cache1; do + for table in clustered nonclustered clustered_cache1 nonclustered_cache1 nonclustered_cache1_shard_autorowid; do run_sql "select count(*) from auto_incr_id.$table" check_contains 'count(*): 3' # insert should work diff --git a/lightning/tests/lightning_tidb_rowid/data/rowid.specific_auto_inc-schema.sql b/lightning/tests/lightning_tidb_rowid/data/rowid.specific_auto_inc-schema.sql index a69f5bf4350eb..c291100c82d2c 100644 --- a/lightning/tests/lightning_tidb_rowid/data/rowid.specific_auto_inc-schema.sql +++ b/lightning/tests/lightning_tidb_rowid/data/rowid.specific_auto_inc-schema.sql @@ -1 +1,3 @@ -create table specific_auto_inc (a varchar(6) primary key /*T![clustered_index] NONCLUSTERED */, b int unique auto_increment) auto_increment=80000; +create table specific_auto_inc ( + a varchar(6) primary key /*T![clustered_index] NONCLUSTERED */, + b int unique auto_increment) auto_increment=80000; diff --git a/lightning/tests/lightning_tool_1472/run.sh b/lightning/tests/lightning_tool_1472/run.sh index e8c2c5a8cfd08..71efe4a079a56 100755 --- a/lightning/tests/lightning_tool_1472/run.sh +++ b/lightning/tests/lightning_tool_1472/run.sh @@ -25,8 +25,8 @@ run_lightning run_sql 'insert into EE1472.pk values ();' run_sql 'select count(a), max(a) from EE1472.pk;' check_contains 'count(a): 3' -check_contains 'max(a): 6' +check_contains 'max(a): 5' run_sql 'insert into EE1472.notpk (a) values (3333);' run_sql 'select b from EE1472.notpk where a = 3333;' -check_contains 'b: 11' +check_contains 'b: 10' diff --git a/pkg/disttask/importinto/planner.go b/pkg/disttask/importinto/planner.go index 7ab081700145d..83d2294a96853 100644 --- a/pkg/disttask/importinto/planner.go +++ b/pkg/disttask/importinto/planner.go @@ -238,7 +238,7 @@ func buildControllerForPlan(p *LogicalPlan) (*importer.LoadDataController, error } func buildController(plan *importer.Plan, stmt string) (*importer.LoadDataController, error) { - idAlloc := kv.NewPanickingAllocators(0) + idAlloc := kv.NewPanickingAllocators(plan.TableInfo.SepAutoInc(), 0) tbl, err := tables.TableFromMeta(idAlloc, plan.TableInfo) if err != nil { return nil, err diff --git a/pkg/disttask/importinto/task_executor.go b/pkg/disttask/importinto/task_executor.go index a30dd3a0360a4..aba2028a1cbd5 100644 --- a/pkg/disttask/importinto/task_executor.go +++ b/pkg/disttask/importinto/task_executor.go @@ -72,7 +72,7 @@ func getTableImporter( taskMeta *TaskMeta, store tidbkv.Storage, ) (*importer.TableImporter, error) { - idAlloc := kv.NewPanickingAllocators(0) + idAlloc := kv.NewPanickingAllocators(taskMeta.Plan.TableInfo.SepAutoInc(), 0) tbl, err := tables.TableFromMeta(idAlloc, taskMeta.Plan.TableInfo) if err != nil { return nil, err diff --git a/pkg/executor/importer/table_import.go b/pkg/executor/importer/table_import.go index 8010c07036162..3fb097261363c 100644 --- a/pkg/executor/importer/table_import.go +++ b/pkg/executor/importer/table_import.go @@ -155,7 +155,7 @@ func NewTableImporter( id string, kvStore tidbkv.Storage, ) (ti *TableImporter, err error) { - idAlloc := kv.NewPanickingAllocators(0) + idAlloc := kv.NewPanickingAllocators(e.Table.Meta().SepAutoInc(), 0) tbl, err := tables.TableFromMeta(idAlloc, e.Table.Meta()) if err != nil { return nil, errors.Annotatef(err, "failed to tables.TableFromMeta %s", e.Table.Meta().Name) @@ -235,7 +235,7 @@ type TableImporter struct { // NewTableImporterForTest creates a new table importer for test. func NewTableImporterForTest(ctx context.Context, e *LoadDataController, id string, helper local.StoreHelper) (*TableImporter, error) { - idAlloc := kv.NewPanickingAllocators(0) + idAlloc := kv.NewPanickingAllocators(e.Table.Meta().SepAutoInc(), 0) tbl, err := tables.TableFromMeta(idAlloc, e.Table.Meta()) if err != nil { return nil, errors.Annotatef(err, "failed to tables.TableFromMeta %s", e.Table.Meta().Name) diff --git a/pkg/lightning/backend/kv/BUILD.bazel b/pkg/lightning/backend/kv/BUILD.bazel index 75523ba9e26a8..f654bc2574d7b 100644 --- a/pkg/lightning/backend/kv/BUILD.bazel +++ b/pkg/lightning/backend/kv/BUILD.bazel @@ -54,6 +54,7 @@ go_test( name = "kv_test", timeout = "short", srcs = [ + "allocator_test.go", "base_test.go", "kv2sql_test.go", "session_internal_test.go", @@ -63,7 +64,7 @@ go_test( embed = [":kv"], flaky = True, race = "on", - shard_count = 18, + shard_count = 19, deps = [ "//pkg/ddl", "//pkg/kv", diff --git a/pkg/lightning/backend/kv/allocator.go b/pkg/lightning/backend/kv/allocator.go index 0741500dda9b4..dfd7027a0e7de 100644 --- a/pkg/lightning/backend/kv/allocator.go +++ b/pkg/lightning/backend/kv/allocator.go @@ -26,7 +26,7 @@ import ( // panickingAllocator is an ID allocator which panics on all operations except Rebase type panickingAllocator struct { autoid.Allocator - base *int64 + base atomic.Int64 ty autoid.AllocatorType } @@ -34,26 +34,30 @@ type panickingAllocator struct { // we use this to collect the max id(either _tidb_rowid or auto_increment id or auto_random) used // during import, and we will use this info to do ALTER TABLE xxx AUTO_RANDOM_BASE or AUTO_INCREMENT // on post-process phase. -// we share the same base among all allocators, so the AllocatorType doesn't matter here. -func NewPanickingAllocators(base int64) autoid.Allocators { - sharedBase := &base - return autoid.NewAllocators( - false, - &panickingAllocator{base: sharedBase, ty: autoid.RowIDAllocType}, - &panickingAllocator{base: sharedBase, ty: autoid.AutoIncrementType}, - &panickingAllocator{base: sharedBase, ty: autoid.AutoRandomType}, - ) +// TODO: support save all bases in checkpoint. +func NewPanickingAllocators(sepAutoInc bool, base int64) autoid.Allocators { + allocs := make([]autoid.Allocator, 0, 3) + for _, t := range []autoid.AllocatorType{ + autoid.RowIDAllocType, + autoid.AutoIncrementType, + autoid.AutoRandomType, + } { + pa := &panickingAllocator{ty: t} + pa.base.Store(base) + allocs = append(allocs, pa) + } + return autoid.NewAllocators(sepAutoInc, allocs...) } // Rebase implements the autoid.Allocator interface func (alloc *panickingAllocator) Rebase(_ context.Context, newBase int64, _ bool) error { // CAS for { - oldBase := atomic.LoadInt64(alloc.base) + oldBase := alloc.base.Load() if newBase <= oldBase { break } - if atomic.CompareAndSwapInt64(alloc.base, oldBase, newBase) { + if alloc.base.CompareAndSwap(oldBase, newBase) { break } } @@ -62,7 +66,7 @@ func (alloc *panickingAllocator) Rebase(_ context.Context, newBase int64, _ bool // Base implements the autoid.Allocator interface func (alloc *panickingAllocator) Base() int64 { - return atomic.LoadInt64(alloc.base) + return alloc.base.Load() } func (alloc *panickingAllocator) GetType() autoid.AllocatorType { diff --git a/pkg/lightning/backend/kv/allocator_test.go b/pkg/lightning/backend/kv/allocator_test.go new file mode 100644 index 0000000000000..a88b54f4b5032 --- /dev/null +++ b/pkg/lightning/backend/kv/allocator_test.go @@ -0,0 +1,35 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kv + +import ( + "testing" + + "github.com/pingcap/tidb/pkg/meta/autoid" + "github.com/stretchr/testify/require" +) + +func TestAllocator(t *testing.T) { + alloc := NewPanickingAllocators(true, 0) + require.NoError(t, alloc.Get(autoid.RowIDAllocType).Rebase(nil, 123, false)) + // cannot revert back + require.NoError(t, alloc.Get(autoid.RowIDAllocType).Rebase(nil, 100, false)) + require.NoError(t, alloc.Get(autoid.AutoIncrementType).Rebase(nil, 456, false)) + require.NoError(t, alloc.Get(autoid.AutoRandomType).Rebase(nil, 789, false)) + + require.EqualValues(t, 123, alloc.Get(autoid.RowIDAllocType).Base()) + require.EqualValues(t, 456, alloc.Get(autoid.AutoIncrementType).Base()) + require.EqualValues(t, 789, alloc.Get(autoid.AutoRandomType).Base()) +} diff --git a/pkg/lightning/backend/kv/base_test.go b/pkg/lightning/backend/kv/base_test.go index 2ab2a454fb1c5..b373085dfdec1 100644 --- a/pkg/lightning/backend/kv/base_test.go +++ b/pkg/lightning/backend/kv/base_test.go @@ -43,7 +43,7 @@ func TestLogKVConvertFailed(t *testing.T) { cols := []*model.ColumnInfo{c1} tblInfo := &model.TableInfo{ID: 1, Columns: cols, PKIsHandle: false, State: model.StatePublic} var tbl table.Table - tbl, err = tables.TableFromMeta(NewPanickingAllocators(0), tblInfo) + tbl, err = tables.TableFromMeta(NewPanickingAllocators(tblInfo.SepAutoInc(), 0), tblInfo) require.NoError(t, err) var baseKVEncoder *BaseKVEncoder diff --git a/pkg/lightning/backend/kv/kv2sql_test.go b/pkg/lightning/backend/kv/kv2sql_test.go index fc422609e876e..e4945e7c6dd0e 100644 --- a/pkg/lightning/backend/kv/kv2sql_test.go +++ b/pkg/lightning/backend/kv/kv2sql_test.go @@ -42,7 +42,7 @@ func TestIterRawIndexKeysClusteredPK(t *testing.T) { require.NoError(t, err) info.State = model.StatePublic require.True(t, info.IsCommonHandle) - tbl, err := tables.TableFromMeta(kv.NewPanickingAllocators(0), info) + tbl, err := tables.TableFromMeta(kv.NewPanickingAllocators(info.SepAutoInc(), 0), info) require.NoError(t, err) sessionOpts := &encode.SessionOptions{ @@ -81,7 +81,7 @@ func TestIterRawIndexKeysIntPK(t *testing.T) { require.NoError(t, err) info.State = model.StatePublic require.True(t, info.PKIsHandle) - tbl, err := tables.TableFromMeta(kv.NewPanickingAllocators(0), info) + tbl, err := tables.TableFromMeta(kv.NewPanickingAllocators(info.SepAutoInc(), 0), info) require.NoError(t, err) sessionOpts := &encode.SessionOptions{ diff --git a/pkg/lightning/backend/kv/sql2kv_test.go b/pkg/lightning/backend/kv/sql2kv_test.go index 7b88ccf95bc5e..5780be319a3a6 100644 --- a/pkg/lightning/backend/kv/sql2kv_test.go +++ b/pkg/lightning/backend/kv/sql2kv_test.go @@ -79,7 +79,7 @@ func TestEncode(t *testing.T) { c1 := &model.ColumnInfo{ID: 1, Name: model.NewCIStr("c1"), State: model.StatePublic, Offset: 0, FieldType: *types.NewFieldType(mysql.TypeTiny)} cols := []*model.ColumnInfo{c1} tblInfo := &model.TableInfo{ID: 1, Columns: cols, PKIsHandle: false, State: model.StatePublic} - tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(0), tblInfo) + tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(tblInfo.SepAutoInc(), 0), tblInfo) require.NoError(t, err) logger := log.Logger{Logger: zap.NewNop()} @@ -162,7 +162,7 @@ func TestDecode(t *testing.T) { c1 := &model.ColumnInfo{ID: 1, Name: model.NewCIStr("c1"), State: model.StatePublic, Offset: 0, FieldType: *types.NewFieldType(mysql.TypeTiny)} cols := []*model.ColumnInfo{c1} tblInfo := &model.TableInfo{ID: 1, Columns: cols, PKIsHandle: false, State: model.StatePublic} - tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(0), tblInfo) + tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(tblInfo.SepAutoInc(), 0), tblInfo) require.NoError(t, err) decoder, err := lkv.NewTableKVDecoder(tbl, "`test`.`c1`", &encode.SessionOptions{ SQLMode: mysql.ModeStrictAllTables, @@ -216,7 +216,7 @@ func TestDecodeIndex(t *testing.T) { State: model.StatePublic, PKIsHandle: false, } - tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(0), tblInfo) + tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(tblInfo.SepAutoInc(), 0), tblInfo) if err != nil { fmt.Printf("error: %v", err.Error()) } @@ -261,7 +261,7 @@ func TestEncodeRowFormatV2(t *testing.T) { c1 := &model.ColumnInfo{ID: 1, Name: model.NewCIStr("c1"), State: model.StatePublic, Offset: 0, FieldType: *types.NewFieldType(mysql.TypeTiny)} cols := []*model.ColumnInfo{c1} tblInfo := &model.TableInfo{ID: 1, Columns: cols, PKIsHandle: false, State: model.StatePublic} - tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(0), tblInfo) + tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(tblInfo.SepAutoInc(), 0), tblInfo) require.NoError(t, err) rows := []types.Datum{ @@ -312,7 +312,7 @@ func TestEncodeTimestamp(t *testing.T) { } cols := []*model.ColumnInfo{c1} tblInfo := &model.TableInfo{ID: 1, Columns: cols, PKIsHandle: false, State: model.StatePublic} - tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(0), tblInfo) + tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(tblInfo.SepAutoInc(), 0), tblInfo) require.NoError(t, err) encoder, err := lkv.NewTableKVEncoder(&encode.EncodingConfig{ @@ -341,7 +341,7 @@ func TestEncodeTimestamp(t *testing.T) { func TestEncodeDoubleAutoIncrement(t *testing.T) { tblInfo := mockTableInfo(t, "create table t (id double not null auto_increment, unique key `u_id` (`id`));") - tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(0), tblInfo) + tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(tblInfo.SepAutoInc(), 0), tblInfo) require.NoError(t, err) encoder, err := lkv.NewTableKVEncoder(&encode.EncodingConfig{ @@ -405,7 +405,7 @@ func TestEncodeMissingAutoValue(t *testing.T) { }, } { tblInfo := mockTableInfo(t, testTblInfo.CreateStmt) - tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(0), tblInfo) + tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(tblInfo.SepAutoInc(), 0), tblInfo) require.NoError(t, err) encoder, err := lkv.NewTableKVEncoder(&encode.EncodingConfig{ @@ -457,7 +457,7 @@ func TestEncodeMissingAutoValue(t *testing.T) { func TestEncodeExpressionColumn(t *testing.T) { tblInfo := mockTableInfo(t, "create table t (id varchar(40) not null DEFAULT uuid(), unique key `u_id` (`id`));") - tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(0), tblInfo) + tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(tblInfo.SepAutoInc(), 0), tblInfo) require.NoError(t, err) encoder, err := lkv.NewTableKVEncoder(&encode.EncodingConfig{ @@ -502,7 +502,7 @@ func mockTableInfo(t *testing.T, createSQL string) *model.TableInfo { func TestDefaultAutoRandoms(t *testing.T) { tblInfo := mockTableInfo(t, "create table t (id bigint unsigned NOT NULL auto_random primary key clustered, a varchar(100));") - tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(0), tblInfo) + tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(tblInfo.SepAutoInc(), 0), tblInfo) require.NoError(t, err) encoder, err := lkv.NewTableKVEncoder(&encode.EncodingConfig{ Table: tbl, @@ -540,7 +540,7 @@ func TestDefaultAutoRandoms(t *testing.T) { func TestShardRowId(t *testing.T) { tblInfo := mockTableInfo(t, "create table t (s varchar(16)) shard_row_id_bits = 3;") - tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(0), tblInfo) + tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(tblInfo.SepAutoInc(), 0), tblInfo) require.NoError(t, err) encoder, err := lkv.NewTableKVEncoder(&encode.EncodingConfig{ Table: tbl, @@ -655,7 +655,7 @@ func SetUpTest(b *testing.B) *benchSQL2KVSuite { tableInfo.State = model.StatePublic // Construct the corresponding KV encoder. - tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(0), tableInfo) + tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(tableInfo.SepAutoInc(), 0), tableInfo) require.NoError(b, err) encoder, err := lkv.NewTableKVEncoder(&encode.EncodingConfig{ Table: tbl, diff --git a/pkg/lightning/backend/local/duplicate_test.go b/pkg/lightning/backend/local/duplicate_test.go index 7935b58185290..a53f58eafdce1 100644 --- a/pkg/lightning/backend/local/duplicate_test.go +++ b/pkg/lightning/backend/local/duplicate_test.go @@ -44,7 +44,7 @@ func TestBuildDupTask(t *testing.T) { info, err := ddl.MockTableInfo(mock.NewContext(), node[0].(*ast.CreateTableStmt), 1) require.NoError(t, err) info.State = model.StatePublic - tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(0), info) + tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(info.SepAutoInc(), 0), info) require.NoError(t, err) // Test build duplicate detecting task. @@ -80,7 +80,7 @@ func buildTableForTestConvertToErrFoundConflictRecords(t *testing.T, node []ast. info, err := ddl.MockTableInfo(mockSctx, node[0].(*ast.CreateTableStmt), 108) require.NoError(t, err) info.State = model.StatePublic - tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(0), info) + tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(info.SepAutoInc(), 0), info) require.NoError(t, err) sessionOpts := encode.SessionOptions{ diff --git a/pkg/lightning/backend/tidb/tidb_test.go b/pkg/lightning/backend/tidb/tidb_test.go index aa9251dab9353..c313074eba63e 100644 --- a/pkg/lightning/backend/tidb/tidb_test.go +++ b/pkg/lightning/backend/tidb/tidb_test.go @@ -67,7 +67,7 @@ func createMysqlSuite(t *testing.T) *mysqlSuite { cols = append(cols, col) } tblInfo := &model.TableInfo{ID: 1, Columns: cols, PKIsHandle: false, State: model.StatePublic} - tbl, err := tables.TableFromMeta(kv.NewPanickingAllocators(0), tblInfo) + tbl, err := tables.TableFromMeta(kv.NewPanickingAllocators(tblInfo.SepAutoInc(), 0), tblInfo) require.NoError(t, err) cfg := config.NewConfig() cfg.Conflict.Strategy = config.ReplaceOnDup @@ -286,7 +286,7 @@ func testStrictMode(t *testing.T) { ft.SetCharset(charset.CharsetASCII) col1 := &model.ColumnInfo{ID: 2, Name: model.NewCIStr("s1"), State: model.StatePublic, Offset: 1, FieldType: ft} tblInfo := &model.TableInfo{ID: 1, Columns: []*model.ColumnInfo{col0, col1}, PKIsHandle: false, State: model.StatePublic} - tbl, err := tables.TableFromMeta(kv.NewPanickingAllocators(0), tblInfo) + tbl, err := tables.TableFromMeta(kv.NewPanickingAllocators(tblInfo.SepAutoInc(), 0), tblInfo) require.NoError(t, err) ctx := context.Background() diff --git a/pkg/lightning/errormanager/errormanager_test.go b/pkg/lightning/errormanager/errormanager_test.go index 1fee5121829f0..2b79b4eb54531 100644 --- a/pkg/lightning/errormanager/errormanager_test.go +++ b/pkg/lightning/errormanager/errormanager_test.go @@ -215,7 +215,7 @@ func TestReplaceConflictOneKey(t *testing.T) { State: model.StatePublic, } - tbl, err := tables.TableFromMeta(tidbkv.NewPanickingAllocators(0), table) + tbl, err := tables.TableFromMeta(tidbkv.NewPanickingAllocators(table.SepAutoInc(), 0), table) require.NoError(t, err) sessionOpts := encode.SessionOptions{ @@ -397,7 +397,7 @@ func TestReplaceConflictOneUniqueKey(t *testing.T) { State: model.StatePublic, } - tbl, err := tables.TableFromMeta(tidbkv.NewPanickingAllocators(0), table) + tbl, err := tables.TableFromMeta(tidbkv.NewPanickingAllocators(table.SepAutoInc(), 0), table) require.NoError(t, err) sessionOpts := encode.SessionOptions{ diff --git a/pkg/lightning/errormanager/resolveconflict_test.go b/pkg/lightning/errormanager/resolveconflict_test.go index 535cff5b2666d..4d09217f3f3b0 100644 --- a/pkg/lightning/errormanager/resolveconflict_test.go +++ b/pkg/lightning/errormanager/resolveconflict_test.go @@ -52,7 +52,7 @@ func TestReplaceConflictMultipleKeysNonclusteredPk(t *testing.T) { require.NoError(t, err) info.State = model.StatePublic require.False(t, info.PKIsHandle) - tbl, err := tables.TableFromMeta(tidbkv.NewPanickingAllocators(0), info) + tbl, err := tables.TableFromMeta(tidbkv.NewPanickingAllocators(info.SepAutoInc(), 0), info) require.NoError(t, err) require.False(t, tbl.Meta().HasClusteredIndex()) @@ -262,7 +262,7 @@ func TestReplaceConflictOneKeyNonclusteredPk(t *testing.T) { require.NoError(t, err) info.State = model.StatePublic require.False(t, info.PKIsHandle) - tbl, err := tables.TableFromMeta(tidbkv.NewPanickingAllocators(0), info) + tbl, err := tables.TableFromMeta(tidbkv.NewPanickingAllocators(info.SepAutoInc(), 0), info) require.NoError(t, err) require.False(t, tbl.Meta().HasClusteredIndex()) @@ -415,7 +415,7 @@ func TestReplaceConflictOneUniqueKeyNonclusteredPk(t *testing.T) { require.NoError(t, err) info.State = model.StatePublic require.False(t, info.PKIsHandle) - tbl, err := tables.TableFromMeta(tidbkv.NewPanickingAllocators(0), info) + tbl, err := tables.TableFromMeta(tidbkv.NewPanickingAllocators(info.SepAutoInc(), 0), info) require.NoError(t, err) require.False(t, tbl.Meta().HasClusteredIndex()) @@ -614,7 +614,7 @@ func TestReplaceConflictOneUniqueKeyNonclusteredVarcharPk(t *testing.T) { require.NoError(t, err) info.State = model.StatePublic require.False(t, info.PKIsHandle) - tbl, err := tables.TableFromMeta(tidbkv.NewPanickingAllocators(0), info) + tbl, err := tables.TableFromMeta(tidbkv.NewPanickingAllocators(info.SepAutoInc(), 0), info) require.NoError(t, err) require.False(t, tbl.Meta().HasClusteredIndex()) diff --git a/pkg/table/tables/index_test.go b/pkg/table/tables/index_test.go index 6b5a31df1c53f..a5dd353343c64 100644 --- a/pkg/table/tables/index_test.go +++ b/pkg/table/tables/index_test.go @@ -175,7 +175,7 @@ func buildTableInfo(t *testing.T, sql string) *model.TableInfo { func TestGenIndexValueFromIndex(t *testing.T) { tblInfo := buildTableInfo(t, "create table a (a int primary key, b int not null, c text, unique key key_b(b));") tblInfo.State = model.StatePublic - tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(0), tblInfo) + tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(tblInfo.SepAutoInc(), 0), tblInfo) require.NoError(t, err) sessionOpts := encode.SessionOptions{ diff --git a/tests/realtikvtest/importintotest2/write_after_import_test.go b/tests/realtikvtest/importintotest2/write_after_import_test.go index 904a5f5b4b5b6..75e45da6869af 100644 --- a/tests/realtikvtest/importintotest2/write_after_import_test.go +++ b/tests/realtikvtest/importintotest2/write_after_import_test.go @@ -62,11 +62,11 @@ func (s *mockGCSSuite) testWriteAfterImport(importSQL string, sourceType importe caseResults: map[importer.DataSourceType]caseResult{ importer.DataSourceTypeFile: { insertedData: "8 1", - nextGlobalAutoID: []int64{8, 8}, + nextGlobalAutoID: []int64{8, 1}, }, importer.DataSourceTypeQuery: { insertedData: "8 1", - nextGlobalAutoID: []int64{8, 8}, + nextGlobalAutoID: []int64{8, 1}, }, }, autoIDCache1: true, @@ -91,11 +91,11 @@ func (s *mockGCSSuite) testWriteAfterImport(importSQL string, sourceType importe caseResults: map[importer.DataSourceType]caseResult{ importer.DataSourceTypeFile: { insertedData: "12 1", - nextGlobalAutoID: []int64{12, 12}, + nextGlobalAutoID: []int64{8, 12}, }, importer.DataSourceTypeQuery: { insertedData: "8 1", - nextGlobalAutoID: []int64{8, 8}, + nextGlobalAutoID: []int64{8, 5}, }, }, autoIDCache1: true,