From 965454fd16f1ce88f29948edea14aa33dce9363b Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Tue, 26 Jul 2022 11:15:10 +0800 Subject: [PATCH 1/4] executor: optimize BCE in the memtable_reader (#34663) ref pingcap/tidb#34669 --- executor/memtable_reader.go | 9 ++++----- infoschema/tables.go | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/executor/memtable_reader.go b/executor/memtable_reader.go index 7bd2cf3b63be3..d71680bfd13a8 100644 --- a/executor/memtable_reader.go +++ b/executor/memtable_reader.go @@ -420,9 +420,9 @@ func parseFailpointServerInfo(s string) []infoschema.ServerInfo { for _, server := range servers { parts := strings.Split(server, ",") serversInfo = append(serversInfo, infoschema.ServerInfo{ - ServerType: parts[0], - Address: parts[1], StatusAddr: parts[2], + Address: parts[1], + ServerType: parts[0], }) } return serversInfo @@ -947,8 +947,7 @@ func (e *hotRegionsHistoryRetriver) getHotRegionRowWithSchemaInfo( updateTimestamp.In(tz) } updateTime := types.NewTime(types.FromGoTime(updateTimestamp), mysql.TypeTimestamp, types.MinFsp) - row := make([]types.Datum, len(infoschema.TableTiDBHotRegionsHistoryCols)) - + row := make([]types.Datum, len(infoschema.GetTableTiDBHotRegionsHistoryCols())) row[0].SetMysqlTime(updateTime) row[1].SetString(strings.ToUpper(tableInfo.DB.Name.O), mysql.DefaultCollationName) row[2].SetString(strings.ToUpper(tableInfo.Table.Name.O), mysql.DefaultCollationName) @@ -1088,7 +1087,7 @@ func (e *tikvRegionPeersRetriever) packTiKVRegionPeersRows( continue } - row := make([]types.Datum, len(infoschema.TableTiKVRegionPeersCols)) + row := make([]types.Datum, len(infoschema.GetTableTiKVRegionPeersCols())) row[0].SetInt64(region.ID) row[1].SetInt64(peer.ID) row[2].SetInt64(peer.StoreID) diff --git a/infoschema/tables.go b/infoschema/tables.go index f8584724b2704..e04e1bb232b48 100644 --- a/infoschema/tables.go +++ b/infoschema/tables.go @@ -927,6 +927,14 @@ var TableTiDBHotRegionsHistoryCols = []columnInfo{ {name: "QUERY_RATE", tp: mysql.TypeDouble, size: 22}, } +// GetTableTiDBHotRegionsHistoryCols is to get TableTiDBHotRegionsHistoryCols. +// It is an optimization because Go does’t support const arrays. The solution is to use initialization functions. +// It is useful in the BCE optimization. +// https://go101.org/article/bounds-check-elimination.html +func GetTableTiDBHotRegionsHistoryCols() []columnInfo { + return TableTiDBHotRegionsHistoryCols +} + // TableTiKVStoreStatusCols is TiDB kv store status columns. var TableTiKVStoreStatusCols = []columnInfo{ {name: "STORE_ID", tp: mysql.TypeLonglong, size: 21}, @@ -996,6 +1004,14 @@ var TableTiKVRegionPeersCols = []columnInfo{ {name: "DOWN_SECONDS", tp: mysql.TypeLonglong, size: 21, deflt: 0}, } +// GetTableTiKVRegionPeersCols is to get TableTiKVRegionPeersCols. +// It is an optimization because Go does’t support const arrays. The solution is to use initialization functions. +// It is useful in the BCE optimization. +// https://go101.org/article/bounds-check-elimination.html +func GetTableTiKVRegionPeersCols() []columnInfo { + return TableTiKVRegionPeersCols +} + var tableTiDBServersInfoCols = []columnInfo{ {name: "DDL_ID", tp: mysql.TypeVarchar, size: 64}, {name: "IP", tp: mysql.TypeVarchar, size: 64}, From db179bf72180885ba4a7099a8b386a80e4be1f24 Mon Sep 17 00:00:00 2001 From: tangenta Date: Tue, 26 Jul 2022 11:55:10 +0800 Subject: [PATCH 2/4] Makefile: remove target `gotest` and use `ut` instead (#36530) close pingcap/tidb#36493 --- Makefile | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index c8a1d29ea761b..60175532853e0 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ include Makefile.common -.PHONY: all clean test gotest server dev benchkv benchraw check checklist parser tidy ddltest build_br build_lightning build_lightning-ctl build_dumpling ut bazel_build bazel_prepare bazel_test +.PHONY: all clean test server dev benchkv benchraw check checklist parser tidy ddltest build_br build_lightning build_lightning-ctl build_dumpling ut bazel_build bazel_prepare bazel_test default: server buildsucc @@ -87,7 +87,7 @@ test: test_part_1 test_part_2 test_part_1: checklist explaintest -test_part_2: test_part_parser gotest gogenerate br_unit_test dumpling_unit_test +test_part_2: test_part_parser ut gogenerate br_unit_test dumpling_unit_test test_part_parser: parser_yacc test_part_parser_dev @@ -122,12 +122,6 @@ ut: tools/bin/ut tools/bin/xprog failpoint-enable @$(FAILPOINT_DISABLE) @$(CLEAN_UT_BINARY) -gotest: failpoint-enable - @echo "Running in native mode." - @export log_level=info; export TZ='Asia/Shanghai'; \ - $(GOTEST) -ldflags '$(TEST_LDFLAGS)' $(EXTRA_TEST_ARGS) -timeout 20m -cover $(PACKAGES_TIDB_TESTS) -coverprofile=coverage.txt > gotest.log || { $(FAILPOINT_DISABLE); cat 'gotest.log'; exit 1; } - @$(FAILPOINT_DISABLE) - gotest_in_verify_ci: tools/bin/xprog tools/bin/ut failpoint-enable @echo "Running gotest_in_verify_ci" @mkdir -p $(TEST_COVERAGE_DIR) From 87c5b5068ab8607b6f70f4ce12eab31ebed9fe64 Mon Sep 17 00:00:00 2001 From: ekexium Date: Tue, 26 Jul 2022 13:15:10 +0800 Subject: [PATCH 3/4] executor: do not acqurie pessimistic lock for non-unique index keys (#36229) close pingcap/tidb#36235 --- session/tidb_test.go | 25 ++++++++++++++----------- session/txn.go | 9 ++++++--- tablecodec/tablecodec.go | 13 +++++++++++++ 3 files changed, 33 insertions(+), 14 deletions(-) diff --git a/session/tidb_test.go b/session/tidb_test.go index 2023921445026..b88d1fd19873d 100644 --- a/session/tidb_test.go +++ b/session/tidb_test.go @@ -69,7 +69,8 @@ func TestParseErrorWarn(t *testing.T) { func TestKeysNeedLock(t *testing.T) { rowKey := tablecodec.EncodeRowKeyWithHandle(1, kv.IntHandle(1)) - indexKey := tablecodec.EncodeIndexSeekKey(1, 1, []byte{1}) + uniqueIndexKey := tablecodec.EncodeIndexSeekKey(1, 1, []byte{1}) + nonUniqueIndexKey := tablecodec.EncodeIndexSeekKey(1, 2, []byte{1}) uniqueValue := make([]byte, 8) uniqueUntouched := append(uniqueValue, '1') nonUniqueVal := []byte{'0'} @@ -83,18 +84,20 @@ func TestKeysNeedLock(t *testing.T) { }{ {rowKey, rowVal, true}, {rowKey, deleteVal, true}, - {indexKey, nonUniqueVal, false}, - {indexKey, nonUniqueUntouched, false}, - {indexKey, uniqueValue, true}, - {indexKey, uniqueUntouched, false}, - {indexKey, deleteVal, false}, + {nonUniqueIndexKey, nonUniqueVal, false}, + {nonUniqueIndexKey, nonUniqueUntouched, false}, + {uniqueIndexKey, uniqueValue, true}, + {uniqueIndexKey, uniqueUntouched, false}, + {uniqueIndexKey, deleteVal, false}, } for _, test := range tests { - require.Equal(t, test.need, keyNeedToLock(test.key, test.val, 0)) - } + need := keyNeedToLock(test.key, test.val, 0) + require.Equal(t, test.need, need) - flag := kv.KeyFlags(1) - require.True(t, flag.HasPresumeKeyNotExists()) - require.True(t, keyNeedToLock(indexKey, deleteVal, flag)) + flag := kv.KeyFlags(1) + need = keyNeedToLock(test.key, test.val, flag) + require.True(t, flag.HasPresumeKeyNotExists()) + require.True(t, need) + } } diff --git a/session/txn.go b/session/txn.go index cdb1339ddac43..c41d4c161ebdd 100644 --- a/session/txn.go +++ b/session/txn.go @@ -506,9 +506,12 @@ func keyNeedToLock(k, v []byte, flags kv.KeyFlags) bool { if tablecodec.IsUntouchedIndexKValue(k, v) { return false } - isNonUniqueIndex := tablecodec.IsIndexKey(k) && len(v) == 1 - // Put row key and unique index need to lock. - return !isNonUniqueIndex + + if !tablecodec.IsIndexKey(k) { + return true + } + + return tablecodec.IndexKVIsUnique(v) } func getBinlogMutation(ctx sessionctx.Context, tableID int64) *binlog.TableMutation { diff --git a/tablecodec/tablecodec.go b/tablecodec/tablecodec.go index df7f4d188efff..0b06a15cd22b4 100644 --- a/tablecodec/tablecodec.go +++ b/tablecodec/tablecodec.go @@ -1580,3 +1580,16 @@ func decodeIndexKvGeneral(key, value []byte, colsLen int, hdStatus HandleStatus, } return resultValues, nil } + +// IndexKVIsUnique uses to judge if an index is unique, it can handle the KV committed by txn already, it doesn't consider the untouched flag. +func IndexKVIsUnique(value []byte) bool { + if len(value) <= MaxOldEncodeValueLen { + return len(value) == 8 + } + if getIndexVersion(value) == 1 { + segs := SplitIndexValueForClusteredIndexVersion1(value) + return segs.CommonHandle != nil + } + segs := SplitIndexValue(value) + return segs.IntHandle != nil || segs.CommonHandle != nil +} From 1b848f40e2eebfb4133e92886efc220bc68e8bea Mon Sep 17 00:00:00 2001 From: Arenatlx <314806019@qq.com> Date: Tue, 26 Jul 2022 14:19:10 +0800 Subject: [PATCH 4/4] planner: just pop cte's handleHelper map out since it shouldn't be considered (#35854) close pingcap/tidb#35758 --- cmd/explaintest/r/explain_cte.result | 74 ++++++++++++++++++++++++++++ cmd/explaintest/t/explain_cte.test | 18 +++++++ planner/core/logical_plan_builder.go | 2 + 3 files changed, 94 insertions(+) diff --git a/cmd/explaintest/r/explain_cte.result b/cmd/explaintest/r/explain_cte.result index e4837878d3054..16249f475a6b6 100644 --- a/cmd/explaintest/r/explain_cte.result +++ b/cmd/explaintest/r/explain_cte.result @@ -498,3 +498,77 @@ CTE_1 10000.00 root Non-Recursive CTE CTE_0 10000.00 root Non-Recursive CTE └─TableReader_22(Seed Part) 10000.00 root data:TableFullScan_21 └─TableFullScan_21 10000.00 cop[tikv] table:tbl keep order:false, stats:pseudo +drop table if exists t1, t2, t3; +create table t1 (a int, b int); +create table t2 (c int, d int); +create table t3 (e int, f int); +insert into t1 values(1,1); +insert into t2 values(1,1); +insert into t3 values(1,1234); +explain update t1 inner join (select t2.c from t2 inner join (with temp as (select e from t3 where t3.f = 1234) select e from temp) tt on t2.d = tt.e) t on t1.a = t.c set t1.b = 4321; +id estRows task access object operator info +Update_14 N/A root N/A +└─HashJoin_25 12.49 root inner join, equal:[eq(test.t1.a, test.t2.c)] + ├─HashJoin_33(Build) 9.99 root inner join, equal:[eq(test.t3.e, test.t2.d)] + │ ├─Selection_35(Build) 7.99 root not(isnull(test.t3.e)) + │ │ └─CTEFullScan_36 9.99 root CTE:temp data:CTE_0 + │ └─TableReader_39(Probe) 9980.01 root data:Selection_38 + │ └─Selection_38 9980.01 cop[tikv] not(isnull(test.t2.c)), not(isnull(test.t2.d)) + │ └─TableFullScan_37 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo + └─TableReader_29(Probe) 9990.00 root data:Selection_28 + └─Selection_28 9990.00 cop[tikv] not(isnull(test.t1.a)) + └─TableFullScan_27 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo +CTE_0 9.99 root Non-Recursive CTE +└─Projection_17(Seed Part) 9.99 root test.t3.e + └─TableReader_20 9.99 root data:Selection_19 + └─Selection_19 9.99 cop[tikv] eq(test.t3.f, 1234), not(isnull(test.t3.e)) + └─TableFullScan_18 10000.00 cop[tikv] table:t3 keep order:false, stats:pseudo +update t1 inner join (select t2.c from t2 inner join (with temp as (select e from t3 where t3.f = 1234) select e from temp) tt on t2.d = tt.e) t on t1.a = t.c set t1.b = 4321; +select * from t1; +a b +1 4321 +explain insert into t1 select t1.a, t1.b from t1 inner join (select t2.c from t2 inner join (with temp as (select e from t3 where t3.f = 1234) select e from temp) tt on t2.d = tt.e) t on t1.a = t.c; +id estRows task access object operator info +Insert_1 N/A root N/A +└─HashJoin_28 12.49 root inner join, equal:[eq(test.t2.c, test.t1.a)] + ├─HashJoin_30(Build) 9.99 root inner join, equal:[eq(test.t3.e, test.t2.d)] + │ ├─Selection_31(Build) 7.99 root not(isnull(test.t3.e)) + │ │ └─CTEFullScan_32 9.99 root CTE:temp data:CTE_0 + │ └─TableReader_35(Probe) 9980.01 root data:Selection_34 + │ └─Selection_34 9980.01 cop[tikv] not(isnull(test.t2.c)), not(isnull(test.t2.d)) + │ └─TableFullScan_33 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo + └─TableReader_38(Probe) 9990.00 root data:Selection_37 + └─Selection_37 9990.00 cop[tikv] not(isnull(test.t1.a)) + └─TableFullScan_36 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo +CTE_0 9.99 root Non-Recursive CTE +└─Projection_18(Seed Part) 9.99 root test.t3.e + └─TableReader_21 9.99 root data:Selection_20 + └─Selection_20 9.99 cop[tikv] eq(test.t3.f, 1234), not(isnull(test.t3.e)) + └─TableFullScan_19 10000.00 cop[tikv] table:t3 keep order:false, stats:pseudo +insert into t1 select t1.a, t1.b from t1 inner join (select t2.c from t2 inner join (with temp as (select e from t3 where t3.f = 1234) select e from temp) tt on t2.d = tt.e) t on t1.a = t.c; +select * from t1; +a b +1 4321 +1 4321 +explain delete from t1 using t1 inner join (select t2.c from t2 inner join (with temp as (select e from t3 where t3.f = 1234) select e from temp) tt on t2.d = tt.e) t on t1.a = t.c; +id estRows task access object operator info +Delete_14 N/A root N/A +└─Projection_25 12.49 root test.t1.a, test.t1.b, test.t1._tidb_rowid, test.t2.c + └─HashJoin_27 12.49 root inner join, equal:[eq(test.t2.c, test.t1.a)] + ├─HashJoin_29(Build) 9.99 root inner join, equal:[eq(test.t3.e, test.t2.d)] + │ ├─Selection_30(Build) 7.99 root not(isnull(test.t3.e)) + │ │ └─CTEFullScan_31 9.99 root CTE:temp data:CTE_0 + │ └─TableReader_34(Probe) 9980.01 root data:Selection_33 + │ └─Selection_33 9980.01 cop[tikv] not(isnull(test.t2.c)), not(isnull(test.t2.d)) + │ └─TableFullScan_32 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo + └─TableReader_37(Probe) 9990.00 root data:Selection_36 + └─Selection_36 9990.00 cop[tikv] not(isnull(test.t1.a)) + └─TableFullScan_35 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo +CTE_0 9.99 root Non-Recursive CTE +└─Projection_17(Seed Part) 9.99 root test.t3.e + └─TableReader_20 9.99 root data:Selection_19 + └─Selection_19 9.99 cop[tikv] eq(test.t3.f, 1234), not(isnull(test.t3.e)) + └─TableFullScan_18 10000.00 cop[tikv] table:t3 keep order:false, stats:pseudo +delete from t1 using t1 inner join (select t2.c from t2 inner join (with temp as (select e from t3 where t3.f = 1234) select e from temp) tt on t2.d = tt.e) t on t1.a = t.c; +select * from t1; +a b diff --git a/cmd/explaintest/t/explain_cte.test b/cmd/explaintest/t/explain_cte.test index 87ea10662e1c2..c97643115d880 100644 --- a/cmd/explaintest/t/explain_cte.test +++ b/cmd/explaintest/t/explain_cte.test @@ -263,3 +263,21 @@ where v1.bench_type =v2.bench_type; drop table if exists tbl; create table tbl (id int); explain with t1 as (select id from tbl), t2 as (select a.id from t1 a join t1 b on a.id = b.id) select * from t2 where id in (select id from t2); + +# issue 35758 +drop table if exists t1, t2, t3; +create table t1 (a int, b int); +create table t2 (c int, d int); +create table t3 (e int, f int); +insert into t1 values(1,1); +insert into t2 values(1,1); +insert into t3 values(1,1234); +explain update t1 inner join (select t2.c from t2 inner join (with temp as (select e from t3 where t3.f = 1234) select e from temp) tt on t2.d = tt.e) t on t1.a = t.c set t1.b = 4321; +update t1 inner join (select t2.c from t2 inner join (with temp as (select e from t3 where t3.f = 1234) select e from temp) tt on t2.d = tt.e) t on t1.a = t.c set t1.b = 4321; +select * from t1; +explain insert into t1 select t1.a, t1.b from t1 inner join (select t2.c from t2 inner join (with temp as (select e from t3 where t3.f = 1234) select e from temp) tt on t2.d = tt.e) t on t1.a = t.c; +insert into t1 select t1.a, t1.b from t1 inner join (select t2.c from t2 inner join (with temp as (select e from t3 where t3.f = 1234) select e from temp) tt on t2.d = tt.e) t on t1.a = t.c; +select * from t1; +explain delete from t1 using t1 inner join (select t2.c from t2 inner join (with temp as (select e from t3 where t3.f = 1234) select e from temp) tt on t2.d = tt.e) t on t1.a = t.c; +delete from t1 using t1 inner join (select t2.c from t2 inner join (with temp as (select e from t3 where t3.f = 1234) select e from temp) tt on t2.d = tt.e) t on t1.a = t.c; +select * from t1; diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 3744c4477aa28..f34c79052994c 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -6944,6 +6944,8 @@ func (b *PlanBuilder) buildWith(ctx context.Context, w *ast.WithClause) error { b.outerCTEs[len(b.outerCTEs)-1].optFlag = b.optFlag b.outerCTEs[len(b.outerCTEs)-1].isBuilding = false b.optFlag = saveFlag + // each cte (select statement) will generate a handle map, pop it out here. + b.handleHelper.popMap() } return nil }