Merge branch 'master' into use_one_shot_session_in_glue

Makefile

@@ -14,7 +14,7 @@
 
 include Makefile.common
 
-.PHONY: all clean test gotest server dev benchkv benchraw check checklist parser tidy ddltest build_br build_lightning build_lightning-ctl build_dumpling ut bazel_build bazel_prepare bazel_test
+.PHONY: all clean test server dev benchkv benchraw check checklist parser tidy ddltest build_br build_lightning build_lightning-ctl build_dumpling ut bazel_build bazel_prepare bazel_test
 
 default: server buildsucc
 
@@ -87,7 +87,7 @@ test: test_part_1 test_part_2
 
 test_part_1: checklist explaintest
 
-test_part_2: test_part_parser gotest gogenerate br_unit_test dumpling_unit_test
+test_part_2: test_part_parser ut gogenerate br_unit_test dumpling_unit_test
 
 test_part_parser: parser_yacc test_part_parser_dev
 
@@ -122,12 +122,6 @@ ut: tools/bin/ut tools/bin/xprog failpoint-enable
 	@$(FAILPOINT_DISABLE)
 	@$(CLEAN_UT_BINARY)
 
-gotest: failpoint-enable
-	@echo "Running in native mode."
-	@export log_level=info; export TZ='Asia/Shanghai'; \
-	$(GOTEST) -ldflags '$(TEST_LDFLAGS)' $(EXTRA_TEST_ARGS) -timeout 20m -cover $(PACKAGES_TIDB_TESTS) -coverprofile=coverage.txt > gotest.log || { $(FAILPOINT_DISABLE); cat 'gotest.log'; exit 1; }
-	@$(FAILPOINT_DISABLE)
-
 gotest_in_verify_ci: tools/bin/xprog tools/bin/ut failpoint-enable
 	@echo "Running gotest_in_verify_ci"
 	@mkdir -p $(TEST_COVERAGE_DIR)

cmd/explaintest/r/explain_cte.result

@@ -498,3 +498,77 @@ CTE_1	10000.00	root		Non-Recursive CTE
 CTE_0	10000.00	root		Non-Recursive CTE
 └─TableReader_22(Seed Part)	10000.00	root		data:TableFullScan_21
   └─TableFullScan_21	10000.00	cop[tikv]	table:tbl	keep order:false, stats:pseudo
+drop table if exists t1, t2, t3;
+create table t1 (a int, b int);
+create table t2 (c int, d int);
+create table t3 (e int, f int);
+insert into t1 values(1,1);
+insert into t2 values(1,1);
+insert into t3 values(1,1234);
+explain update t1 inner join (select t2.c from t2 inner join (with temp as (select e from t3 where t3.f = 1234) select e from temp) tt on t2.d = tt.e) t on t1.a = t.c set t1.b = 4321;
+id	estRows	task	access object	operator info
+Update_14	N/A	root		N/A
+└─HashJoin_25	12.49	root		inner join, equal:[eq(test.t1.a, test.t2.c)]
+  ├─HashJoin_33(Build)	9.99	root		inner join, equal:[eq(test.t3.e, test.t2.d)]
+  │ ├─Selection_35(Build)	7.99	root		not(isnull(test.t3.e))
+  │ │ └─CTEFullScan_36	9.99	root	CTE:temp	data:CTE_0
+  │ └─TableReader_39(Probe)	9980.01	root		data:Selection_38
+  │   └─Selection_38	9980.01	cop[tikv]		not(isnull(test.t2.c)), not(isnull(test.t2.d))
+  │     └─TableFullScan_37	10000.00	cop[tikv]	table:t2	keep order:false, stats:pseudo
+  └─TableReader_29(Probe)	9990.00	root		data:Selection_28
+    └─Selection_28	9990.00	cop[tikv]		not(isnull(test.t1.a))
+      └─TableFullScan_27	10000.00	cop[tikv]	table:t1	keep order:false, stats:pseudo
+CTE_0	9.99	root		Non-Recursive CTE
+└─Projection_17(Seed Part)	9.99	root		test.t3.e
+  └─TableReader_20	9.99	root		data:Selection_19
+    └─Selection_19	9.99	cop[tikv]		eq(test.t3.f, 1234), not(isnull(test.t3.e))
+      └─TableFullScan_18	10000.00	cop[tikv]	table:t3	keep order:false, stats:pseudo
+update t1 inner join (select t2.c from t2 inner join (with temp as (select e from t3 where t3.f = 1234) select e from temp) tt on t2.d = tt.e) t on t1.a = t.c set t1.b = 4321;
+select * from t1;
+a	b
+1	4321
+explain insert into t1  select t1.a, t1.b  from  t1 inner join (select t2.c from t2 inner join (with temp as (select e from t3 where t3.f = 1234) select e from temp) tt on t2.d = tt.e) t on t1.a = t.c;
+id	estRows	task	access object	operator info
+Insert_1	N/A	root		N/A
+└─HashJoin_28	12.49	root		inner join, equal:[eq(test.t2.c, test.t1.a)]
+  ├─HashJoin_30(Build)	9.99	root		inner join, equal:[eq(test.t3.e, test.t2.d)]
+  │ ├─Selection_31(Build)	7.99	root		not(isnull(test.t3.e))
+  │ │ └─CTEFullScan_32	9.99	root	CTE:temp	data:CTE_0
+  │ └─TableReader_35(Probe)	9980.01	root		data:Selection_34
+  │   └─Selection_34	9980.01	cop[tikv]		not(isnull(test.t2.c)), not(isnull(test.t2.d))
+  │     └─TableFullScan_33	10000.00	cop[tikv]	table:t2	keep order:false, stats:pseudo
+  └─TableReader_38(Probe)	9990.00	root		data:Selection_37
+    └─Selection_37	9990.00	cop[tikv]		not(isnull(test.t1.a))
+      └─TableFullScan_36	10000.00	cop[tikv]	table:t1	keep order:false, stats:pseudo
+CTE_0	9.99	root		Non-Recursive CTE
+└─Projection_18(Seed Part)	9.99	root		test.t3.e
+  └─TableReader_21	9.99	root		data:Selection_20
+    └─Selection_20	9.99	cop[tikv]		eq(test.t3.f, 1234), not(isnull(test.t3.e))
+      └─TableFullScan_19	10000.00	cop[tikv]	table:t3	keep order:false, stats:pseudo
+insert into t1  select t1.a, t1.b  from  t1 inner join (select t2.c from t2 inner join (with temp as (select e from t3 where t3.f = 1234) select e from temp) tt on t2.d = tt.e) t on t1.a = t.c;
+select * from t1;
+a	b
+1	4321
+1	4321
+explain delete from  t1 using t1 inner join (select t2.c from t2 inner join (with temp as (select e from t3 where t3.f = 1234) select e from temp) tt on t2.d = tt.e) t on t1.a = t.c;
+id	estRows	task	access object	operator info
+Delete_14	N/A	root		N/A
+└─Projection_25	12.49	root		test.t1.a, test.t1.b, test.t1._tidb_rowid, test.t2.c
+  └─HashJoin_27	12.49	root		inner join, equal:[eq(test.t2.c, test.t1.a)]
+    ├─HashJoin_29(Build)	9.99	root		inner join, equal:[eq(test.t3.e, test.t2.d)]
+    │ ├─Selection_30(Build)	7.99	root		not(isnull(test.t3.e))
+    │ │ └─CTEFullScan_31	9.99	root	CTE:temp	data:CTE_0
+    │ └─TableReader_34(Probe)	9980.01	root		data:Selection_33
+    │   └─Selection_33	9980.01	cop[tikv]		not(isnull(test.t2.c)), not(isnull(test.t2.d))
+    │     └─TableFullScan_32	10000.00	cop[tikv]	table:t2	keep order:false, stats:pseudo
+    └─TableReader_37(Probe)	9990.00	root		data:Selection_36
+      └─Selection_36	9990.00	cop[tikv]		not(isnull(test.t1.a))
+        └─TableFullScan_35	10000.00	cop[tikv]	table:t1	keep order:false, stats:pseudo
+CTE_0	9.99	root		Non-Recursive CTE
+└─Projection_17(Seed Part)	9.99	root		test.t3.e
+  └─TableReader_20	9.99	root		data:Selection_19
+    └─Selection_19	9.99	cop[tikv]		eq(test.t3.f, 1234), not(isnull(test.t3.e))
+      └─TableFullScan_18	10000.00	cop[tikv]	table:t3	keep order:false, stats:pseudo
+delete from  t1 using t1 inner join (select t2.c from t2 inner join (with temp as (select e from t3 where t3.f = 1234) select e from temp) tt on t2.d = tt.e) t on t1.a = t.c;
+select * from t1;
+a	b

cmd/explaintest/t/explain_cte.test

@@ -263,3 +263,21 @@ where v1.bench_type =v2.bench_type;
 drop table if exists tbl;
 create table tbl (id int);
 explain with t1 as (select id from tbl), t2 as (select a.id from t1 a join t1 b on a.id = b.id) select * from t2 where id in (select id from t2);
+
+# issue 35758
+drop table if exists t1, t2, t3;
+create table t1 (a int, b int);
+create table t2 (c int, d int);
+create table t3 (e int, f int);
+insert into t1 values(1,1);
+insert into t2 values(1,1);
+insert into t3 values(1,1234);
+explain update t1 inner join (select t2.c from t2 inner join (with temp as (select e from t3 where t3.f = 1234) select e from temp) tt on t2.d = tt.e) t on t1.a = t.c set t1.b = 4321;
+update t1 inner join (select t2.c from t2 inner join (with temp as (select e from t3 where t3.f = 1234) select e from temp) tt on t2.d = tt.e) t on t1.a = t.c set t1.b = 4321;
+select * from t1;
+explain insert into t1  select t1.a, t1.b  from  t1 inner join (select t2.c from t2 inner join (with temp as (select e from t3 where t3.f = 1234) select e from temp) tt on t2.d = tt.e) t on t1.a = t.c;
+insert into t1  select t1.a, t1.b  from  t1 inner join (select t2.c from t2 inner join (with temp as (select e from t3 where t3.f = 1234) select e from temp) tt on t2.d = tt.e) t on t1.a = t.c;
+select * from t1;
+explain delete from  t1 using t1 inner join (select t2.c from t2 inner join (with temp as (select e from t3 where t3.f = 1234) select e from temp) tt on t2.d = tt.e) t on t1.a = t.c;
+delete from  t1 using t1 inner join (select t2.c from t2 inner join (with temp as (select e from t3 where t3.f = 1234) select e from temp) tt on t2.d = tt.e) t on t1.a = t.c;
+select * from t1;

executor/memtable_reader.go

@@ -420,9 +420,9 @@ func parseFailpointServerInfo(s string) []infoschema.ServerInfo {
 	for _, server := range servers {
 		parts := strings.Split(server, ",")
 		serversInfo = append(serversInfo, infoschema.ServerInfo{
-			ServerType: parts[0],
-			Address:    parts[1],
 			StatusAddr: parts[2],
+			Address:    parts[1],
+			ServerType: parts[0],
 		})
 	}
 	return serversInfo
@@ -947,8 +947,7 @@ func (e *hotRegionsHistoryRetriver) getHotRegionRowWithSchemaInfo(
 				updateTimestamp.In(tz)
 			}
 			updateTime := types.NewTime(types.FromGoTime(updateTimestamp), mysql.TypeTimestamp, types.MinFsp)
-			row := make([]types.Datum, len(infoschema.TableTiDBHotRegionsHistoryCols))
-
+			row := make([]types.Datum, len(infoschema.GetTableTiDBHotRegionsHistoryCols()))
 			row[0].SetMysqlTime(updateTime)
 			row[1].SetString(strings.ToUpper(tableInfo.DB.Name.O), mysql.DefaultCollationName)
 			row[2].SetString(strings.ToUpper(tableInfo.Table.Name.O), mysql.DefaultCollationName)
@@ -1088,7 +1087,7 @@ func (e *tikvRegionPeersRetriever) packTiKVRegionPeersRows(
 				continue
 			}
 
-			row := make([]types.Datum, len(infoschema.TableTiKVRegionPeersCols))
+			row := make([]types.Datum, len(infoschema.GetTableTiKVRegionPeersCols()))
 			row[0].SetInt64(region.ID)
 			row[1].SetInt64(peer.ID)
 			row[2].SetInt64(peer.StoreID)

infoschema/tables.go

@@ -927,6 +927,14 @@ var TableTiDBHotRegionsHistoryCols = []columnInfo{
 	{name: "QUERY_RATE", tp: mysql.TypeDouble, size: 22},
 }
 
+// GetTableTiDBHotRegionsHistoryCols is to get TableTiDBHotRegionsHistoryCols.
+// It is an optimization because Go does’t support const arrays. The solution  is to use initialization functions.
+// It is useful in the BCE optimization.
+// https://go101.org/article/bounds-check-elimination.html
+func GetTableTiDBHotRegionsHistoryCols() []columnInfo {
+	return TableTiDBHotRegionsHistoryCols
+}
+
 // TableTiKVStoreStatusCols is TiDB kv store status columns.
 var TableTiKVStoreStatusCols = []columnInfo{
 	{name: "STORE_ID", tp: mysql.TypeLonglong, size: 21},
@@ -996,6 +1004,14 @@ var TableTiKVRegionPeersCols = []columnInfo{
 	{name: "DOWN_SECONDS", tp: mysql.TypeLonglong, size: 21, deflt: 0},
 }
 
+// GetTableTiKVRegionPeersCols is to get TableTiKVRegionPeersCols.
+// It is an optimization because Go does’t support const arrays. The solution  is to use initialization functions.
+// It is useful in the BCE optimization.
+// https://go101.org/article/bounds-check-elimination.html
+func GetTableTiKVRegionPeersCols() []columnInfo {
+	return TableTiKVRegionPeersCols
+}
+
 var tableTiDBServersInfoCols = []columnInfo{
 	{name: "DDL_ID", tp: mysql.TypeVarchar, size: 64},
 	{name: "IP", tp: mysql.TypeVarchar, size: 64},

planner/core/logical_plan_builder.go

@@ -6944,6 +6944,8 @@ func (b *PlanBuilder) buildWith(ctx context.Context, w *ast.WithClause) error {
 		b.outerCTEs[len(b.outerCTEs)-1].optFlag = b.optFlag
 		b.outerCTEs[len(b.outerCTEs)-1].isBuilding = false
 		b.optFlag = saveFlag
+		// each cte (select statement) will generate a handle map, pop it out here.
+		b.handleHelper.popMap()
 	}
 	return nil
 }

session/tidb_test.go

@@ -69,7 +69,8 @@ func TestParseErrorWarn(t *testing.T) {
 
 func TestKeysNeedLock(t *testing.T) {
 	rowKey := tablecodec.EncodeRowKeyWithHandle(1, kv.IntHandle(1))
-	indexKey := tablecodec.EncodeIndexSeekKey(1, 1, []byte{1})
+	uniqueIndexKey := tablecodec.EncodeIndexSeekKey(1, 1, []byte{1})
+	nonUniqueIndexKey := tablecodec.EncodeIndexSeekKey(1, 2, []byte{1})
 	uniqueValue := make([]byte, 8)
 	uniqueUntouched := append(uniqueValue, '1')
 	nonUniqueVal := []byte{'0'}
@@ -83,18 +84,20 @@ func TestKeysNeedLock(t *testing.T) {
 	}{
 		{rowKey, rowVal, true},
 		{rowKey, deleteVal, true},
-		{indexKey, nonUniqueVal, false},
-		{indexKey, nonUniqueUntouched, false},
-		{indexKey, uniqueValue, true},
-		{indexKey, uniqueUntouched, false},
-		{indexKey, deleteVal, false},
+		{nonUniqueIndexKey, nonUniqueVal, false},
+		{nonUniqueIndexKey, nonUniqueUntouched, false},
+		{uniqueIndexKey, uniqueValue, true},
+		{uniqueIndexKey, uniqueUntouched, false},
+		{uniqueIndexKey, deleteVal, false},
 	}
 
 	for _, test := range tests {
-		require.Equal(t, test.need, keyNeedToLock(test.key, test.val, 0))
-	}
+		need := keyNeedToLock(test.key, test.val, 0)
+		require.Equal(t, test.need, need)
 
-	flag := kv.KeyFlags(1)
-	require.True(t, flag.HasPresumeKeyNotExists())
-	require.True(t, keyNeedToLock(indexKey, deleteVal, flag))
+		flag := kv.KeyFlags(1)
+		need = keyNeedToLock(test.key, test.val, flag)
+		require.True(t, flag.HasPresumeKeyNotExists())
+		require.True(t, need)
+	}
 }

session/txn.go

@@ -506,9 +506,12 @@ func keyNeedToLock(k, v []byte, flags kv.KeyFlags) bool {
 	if tablecodec.IsUntouchedIndexKValue(k, v) {
 		return false
 	}
-	isNonUniqueIndex := tablecodec.IsIndexKey(k) && len(v) == 1
-	// Put row key and unique index need to lock.
-	return !isNonUniqueIndex
+
+	if !tablecodec.IsIndexKey(k) {
+		return true
+	}
+
+	return tablecodec.IndexKVIsUnique(v)
 }
 
 func getBinlogMutation(ctx sessionctx.Context, tableID int64) *binlog.TableMutation {

tablecodec/tablecodec.go

@@ -1580,3 +1580,16 @@ func decodeIndexKvGeneral(key, value []byte, colsLen int, hdStatus HandleStatus,
 	}
 	return resultValues, nil
 }
+
+// IndexKVIsUnique uses to judge if an index is unique, it can handle the KV committed by txn already, it doesn't consider the untouched flag.
+func IndexKVIsUnique(value []byte) bool {
+	if len(value) <= MaxOldEncodeValueLen {
+		return len(value) == 8
+	}
+	if getIndexVersion(value) == 1 {
+		segs := SplitIndexValueForClusteredIndexVersion1(value)
+		return segs.CommonHandle != nil
+	}
+	segs := SplitIndexValue(value)
+	return segs.IntHandle != nil || segs.CommonHandle != nil
+}