From b4c9580bd6dee7d5e39b901df29eb79594b22fbb Mon Sep 17 00:00:00 2001 From: Calvin Neo Date: Wed, 27 Apr 2022 11:08:51 +0800 Subject: [PATCH 1/3] ddl: eliminate too much updateTiFlashStores logs (#34230) close pingcap/tidb#34223 --- ddl/ddl_tiflash_api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddl/ddl_tiflash_api.go b/ddl/ddl_tiflash_api.go index 5638cc4bbb39d..6ef17755a5fcd 100644 --- a/ddl/ddl_tiflash_api.go +++ b/ddl/ddl_tiflash_api.go @@ -343,7 +343,7 @@ func updateTiFlashStores(pollTiFlashContext *TiFlashManagementContext) error { } } } - logutil.BgLogger().Info("updateTiFlashStores finished", zap.Int("TiFlash store count", len(pollTiFlashContext.TiFlashStores))) + logutil.BgLogger().Debug("updateTiFlashStores finished", zap.Int("TiFlash store count", len(pollTiFlashContext.TiFlashStores))) return nil } From 591c2050a7b32b35b34de2f13e25d1318776979b Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Wed, 27 Apr 2022 11:22:51 +0800 Subject: [PATCH 2/3] planner: let apply be paralleled though there's order property can be used (#34238) close pingcap/tidb#34237 --- executor/parallel_apply_test.go | 6 ++++++ planner/core/exhaust_physical_plans.go | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/executor/parallel_apply_test.go b/executor/parallel_apply_test.go index 1d775fcc2218a..63e439cbc661f 100644 --- a/executor/parallel_apply_test.go +++ b/executor/parallel_apply_test.go @@ -66,6 +66,12 @@ func TestParallelApplyPlan(t *testing.T) { q2 := "select * from t t0 where t0.b <= (select max(t1.b) from t t1 where t1.b > (select max(b) from t t2 where t1.a > t2.a and t0.a > t2.a));" checkApplyPlan(t, tk, q2, 1) // only the outside apply can be parallel tk.MustQuery(q2).Sort().Check(testkit.Rows("1 1", "2 2", "3 3", "4 4", "5 5", "6 6", "7 7", "8 8", "9 9")) + q3 := "select t1.b from t t1 where t1.b > (select max(b) from t t2 where t1.a > t2.a) order by t1.a" + checkApplyPlan(t, tk, q3, 0) + tk.MustExec("alter table t add index idx(a)") + checkApplyPlan(t, tk, q3, 1) + tk.MustQuery(q3).Sort().Check(testkit.Rows("1", "2", "3", "4", "5", "6", "7", "8", "9")) + tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 Parallel Apply rejects the possible order properties of its outer child currently")) } func TestApplyColumnType(t *testing.T) { diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index 4f2e83bc20cae..ef8507470cf2e 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -2175,6 +2175,10 @@ func (la *LogicalApply) exhaustPhysicalPlans(prop *property.PhysicalProperty) ([ "MPP mode may be blocked because operator `Apply` is not supported now.") return nil, true, nil } + if !prop.IsEmpty() && la.SCtx().GetSessionVars().EnableParallelApply { + la.ctx.GetSessionVars().StmtCtx.AppendWarning(errors.Errorf("Parallel Apply rejects the possible order properties of its outer child currently")) + return nil, true, nil + } disableAggPushDownToCop(la.children[0]) join := la.GetHashJoin(prop) var columns = make([]*expression.Column, 0, len(la.CorCols)) From 1a19f959f4543a5cdf4b442f71588c6e68275ae8 Mon Sep 17 00:00:00 2001 From: Lynn Date: Wed, 27 Apr 2022 15:20:51 +0800 Subject: [PATCH 3/3] session, table: fix listColumnPartition data race (#33199) close pingcap/tidb#33030 --- session/bootstrap.go | 31 +++++++++++++++++++++++++++ session/session.go | 2 ++ table/tables/partition.go | 44 ++++++++++++++++++--------------------- 3 files changed, 53 insertions(+), 24 deletions(-) diff --git a/session/bootstrap.go b/session/bootstrap.go index a4e34493dae36..9f5aca1b606f6 100644 --- a/session/bootstrap.go +++ b/session/bootstrap.go @@ -37,10 +37,12 @@ import ( "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/parser" "github.com/pingcap/tidb/parser/auth" + "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/parser/mysql" "github.com/pingcap/tidb/parser/terror" "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/sessionctx/variable" + "github.com/pingcap/tidb/table/tables" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/dbterror" @@ -2007,3 +2009,32 @@ func oldPasswordUpgrade(pass string) (string, error) { newpass := fmt.Sprintf("*%X", hash2) return newpass, nil } + +// rebuildAllPartitionValueMapAndSorted rebuilds all value map and sorted info for list column partitions with InfoSchema. +func rebuildAllPartitionValueMapAndSorted(s *session) { + type partitionExpr interface { + PartitionExpr() (*tables.PartitionExpr, error) + } + + p := parser.New() + is := s.GetInfoSchema().(infoschema.InfoSchema) + for _, dbInfo := range is.AllSchemas() { + for _, t := range is.SchemaTables(dbInfo.Name) { + pi := t.Meta().GetPartitionInfo() + if pi == nil || pi.Type != model.PartitionTypeList { + continue + } + + pe, err := t.(partitionExpr).PartitionExpr() + if err != nil { + panic("partition table gets partition expression failed") + } + for _, cp := range pe.ColPrunes { + if err = cp.RebuildPartitionValueMapAndSorted(p); err != nil { + logutil.BgLogger().Warn("build list column partition value map and sorted failed") + break + } + } + } + } +} diff --git a/session/session.go b/session/session.go index ba76f7bc2c360..b69fa2ded1ca9 100644 --- a/session/session.go +++ b/session/session.go @@ -2795,6 +2795,8 @@ func BootstrapSession(store kv.Storage) (*domain.Domain, error) { return nil, err } collate.SetNewCollationEnabledForTest(newCollationEnabled) + // To deal with the location partition failure caused by inconsistent NewCollationEnabled values(see issue #32416). + rebuildAllPartitionValueMapAndSorted(ses[0]) err = updateMemoryConfigAndSysVar(ses[0]) if err != nil { diff --git a/table/tables/partition.go b/table/tables/partition.go index 6f37818cb13cd..9f84333213a82 100644 --- a/table/tables/partition.go +++ b/table/tables/partition.go @@ -252,7 +252,6 @@ type ForListColumnPruning struct { ExprCol *expression.Column valueTp *types.FieldType valueMap map[string]ListPartitionLocation - mu sync.RWMutex sorted *btree.BTree // To deal with the location partition failure caused by inconsistent NewCollationEnabled values(see issue #32416). @@ -658,6 +657,7 @@ func (lp *ForListPruning) buildListColumnsPruner(ctx sessionctx.Context, tblInfo columns []*expression.Column, names types.NameSlice) error { pi := tblInfo.GetPartitionInfo() schema := expression.NewSchema(columns...) + p := parser.New() colPrunes := make([]*ForListColumnPruning, 0, len(pi.Columns)) for colIdx := range pi.Columns { colInfo := model.FindColumnInfo(tblInfo.Columns, pi.Columns[colIdx].L) @@ -679,7 +679,10 @@ func (lp *ForListPruning) buildListColumnsPruner(ctx sessionctx.Context, tblInfo valueMap: make(map[string]ListPartitionLocation), sorted: btree.New(btreeDegree), } - + err := colPrune.buildPartitionValueMapAndSorted(p) + if err != nil { + return err + } colPrunes = append(colPrunes, colPrune) } lp.ColPrunes = colPrunes @@ -760,22 +763,28 @@ func (lp *ForListPruning) locateListColumnsPartitionByRow(ctx sessionctx.Context return location[0].PartIdx, nil } -// buildListPartitionValueMapAndSorted builds list columns partition value map for the specified column. -// it also builds list columns partition value btree for the specified column. +// buildPartitionValueMapAndSorted builds list columns partition value map for the specified column. +// It also builds list columns partition value btree for the specified column. // colIdx is the specified column index in the list columns. -func (lp *ForListColumnPruning) buildPartitionValueMapAndSorted() error { - lp.mu.RLock() +func (lp *ForListColumnPruning) buildPartitionValueMapAndSorted(p *parser.Parser) error { l := len(lp.valueMap) - lp.mu.RUnlock() if l != 0 { return nil } - p := parser.New() + return lp.buildListPartitionValueMapAndSorted(p) +} + +// RebuildPartitionValueMapAndSorted rebuilds list columns partition value map for the specified column. +func (lp *ForListColumnPruning) RebuildPartitionValueMapAndSorted(p *parser.Parser) error { + lp.valueMap = make(map[string]ListPartitionLocation, len(lp.valueMap)) + lp.sorted.Clear(false) + return lp.buildListPartitionValueMapAndSorted(p) +} + +func (lp *ForListColumnPruning) buildListPartitionValueMapAndSorted(p *parser.Parser) error { pi := lp.tblInfo.GetPartitionInfo() sc := lp.ctx.GetSessionVars().StmtCtx - lp.mu.Lock() - defer lp.mu.Unlock() for partitionIdx, def := range pi.Definitions { for groupIdx, vs := range def.InValues { keyBytes, err := lp.genConstExprKey(lp.ctx, sc, vs[lp.colIdx], lp.schema, lp.names, p) @@ -830,19 +839,11 @@ func (lp *ForListColumnPruning) genKey(sc *stmtctx.StatementContext, v types.Dat // LocatePartition locates partition by the column value func (lp *ForListColumnPruning) LocatePartition(sc *stmtctx.StatementContext, v types.Datum) (ListPartitionLocation, error) { - // To deal with the location partition failure caused by inconsistent NewCollationEnabled values(see issue #32416). - err := lp.buildPartitionValueMapAndSorted() - if err != nil { - return nil, err - } - key, err := lp.genKey(sc, v) if err != nil { return nil, errors.Trace(err) } - lp.mu.RLock() location, ok := lp.valueMap[string(key)] - lp.mu.RUnlock() if !ok { return nil, nil } @@ -851,13 +852,8 @@ func (lp *ForListColumnPruning) LocatePartition(sc *stmtctx.StatementContext, v // LocateRanges locates partition ranges by the column range func (lp *ForListColumnPruning) LocateRanges(sc *stmtctx.StatementContext, r *ranger.Range) ([]ListPartitionLocation, error) { - // To deal with the location partition failure caused by inconsistent NewCollationEnabled values(see issue #32416). - err := lp.buildPartitionValueMapAndSorted() - if err != nil { - return nil, err - } - var lowKey, highKey []byte + var err error lowVal := r.LowVal[0] if r.LowVal[0].Kind() == types.KindMinNotNull { lowVal = types.GetMinValue(lp.ExprCol.GetType())