From 56bf325a73f1dba0a4559e2e3e893a04bb0eeb07 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Wed, 1 Dec 2021 10:31:09 +0800 Subject: [PATCH 01/23] planner: fix index merge plan when expr has not been pushed to tikv. Signed-off-by: guo-shaoge --- planner/core/find_best_task.go | 49 +++++++++++++++++++++++++------- planner/core/integration_test.go | 26 +++++++++++++++++ planner/core/stats.go | 10 +++---- planner/core/task.go | 8 ++++-- 4 files changed, 75 insertions(+), 18 deletions(-) diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index 5e44bc2111870..98b0b41a2b499 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -15,6 +15,7 @@ package core import ( + "bytes" "fmt" "math" "strings" @@ -972,7 +973,7 @@ func (ds *DataSource) convertToIndexMergeScan(prop *property.PhysicalProperty, c if prop.ExpectedCnt < ds.stats.RowCount { totalRowCount *= prop.ExpectedCnt / ds.stats.RowCount } - ts, partialCost, err := ds.buildIndexMergeTableScan(prop, path.TableFilters, totalRowCount) + ts, partialCost, remainedFilters, err := ds.buildIndexMergeTableScan(prop, path.TableFilters, totalRowCount) if err != nil { return nil, err } @@ -980,6 +981,9 @@ func (ds *DataSource) convertToIndexMergeScan(prop *property.PhysicalProperty, c cop.tablePlan = ts cop.idxMergePartPlans = scans cop.cst = totalCost + if remainedFilters != nil { + cop.rootTaskConds = remainedFilters + } task = cop.convertToRootTask(ds.ctx) ds.addSelection4PlanCache(task.(*rootTask), ds.tableStats.ScaleByExpectCnt(totalRowCount), prop) return task, nil @@ -1092,7 +1096,7 @@ func setIndexMergeTableScanHandleCols(ds *DataSource, ts *PhysicalTableScan) (er } func (ds *DataSource) buildIndexMergeTableScan(prop *property.PhysicalProperty, tableFilters []expression.Expression, - totalRowCount float64) (PhysicalPlan, float64, error) { + totalRowCount float64) (PhysicalPlan, float64, []expression.Expression, error) { var partialCost float64 sessVars := ds.ctx.GetSessionVars() ts := PhysicalTableScan{ @@ -1107,7 +1111,7 @@ func (ds *DataSource) buildIndexMergeTableScan(prop *property.PhysicalProperty, ts.SetSchema(ds.schema.Clone()) err := setIndexMergeTableScanHandleCols(ds, ts) if err != nil { - return nil, 0, err + return nil, 0, nil, err } if ts.Table.PKIsHandle { if pkColInfo := ts.Table.GetPkColInfo(); pkColInfo != nil { @@ -1123,17 +1127,40 @@ func (ds *DataSource) buildIndexMergeTableScan(prop *property.PhysicalProperty, ts.stats.StatsVersion = statistics.PseudoVersion } if len(tableFilters) > 0 { + // Here we want to add a Selection for exprs that cannot be pushed to TiKV, but some of them has already been put + // in Selection above DataSource, so we need to filter these exprs. + _, filtersInSelection := expression.PushDownExprs(sessVars.StmtCtx, tableFilters, ds.ctx.GetClient(), kv.UnSpecified) + pushedFilters, remainedFilters := expression.PushDownExprs(sessVars.StmtCtx, tableFilters, ds.ctx.GetClient(), kv.TiKV) + remainedFilters = removeExprsInSelection(sessVars.StmtCtx, remainedFilters, filtersInSelection) partialCost += totalRowCount * sessVars.CopCPUFactor - selectivity, _, err := ds.tableStats.HistColl.Selectivity(ds.ctx, tableFilters, nil) - if err != nil { - logutil.BgLogger().Debug("calculate selectivity failed, use selection factor", zap.Error(err)) - selectivity = SelectionFactor + if len(pushedFilters) != 0 { + selectivity, _, err := ds.tableStats.HistColl.Selectivity(ds.ctx, pushedFilters, nil) + if err != nil { + logutil.BgLogger().Debug("calculate selectivity failed, use selection factor", zap.Error(err)) + selectivity = SelectionFactor + } + sel := PhysicalSelection{Conditions: pushedFilters}.Init(ts.ctx, ts.stats.ScaleByExpectCnt(selectivity*totalRowCount), ts.blockOffset) + sel.SetChildren(ts) + return sel, partialCost, remainedFilters, nil + } + return ts, partialCost, remainedFilters, nil + } + return ts, partialCost, nil, nil +} + +func removeExprsInSelection(sc *stmtctx.StatementContext, remainedFilters []expression.Expression, filtersInSelection []expression.Expression) (res []expression.Expression) { + for _, e1 := range remainedFilters { + remove := false + for _, e2 := range filtersInSelection { + if bytes.Equal(e1.HashCode(sc), e2.HashCode(sc)) { + remove = true + } + } + if !remove { + res = append(res, e1) } - sel := PhysicalSelection{Conditions: tableFilters}.Init(ts.ctx, ts.stats.ScaleByExpectCnt(selectivity*totalRowCount), ts.blockOffset) - sel.SetChildren(ts) - return sel, partialCost, nil } - return ts, partialCost, nil + return res } func indexCoveringCol(col *expression.Column, indexCols []*expression.Column, idxColLens []int) bool { diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 351a20ba4c552..e42c3524f4f44 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -4897,3 +4897,29 @@ func (s *testIntegrationSuite) TestIssue30094(c *C) { " └─TableFullScan 10000.00 cop[tikv] table:t30094 keep order:false, stats:pseudo", )) } + +func (s *testIntegrationSuite) TestIssue30200(c *C) { + tk := testkit.NewTestKit(c, s.store) + + tk.MustExec("use test") + tk.MustExec("drop table if exists t1;") + tk.MustExec("create table t1(c1 varchar(100), c2 varchar(100), key(c1), key(c2), c3 varchar(100));") + tk.MustExec("insert into t1 values('ab', '10', '10');") + // lpad has not been pushed to TiKV or TiFlash. + tk.MustQuery("explain select /*+ use_index_merge(t1) */ * from t1 where c1 = 'ab' or c2 = '10' and char_length(lpad(c1, 10, 'a')) = 10;").Check(testkit.Rows( + "Selection_5 8000.00 root or(eq(test.t1.c1, \"ab\"), and(eq(test.t1.c2, \"10\"), eq(char_length(lpad(test.t1.c1, 10, \"a\")), 10)))", + "└─IndexMerge_9 19.99 root ", + " ├─IndexRangeScan_6(Build) 10.00 cop[tikv] table:t1, index:c1(c1) range:[\"ab\",\"ab\"], keep order:false, stats:pseudo", + " ├─IndexRangeScan_7(Build) 10.00 cop[tikv] table:t1, index:c2(c2) range:[\"10\",\"10\"], keep order:false, stats:pseudo", + " └─TableRowIDScan_8(Probe) 19.99 cop[tikv] table:t1 keep order:false, stats:pseudo")) + tk.MustQuery("select /*+ use_index_merge(t1) */ 1 from t1 where c1 = 'de' or c2 = '10' and char_length(lpad(c1, 10, 'a')) = 10;").Check(testkit.Rows("1")) + + // left has not been pushed to TiKV, but it has been pushed to TiFlash. + tk.MustQuery("explain select /*+ use_index_merge(t1) */ * from t1 where c1 = 'ab' or c2 = '10' and char_length(left(c1, 10)) = 10;").Check(testkit.Rows( + "Selection_9 0.04 root or(eq(test.t1.c1, \"ab\"), and(eq(test.t1.c2, \"10\"), eq(char_length(left(test.t1.c1, 10)), 10)))", + "└─IndexMerge_8 19.99 root ", + " ├─IndexRangeScan_5(Build) 10.00 cop[tikv] table:t1, index:c1(c1) range:[\"ab\",\"ab\"], keep order:false, stats:pseudo", + " ├─IndexRangeScan_6(Build) 10.00 cop[tikv] table:t1, index:c2(c2) range:[\"10\",\"10\"], keep order:false, stats:pseudo", + " └─TableRowIDScan_7(Probe) 19.99 cop[tikv] table:t1 keep order:false, stats:pseudo")) + tk.MustQuery("select /*+ use_index_merge(t1) */ 1 from t1 where c1 = 'ab' or c2 = '10' and char_length(left(c1, 10)) = 10;").Check(testkit.Rows("1")) +} diff --git a/planner/core/stats.go b/planner/core/stats.go index 8dabb4a648621..ba2b1b31449ab 100644 --- a/planner/core/stats.go +++ b/planner/core/stats.go @@ -418,7 +418,7 @@ func (ds *DataSource) DeriveStats(childStats []*property.StatsInfo, selfSchema * isReadOnlyTxn = false } // Consider the IndexMergePath. Now, we just generate `IndexMergePath` in DNF case. - isPossibleIdxMerge := len(ds.pushedDownConds) > 0 && len(ds.possibleAccessPaths) > 1 + isPossibleIdxMerge := len(ds.allConds) > 0 && len(ds.possibleAccessPaths) > 1 sessionAndStmtPermission := (ds.ctx.GetSessionVars().GetEnableIndexMerge() || len(ds.indexMergeHints) > 0) && !ds.ctx.GetSessionVars().StmtCtx.NoIndexMergeHint // If there is an index path, we current do not consider `IndexMergePath`. needConsiderIndexMerge := true @@ -520,7 +520,7 @@ func (is *LogicalIndexScan) DeriveStats(childStats []*property.StatsInfo, selfSc // getIndexMergeOrPath generates all possible IndexMergeOrPaths. func (ds *DataSource) generateIndexMergeOrPaths() error { usedIndexCount := len(ds.possibleAccessPaths) - for i, cond := range ds.pushedDownConds { + for i, cond := range ds.allConds { sf, ok := cond.(*expression.ScalarFunction) if !ok || sf.FuncName.L != ast.LogicOr { continue @@ -696,12 +696,12 @@ func (ds *DataSource) buildIndexMergePartialPath(indexAccessPaths []*util.Access // buildIndexMergeOrPath generates one possible IndexMergePath. func (ds *DataSource) buildIndexMergeOrPath(partialPaths []*util.AccessPath, current int) *util.AccessPath { indexMergePath := &util.AccessPath{PartialIndexPaths: partialPaths} - indexMergePath.TableFilters = append(indexMergePath.TableFilters, ds.pushedDownConds[:current]...) - indexMergePath.TableFilters = append(indexMergePath.TableFilters, ds.pushedDownConds[current+1:]...) + indexMergePath.TableFilters = append(indexMergePath.TableFilters, ds.allConds[:current]...) + indexMergePath.TableFilters = append(indexMergePath.TableFilters, ds.allConds[current+1:]...) for _, path := range partialPaths { // If any partial path contains table filters, we need to keep the whole DNF filter in the Selection. if len(path.TableFilters) > 0 { - indexMergePath.TableFilters = append(indexMergePath.TableFilters, ds.pushedDownConds[current]) + indexMergePath.TableFilters = append(indexMergePath.TableFilters, ds.allConds[current]) break } } diff --git a/planner/core/task.go b/planner/core/task.go index a6af6c136a9d4..b7b2a5882b43b 100644 --- a/planner/core/task.go +++ b/planner/core/task.go @@ -1005,6 +1005,7 @@ func (t *copTask) convertToRootTaskImpl(ctx sessionctx.Context) *rootTask { setTableScanToTableRowIDScan(p.tablePlan) newTask.p = p p.cost = newTask.cost() + t.handleRootTaskConds(ctx, newTask) if t.needExtraProj { schema := t.originSchema proj := PhysicalProjection{Exprs: expression.Column2Exprs(schema.Columns)}.Init(ctx, p.stats, t.idxMergePartPlans[0].SelectBlockOffset(), nil) @@ -1066,6 +1067,11 @@ func (t *copTask) convertToRootTaskImpl(ctx sessionctx.Context) *rootTask { } } + t.handleRootTaskConds(ctx, newTask) + return newTask +} + +func (t *copTask) handleRootTaskConds(ctx sessionctx.Context, newTask *rootTask) { if len(t.rootTaskConds) > 0 { selectivity, _, err := t.tblColHists.Selectivity(ctx, t.rootTaskConds, nil) if err != nil { @@ -1077,8 +1083,6 @@ func (t *copTask) convertToRootTaskImpl(ctx sessionctx.Context) *rootTask { newTask.p = sel sel.cost = newTask.cost() } - - return newTask } // setTableScanToTableRowIDScan is to update the isChildOfIndexLookUp attribute of PhysicalTableScan child From 480045723af4786482aa8e9209892a356bed8598 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Wed, 8 Dec 2021 21:01:16 +0800 Subject: [PATCH 02/23] fix comment Signed-off-by: guo-shaoge --- planner/core/integration_test.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 66f2ff3ef6175..ec8ecb917f7c3 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -4962,21 +4962,21 @@ func (s *testIntegrationSuite) TestIssue30200(c *C) { tk.MustExec("create table t1(c1 varchar(100), c2 varchar(100), key(c1), key(c2), c3 varchar(100));") tk.MustExec("insert into t1 values('ab', '10', '10');") // lpad has not been pushed to TiKV or TiFlash. - tk.MustQuery("explain select /*+ use_index_merge(t1) */ * from t1 where c1 = 'ab' or c2 = '10' and char_length(lpad(c1, 10, 'a')) = 10;").Check(testkit.Rows( - "Selection_5 8000.00 root or(eq(test.t1.c1, \"ab\"), and(eq(test.t1.c2, \"10\"), eq(char_length(lpad(test.t1.c1, 10, \"a\")), 10)))", - "└─IndexMerge_9 19.99 root ", - " ├─IndexRangeScan_6(Build) 10.00 cop[tikv] table:t1, index:c1(c1) range:[\"ab\",\"ab\"], keep order:false, stats:pseudo", - " ├─IndexRangeScan_7(Build) 10.00 cop[tikv] table:t1, index:c2(c2) range:[\"10\",\"10\"], keep order:false, stats:pseudo", - " └─TableRowIDScan_8(Probe) 19.99 cop[tikv] table:t1 keep order:false, stats:pseudo")) + tk.MustQuery("explain format=brief select /*+ use_index_merge(t1) */ * from t1 where c1 = 'ab' or c2 = '10' and char_length(lpad(c1, 10, 'a')) = 10;").Check(testkit.Rows( + "Selection 8000.00 root or(eq(test.t1.c1, \"ab\"), and(eq(test.t1.c2, \"10\"), eq(char_length(lpad(test.t1.c1, 10, \"a\")), 10)))", + "└─IndexMerge 19.99 root ", + " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t1, index:c1(c1) range:[\"ab\",\"ab\"], keep order:false, stats:pseudo", + " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t1, index:c2(c2) range:[\"10\",\"10\"], keep order:false, stats:pseudo", + " └─TableRowIDScan(Probe) 19.99 cop[tikv] table:t1 keep order:false, stats:pseudo")) tk.MustQuery("select /*+ use_index_merge(t1) */ 1 from t1 where c1 = 'de' or c2 = '10' and char_length(lpad(c1, 10, 'a')) = 10;").Check(testkit.Rows("1")) // left has not been pushed to TiKV, but it has been pushed to TiFlash. - tk.MustQuery("explain select /*+ use_index_merge(t1) */ * from t1 where c1 = 'ab' or c2 = '10' and char_length(left(c1, 10)) = 10;").Check(testkit.Rows( - "Selection_9 0.04 root or(eq(test.t1.c1, \"ab\"), and(eq(test.t1.c2, \"10\"), eq(char_length(left(test.t1.c1, 10)), 10)))", - "└─IndexMerge_8 19.99 root ", - " ├─IndexRangeScan_5(Build) 10.00 cop[tikv] table:t1, index:c1(c1) range:[\"ab\",\"ab\"], keep order:false, stats:pseudo", - " ├─IndexRangeScan_6(Build) 10.00 cop[tikv] table:t1, index:c2(c2) range:[\"10\",\"10\"], keep order:false, stats:pseudo", - " └─TableRowIDScan_7(Probe) 19.99 cop[tikv] table:t1 keep order:false, stats:pseudo")) + tk.MustQuery("explain format=brief select /*+ use_index_merge(t1) */ * from t1 where c1 = 'ab' or c2 = '10' and char_length(left(c1, 10)) = 10;").Check(testkit.Rows( + "Selection 0.04 root or(eq(test.t1.c1, \"ab\"), and(eq(test.t1.c2, \"10\"), eq(char_length(left(test.t1.c1, 10)), 10)))", + "└─IndexMerge 19.99 root ", + " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t1, index:c1(c1) range:[\"ab\",\"ab\"], keep order:false, stats:pseudo", + " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t1, index:c2(c2) range:[\"10\",\"10\"], keep order:false, stats:pseudo", + " └─TableRowIDScan(Probe) 19.99 cop[tikv] table:t1 keep order:false, stats:pseudo")) tk.MustQuery("select /*+ use_index_merge(t1) */ 1 from t1 where c1 = 'ab' or c2 = '10' and char_length(left(c1, 10)) = 10;").Check(testkit.Rows("1")) } From b4654ea4aef265fcc6e19aa5940ada350b309556 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Sun, 12 Dec 2021 04:19:59 +0800 Subject: [PATCH 03/23] fix comment Signed-off-by: guo-shaoge --- planner/core/find_best_task.go | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index 2c5f39009ab1e..53c5277ec7036 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -15,7 +15,6 @@ package core import ( - "bytes" "fmt" "math" "strings" @@ -1128,11 +1127,7 @@ func (ds *DataSource) buildIndexMergeTableScan(prop *property.PhysicalProperty, ts.stats.StatsVersion = statistics.PseudoVersion } if len(tableFilters) > 0 { - // Here we want to add a Selection for exprs that cannot be pushed to TiKV, but some of them has already been put - // in Selection above DataSource, so we need to filter these exprs. - _, filtersInSelection := expression.PushDownExprs(sessVars.StmtCtx, tableFilters, ds.ctx.GetClient(), kv.UnSpecified) - pushedFilters, remainedFilters := expression.PushDownExprs(sessVars.StmtCtx, tableFilters, ds.ctx.GetClient(), kv.TiKV) - remainedFilters = removeExprsInSelection(sessVars.StmtCtx, remainedFilters, filtersInSelection) + pushedFilters, remainedFilters := extraceFiltersForIndexMerge(sessVars.StmtCtx, ds.ctx.GetClient(), tableFilters) partialCost += totalRowCount * sessVars.CopCPUFactor if len(pushedFilters) != 0 { selectivity, _, err := ds.tableStats.HistColl.Selectivity(ds.ctx, pushedFilters, nil) @@ -1149,19 +1144,23 @@ func (ds *DataSource) buildIndexMergeTableScan(prop *property.PhysicalProperty, return ts, partialCost, nil, nil } -func removeExprsInSelection(sc *stmtctx.StatementContext, remainedFilters []expression.Expression, filtersInSelection []expression.Expression) (res []expression.Expression) { - for _, e1 := range remainedFilters { - remove := false - for _, e2 := range filtersInSelection { - if bytes.Equal(e1.HashCode(sc), e2.HashCode(sc)) { - remove = true - } +// extraceFiltersForIndexMerge returns: +// 1. exprs that can be pushed to TiKV. +// 2. exprs that cannot be pushed to TiKV but can be pushed to other storages. +// Why: IndexMerge only works on TiKV, so we need to find all exprs that cannot be pushed to TiKV, and add a new Selection above IndexMergeReader. +// But the new Selection should exclude exprs that cannot be pushed to other storages either. +// Because these exprs have already been put in another Selection(check rule_predicate_push_down). +func extraceFiltersForIndexMerge(sc *stmtctx.StatementContext, client kv.Client, filters []expression.Expression) (pushed []expression.Expression, remained []expression.Expression) { + for _, expr := range filters { + if expression.CanExprsPushDown(sc, []expression.Expression{expr}, client, kv.TiKV) { + pushed = append(pushed, expr) + continue } - if !remove { - res = append(res, e1) + if expression.CanExprsPushDown(sc, []expression.Expression{expr}, client, kv.UnSpecified) { + remained = append(remained, expr) } } - return res + return } func indexCoveringCol(col *expression.Column, indexCols []*expression.Column, idxColLens []int) bool { From cc74a3e38fb9efe224058b70e9de3ac128f026c4 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Sun, 12 Dec 2021 10:55:34 +0800 Subject: [PATCH 04/23] fix cannot be pushed expr in partial_path.IndexFilters Signed-off-by: guo-shaoge --- planner/core/find_best_task.go | 6 +++--- planner/core/integration_test.go | 15 +++++++++++++++ planner/core/stats.go | 14 ++++++++++++-- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index 53c5277ec7036..c86c09e746674 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -1127,7 +1127,7 @@ func (ds *DataSource) buildIndexMergeTableScan(prop *property.PhysicalProperty, ts.stats.StatsVersion = statistics.PseudoVersion } if len(tableFilters) > 0 { - pushedFilters, remainedFilters := extraceFiltersForIndexMerge(sessVars.StmtCtx, ds.ctx.GetClient(), tableFilters) + pushedFilters, remainedFilters := extractFiltersForIndexMerge(sessVars.StmtCtx, ds.ctx.GetClient(), tableFilters) partialCost += totalRowCount * sessVars.CopCPUFactor if len(pushedFilters) != 0 { selectivity, _, err := ds.tableStats.HistColl.Selectivity(ds.ctx, pushedFilters, nil) @@ -1144,13 +1144,13 @@ func (ds *DataSource) buildIndexMergeTableScan(prop *property.PhysicalProperty, return ts, partialCost, nil, nil } -// extraceFiltersForIndexMerge returns: +// extractFiltersForIndexMerge returns: // 1. exprs that can be pushed to TiKV. // 2. exprs that cannot be pushed to TiKV but can be pushed to other storages. // Why: IndexMerge only works on TiKV, so we need to find all exprs that cannot be pushed to TiKV, and add a new Selection above IndexMergeReader. // But the new Selection should exclude exprs that cannot be pushed to other storages either. // Because these exprs have already been put in another Selection(check rule_predicate_push_down). -func extraceFiltersForIndexMerge(sc *stmtctx.StatementContext, client kv.Client, filters []expression.Expression) (pushed []expression.Expression, remained []expression.Expression) { +func extractFiltersForIndexMerge(sc *stmtctx.StatementContext, client kv.Client, filters []expression.Expression) (pushed []expression.Expression, remained []expression.Expression) { for _, expr := range filters { if expression.CanExprsPushDown(sc, []expression.Expression{expr}, client, kv.TiKV) { pushed = append(pushed, expr) diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index ec8ecb917f7c3..3bf623f424259 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -4978,6 +4978,21 @@ func (s *testIntegrationSuite) TestIssue30200(c *C) { " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t1, index:c2(c2) range:[\"10\",\"10\"], keep order:false, stats:pseudo", " └─TableRowIDScan(Probe) 19.99 cop[tikv] table:t1 keep order:false, stats:pseudo")) tk.MustQuery("select /*+ use_index_merge(t1) */ 1 from t1 where c1 = 'ab' or c2 = '10' and char_length(left(c1, 10)) = 10;").Check(testkit.Rows("1")) + + tk.MustExec("use test") + tk.MustExec("drop table if exists t1;") + tk.MustExec("create table t1(c1 varchar(100), c2 varchar(100), c3 varchar(100), c4 varchar(100), key idx_0(c1), key idx_1(c2, c3));") + tk.MustExec("insert into t1 values('ab', '10', '10', '10');") + // c3 is part of idx_1, so it will be put in partial_path's IndexFilters instead of TableFilters. + // But it still cannot be pushed to TiKV. + tk.MustQuery("explain select /*+ use_index_merge(t1) */ 1 from t1 where c1 = 'de' or c2 = '10' and char_length(lpad(c3, 10, 'a')) = 10;").Check(testkit.Rows( + "Projection_4 8000.00 root 1->Column#6", + "└─Selection_5 8000.00 root or(eq(test.t1.c1, \"de\"), and(eq(test.t1.c2, \"10\"), eq(char_length(lpad(test.t1.c3, 10, \"a\")), 10)))", + " └─IndexMerge_9 19.99 root ", + " ├─IndexRangeScan_6(Build) 10.00 cop[tikv] table:t1, index:idx_0(c1) range:[\"de\",\"de\"], keep order:false, stats:pseudo", + " ├─IndexRangeScan_7(Build) 10.00 cop[tikv] table:t1, index:idx_1(c2, c3) range:[\"10\",\"10\"], keep order:false, stats:pseudo", + " └─TableRowIDScan_8(Probe) 19.99 cop[tikv] table:t1 keep order:false, stats:pseudo")) + tk.MustQuery("select /*+ use_index_merge(t1) */ 1 from t1 where c1 = 'de' or c2 = '10' and char_length(lpad(c3, 10, 'a')) = 10;").Check(testkit.Rows("1")) } func (s *testIntegrationSuite) TestIssue29705(c *C) { diff --git a/planner/core/stats.go b/planner/core/stats.go index ba2b1b31449ab..0e485aecfc4f6 100644 --- a/planner/core/stats.go +++ b/planner/core/stats.go @@ -23,6 +23,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/tidb/expression" + "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/parser/ast" "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/parser/mysql" @@ -698,13 +699,22 @@ func (ds *DataSource) buildIndexMergeOrPath(partialPaths []*util.AccessPath, cur indexMergePath := &util.AccessPath{PartialIndexPaths: partialPaths} indexMergePath.TableFilters = append(indexMergePath.TableFilters, ds.allConds[:current]...) indexMergePath.TableFilters = append(indexMergePath.TableFilters, ds.allConds[current+1:]...) + var addCurrentFilter bool for _, path := range partialPaths { // If any partial path contains table filters, we need to keep the whole DNF filter in the Selection. if len(path.TableFilters) > 0 { - indexMergePath.TableFilters = append(indexMergePath.TableFilters, ds.allConds[current]) - break + addCurrentFilter = true + } + // If any partial path's index filter cannot be pushed to TiKV, we should keep the whole DNF filter. + if len(path.IndexFilters) != 0 && !expression.CanExprsPushDown(ds.ctx.GetSessionVars().StmtCtx, path.IndexFilters, ds.ctx.GetClient(), kv.TiKV) { + addCurrentFilter = true + // Clear IndexFilter, the whole filter will be put in indexMergePath.TableFilters. + path.IndexFilters = nil } } + if addCurrentFilter { + indexMergePath.TableFilters = append(indexMergePath.TableFilters, ds.allConds[current]) + } return indexMergePath } From 5d98c2cf4c70dd402c46900d4244e2a7a0cc6ce0 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Sun, 12 Dec 2021 23:00:50 +0800 Subject: [PATCH 05/23] fix cost Signed-off-by: guo-shaoge --- planner/core/find_best_task.go | 2 +- planner/core/logical_plans.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index c86c09e746674..8253fe9927394 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -1128,8 +1128,8 @@ func (ds *DataSource) buildIndexMergeTableScan(prop *property.PhysicalProperty, } if len(tableFilters) > 0 { pushedFilters, remainedFilters := extractFiltersForIndexMerge(sessVars.StmtCtx, ds.ctx.GetClient(), tableFilters) - partialCost += totalRowCount * sessVars.CopCPUFactor if len(pushedFilters) != 0 { + partialCost += totalRowCount * sessVars.CopCPUFactor selectivity, _, err := ds.tableStats.HistColl.Selectivity(ds.ctx, pushedFilters, nil) if err != nil { logutil.BgLogger().Debug("calculate selectivity failed, use selection factor", zap.Error(err)) diff --git a/planner/core/logical_plans.go b/planner/core/logical_plans.go index 212f10d65346a..221b896e6de23 100644 --- a/planner/core/logical_plans.go +++ b/planner/core/logical_plans.go @@ -549,7 +549,7 @@ type DataSource struct { // pushedDownConds are the conditions that will be pushed down to coprocessor. pushedDownConds []expression.Expression // allConds contains all the filters on this table. For now it's maintained - // in predicate push down and used only in partition pruning. + // in predicate push down and used in partition pruning/index merge. allConds []expression.Expression statisticTable *statistics.Table From 7f00244fa8ab905d8f80cee93f9fd5500969b788 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Mon, 13 Dec 2021 13:23:00 +0800 Subject: [PATCH 06/23] Update planner/core/find_best_task.go Co-authored-by: HuaiyuXu --- planner/core/find_best_task.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index 8253fe9927394..91c226aeeb0e5 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -1145,8 +1145,8 @@ func (ds *DataSource) buildIndexMergeTableScan(prop *property.PhysicalProperty, } // extractFiltersForIndexMerge returns: -// 1. exprs that can be pushed to TiKV. -// 2. exprs that cannot be pushed to TiKV but can be pushed to other storages. +// `pushed`: exprs that can be pushed to TiKV. +// `remained`: exprs that can NOT be pushed to TiKV but can be pushed to other storage engines. // Why: IndexMerge only works on TiKV, so we need to find all exprs that cannot be pushed to TiKV, and add a new Selection above IndexMergeReader. // But the new Selection should exclude exprs that cannot be pushed to other storages either. // Because these exprs have already been put in another Selection(check rule_predicate_push_down). From eba2cbd23bf6d8e8dcc3759c51af0ddfea9b5837 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Mon, 13 Dec 2021 13:23:17 +0800 Subject: [PATCH 07/23] Update planner/core/find_best_task.go Co-authored-by: HuaiyuXu --- planner/core/find_best_task.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index 91c226aeeb0e5..0ff432ab93ede 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -1147,7 +1147,8 @@ func (ds *DataSource) buildIndexMergeTableScan(prop *property.PhysicalProperty, // extractFiltersForIndexMerge returns: // `pushed`: exprs that can be pushed to TiKV. // `remained`: exprs that can NOT be pushed to TiKV but can be pushed to other storage engines. -// Why: IndexMerge only works on TiKV, so we need to find all exprs that cannot be pushed to TiKV, and add a new Selection above IndexMergeReader. +// Why do we need this func? +// IndexMerge only works on TiKV, so we need to find all exprs that cannot be pushed to TiKV, and add a new Selection above IndexMergeReader. // But the new Selection should exclude exprs that cannot be pushed to other storages either. // Because these exprs have already been put in another Selection(check rule_predicate_push_down). func extractFiltersForIndexMerge(sc *stmtctx.StatementContext, client kv.Client, filters []expression.Expression) (pushed []expression.Expression, remained []expression.Expression) { From 9b4ec81bbadca79f6c224585ffe65669b6cd5fd8 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Mon, 13 Dec 2021 13:25:31 +0800 Subject: [PATCH 08/23] Update planner/core/find_best_task.go Co-authored-by: HuaiyuXu --- planner/core/find_best_task.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index 0ff432ab93ede..073bd10973ffb 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -1149,7 +1149,7 @@ func (ds *DataSource) buildIndexMergeTableScan(prop *property.PhysicalProperty, // `remained`: exprs that can NOT be pushed to TiKV but can be pushed to other storage engines. // Why do we need this func? // IndexMerge only works on TiKV, so we need to find all exprs that cannot be pushed to TiKV, and add a new Selection above IndexMergeReader. -// But the new Selection should exclude exprs that cannot be pushed to other storages either. +// But the new Selection should exclude the exprs that can NOT be pushed to ALL the storage engines. // Because these exprs have already been put in another Selection(check rule_predicate_push_down). func extractFiltersForIndexMerge(sc *stmtctx.StatementContext, client kv.Client, filters []expression.Expression) (pushed []expression.Expression, remained []expression.Expression) { for _, expr := range filters { From b59caf3c60f33a5d0791419f980d2505837daa68 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Tue, 14 Dec 2021 21:51:33 +0800 Subject: [PATCH 09/23] check partial path's TableFilters can be pushed to TiKV or not. Signed-off-by: guo-shaoge --- planner/core/integration_test.go | 12 ++++++++++++ planner/core/stats.go | 4 ++++ 2 files changed, 16 insertions(+) diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 980f2f13e6ac8..d9d6b0bbadeda 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -5031,6 +5031,18 @@ func (s *testIntegrationSuite) TestIssue30200(c *C) { " ├─IndexRangeScan_7(Build) 10.00 cop[tikv] table:t1, index:idx_1(c2, c3) range:[\"10\",\"10\"], keep order:false, stats:pseudo", " └─TableRowIDScan_8(Probe) 19.99 cop[tikv] table:t1 keep order:false, stats:pseudo")) tk.MustQuery("select /*+ use_index_merge(t1) */ 1 from t1 where c1 = 'de' or c2 = '10' and char_length(lpad(c3, 10, 'a')) = 10;").Check(testkit.Rows("1")) + + tk.MustExec("drop table if exists t1;") + tk.MustExec("create table t1 (c1 int , pk int, primary key( pk ) , unique key( c1));") + tk.MustExec("insert into t1 values(-3896405, -1), (-2, 1), (-1, -2);") + // to_base64(left(pk, 5)) is in partial_path's TableFilters. But it cannot be pushed to TiKV. So it should be executed in TiDB. + tk.MustQuery("explain select /*+ use_index_merge( t1 ) */ * from t1 where t1.c1 in (-3896405) or t1.pk in (1, 53330) and to_base64(left(pk, 5));").Check(testkit.Rows( + "Selection_5 8000.00 root or(eq(test.t1.c1, -3896405), and(in(test.t1.pk, 1, 53330), istrue_with_null(cast(to_base64(left(cast(test.t1.pk, var_string(20)), 5)), double BINARY))))", + "└─IndexMerge_9 3.00 root ", + " ├─IndexRangeScan_6(Build) 1.00 cop[tikv] table:t1, index:c1(c1) range:[-3896405,-3896405], keep order:false, stats:pseudo", + " ├─TableRangeScan_7(Build) 2.00 cop[tikv] table:t1 range:[1,1], [53330,53330], keep order:false, stats:pseudo", + " └─TableRowIDScan_8(Probe) 3.00 cop[tikv] table:t1 keep order:false, stats:pseudo")) + tk.MustQuery("select /*+ use_index_merge( t1 ) */ * from t1 where t1.c1 in (-3896405) or t1.pk in (1, 53330) and to_base64(left(pk, 5));").Check(testkit.Rows("-3896405 -1")) } func (s *testIntegrationSuite) TestIssue29705(c *C) { diff --git a/planner/core/stats.go b/planner/core/stats.go index 189e0656c29bc..fad1bd781f861 100644 --- a/planner/core/stats.go +++ b/planner/core/stats.go @@ -703,6 +703,10 @@ func (ds *DataSource) buildIndexMergeOrPath(partialPaths []*util.AccessPath, cur // Clear IndexFilter, the whole filter will be put in indexMergePath.TableFilters. path.IndexFilters = nil } + if len(path.TableFilters) != 0 && !expression.CanExprsPushDown(ds.ctx.GetSessionVars().StmtCtx, path.TableFilters, ds.ctx.GetClient(), kv.TiKV) { + addCurrentFilter = true + path.TableFilters = nil + } } if addCurrentFilter { indexMergePath.TableFilters = append(indexMergePath.TableFilters, ds.allConds[current]) From bc8c836081f16fe79cdc9ff0864ee392a569e5a2 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Wed, 15 Dec 2021 00:17:27 +0800 Subject: [PATCH 10/23] fix Signed-off-by: guo-shaoge --- planner/core/find_best_task.go | 1 + 1 file changed, 1 insertion(+) diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index 8b1c3e1cd1e3e..ba81bc9478482 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -27,6 +27,7 @@ import ( "github.com/pingcap/tidb/parser/mysql" "github.com/pingcap/tidb/planner/property" "github.com/pingcap/tidb/planner/util" + "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/statistics" "github.com/pingcap/tidb/types" From 9187957c8031188b7a47cf43b19f4cfada10465c Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Wed, 15 Dec 2021 02:19:31 +0800 Subject: [PATCH 11/23] fix didn't push not expr Signed-off-by: guo-shaoge --- planner/core/stats.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/planner/core/stats.go b/planner/core/stats.go index 6782b91eb5490..d368088fa2915 100644 --- a/planner/core/stats.go +++ b/planner/core/stats.go @@ -512,7 +512,13 @@ func (is *LogicalIndexScan) DeriveStats(childStats []*property.StatsInfo, selfSc // getIndexMergeOrPath generates all possible IndexMergeOrPaths. func (ds *DataSource) generateIndexMergeOrPaths() error { + for i, expr := range ds.allConds { + ds.allConds[i] = expression.PushDownNot(ds.ctx, expr) + } usedIndexCount := len(ds.possibleAccessPaths) + // Use allConds instread of pushedDownConds, + // because we want to use IndexMerge even if some expr cannot be pushed to TiKV. + // We will create new Selection for exprs that cannot be pushed in convertToIndexMergeScan. for i, cond := range ds.allConds { sf, ok := cond.(*expression.ScalarFunction) if !ok || sf.FuncName.L != ast.LogicOr { From b89e269f4960d7a3abf9a60e6d5f8fb403b70b72 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Sun, 19 Dec 2021 14:03:56 +0800 Subject: [PATCH 12/23] fix est row Signed-off-by: guo-shaoge --- planner/core/find_best_task.go | 2 +- planner/core/integration_test.go | 8 +++--- planner/core/stats.go | 45 ++++++++++++++++++++------------ 3 files changed, 33 insertions(+), 22 deletions(-) diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index ba81bc9478482..f35ba9d4bbeb5 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -27,8 +27,8 @@ import ( "github.com/pingcap/tidb/parser/mysql" "github.com/pingcap/tidb/planner/property" "github.com/pingcap/tidb/planner/util" - "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/statistics" "github.com/pingcap/tidb/types" tidbutil "github.com/pingcap/tidb/util" diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index d9d6b0bbadeda..3a007b44d64b3 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -5001,7 +5001,7 @@ func (s *testIntegrationSuite) TestIssue30200(c *C) { tk.MustExec("insert into t1 values('ab', '10', '10');") // lpad has not been pushed to TiKV or TiFlash. tk.MustQuery("explain format=brief select /*+ use_index_merge(t1) */ * from t1 where c1 = 'ab' or c2 = '10' and char_length(lpad(c1, 10, 'a')) = 10;").Check(testkit.Rows( - "Selection 8000.00 root or(eq(test.t1.c1, \"ab\"), and(eq(test.t1.c2, \"10\"), eq(char_length(lpad(test.t1.c1, 10, \"a\")), 10)))", + "Selection 15.99 root or(eq(test.t1.c1, \"ab\"), and(eq(test.t1.c2, \"10\"), eq(char_length(lpad(test.t1.c1, 10, \"a\")), 10)))", "└─IndexMerge 19.99 root ", " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t1, index:c1(c1) range:[\"ab\",\"ab\"], keep order:false, stats:pseudo", " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t1, index:c2(c2) range:[\"10\",\"10\"], keep order:false, stats:pseudo", @@ -5024,8 +5024,8 @@ func (s *testIntegrationSuite) TestIssue30200(c *C) { // c3 is part of idx_1, so it will be put in partial_path's IndexFilters instead of TableFilters. // But it still cannot be pushed to TiKV. tk.MustQuery("explain select /*+ use_index_merge(t1) */ 1 from t1 where c1 = 'de' or c2 = '10' and char_length(lpad(c3, 10, 'a')) = 10;").Check(testkit.Rows( - "Projection_4 8000.00 root 1->Column#6", - "└─Selection_5 8000.00 root or(eq(test.t1.c1, \"de\"), and(eq(test.t1.c2, \"10\"), eq(char_length(lpad(test.t1.c3, 10, \"a\")), 10)))", + "Projection_4 15.99 root 1->Column#6", + "└─Selection_5 15.99 root or(eq(test.t1.c1, \"de\"), and(eq(test.t1.c2, \"10\"), eq(char_length(lpad(test.t1.c3, 10, \"a\")), 10)))", " └─IndexMerge_9 19.99 root ", " ├─IndexRangeScan_6(Build) 10.00 cop[tikv] table:t1, index:idx_0(c1) range:[\"de\",\"de\"], keep order:false, stats:pseudo", " ├─IndexRangeScan_7(Build) 10.00 cop[tikv] table:t1, index:idx_1(c2, c3) range:[\"10\",\"10\"], keep order:false, stats:pseudo", @@ -5037,7 +5037,7 @@ func (s *testIntegrationSuite) TestIssue30200(c *C) { tk.MustExec("insert into t1 values(-3896405, -1), (-2, 1), (-1, -2);") // to_base64(left(pk, 5)) is in partial_path's TableFilters. But it cannot be pushed to TiKV. So it should be executed in TiDB. tk.MustQuery("explain select /*+ use_index_merge( t1 ) */ * from t1 where t1.c1 in (-3896405) or t1.pk in (1, 53330) and to_base64(left(pk, 5));").Check(testkit.Rows( - "Selection_5 8000.00 root or(eq(test.t1.c1, -3896405), and(in(test.t1.pk, 1, 53330), istrue_with_null(cast(to_base64(left(cast(test.t1.pk, var_string(20)), 5)), double BINARY))))", + "Selection_5 2.40 root or(eq(test.t1.c1, -3896405), and(in(test.t1.pk, 1, 53330), istrue_with_null(cast(to_base64(left(cast(test.t1.pk, var_string(20)), 5)), double BINARY))))", "└─IndexMerge_9 3.00 root ", " ├─IndexRangeScan_6(Build) 1.00 cop[tikv] table:t1, index:c1(c1) range:[-3896405,-3896405], keep order:false, stats:pseudo", " ├─TableRangeScan_7(Build) 2.00 cop[tikv] table:t1 range:[1,1], [53330,53330], keep order:false, stats:pseudo", diff --git a/planner/core/stats.go b/planner/core/stats.go index d368088fa2915..2a52e14fec83b 100644 --- a/planner/core/stats.go +++ b/planner/core/stats.go @@ -410,7 +410,17 @@ func (ds *DataSource) DeriveStats(childStats []*property.StatsInfo, selfSchema * } // Consider the IndexMergePath. Now, we just generate `IndexMergePath` in DNF case. - isPossibleIdxMerge := len(ds.allConds) > 0 && len(ds.possibleAccessPaths) > 1 + indexMergeConds := ds.pushedDownConds + if len(ds.indexMergeHints) > 0 { + // Use allConds instread of pushedDownConds, + // because we want to use IndexMerge even if some expr cannot be pushed to TiKV. + // We will create new Selection for exprs that cannot be pushed in convertToIndexMergeScan. + indexMergeConds = indexMergeConds[:0] + for _, expr := range ds.allConds { + indexMergeConds = append(indexMergeConds, expression.PushDownNot(ds.ctx, expr)) + } + } + isPossibleIdxMerge := len(indexMergeConds) > 0 && len(ds.possibleAccessPaths) > 1 sessionAndStmtPermission := (ds.ctx.GetSessionVars().GetEnableIndexMerge() || len(ds.indexMergeHints) > 0) && !ds.ctx.GetSessionVars().StmtCtx.NoIndexMergeHint // If there is an index path, we current do not consider `IndexMergePath`. needConsiderIndexMerge := true @@ -425,7 +435,7 @@ func (ds *DataSource) DeriveStats(childStats []*property.StatsInfo, selfSchema * readFromTableCache := ds.ctx.GetSessionVars().StmtCtx.ReadFromTableCache if isPossibleIdxMerge && sessionAndStmtPermission && needConsiderIndexMerge && ds.tableInfo.TempTableType != model.TempTableLocal && !readFromTableCache { - err := ds.generateAndPruneIndexMergePath(ds.indexMergeHints != nil) + err := ds.generateAndPruneIndexMergePath(indexMergeConds, ds.indexMergeHints != nil) if err != nil { return nil, err } @@ -436,9 +446,9 @@ func (ds *DataSource) DeriveStats(childStats []*property.StatsInfo, selfSchema * return ds.stats, nil } -func (ds *DataSource) generateAndPruneIndexMergePath(needPrune bool) error { +func (ds *DataSource) generateAndPruneIndexMergePath(indexMergeConds []expression.Expression, needPrune bool) error { regularPathCount := len(ds.possibleAccessPaths) - err := ds.generateIndexMergeOrPaths() + err := ds.generateIndexMergeOrPaths(indexMergeConds) if err != nil { return err } @@ -455,6 +465,13 @@ func (ds *DataSource) generateAndPruneIndexMergePath(needPrune bool) error { // Do not need to consider the regular paths in find_best_task(). if needPrune { ds.possibleAccessPaths = ds.possibleAccessPaths[regularPathCount:] + minRowCount := ds.possibleAccessPaths[0].CountAfterAccess + for _, path := range ds.possibleAccessPaths { + if minRowCount < path.CountAfterAccess { + minRowCount = path.CountAfterAccess + } + } + ds.stats = ds.tableStats.ScaleByExpectCnt(minRowCount) } return nil } @@ -511,15 +528,9 @@ func (is *LogicalIndexScan) DeriveStats(childStats []*property.StatsInfo, selfSc } // getIndexMergeOrPath generates all possible IndexMergeOrPaths. -func (ds *DataSource) generateIndexMergeOrPaths() error { - for i, expr := range ds.allConds { - ds.allConds[i] = expression.PushDownNot(ds.ctx, expr) - } +func (ds *DataSource) generateIndexMergeOrPaths(filters []expression.Expression) error { usedIndexCount := len(ds.possibleAccessPaths) - // Use allConds instread of pushedDownConds, - // because we want to use IndexMerge even if some expr cannot be pushed to TiKV. - // We will create new Selection for exprs that cannot be pushed in convertToIndexMergeScan. - for i, cond := range ds.allConds { + for i, cond := range filters { sf, ok := cond.(*expression.ScalarFunction) if !ok || sf.FuncName.L != ast.LogicOr { continue @@ -555,7 +566,7 @@ func (ds *DataSource) generateIndexMergeOrPaths() error { continue } if len(partialPaths) > 1 { - possiblePath := ds.buildIndexMergeOrPath(partialPaths, i) + possiblePath := ds.buildIndexMergeOrPath(filters, partialPaths, i) if possiblePath == nil { return nil } @@ -693,10 +704,10 @@ func (ds *DataSource) buildIndexMergePartialPath(indexAccessPaths []*util.Access } // buildIndexMergeOrPath generates one possible IndexMergePath. -func (ds *DataSource) buildIndexMergeOrPath(partialPaths []*util.AccessPath, current int) *util.AccessPath { +func (ds *DataSource) buildIndexMergeOrPath(filters []expression.Expression, partialPaths []*util.AccessPath, current int) *util.AccessPath { indexMergePath := &util.AccessPath{PartialIndexPaths: partialPaths} - indexMergePath.TableFilters = append(indexMergePath.TableFilters, ds.allConds[:current]...) - indexMergePath.TableFilters = append(indexMergePath.TableFilters, ds.allConds[current+1:]...) + indexMergePath.TableFilters = append(indexMergePath.TableFilters, filters[:current]...) + indexMergePath.TableFilters = append(indexMergePath.TableFilters, filters[current+1:]...) var addCurrentFilter bool for _, path := range partialPaths { // If any partial path contains table filters, we need to keep the whole DNF filter in the Selection. @@ -715,7 +726,7 @@ func (ds *DataSource) buildIndexMergeOrPath(partialPaths []*util.AccessPath, cur } } if addCurrentFilter { - indexMergePath.TableFilters = append(indexMergePath.TableFilters, ds.allConds[current]) + indexMergePath.TableFilters = append(indexMergePath.TableFilters, filters[current]) } return indexMergePath } From fb21ec4151bfe39c578c3282cffcd58d82067ab3 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Sun, 19 Dec 2021 14:23:33 +0800 Subject: [PATCH 13/23] check expr can be pushed in ds.pushedDownConds Signed-off-by: guo-shaoge --- planner/core/integration_test.go | 14 +++++++++++++- planner/core/stats.go | 6 ++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 3a007b44d64b3..6f259cd6ff7aa 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -4999,6 +4999,7 @@ func (s *testIntegrationSuite) TestIssue30200(c *C) { tk.MustExec("drop table if exists t1;") tk.MustExec("create table t1(c1 varchar(100), c2 varchar(100), key(c1), key(c2), c3 varchar(100));") tk.MustExec("insert into t1 values('ab', '10', '10');") + // lpad has not been pushed to TiKV or TiFlash. tk.MustQuery("explain format=brief select /*+ use_index_merge(t1) */ * from t1 where c1 = 'ab' or c2 = '10' and char_length(lpad(c1, 10, 'a')) = 10;").Check(testkit.Rows( "Selection 15.99 root or(eq(test.t1.c1, \"ab\"), and(eq(test.t1.c2, \"10\"), eq(char_length(lpad(test.t1.c1, 10, \"a\")), 10)))", @@ -5008,7 +5009,7 @@ func (s *testIntegrationSuite) TestIssue30200(c *C) { " └─TableRowIDScan(Probe) 19.99 cop[tikv] table:t1 keep order:false, stats:pseudo")) tk.MustQuery("select /*+ use_index_merge(t1) */ 1 from t1 where c1 = 'de' or c2 = '10' and char_length(lpad(c1, 10, 'a')) = 10;").Check(testkit.Rows("1")) - // left has not been pushed to TiKV, but it has been pushed to TiFlash. + // `left` has not been pushed to TiKV, but it has been pushed to TiFlash. tk.MustQuery("explain format=brief select /*+ use_index_merge(t1) */ * from t1 where c1 = 'ab' or c2 = '10' and char_length(left(c1, 10)) = 10;").Check(testkit.Rows( "Selection 0.04 root or(eq(test.t1.c1, \"ab\"), and(eq(test.t1.c2, \"10\"), eq(char_length(left(test.t1.c1, 10)), 10)))", "└─IndexMerge 19.99 root ", @@ -5017,6 +5018,17 @@ func (s *testIntegrationSuite) TestIssue30200(c *C) { " └─TableRowIDScan(Probe) 19.99 cop[tikv] table:t1 keep order:false, stats:pseudo")) tk.MustQuery("select /*+ use_index_merge(t1) */ 1 from t1 where c1 = 'ab' or c2 = '10' and char_length(left(c1, 10)) = 10;").Check(testkit.Rows("1")) + // `left` cannot be pushed to TiKV, and there is no hint, so we cannot use index merge. + oriIndexMergeSwitcher := tk.MustQuery("select @@tidb_enable_index_merge;").Rows()[0][0].(string) + tk.MustExec("set tidb_enable_index_merge = on;") + defer func() { + tk.MustExec(fmt.Sprintf("set tidb_enable_index_merge = %s;", oriIndexMergeSwitcher)) + }() + tk.MustQuery("explain format=brief select * from t1 where c1 = 'ab' or c2 = '10' and length(left(c1, 10)) = 10;").Check(testkit.Rows( + "Selection 17.99 root or(eq(test.t1.c1, \"ab\"), and(eq(test.t1.c2, \"10\"), eq(length(left(test.t1.c1, 10)), 10)))", + "└─TableReader 10000.00 root data:TableFullScan", + " └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo")) + tk.MustExec("use test") tk.MustExec("drop table if exists t1;") tk.MustExec("create table t1(c1 varchar(100), c2 varchar(100), c3 varchar(100), c4 varchar(100), key idx_0(c1), key idx_1(c2, c3));") diff --git a/planner/core/stats.go b/planner/core/stats.go index 2a52e14fec83b..8334b189e2ab4 100644 --- a/planner/core/stats.go +++ b/planner/core/stats.go @@ -410,15 +410,17 @@ func (ds *DataSource) DeriveStats(childStats []*property.StatsInfo, selfSchema * } // Consider the IndexMergePath. Now, we just generate `IndexMergePath` in DNF case. - indexMergeConds := ds.pushedDownConds + var indexMergeConds []expression.Expression if len(ds.indexMergeHints) > 0 { // Use allConds instread of pushedDownConds, // because we want to use IndexMerge even if some expr cannot be pushed to TiKV. // We will create new Selection for exprs that cannot be pushed in convertToIndexMergeScan. - indexMergeConds = indexMergeConds[:0] for _, expr := range ds.allConds { indexMergeConds = append(indexMergeConds, expression.PushDownNot(ds.ctx, expr)) } + } else { + // Make sure index merge only handle exprs that can be pushed down to tikv. + indexMergeConds, _ = expression.PushDownExprs(ds.ctx.GetSessionVars().StmtCtx, ds.pushedDownConds, ds.ctx.GetClient(), kv.TiKV) } isPossibleIdxMerge := len(indexMergeConds) > 0 && len(ds.possibleAccessPaths) > 1 sessionAndStmtPermission := (ds.ctx.GetSessionVars().GetEnableIndexMerge() || len(ds.indexMergeHints) > 0) && !ds.ctx.GetSessionVars().StmtCtx.NoIndexMergeHint From 19ffe5514c78e4f8b671f8602b20bed6f94dc39c Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Sun, 19 Dec 2021 14:33:16 +0800 Subject: [PATCH 14/23] add more comment Signed-off-by: guo-shaoge --- planner/core/find_best_task.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index f35ba9d4bbeb5..b50ab3b729c44 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -1096,6 +1096,8 @@ func setIndexMergeTableScanHandleCols(ds *DataSource, ts *PhysicalTableScan) (er return } +// buildIndexMergeTableScan will return Selection that will be pushed to TiKV. +// Filters that cannot be pushed to TiKV is also returned, and an extra Selection above IndexMergeReader will be constructed. func (ds *DataSource) buildIndexMergeTableScan(prop *property.PhysicalProperty, tableFilters []expression.Expression, totalRowCount float64) (PhysicalPlan, float64, []expression.Expression, error) { var partialCost float64 From e3f3e2eeadf726fd4081dbd3650ba06e04fd6a83 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Sun, 19 Dec 2021 14:55:14 +0800 Subject: [PATCH 15/23] fix index on virtual column for index merge Signed-off-by: guo-shaoge --- planner/core/find_best_task.go | 21 ++++++++++++--------- planner/core/integration_test.go | 11 +++++++++++ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index b50ab3b729c44..8ff947d73f275 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -974,7 +974,7 @@ func (ds *DataSource) convertToIndexMergeScan(prop *property.PhysicalProperty, c if prop.ExpectedCnt < ds.stats.RowCount { totalRowCount *= prop.ExpectedCnt / ds.stats.RowCount } - ts, partialCost, remainedFilters, err := ds.buildIndexMergeTableScan(prop, path.TableFilters, totalRowCount) + ts, partialCost, remainingFilters, err := ds.buildIndexMergeTableScan(prop, path.TableFilters, totalRowCount) if err != nil { return nil, err } @@ -982,8 +982,8 @@ func (ds *DataSource) convertToIndexMergeScan(prop *property.PhysicalProperty, c cop.tablePlan = ts cop.idxMergePartPlans = scans cop.cst = totalCost - if remainedFilters != nil { - cop.rootTaskConds = remainedFilters + if remainingFilters != nil { + cop.rootTaskConds = remainingFilters } task = cop.convertToRootTask(ds.ctx) ds.addSelection4PlanCache(task.(*rootTask), ds.tableStats.ScaleByExpectCnt(totalRowCount), prop) @@ -1130,7 +1130,10 @@ func (ds *DataSource) buildIndexMergeTableScan(prop *property.PhysicalProperty, ts.stats.StatsVersion = statistics.PseudoVersion } if len(tableFilters) > 0 { - pushedFilters, remainedFilters := extractFiltersForIndexMerge(sessVars.StmtCtx, ds.ctx.GetClient(), tableFilters) + pushedFilters, remainingFilters := extractFiltersForIndexMerge(sessVars.StmtCtx, ds.ctx.GetClient(), tableFilters) + pushedFilters1, remainingFilters1 := SplitSelCondsWithVirtualColumn(pushedFilters) + pushedFilters = pushedFilters1 + remainingFilters = append(remainingFilters, remainingFilters1...) if len(pushedFilters) != 0 { partialCost += totalRowCount * sessVars.CopCPUFactor selectivity, _, err := ds.tableStats.HistColl.Selectivity(ds.ctx, pushedFilters, nil) @@ -1140,28 +1143,28 @@ func (ds *DataSource) buildIndexMergeTableScan(prop *property.PhysicalProperty, } sel := PhysicalSelection{Conditions: pushedFilters}.Init(ts.ctx, ts.stats.ScaleByExpectCnt(selectivity*totalRowCount), ts.blockOffset) sel.SetChildren(ts) - return sel, partialCost, remainedFilters, nil + return sel, partialCost, remainingFilters, nil } - return ts, partialCost, remainedFilters, nil + return ts, partialCost, remainingFilters, nil } return ts, partialCost, nil, nil } // extractFiltersForIndexMerge returns: // `pushed`: exprs that can be pushed to TiKV. -// `remained`: exprs that can NOT be pushed to TiKV but can be pushed to other storage engines. +// `remaing`: exprs that can NOT be pushed to TiKV but can be pushed to other storage engines. // Why do we need this func? // IndexMerge only works on TiKV, so we need to find all exprs that cannot be pushed to TiKV, and add a new Selection above IndexMergeReader. // But the new Selection should exclude the exprs that can NOT be pushed to ALL the storage engines. // Because these exprs have already been put in another Selection(check rule_predicate_push_down). -func extractFiltersForIndexMerge(sc *stmtctx.StatementContext, client kv.Client, filters []expression.Expression) (pushed []expression.Expression, remained []expression.Expression) { +func extractFiltersForIndexMerge(sc *stmtctx.StatementContext, client kv.Client, filters []expression.Expression) (pushed []expression.Expression, remaing []expression.Expression) { for _, expr := range filters { if expression.CanExprsPushDown(sc, []expression.Expression{expr}, client, kv.TiKV) { pushed = append(pushed, expr) continue } if expression.CanExprsPushDown(sc, []expression.Expression{expr}, client, kv.UnSpecified) { - remained = append(remained, expr) + remaing = append(remaing, expr) } } return diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 6f259cd6ff7aa..3f88cdcbc73ba 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -5055,6 +5055,17 @@ func (s *testIntegrationSuite) TestIssue30200(c *C) { " ├─TableRangeScan_7(Build) 2.00 cop[tikv] table:t1 range:[1,1], [53330,53330], keep order:false, stats:pseudo", " └─TableRowIDScan_8(Probe) 3.00 cop[tikv] table:t1 keep order:false, stats:pseudo")) tk.MustQuery("select /*+ use_index_merge( t1 ) */ * from t1 where t1.c1 in (-3896405) or t1.pk in (1, 53330) and to_base64(left(pk, 5));").Check(testkit.Rows("-3896405 -1")) + + tk.MustExec("drop table if exists t1;") + tk.MustExec("create table t1(c1 int, c2 int, c3 int as (c1 + c2), key(c1), key(c2), key(c3));") + tk.MustExec("insert into t1(c1, c2) values(1, 1);") + tk.MustQuery("explain format=brief select /*+ use_index_merge(t1) */ * from t1 where c1 < -10 or c2 < 10 and reverse(c3) = '2';").Check(testkit.Rows( + "Selection 2825.66 root or(lt(test.t1.c1, -10), and(lt(test.t1.c2, 10), eq(reverse(cast(test.t1.c3, var_string(20))), \"2\")))", + "└─IndexMerge 5542.21 root ", + " ├─IndexRangeScan(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,-10), keep order:false, stats:pseudo", + " ├─IndexRangeScan(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo", + " └─TableRowIDScan(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo")) + tk.MustQuery("select /*+ use_index_merge(t1) */ * from t1 where c1 < -10 or c2 < 10 and reverse(c3) = '2';").Check(testkit.Rows("1 1 2")) } func (s *testIntegrationSuite) TestIssue29705(c *C) { From e249201f677052ff6cd5f1430210a887d91e96ea Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Sun, 19 Dec 2021 15:03:35 +0800 Subject: [PATCH 16/23] fix typo Signed-off-by: guo-shaoge --- planner/core/find_best_task.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index 8ff947d73f275..d15367d9b963e 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -1152,19 +1152,19 @@ func (ds *DataSource) buildIndexMergeTableScan(prop *property.PhysicalProperty, // extractFiltersForIndexMerge returns: // `pushed`: exprs that can be pushed to TiKV. -// `remaing`: exprs that can NOT be pushed to TiKV but can be pushed to other storage engines. +// `remaining`: exprs that can NOT be pushed to TiKV but can be pushed to other storage engines. // Why do we need this func? // IndexMerge only works on TiKV, so we need to find all exprs that cannot be pushed to TiKV, and add a new Selection above IndexMergeReader. // But the new Selection should exclude the exprs that can NOT be pushed to ALL the storage engines. // Because these exprs have already been put in another Selection(check rule_predicate_push_down). -func extractFiltersForIndexMerge(sc *stmtctx.StatementContext, client kv.Client, filters []expression.Expression) (pushed []expression.Expression, remaing []expression.Expression) { +func extractFiltersForIndexMerge(sc *stmtctx.StatementContext, client kv.Client, filters []expression.Expression) (pushed []expression.Expression, remaining []expression.Expression) { for _, expr := range filters { if expression.CanExprsPushDown(sc, []expression.Expression{expr}, client, kv.TiKV) { pushed = append(pushed, expr) continue } if expression.CanExprsPushDown(sc, []expression.Expression{expr}, client, kv.UnSpecified) { - remaing = append(remaing, expr) + remaining = append(remaining, expr) } } return From f47ebc973b9213992962a320523606e73998b98a Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Tue, 21 Dec 2021 16:24:27 +0800 Subject: [PATCH 17/23] fix case in index_merge_reader_test.go Signed-off-by: guo-shaoge --- planner/core/stats.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/planner/core/stats.go b/planner/core/stats.go index 732938d78fe1a..6e08fa7e96f3b 100644 --- a/planner/core/stats.go +++ b/planner/core/stats.go @@ -465,6 +465,7 @@ func (ds *DataSource) generateAndPruneIndexMergePath(indexMergeConds []expressio return nil } // Do not need to consider the regular paths in find_best_task(). + // So we can use index merge's row count as DataSource's row count. if needPrune { ds.possibleAccessPaths = ds.possibleAccessPaths[regularPathCount:] minRowCount := ds.possibleAccessPaths[0].CountAfterAccess @@ -473,7 +474,9 @@ func (ds *DataSource) generateAndPruneIndexMergePath(indexMergeConds []expressio minRowCount = path.CountAfterAccess } } - ds.stats = ds.tableStats.ScaleByExpectCnt(minRowCount) + if ds.stats.RowCount > minRowCount { + ds.stats = ds.tableStats.ScaleByExpectCnt(minRowCount) + } } return nil } From 11e5179c9c42d216c4d4721fac0bcdeb5c099800 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Tue, 21 Dec 2021 17:10:17 +0800 Subject: [PATCH 18/23] fix case Signed-off-by: guo-shaoge --- cmd/explaintest/r/index_merge.result | 60 ++++++++++++++-------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/cmd/explaintest/r/index_merge.result b/cmd/explaintest/r/index_merge.result index f790569635b28..261acdb49d8a2 100644 --- a/cmd/explaintest/r/index_merge.result +++ b/cmd/explaintest/r/index_merge.result @@ -237,11 +237,11 @@ insert into t1(c1, c2) values(1, 1), (2, 2), (3, 3), (4, 4), (5, 5); explain select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and c3 < 10 order by 1; id estRows task access object operator info Sort_5 4060.74 root test.t1.c1 -└─IndexMerge_12 2250.55 root - ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo - ├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo - └─Selection_11(Probe) 2250.55 cop[tikv] or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), lt(test.t1.c3, 10))) - └─TableRowIDScan_10 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo +└─Selection_12 2250.55 root or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), lt(test.t1.c3, 10))) + └─IndexMerge_11 5542.21 root + ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo + ├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo + └─TableRowIDScan_10(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and c3 < 10 order by 1; c1 c2 c3 1 1 2 @@ -252,11 +252,11 @@ c1 c2 c3 explain select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and c3 = c1 + c2 order by 1; id estRows task access object operator info Sort_5 5098.44 root test.t1.c1 -└─IndexMerge_12 2825.66 root - ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo - ├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo - └─Selection_11(Probe) 2825.66 cop[tikv] or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), eq(test.t1.c3, plus(test.t1.c1, test.t1.c2)))) - └─TableRowIDScan_10 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo +└─Selection_12 2825.66 root or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), eq(test.t1.c3, plus(test.t1.c1, test.t1.c2)))) + └─IndexMerge_11 5542.21 root + ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo + ├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo + └─TableRowIDScan_10(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and c3 = c1 + c2 order by 1; c1 c2 c3 1 1 2 @@ -267,11 +267,11 @@ c1 c2 c3 explain select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and substring(c3, c2) order by 1; id estRows task access object operator info Sort_5 5098.44 root test.t1.c1 -└─IndexMerge_12 2825.66 root - ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo - ├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo - └─Selection_11(Probe) 2825.66 cop[tikv] or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), istrue_with_null(cast(substring(cast(test.t1.c3, var_string(20)), test.t1.c2), double BINARY)))) - └─TableRowIDScan_10 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo +└─Selection_12 2825.66 root or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), istrue_with_null(cast(substring(cast(test.t1.c3, var_string(20)), test.t1.c2), double BINARY)))) + └─IndexMerge_11 5542.21 root + ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo + ├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo + └─TableRowIDScan_10(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and substring(c3, c2) order by 1; c1 c2 c3 1 1 2 @@ -282,11 +282,11 @@ c1 c2 c3 explain select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and c3 order by 1; id estRows task access object operator info Sort_5 4800.37 root test.t1.c1 -└─IndexMerge_12 2660.47 root - ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo - ├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo - └─Selection_11(Probe) 2660.47 cop[tikv] or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), test.t1.c3)) - └─TableRowIDScan_10 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo +└─Selection_12 2660.47 root or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), test.t1.c3)) + └─IndexMerge_11 5542.21 root + ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo + ├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo + └─TableRowIDScan_10(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and c3 order by 1; c1 c2 c3 1 1 2 @@ -302,11 +302,11 @@ select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and c3 < 10 explain select * from t1 where c1 < 10 or c2 < 10 and c3 < 10 order by 1; id estRows task access object operator info Sort_5 4060.74 root test.t1.c1 -└─IndexMerge_12 2250.55 root - ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo - ├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo - └─Selection_11(Probe) 2250.55 cop[tikv] or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), lt(test.t1.c3, 10))) - └─TableRowIDScan_10 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo +└─Selection_12 2250.55 root or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), lt(test.t1.c3, 10))) + └─IndexMerge_11 5542.21 root + ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo + ├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo + └─TableRowIDScan_10(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo select * from t1 where c1 < 10 or c2 < 10 and c3 < 10 order by 1; c1 c2 c3 1 1 2 @@ -722,11 +722,11 @@ c1 c2 c3 c4 c5 explain select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < 10) and substring(c3, 1, 1) = '1' order by 1; id estRows task access object operator info Sort_5 4433.77 root test.t1.c1 -└─IndexMerge_12 4433.77 root - ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo - ├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo - └─Selection_11(Probe) 4433.77 cop[tikv] eq(substring(cast(test.t1.c3, var_string(20)), 1, 1), "1") - └─TableRowIDScan_10 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo +└─Selection_12 4433.77 root eq(substring(cast(test.t1.c3, var_string(20)), 1, 1), "1") + └─IndexMerge_11 5542.21 root + ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo + ├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo + └─TableRowIDScan_10(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < 10) and substring(c3, 1, 1) = '1' order by 1; c1 c2 c3 c4 c5 1 1 1 1 1 From 887bccdd979e6c34e1f85458708d233a92a2a974 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Tue, 21 Dec 2021 19:36:04 +0800 Subject: [PATCH 19/23] a simple way to do same thing Signed-off-by: guo-shaoge --- planner/core/integration_test.go | 10 +++++----- planner/core/stats.go | 22 +++++++++++----------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index b46ec43f9d982..ac1c1db7680d3 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -5018,16 +5018,16 @@ func (s *testIntegrationSuite) TestIssue30200(c *C) { " └─TableRowIDScan(Probe) 19.99 cop[tikv] table:t1 keep order:false, stats:pseudo")) tk.MustQuery("select /*+ use_index_merge(t1) */ 1 from t1 where c1 = 'ab' or c2 = '10' and char_length(left(c1, 10)) = 10;").Check(testkit.Rows("1")) - // `left` cannot be pushed to TiKV, and there is no hint, so we cannot use index merge. + // If no hint, we cannot use index merge if filter cannot be pushed to any storage. oriIndexMergeSwitcher := tk.MustQuery("select @@tidb_enable_index_merge;").Rows()[0][0].(string) tk.MustExec("set tidb_enable_index_merge = on;") defer func() { tk.MustExec(fmt.Sprintf("set tidb_enable_index_merge = %s;", oriIndexMergeSwitcher)) }() - tk.MustQuery("explain format=brief select * from t1 where c1 = 'ab' or c2 = '10' and length(left(c1, 10)) = 10;").Check(testkit.Rows( - "Selection 17.99 root or(eq(test.t1.c1, \"ab\"), and(eq(test.t1.c2, \"10\"), eq(length(left(test.t1.c1, 10)), 10)))", - "└─TableReader 10000.00 root data:TableFullScan", - " └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo")) + tk.MustQuery("explain format=brief select * from t1 where c1 = 'ab' or c2 = '10' and char_length(lpad(c1, 10, 'a')) = 10;").Check(testkit.Rows( +"Selection 8000.00 root or(eq(test.t1.c1, \"ab\"), and(eq(test.t1.c2, \"10\"), eq(char_length(lpad(test.t1.c1, 10, \"a\")), 10)))", +"└─TableReader 10000.00 root data:TableFullScan", +" └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo")) tk.MustExec("use test") tk.MustExec("drop table if exists t1;") diff --git a/planner/core/stats.go b/planner/core/stats.go index 6e08fa7e96f3b..f8593308c916c 100644 --- a/planner/core/stats.go +++ b/planner/core/stats.go @@ -410,21 +410,17 @@ func (ds *DataSource) DeriveStats(childStats []*property.StatsInfo, selfSchema * } // Consider the IndexMergePath. Now, we just generate `IndexMergePath` in DNF case. + // Use allConds instread of pushedDownConds, + // because we want to use IndexMerge even if some expr cannot be pushed to TiKV. + // We will create new Selection for exprs that cannot be pushed in convertToIndexMergeScan. var indexMergeConds []expression.Expression - if len(ds.indexMergeHints) > 0 { - // Use allConds instread of pushedDownConds, - // because we want to use IndexMerge even if some expr cannot be pushed to TiKV. - // We will create new Selection for exprs that cannot be pushed in convertToIndexMergeScan. - for _, expr := range ds.allConds { - indexMergeConds = append(indexMergeConds, expression.PushDownNot(ds.ctx, expr)) - } - } else { - // Make sure index merge only handle exprs that can be pushed down to tikv. - indexMergeConds, _ = expression.PushDownExprs(ds.ctx.GetSessionVars().StmtCtx, ds.pushedDownConds, ds.ctx.GetClient(), kv.TiKV) + for _, expr := range ds.allConds { + indexMergeConds = append(indexMergeConds, expression.PushDownNot(ds.ctx, expr)) } + isPossibleIdxMerge := len(indexMergeConds) > 0 && len(ds.possibleAccessPaths) > 1 sessionAndStmtPermission := (ds.ctx.GetSessionVars().GetEnableIndexMerge() || len(ds.indexMergeHints) > 0) && !ds.ctx.GetSessionVars().StmtCtx.NoIndexMergeHint - // If there is an index path, we current do not consider `IndexMergePath`. + // If there is an index path or exprs that cannot be pushed down, we current do not consider `IndexMergePath`. needConsiderIndexMerge := true if len(ds.indexMergeHints) == 0 { for i := 1; i < len(ds.possibleAccessPaths); i++ { @@ -433,6 +429,10 @@ func (ds *DataSource) DeriveStats(childStats []*property.StatsInfo, selfSchema * break } } + _, remaining := expression.PushDownExprs(ds.ctx.GetSessionVars().StmtCtx, indexMergeConds, ds.ctx.GetClient(), kv.UnSpecified) + if len(remaining) != 0 { + needConsiderIndexMerge = false + } } readFromTableCache := ds.ctx.GetSessionVars().StmtCtx.ReadFromTableCache From 1d6abd4ac7b94d8369753162259c017f8238fe5b Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Tue, 21 Dec 2021 22:41:15 +0800 Subject: [PATCH 20/23] fix warnings and case Signed-off-by: guo-shaoge --- planner/core/integration_test.go | 6 +++--- planner/core/stats.go | 16 +++++++++++----- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index ac1c1db7680d3..243d886f68ad3 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -5025,9 +5025,9 @@ func (s *testIntegrationSuite) TestIssue30200(c *C) { tk.MustExec(fmt.Sprintf("set tidb_enable_index_merge = %s;", oriIndexMergeSwitcher)) }() tk.MustQuery("explain format=brief select * from t1 where c1 = 'ab' or c2 = '10' and char_length(lpad(c1, 10, 'a')) = 10;").Check(testkit.Rows( -"Selection 8000.00 root or(eq(test.t1.c1, \"ab\"), and(eq(test.t1.c2, \"10\"), eq(char_length(lpad(test.t1.c1, 10, \"a\")), 10)))", -"└─TableReader 10000.00 root data:TableFullScan", -" └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo")) + "Selection 8000.00 root or(eq(test.t1.c1, \"ab\"), and(eq(test.t1.c2, \"10\"), eq(char_length(lpad(test.t1.c1, 10, \"a\")), 10)))", + "└─TableReader 10000.00 root data:TableFullScan", + " └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo")) tk.MustExec("use test") tk.MustExec("drop table if exists t1;") diff --git a/planner/core/stats.go b/planner/core/stats.go index f8593308c916c..f248ee147b4ba 100644 --- a/planner/core/stats.go +++ b/planner/core/stats.go @@ -418,9 +418,12 @@ func (ds *DataSource) DeriveStats(childStats []*property.StatsInfo, selfSchema * indexMergeConds = append(indexMergeConds, expression.PushDownNot(ds.ctx, expr)) } + stmtCtx := ds.ctx.GetSessionVars().StmtCtx isPossibleIdxMerge := len(indexMergeConds) > 0 && len(ds.possibleAccessPaths) > 1 - sessionAndStmtPermission := (ds.ctx.GetSessionVars().GetEnableIndexMerge() || len(ds.indexMergeHints) > 0) && !ds.ctx.GetSessionVars().StmtCtx.NoIndexMergeHint - // If there is an index path or exprs that cannot be pushed down, we current do not consider `IndexMergePath`. + sessionAndStmtPermission := (ds.ctx.GetSessionVars().GetEnableIndexMerge() || len(ds.indexMergeHints) > 0) && !stmtCtx.NoIndexMergeHint + // We current do not consider `IndexMergePath`: + // 1. If there is an index path. + // 2. TODO: If there exists exprs that cannot be pushed down. This is to avoid wrongly estRow of Selection added by rule_predicate_push_down. needConsiderIndexMerge := true if len(ds.indexMergeHints) == 0 { for i := 1; i < len(ds.possibleAccessPaths); i++ { @@ -429,13 +432,16 @@ func (ds *DataSource) DeriveStats(childStats []*property.StatsInfo, selfSchema * break } } - _, remaining := expression.PushDownExprs(ds.ctx.GetSessionVars().StmtCtx, indexMergeConds, ds.ctx.GetClient(), kv.UnSpecified) + // PushDownExprs() will append extra warnings, which annoying. So we reset warnings here. + warnings := stmtCtx.GetWarnings() + _, remaining := expression.PushDownExprs(stmtCtx, indexMergeConds, ds.ctx.GetClient(), kv.UnSpecified) + stmtCtx.SetWarnings(warnings) if len(remaining) != 0 { needConsiderIndexMerge = false } } - readFromTableCache := ds.ctx.GetSessionVars().StmtCtx.ReadFromTableCache + readFromTableCache := stmtCtx.ReadFromTableCache if isPossibleIdxMerge && sessionAndStmtPermission && needConsiderIndexMerge && ds.tableInfo.TempTableType != model.TempTableLocal && !readFromTableCache { err := ds.generateAndPruneIndexMergePath(indexMergeConds, ds.indexMergeHints != nil) if err != nil { @@ -443,7 +449,7 @@ func (ds *DataSource) DeriveStats(childStats []*property.StatsInfo, selfSchema * } } else if len(ds.indexMergeHints) > 0 { ds.indexMergeHints = nil - ds.ctx.GetSessionVars().StmtCtx.AppendWarning(errors.Errorf("IndexMerge is inapplicable or disabled")) + stmtCtx.AppendWarning(errors.Errorf("IndexMerge is inapplicable or disabled")) } return ds.stats, nil } From b75d08774b26971a197a1210b9e75c58ca315f45 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Wed, 22 Dec 2021 15:01:45 +0800 Subject: [PATCH 21/23] fix comment Signed-off-by: guo-shaoge --- planner/core/find_best_task.go | 4 ++-- planner/core/stats.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index e2dcfe8aa2b20..2df5aa73eee34 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -1096,8 +1096,8 @@ func setIndexMergeTableScanHandleCols(ds *DataSource, ts *PhysicalTableScan) (er return } -// buildIndexMergeTableScan will return Selection that will be pushed to TiKV. -// Filters that cannot be pushed to TiKV is also returned, and an extra Selection above IndexMergeReader will be constructed. +// buildIndexMergeTableScan() returns Selection that will be pushed to TiKV. +// Filters that cannot be pushed to TiKV are also returned, and an extra Selection above IndexMergeReader will be constructed later. func (ds *DataSource) buildIndexMergeTableScan(prop *property.PhysicalProperty, tableFilters []expression.Expression, totalRowCount float64) (PhysicalPlan, float64, []expression.Expression, error) { var partialCost float64 diff --git a/planner/core/stats.go b/planner/core/stats.go index f248ee147b4ba..109e1fbe95fbc 100644 --- a/planner/core/stats.go +++ b/planner/core/stats.go @@ -432,7 +432,7 @@ func (ds *DataSource) DeriveStats(childStats []*property.StatsInfo, selfSchema * break } } - // PushDownExprs() will append extra warnings, which annoying. So we reset warnings here. + // PushDownExprs() will append extra warnings, which is annoying. So we reset warnings here. warnings := stmtCtx.GetWarnings() _, remaining := expression.PushDownExprs(stmtCtx, indexMergeConds, ds.ctx.GetClient(), kv.UnSpecified) stmtCtx.SetWarnings(warnings) From ff0fb078afe72cc59dc92ca5fb75eda933c22117 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Wed, 22 Dec 2021 22:13:48 +0800 Subject: [PATCH 22/23] add index merge warnings Signed-off-by: guo-shaoge --- planner/core/integration_test.go | 23 +++++++++++++++++++++++ planner/core/stats.go | 30 +++++++++++++++++++++++------- 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 243d886f68ad3..e2cd9a55b4872 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -5110,3 +5110,26 @@ func (s *testIntegrationSuite) TestIssue30804(c *C) { c.Assert(core.ErrWindowNoSuchWindow.Equal(err), IsTrue) tk.MustExec("select avg(0) over w1 from t1 where b > (select sum(t2.a) over w2 from t2 window w2 as (partition by t2.b)) window w1 as (partition by t1.b)") } + +func (s *testIntegrationSuite) TestIndexMergeWarning(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + + tk.MustExec("drop table if exists t1") + tk.MustExec("create table t1(c1 int, c2 int)") + tk.MustExec("select /*+ use_index_merge(t1) */ * from t1 where c1 < 1 or c2 < 1") + warningMsg := "Warning 1105 IndexMerge is inapplicable or disabled. No available filter or access path." + tk.MustQuery("show warnings").Check(testkit.Rows(warningMsg)) + + tk.MustExec("drop table if exists t1") + tk.MustExec("create table t1(c1 int, c2 int, key(c1), key(c2))") + tk.MustExec("select /*+ use_index_merge(t1), no_index_merge() */ * from t1 where c1 < 1 or c2 < 1") + warningMsg = "Warning 1105 IndexMerge is inapplicable or disabled. Got no_index_merge hint or switcher is off." + tk.MustQuery("show warnings").Check(testkit.Rows(warningMsg)) + + tk.MustExec("drop table if exists t1") + tk.MustExec("create temporary table t1(c1 int, c2 int, key(c1), key(c2))") + tk.MustExec("select /*+ use_index_merge(t1) */ * from t1 where c1 < 1 or c2 < 1") + warningMsg = "Warning 1105 IndexMerge is inapplicable or disabled. Cannot use IndexMerge on temporary table." + tk.MustQuery("show warnings").Check(testkit.Rows(warningMsg)) +} diff --git a/planner/core/stats.go b/planner/core/stats.go index 109e1fbe95fbc..cf79f098e9ac2 100644 --- a/planner/core/stats.go +++ b/planner/core/stats.go @@ -432,12 +432,14 @@ func (ds *DataSource) DeriveStats(childStats []*property.StatsInfo, selfSchema * break } } - // PushDownExprs() will append extra warnings, which is annoying. So we reset warnings here. - warnings := stmtCtx.GetWarnings() - _, remaining := expression.PushDownExprs(stmtCtx, indexMergeConds, ds.ctx.GetClient(), kv.UnSpecified) - stmtCtx.SetWarnings(warnings) - if len(remaining) != 0 { - needConsiderIndexMerge = false + if needConsiderIndexMerge { + // PushDownExprs() will append extra warnings, which is annoying. So we reset warnings here. + warnings := stmtCtx.GetWarnings() + _, remaining := expression.PushDownExprs(stmtCtx, indexMergeConds, ds.ctx.GetClient(), kv.UnSpecified) + stmtCtx.SetWarnings(warnings) + if len(remaining) != 0 { + needConsiderIndexMerge = false + } } } @@ -449,7 +451,21 @@ func (ds *DataSource) DeriveStats(childStats []*property.StatsInfo, selfSchema * } } else if len(ds.indexMergeHints) > 0 { ds.indexMergeHints = nil - stmtCtx.AppendWarning(errors.Errorf("IndexMerge is inapplicable or disabled")) + var msg string + if !isPossibleIdxMerge { + msg = "No available filter or access path." + } else if !sessionAndStmtPermission { + msg = "Got no_index_merge hint or switcher is off." + } else if !needConsiderIndexMerge { + msg = "Got index path or exprs that cannot be pushed." + } else if ds.tableInfo.TempTableType == model.TempTableLocal { + msg = "Cannot use IndexMerge on temporary table." + } else if readFromTableCache { + msg = "Cannot use IndexMerge on TableCache." + } + msg = fmt.Sprintf("IndexMerge is inapplicable or disabled. %s", msg) + stmtCtx.AppendWarning(errors.Errorf(msg)) + logutil.BgLogger().Info(msg) } return ds.stats, nil } From 8b50eec00bda9138d9a8ac5b7a4207a77a40e14f Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Thu, 23 Dec 2021 11:48:04 +0800 Subject: [PATCH 23/23] fix warning case Signed-off-by: guo-shaoge --- planner/core/integration_test.go | 4 ++-- planner/core/stats.go | 10 ++++------ planner/core/testdata/integration_suite_out.json | 4 ++-- session/session_test.go | 4 ++-- table/tables/cache_test.go | 2 +- 5 files changed, 11 insertions(+), 13 deletions(-) diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index e2cd9a55b4872..38048d4d30009 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -5118,13 +5118,13 @@ func (s *testIntegrationSuite) TestIndexMergeWarning(c *C) { tk.MustExec("drop table if exists t1") tk.MustExec("create table t1(c1 int, c2 int)") tk.MustExec("select /*+ use_index_merge(t1) */ * from t1 where c1 < 1 or c2 < 1") - warningMsg := "Warning 1105 IndexMerge is inapplicable or disabled. No available filter or access path." + warningMsg := "Warning 1105 IndexMerge is inapplicable or disabled. No available filter or available index." tk.MustQuery("show warnings").Check(testkit.Rows(warningMsg)) tk.MustExec("drop table if exists t1") tk.MustExec("create table t1(c1 int, c2 int, key(c1), key(c2))") tk.MustExec("select /*+ use_index_merge(t1), no_index_merge() */ * from t1 where c1 < 1 or c2 < 1") - warningMsg = "Warning 1105 IndexMerge is inapplicable or disabled. Got no_index_merge hint or switcher is off." + warningMsg = "Warning 1105 IndexMerge is inapplicable or disabled. Got no_index_merge hint or tidb_enable_index_merge is off." tk.MustQuery("show warnings").Check(testkit.Rows(warningMsg)) tk.MustExec("drop table if exists t1") diff --git a/planner/core/stats.go b/planner/core/stats.go index cf79f098e9ac2..4b7ac3e33d00a 100644 --- a/planner/core/stats.go +++ b/planner/core/stats.go @@ -453,11 +453,9 @@ func (ds *DataSource) DeriveStats(childStats []*property.StatsInfo, selfSchema * ds.indexMergeHints = nil var msg string if !isPossibleIdxMerge { - msg = "No available filter or access path." + msg = "No available filter or available index." } else if !sessionAndStmtPermission { - msg = "Got no_index_merge hint or switcher is off." - } else if !needConsiderIndexMerge { - msg = "Got index path or exprs that cannot be pushed." + msg = "Got no_index_merge hint or tidb_enable_index_merge is off." } else if ds.tableInfo.TempTableType == model.TempTableLocal { msg = "Cannot use IndexMerge on temporary table." } else if readFromTableCache { @@ -465,7 +463,7 @@ func (ds *DataSource) DeriveStats(childStats []*property.StatsInfo, selfSchema * } msg = fmt.Sprintf("IndexMerge is inapplicable or disabled. %s", msg) stmtCtx.AppendWarning(errors.Errorf(msg)) - logutil.BgLogger().Info(msg) + logutil.BgLogger().Debug(msg) } return ds.stats, nil } @@ -483,7 +481,7 @@ func (ds *DataSource) generateAndPruneIndexMergePath(indexMergeConds []expressio // With hints and without generated IndexMerge paths if regularPathCount == len(ds.possibleAccessPaths) { ds.indexMergeHints = nil - ds.ctx.GetSessionVars().StmtCtx.AppendWarning(errors.Errorf("IndexMerge is inapplicable or disabled")) + ds.ctx.GetSessionVars().StmtCtx.AppendWarning(errors.Errorf("IndexMerge is inapplicable.")) return nil } // Do not need to consider the regular paths in find_best_task(). diff --git a/planner/core/testdata/integration_suite_out.json b/planner/core/testdata/integration_suite_out.json index 6c946cdac4d2d..c3bbda1ed3f2c 100644 --- a/planner/core/testdata/integration_suite_out.json +++ b/planner/core/testdata/integration_suite_out.json @@ -1332,7 +1332,7 @@ "└─IndexRangeScan 20.00 cop[tikv] table:tt, index:a(a) range:[10,10], [20,20], keep order:false, stats:pseudo" ], "Warnings": [ - "Warning 1105 IndexMerge is inapplicable or disabled" + "Warning 1105 IndexMerge is inapplicable." ] }, { @@ -1342,7 +1342,7 @@ "└─IndexRangeScan 6666.67 cop[tikv] table:tt, index:a(a) range:[-inf,10), [15,15], (20,+inf], keep order:false, stats:pseudo" ], "Warnings": [ - "Warning 1105 IndexMerge is inapplicable or disabled" + "Warning 1105 IndexMerge is inapplicable." ] } ] diff --git a/session/session_test.go b/session/session_test.go index 4602758eeaa28..d8694348ffe85 100644 --- a/session/session_test.go +++ b/session/session_test.go @@ -5419,7 +5419,7 @@ func (s *testSessionSuite) TestLocalTemporaryTableScan(c *C) { "12 112 1012", "3 113 1003", "14 114 1014", "16 116 1016", "7 117 1007", "18 118 1018", )) - tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 IndexMerge is inapplicable or disabled")) + tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 IndexMerge is inapplicable or disabled. Cannot use IndexMerge on temporary table.")) } doModify := func() { @@ -5458,7 +5458,7 @@ func (s *testSessionSuite) TestLocalTemporaryTableScan(c *C) { "3 113 1003", "14 114 1014", "7 117 9999", "18 118 1018", "12 132 1012", )) - tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 IndexMerge is inapplicable or disabled")) + tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 IndexMerge is inapplicable or disabled. Cannot use IndexMerge on temporary table.")) } assertSelectAsUnModified() diff --git a/table/tables/cache_test.go b/table/tables/cache_test.go index 62e48ccd24c94..30a3c20c3af00 100644 --- a/table/tables/cache_test.go +++ b/table/tables/cache_test.go @@ -142,7 +142,7 @@ func TestCacheTableBasicScan(t *testing.T) { "12 112 1012", "3 113 1003", "14 114 1014", "16 116 1016", "7 117 1007", "18 118 1018", )) - tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 IndexMerge is inapplicable or disabled")) + tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 IndexMerge is inapplicable or disabled. Cannot use IndexMerge on TableCache.")) } assertSelect()