diff --git a/cmd/explaintest/r/index_merge.result b/cmd/explaintest/r/index_merge.result index f790569635b28..261acdb49d8a2 100644 --- a/cmd/explaintest/r/index_merge.result +++ b/cmd/explaintest/r/index_merge.result @@ -237,11 +237,11 @@ insert into t1(c1, c2) values(1, 1), (2, 2), (3, 3), (4, 4), (5, 5); explain select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and c3 < 10 order by 1; id estRows task access object operator info Sort_5 4060.74 root test.t1.c1 -└─IndexMerge_12 2250.55 root - ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo - ├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo - └─Selection_11(Probe) 2250.55 cop[tikv] or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), lt(test.t1.c3, 10))) - └─TableRowIDScan_10 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo +└─Selection_12 2250.55 root or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), lt(test.t1.c3, 10))) + └─IndexMerge_11 5542.21 root + ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo + ├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo + └─TableRowIDScan_10(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and c3 < 10 order by 1; c1 c2 c3 1 1 2 @@ -252,11 +252,11 @@ c1 c2 c3 explain select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and c3 = c1 + c2 order by 1; id estRows task access object operator info Sort_5 5098.44 root test.t1.c1 -└─IndexMerge_12 2825.66 root - ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo - ├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo - └─Selection_11(Probe) 2825.66 cop[tikv] or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), eq(test.t1.c3, plus(test.t1.c1, test.t1.c2)))) - └─TableRowIDScan_10 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo +└─Selection_12 2825.66 root or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), eq(test.t1.c3, plus(test.t1.c1, test.t1.c2)))) + └─IndexMerge_11 5542.21 root + ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo + ├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo + └─TableRowIDScan_10(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and c3 = c1 + c2 order by 1; c1 c2 c3 1 1 2 @@ -267,11 +267,11 @@ c1 c2 c3 explain select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and substring(c3, c2) order by 1; id estRows task access object operator info Sort_5 5098.44 root test.t1.c1 -└─IndexMerge_12 2825.66 root - ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo - ├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo - └─Selection_11(Probe) 2825.66 cop[tikv] or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), istrue_with_null(cast(substring(cast(test.t1.c3, var_string(20)), test.t1.c2), double BINARY)))) - └─TableRowIDScan_10 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo +└─Selection_12 2825.66 root or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), istrue_with_null(cast(substring(cast(test.t1.c3, var_string(20)), test.t1.c2), double BINARY)))) + └─IndexMerge_11 5542.21 root + ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo + ├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo + └─TableRowIDScan_10(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and substring(c3, c2) order by 1; c1 c2 c3 1 1 2 @@ -282,11 +282,11 @@ c1 c2 c3 explain select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and c3 order by 1; id estRows task access object operator info Sort_5 4800.37 root test.t1.c1 -└─IndexMerge_12 2660.47 root - ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo - ├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo - └─Selection_11(Probe) 2660.47 cop[tikv] or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), test.t1.c3)) - └─TableRowIDScan_10 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo +└─Selection_12 2660.47 root or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), test.t1.c3)) + └─IndexMerge_11 5542.21 root + ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo + ├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo + └─TableRowIDScan_10(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and c3 order by 1; c1 c2 c3 1 1 2 @@ -302,11 +302,11 @@ select /*+ use_index_merge(t1) */ * from t1 where c1 < 10 or c2 < 10 and c3 < 10 explain select * from t1 where c1 < 10 or c2 < 10 and c3 < 10 order by 1; id estRows task access object operator info Sort_5 4060.74 root test.t1.c1 -└─IndexMerge_12 2250.55 root - ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo - ├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo - └─Selection_11(Probe) 2250.55 cop[tikv] or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), lt(test.t1.c3, 10))) - └─TableRowIDScan_10 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo +└─Selection_12 2250.55 root or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), lt(test.t1.c3, 10))) + └─IndexMerge_11 5542.21 root + ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo + ├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo + └─TableRowIDScan_10(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo select * from t1 where c1 < 10 or c2 < 10 and c3 < 10 order by 1; c1 c2 c3 1 1 2 @@ -722,11 +722,11 @@ c1 c2 c3 c4 c5 explain select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < 10) and substring(c3, 1, 1) = '1' order by 1; id estRows task access object operator info Sort_5 4433.77 root test.t1.c1 -└─IndexMerge_12 4433.77 root - ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo - ├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo - └─Selection_11(Probe) 4433.77 cop[tikv] eq(substring(cast(test.t1.c3, var_string(20)), 1, 1), "1") - └─TableRowIDScan_10 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo +└─Selection_12 4433.77 root eq(substring(cast(test.t1.c3, var_string(20)), 1, 1), "1") + └─IndexMerge_11 5542.21 root + ├─IndexRangeScan_8(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo + ├─IndexRangeScan_9(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo + └─TableRowIDScan_10(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < 10) and substring(c3, 1, 1) = '1' order by 1; c1 c2 c3 c4 c5 1 1 1 1 1 diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index 166d3adc298b3..2df5aa73eee34 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -28,6 +28,7 @@ import ( "github.com/pingcap/tidb/planner/property" "github.com/pingcap/tidb/planner/util" "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/statistics" "github.com/pingcap/tidb/types" tidbutil "github.com/pingcap/tidb/util" @@ -973,7 +974,7 @@ func (ds *DataSource) convertToIndexMergeScan(prop *property.PhysicalProperty, c if prop.ExpectedCnt < ds.stats.RowCount { totalRowCount *= prop.ExpectedCnt / ds.stats.RowCount } - ts, partialCost, err := ds.buildIndexMergeTableScan(prop, path.TableFilters, totalRowCount) + ts, partialCost, remainingFilters, err := ds.buildIndexMergeTableScan(prop, path.TableFilters, totalRowCount) if err != nil { return nil, err } @@ -981,6 +982,9 @@ func (ds *DataSource) convertToIndexMergeScan(prop *property.PhysicalProperty, c cop.tablePlan = ts cop.idxMergePartPlans = scans cop.cst = totalCost + if remainingFilters != nil { + cop.rootTaskConds = remainingFilters + } task = cop.convertToRootTask(ds.ctx) ds.addSelection4PlanCache(task.(*rootTask), ds.tableStats.ScaleByExpectCnt(totalRowCount), prop) return task, nil @@ -1092,8 +1096,10 @@ func setIndexMergeTableScanHandleCols(ds *DataSource, ts *PhysicalTableScan) (er return } +// buildIndexMergeTableScan() returns Selection that will be pushed to TiKV. +// Filters that cannot be pushed to TiKV are also returned, and an extra Selection above IndexMergeReader will be constructed later. func (ds *DataSource) buildIndexMergeTableScan(prop *property.PhysicalProperty, tableFilters []expression.Expression, - totalRowCount float64) (PhysicalPlan, float64, error) { + totalRowCount float64) (PhysicalPlan, float64, []expression.Expression, error) { var partialCost float64 sessVars := ds.ctx.GetSessionVars() ts := PhysicalTableScan{ @@ -1108,7 +1114,7 @@ func (ds *DataSource) buildIndexMergeTableScan(prop *property.PhysicalProperty, ts.SetSchema(ds.schema.Clone()) err := setIndexMergeTableScanHandleCols(ds, ts) if err != nil { - return nil, 0, err + return nil, 0, nil, err } if ts.Table.PKIsHandle { if pkColInfo := ts.Table.GetPkColInfo(); pkColInfo != nil { @@ -1124,17 +1130,44 @@ func (ds *DataSource) buildIndexMergeTableScan(prop *property.PhysicalProperty, ts.stats.StatsVersion = statistics.PseudoVersion } if len(tableFilters) > 0 { - partialCost += totalRowCount * sessVars.CopCPUFactor - selectivity, _, err := ds.tableStats.HistColl.Selectivity(ds.ctx, tableFilters, nil) - if err != nil { - logutil.BgLogger().Debug("calculate selectivity failed, use selection factor", zap.Error(err)) - selectivity = SelectionFactor + pushedFilters, remainingFilters := extractFiltersForIndexMerge(sessVars.StmtCtx, ds.ctx.GetClient(), tableFilters) + pushedFilters1, remainingFilters1 := SplitSelCondsWithVirtualColumn(pushedFilters) + pushedFilters = pushedFilters1 + remainingFilters = append(remainingFilters, remainingFilters1...) + if len(pushedFilters) != 0 { + partialCost += totalRowCount * sessVars.CopCPUFactor + selectivity, _, err := ds.tableStats.HistColl.Selectivity(ds.ctx, pushedFilters, nil) + if err != nil { + logutil.BgLogger().Debug("calculate selectivity failed, use selection factor", zap.Error(err)) + selectivity = SelectionFactor + } + sel := PhysicalSelection{Conditions: pushedFilters}.Init(ts.ctx, ts.stats.ScaleByExpectCnt(selectivity*totalRowCount), ts.blockOffset) + sel.SetChildren(ts) + return sel, partialCost, remainingFilters, nil + } + return ts, partialCost, remainingFilters, nil + } + return ts, partialCost, nil, nil +} + +// extractFiltersForIndexMerge returns: +// `pushed`: exprs that can be pushed to TiKV. +// `remaining`: exprs that can NOT be pushed to TiKV but can be pushed to other storage engines. +// Why do we need this func? +// IndexMerge only works on TiKV, so we need to find all exprs that cannot be pushed to TiKV, and add a new Selection above IndexMergeReader. +// But the new Selection should exclude the exprs that can NOT be pushed to ALL the storage engines. +// Because these exprs have already been put in another Selection(check rule_predicate_push_down). +func extractFiltersForIndexMerge(sc *stmtctx.StatementContext, client kv.Client, filters []expression.Expression) (pushed []expression.Expression, remaining []expression.Expression) { + for _, expr := range filters { + if expression.CanExprsPushDown(sc, []expression.Expression{expr}, client, kv.TiKV) { + pushed = append(pushed, expr) + continue + } + if expression.CanExprsPushDown(sc, []expression.Expression{expr}, client, kv.UnSpecified) { + remaining = append(remaining, expr) } - sel := PhysicalSelection{Conditions: tableFilters}.Init(ts.ctx, ts.stats.ScaleByExpectCnt(selectivity*totalRowCount), ts.blockOffset) - sel.SetChildren(ts) - return sel, partialCost, nil } - return ts, partialCost, nil + return } func indexCoveringCol(col *expression.Column, indexCols []*expression.Column, idxColLens []int) bool { diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 0dfc4532a89bc..38048d4d30009 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -4992,6 +4992,82 @@ func (s *testIntegrationSuite) TestIssue30094(c *C) { )) } +func (s *testIntegrationSuite) TestIssue30200(c *C) { + tk := testkit.NewTestKit(c, s.store) + + tk.MustExec("use test") + tk.MustExec("drop table if exists t1;") + tk.MustExec("create table t1(c1 varchar(100), c2 varchar(100), key(c1), key(c2), c3 varchar(100));") + tk.MustExec("insert into t1 values('ab', '10', '10');") + + // lpad has not been pushed to TiKV or TiFlash. + tk.MustQuery("explain format=brief select /*+ use_index_merge(t1) */ * from t1 where c1 = 'ab' or c2 = '10' and char_length(lpad(c1, 10, 'a')) = 10;").Check(testkit.Rows( + "Selection 15.99 root or(eq(test.t1.c1, \"ab\"), and(eq(test.t1.c2, \"10\"), eq(char_length(lpad(test.t1.c1, 10, \"a\")), 10)))", + "└─IndexMerge 19.99 root ", + " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t1, index:c1(c1) range:[\"ab\",\"ab\"], keep order:false, stats:pseudo", + " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t1, index:c2(c2) range:[\"10\",\"10\"], keep order:false, stats:pseudo", + " └─TableRowIDScan(Probe) 19.99 cop[tikv] table:t1 keep order:false, stats:pseudo")) + tk.MustQuery("select /*+ use_index_merge(t1) */ 1 from t1 where c1 = 'de' or c2 = '10' and char_length(lpad(c1, 10, 'a')) = 10;").Check(testkit.Rows("1")) + + // `left` has not been pushed to TiKV, but it has been pushed to TiFlash. + tk.MustQuery("explain format=brief select /*+ use_index_merge(t1) */ * from t1 where c1 = 'ab' or c2 = '10' and char_length(left(c1, 10)) = 10;").Check(testkit.Rows( + "Selection 0.04 root or(eq(test.t1.c1, \"ab\"), and(eq(test.t1.c2, \"10\"), eq(char_length(left(test.t1.c1, 10)), 10)))", + "└─IndexMerge 19.99 root ", + " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t1, index:c1(c1) range:[\"ab\",\"ab\"], keep order:false, stats:pseudo", + " ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t1, index:c2(c2) range:[\"10\",\"10\"], keep order:false, stats:pseudo", + " └─TableRowIDScan(Probe) 19.99 cop[tikv] table:t1 keep order:false, stats:pseudo")) + tk.MustQuery("select /*+ use_index_merge(t1) */ 1 from t1 where c1 = 'ab' or c2 = '10' and char_length(left(c1, 10)) = 10;").Check(testkit.Rows("1")) + + // If no hint, we cannot use index merge if filter cannot be pushed to any storage. + oriIndexMergeSwitcher := tk.MustQuery("select @@tidb_enable_index_merge;").Rows()[0][0].(string) + tk.MustExec("set tidb_enable_index_merge = on;") + defer func() { + tk.MustExec(fmt.Sprintf("set tidb_enable_index_merge = %s;", oriIndexMergeSwitcher)) + }() + tk.MustQuery("explain format=brief select * from t1 where c1 = 'ab' or c2 = '10' and char_length(lpad(c1, 10, 'a')) = 10;").Check(testkit.Rows( + "Selection 8000.00 root or(eq(test.t1.c1, \"ab\"), and(eq(test.t1.c2, \"10\"), eq(char_length(lpad(test.t1.c1, 10, \"a\")), 10)))", + "└─TableReader 10000.00 root data:TableFullScan", + " └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo")) + + tk.MustExec("use test") + tk.MustExec("drop table if exists t1;") + tk.MustExec("create table t1(c1 varchar(100), c2 varchar(100), c3 varchar(100), c4 varchar(100), key idx_0(c1), key idx_1(c2, c3));") + tk.MustExec("insert into t1 values('ab', '10', '10', '10');") + // c3 is part of idx_1, so it will be put in partial_path's IndexFilters instead of TableFilters. + // But it still cannot be pushed to TiKV. + tk.MustQuery("explain select /*+ use_index_merge(t1) */ 1 from t1 where c1 = 'de' or c2 = '10' and char_length(lpad(c3, 10, 'a')) = 10;").Check(testkit.Rows( + "Projection_4 15.99 root 1->Column#6", + "└─Selection_5 15.99 root or(eq(test.t1.c1, \"de\"), and(eq(test.t1.c2, \"10\"), eq(char_length(lpad(test.t1.c3, 10, \"a\")), 10)))", + " └─IndexMerge_9 19.99 root ", + " ├─IndexRangeScan_6(Build) 10.00 cop[tikv] table:t1, index:idx_0(c1) range:[\"de\",\"de\"], keep order:false, stats:pseudo", + " ├─IndexRangeScan_7(Build) 10.00 cop[tikv] table:t1, index:idx_1(c2, c3) range:[\"10\",\"10\"], keep order:false, stats:pseudo", + " └─TableRowIDScan_8(Probe) 19.99 cop[tikv] table:t1 keep order:false, stats:pseudo")) + tk.MustQuery("select /*+ use_index_merge(t1) */ 1 from t1 where c1 = 'de' or c2 = '10' and char_length(lpad(c3, 10, 'a')) = 10;").Check(testkit.Rows("1")) + + tk.MustExec("drop table if exists t1;") + tk.MustExec("create table t1 (c1 int , pk int, primary key( pk ) , unique key( c1));") + tk.MustExec("insert into t1 values(-3896405, -1), (-2, 1), (-1, -2);") + // to_base64(left(pk, 5)) is in partial_path's TableFilters. But it cannot be pushed to TiKV. So it should be executed in TiDB. + tk.MustQuery("explain select /*+ use_index_merge( t1 ) */ * from t1 where t1.c1 in (-3896405) or t1.pk in (1, 53330) and to_base64(left(pk, 5));").Check(testkit.Rows( + "Selection_5 2.40 root or(eq(test.t1.c1, -3896405), and(in(test.t1.pk, 1, 53330), istrue_with_null(cast(to_base64(left(cast(test.t1.pk, var_string(20)), 5)), double BINARY))))", + "└─IndexMerge_9 3.00 root ", + " ├─IndexRangeScan_6(Build) 1.00 cop[tikv] table:t1, index:c1(c1) range:[-3896405,-3896405], keep order:false, stats:pseudo", + " ├─TableRangeScan_7(Build) 2.00 cop[tikv] table:t1 range:[1,1], [53330,53330], keep order:false, stats:pseudo", + " └─TableRowIDScan_8(Probe) 3.00 cop[tikv] table:t1 keep order:false, stats:pseudo")) + tk.MustQuery("select /*+ use_index_merge( t1 ) */ * from t1 where t1.c1 in (-3896405) or t1.pk in (1, 53330) and to_base64(left(pk, 5));").Check(testkit.Rows("-3896405 -1")) + + tk.MustExec("drop table if exists t1;") + tk.MustExec("create table t1(c1 int, c2 int, c3 int as (c1 + c2), key(c1), key(c2), key(c3));") + tk.MustExec("insert into t1(c1, c2) values(1, 1);") + tk.MustQuery("explain format=brief select /*+ use_index_merge(t1) */ * from t1 where c1 < -10 or c2 < 10 and reverse(c3) = '2';").Check(testkit.Rows( + "Selection 2825.66 root or(lt(test.t1.c1, -10), and(lt(test.t1.c2, 10), eq(reverse(cast(test.t1.c3, var_string(20))), \"2\")))", + "└─IndexMerge 5542.21 root ", + " ├─IndexRangeScan(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,-10), keep order:false, stats:pseudo", + " ├─IndexRangeScan(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo", + " └─TableRowIDScan(Probe) 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo")) + tk.MustQuery("select /*+ use_index_merge(t1) */ * from t1 where c1 < -10 or c2 < 10 and reverse(c3) = '2';").Check(testkit.Rows("1 1 2")) +} + func (s *testIntegrationSuite) TestIssue29705(c *C) { tk := testkit.NewTestKit(c, s.store) origin := tk.MustQuery("SELECT @@session.tidb_partition_prune_mode") @@ -5034,3 +5110,26 @@ func (s *testIntegrationSuite) TestIssue30804(c *C) { c.Assert(core.ErrWindowNoSuchWindow.Equal(err), IsTrue) tk.MustExec("select avg(0) over w1 from t1 where b > (select sum(t2.a) over w2 from t2 window w2 as (partition by t2.b)) window w1 as (partition by t1.b)") } + +func (s *testIntegrationSuite) TestIndexMergeWarning(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + + tk.MustExec("drop table if exists t1") + tk.MustExec("create table t1(c1 int, c2 int)") + tk.MustExec("select /*+ use_index_merge(t1) */ * from t1 where c1 < 1 or c2 < 1") + warningMsg := "Warning 1105 IndexMerge is inapplicable or disabled. No available filter or available index." + tk.MustQuery("show warnings").Check(testkit.Rows(warningMsg)) + + tk.MustExec("drop table if exists t1") + tk.MustExec("create table t1(c1 int, c2 int, key(c1), key(c2))") + tk.MustExec("select /*+ use_index_merge(t1), no_index_merge() */ * from t1 where c1 < 1 or c2 < 1") + warningMsg = "Warning 1105 IndexMerge is inapplicable or disabled. Got no_index_merge hint or tidb_enable_index_merge is off." + tk.MustQuery("show warnings").Check(testkit.Rows(warningMsg)) + + tk.MustExec("drop table if exists t1") + tk.MustExec("create temporary table t1(c1 int, c2 int, key(c1), key(c2))") + tk.MustExec("select /*+ use_index_merge(t1) */ * from t1 where c1 < 1 or c2 < 1") + warningMsg = "Warning 1105 IndexMerge is inapplicable or disabled. Cannot use IndexMerge on temporary table." + tk.MustQuery("show warnings").Check(testkit.Rows(warningMsg)) +} diff --git a/planner/core/logical_plans.go b/planner/core/logical_plans.go index 5fe0426b5c15b..1b0f6c4543985 100644 --- a/planner/core/logical_plans.go +++ b/planner/core/logical_plans.go @@ -549,7 +549,7 @@ type DataSource struct { // pushedDownConds are the conditions that will be pushed down to coprocessor. pushedDownConds []expression.Expression // allConds contains all the filters on this table. For now it's maintained - // in predicate push down and used only in partition pruning. + // in predicate push down and used in partition pruning/index merge. allConds []expression.Expression statisticTable *statistics.Table diff --git a/planner/core/stats.go b/planner/core/stats.go index d3f23427b2f40..4b7ac3e33d00a 100644 --- a/planner/core/stats.go +++ b/planner/core/stats.go @@ -23,6 +23,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/tidb/expression" + "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/parser/ast" "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/parser/mysql" @@ -409,9 +410,20 @@ func (ds *DataSource) DeriveStats(childStats []*property.StatsInfo, selfSchema * } // Consider the IndexMergePath. Now, we just generate `IndexMergePath` in DNF case. - isPossibleIdxMerge := len(ds.pushedDownConds) > 0 && len(ds.possibleAccessPaths) > 1 - sessionAndStmtPermission := (ds.ctx.GetSessionVars().GetEnableIndexMerge() || len(ds.indexMergeHints) > 0) && !ds.ctx.GetSessionVars().StmtCtx.NoIndexMergeHint - // If there is an index path, we current do not consider `IndexMergePath`. + // Use allConds instread of pushedDownConds, + // because we want to use IndexMerge even if some expr cannot be pushed to TiKV. + // We will create new Selection for exprs that cannot be pushed in convertToIndexMergeScan. + var indexMergeConds []expression.Expression + for _, expr := range ds.allConds { + indexMergeConds = append(indexMergeConds, expression.PushDownNot(ds.ctx, expr)) + } + + stmtCtx := ds.ctx.GetSessionVars().StmtCtx + isPossibleIdxMerge := len(indexMergeConds) > 0 && len(ds.possibleAccessPaths) > 1 + sessionAndStmtPermission := (ds.ctx.GetSessionVars().GetEnableIndexMerge() || len(ds.indexMergeHints) > 0) && !stmtCtx.NoIndexMergeHint + // We current do not consider `IndexMergePath`: + // 1. If there is an index path. + // 2. TODO: If there exists exprs that cannot be pushed down. This is to avoid wrongly estRow of Selection added by rule_predicate_push_down. needConsiderIndexMerge := true if len(ds.indexMergeHints) == 0 { for i := 1; i < len(ds.possibleAccessPaths); i++ { @@ -420,24 +432,45 @@ func (ds *DataSource) DeriveStats(childStats []*property.StatsInfo, selfSchema * break } } + if needConsiderIndexMerge { + // PushDownExprs() will append extra warnings, which is annoying. So we reset warnings here. + warnings := stmtCtx.GetWarnings() + _, remaining := expression.PushDownExprs(stmtCtx, indexMergeConds, ds.ctx.GetClient(), kv.UnSpecified) + stmtCtx.SetWarnings(warnings) + if len(remaining) != 0 { + needConsiderIndexMerge = false + } + } } - readFromTableCache := ds.ctx.GetSessionVars().StmtCtx.ReadFromTableCache + readFromTableCache := stmtCtx.ReadFromTableCache if isPossibleIdxMerge && sessionAndStmtPermission && needConsiderIndexMerge && ds.tableInfo.TempTableType != model.TempTableLocal && !readFromTableCache { - err := ds.generateAndPruneIndexMergePath(ds.indexMergeHints != nil) + err := ds.generateAndPruneIndexMergePath(indexMergeConds, ds.indexMergeHints != nil) if err != nil { return nil, err } } else if len(ds.indexMergeHints) > 0 { ds.indexMergeHints = nil - ds.ctx.GetSessionVars().StmtCtx.AppendWarning(errors.Errorf("IndexMerge is inapplicable or disabled")) + var msg string + if !isPossibleIdxMerge { + msg = "No available filter or available index." + } else if !sessionAndStmtPermission { + msg = "Got no_index_merge hint or tidb_enable_index_merge is off." + } else if ds.tableInfo.TempTableType == model.TempTableLocal { + msg = "Cannot use IndexMerge on temporary table." + } else if readFromTableCache { + msg = "Cannot use IndexMerge on TableCache." + } + msg = fmt.Sprintf("IndexMerge is inapplicable or disabled. %s", msg) + stmtCtx.AppendWarning(errors.Errorf(msg)) + logutil.BgLogger().Debug(msg) } return ds.stats, nil } -func (ds *DataSource) generateAndPruneIndexMergePath(needPrune bool) error { +func (ds *DataSource) generateAndPruneIndexMergePath(indexMergeConds []expression.Expression, needPrune bool) error { regularPathCount := len(ds.possibleAccessPaths) - err := ds.generateIndexMergeOrPaths() + err := ds.generateIndexMergeOrPaths(indexMergeConds) if err != nil { return err } @@ -448,12 +481,22 @@ func (ds *DataSource) generateAndPruneIndexMergePath(needPrune bool) error { // With hints and without generated IndexMerge paths if regularPathCount == len(ds.possibleAccessPaths) { ds.indexMergeHints = nil - ds.ctx.GetSessionVars().StmtCtx.AppendWarning(errors.Errorf("IndexMerge is inapplicable or disabled")) + ds.ctx.GetSessionVars().StmtCtx.AppendWarning(errors.Errorf("IndexMerge is inapplicable.")) return nil } // Do not need to consider the regular paths in find_best_task(). + // So we can use index merge's row count as DataSource's row count. if needPrune { ds.possibleAccessPaths = ds.possibleAccessPaths[regularPathCount:] + minRowCount := ds.possibleAccessPaths[0].CountAfterAccess + for _, path := range ds.possibleAccessPaths { + if minRowCount < path.CountAfterAccess { + minRowCount = path.CountAfterAccess + } + } + if ds.stats.RowCount > minRowCount { + ds.stats = ds.tableStats.ScaleByExpectCnt(minRowCount) + } } return nil } @@ -510,9 +553,9 @@ func (is *LogicalIndexScan) DeriveStats(childStats []*property.StatsInfo, selfSc } // getIndexMergeOrPath generates all possible IndexMergeOrPaths. -func (ds *DataSource) generateIndexMergeOrPaths() error { +func (ds *DataSource) generateIndexMergeOrPaths(filters []expression.Expression) error { usedIndexCount := len(ds.possibleAccessPaths) - for i, cond := range ds.pushedDownConds { + for i, cond := range filters { sf, ok := cond.(*expression.ScalarFunction) if !ok || sf.FuncName.L != ast.LogicOr { continue @@ -548,7 +591,7 @@ func (ds *DataSource) generateIndexMergeOrPaths() error { continue } if len(partialPaths) > 1 { - possiblePath := ds.buildIndexMergeOrPath(partialPaths, i) + possiblePath := ds.buildIndexMergeOrPath(filters, partialPaths, i) if possiblePath == nil { return nil } @@ -686,16 +729,29 @@ func (ds *DataSource) buildIndexMergePartialPath(indexAccessPaths []*util.Access } // buildIndexMergeOrPath generates one possible IndexMergePath. -func (ds *DataSource) buildIndexMergeOrPath(partialPaths []*util.AccessPath, current int) *util.AccessPath { +func (ds *DataSource) buildIndexMergeOrPath(filters []expression.Expression, partialPaths []*util.AccessPath, current int) *util.AccessPath { indexMergePath := &util.AccessPath{PartialIndexPaths: partialPaths} - indexMergePath.TableFilters = append(indexMergePath.TableFilters, ds.pushedDownConds[:current]...) - indexMergePath.TableFilters = append(indexMergePath.TableFilters, ds.pushedDownConds[current+1:]...) + indexMergePath.TableFilters = append(indexMergePath.TableFilters, filters[:current]...) + indexMergePath.TableFilters = append(indexMergePath.TableFilters, filters[current+1:]...) + var addCurrentFilter bool for _, path := range partialPaths { // If any partial path contains table filters, we need to keep the whole DNF filter in the Selection. if len(path.TableFilters) > 0 { - indexMergePath.TableFilters = append(indexMergePath.TableFilters, ds.pushedDownConds[current]) - break + addCurrentFilter = true } + // If any partial path's index filter cannot be pushed to TiKV, we should keep the whole DNF filter. + if len(path.IndexFilters) != 0 && !expression.CanExprsPushDown(ds.ctx.GetSessionVars().StmtCtx, path.IndexFilters, ds.ctx.GetClient(), kv.TiKV) { + addCurrentFilter = true + // Clear IndexFilter, the whole filter will be put in indexMergePath.TableFilters. + path.IndexFilters = nil + } + if len(path.TableFilters) != 0 && !expression.CanExprsPushDown(ds.ctx.GetSessionVars().StmtCtx, path.TableFilters, ds.ctx.GetClient(), kv.TiKV) { + addCurrentFilter = true + path.TableFilters = nil + } + } + if addCurrentFilter { + indexMergePath.TableFilters = append(indexMergePath.TableFilters, filters[current]) } return indexMergePath } diff --git a/planner/core/task.go b/planner/core/task.go index 187140c613aa5..4b9c5692ca29d 100644 --- a/planner/core/task.go +++ b/planner/core/task.go @@ -1005,6 +1005,7 @@ func (t *copTask) convertToRootTaskImpl(ctx sessionctx.Context) *rootTask { setTableScanToTableRowIDScan(p.tablePlan) newTask.p = p p.cost = newTask.cost() + t.handleRootTaskConds(ctx, newTask) if t.needExtraProj { schema := t.originSchema proj := PhysicalProjection{Exprs: expression.Column2Exprs(schema.Columns)}.Init(ctx, p.stats, t.idxMergePartPlans[0].SelectBlockOffset(), nil) @@ -1066,6 +1067,11 @@ func (t *copTask) convertToRootTaskImpl(ctx sessionctx.Context) *rootTask { } } + t.handleRootTaskConds(ctx, newTask) + return newTask +} + +func (t *copTask) handleRootTaskConds(ctx sessionctx.Context, newTask *rootTask) { if len(t.rootTaskConds) > 0 { selectivity, _, err := t.tblColHists.Selectivity(ctx, t.rootTaskConds, nil) if err != nil { @@ -1077,8 +1083,6 @@ func (t *copTask) convertToRootTaskImpl(ctx sessionctx.Context) *rootTask { newTask.p = sel sel.cost = newTask.cost() } - - return newTask } // setTableScanToTableRowIDScan is to update the isChildOfIndexLookUp attribute of PhysicalTableScan child diff --git a/planner/core/testdata/integration_suite_out.json b/planner/core/testdata/integration_suite_out.json index 6c946cdac4d2d..c3bbda1ed3f2c 100644 --- a/planner/core/testdata/integration_suite_out.json +++ b/planner/core/testdata/integration_suite_out.json @@ -1332,7 +1332,7 @@ "└─IndexRangeScan 20.00 cop[tikv] table:tt, index:a(a) range:[10,10], [20,20], keep order:false, stats:pseudo" ], "Warnings": [ - "Warning 1105 IndexMerge is inapplicable or disabled" + "Warning 1105 IndexMerge is inapplicable." ] }, { @@ -1342,7 +1342,7 @@ "└─IndexRangeScan 6666.67 cop[tikv] table:tt, index:a(a) range:[-inf,10), [15,15], (20,+inf], keep order:false, stats:pseudo" ], "Warnings": [ - "Warning 1105 IndexMerge is inapplicable or disabled" + "Warning 1105 IndexMerge is inapplicable." ] } ] diff --git a/session/session_test.go b/session/session_test.go index e70660f172111..67c9fb64aef12 100644 --- a/session/session_test.go +++ b/session/session_test.go @@ -5420,7 +5420,7 @@ func (s *testSessionSuite) TestLocalTemporaryTableScan(c *C) { "12 112 1012", "3 113 1003", "14 114 1014", "16 116 1016", "7 117 1007", "18 118 1018", )) - tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 IndexMerge is inapplicable or disabled")) + tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 IndexMerge is inapplicable or disabled. Cannot use IndexMerge on temporary table.")) } doModify := func() { @@ -5459,7 +5459,7 @@ func (s *testSessionSuite) TestLocalTemporaryTableScan(c *C) { "3 113 1003", "14 114 1014", "7 117 9999", "18 118 1018", "12 132 1012", )) - tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 IndexMerge is inapplicable or disabled")) + tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 IndexMerge is inapplicable or disabled. Cannot use IndexMerge on temporary table.")) } assertSelectAsUnModified() diff --git a/table/tables/cache_test.go b/table/tables/cache_test.go index a4dc5b4d43d68..788ecd48fd1d7 100644 --- a/table/tables/cache_test.go +++ b/table/tables/cache_test.go @@ -142,7 +142,7 @@ func TestCacheTableBasicScan(t *testing.T) { "12 112 1012", "3 113 1003", "14 114 1014", "16 116 1016", "7 117 1007", "18 118 1018", )) - tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 IndexMerge is inapplicable or disabled")) + tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 IndexMerge is inapplicable or disabled. Cannot use IndexMerge on TableCache.")) } assertSelect()