Skip to content

Commit

Permalink
*: re-implement partition pruning for better performance (pingcap#14679)
Browse files Browse the repository at this point in the history
  • Loading branch information
@tiancaiamao
tiancaiamao committed Mar 13, 2020
1 parent f51cdef commit ad569a4
Show file tree
Hide file tree
Showing 6 changed files with 2,517 additions and 968 deletions.
1,698 changes: 771 additions & 927 deletions cmd/explaintest/r/partition_pruning.result
View file

Large diffs are not rendered by default.

2 changes: 0 additions & 2 deletions expression/builtin_time_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
package expression

import (
"fmt"
"math"
"strings"
"time"
Expand Down Expand Up @@ -1714,7 +1713,6 @@ func (s *testEvaluatorSuite) TestUnixTimestamp(c *C) {
}

for _, test := range tests {
fmt.Printf("Begin Test %v\n", test)
expr := s.datumsToConstants([]types.Datum{test.input})
expr[0].GetType().Decimal = test.inputDecimal
resetStmtContext(s.ctx)
Expand Down
229 changes: 197 additions & 32 deletions planner/core/partition_pruning_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"github.com/pingcap/parser/model"
"github.com/pingcap/tidb/ddl"
"github.com/pingcap/tidb/expression"
"github.com/pingcap/tidb/sessionctx"
"github.com/pingcap/tidb/util/mock"
)

Expand All @@ -29,12 +30,6 @@ type testPartitionPruningSuite struct {
}

func (s *testPartitionPruningSuite) TestCanBePrune(c *C) {
p := parser.New()
stmt, err := p.ParseOneStmt("create table t (d datetime not null)", "", "")
c.Assert(err, IsNil)
tblInfo, err := ddl.BuildTableInfoFromAST(stmt.(*ast.CreateTableStmt))
c.Assert(err, IsNil)

// For the following case:
// CREATE TABLE t1 ( recdate DATETIME NOT NULL )
// PARTITION BY RANGE( TO_DAYS(recdate) ) (
Expand All @@ -43,21 +38,18 @@ func (s *testPartitionPruningSuite) TestCanBePrune(c *C) {
// );
// SELECT * FROM t1 WHERE recdate < '2007-03-08 00:00:00';
// SELECT * FROM t1 WHERE recdate > '2018-03-08 00:00:00';

ctx := mock.NewContext()
columns := expression.ColumnInfos2ColumnsWithDBName(ctx, model.NewCIStr("t"), tblInfo.Name, tblInfo.Columns)
schema := expression.NewSchema(columns...)
partitionExpr, err := expression.ParseSimpleExprsWithSchema(ctx, "to_days(d) < to_days('2007-03-08') and to_days(d) >= to_days('2007-03-07')", schema)
c.Assert(err, IsNil)
queryExpr, err := expression.ParseSimpleExprsWithSchema(ctx, "d < '2000-03-08 00:00:00'", schema)
c.Assert(err, IsNil)
succ, err := s.canBePruned(ctx, nil, partitionExpr[0], queryExpr)
tc := prepareTestCtx(c,
"create table t (d datetime not null)",
"to_days(d)",
)
partitionExpr := tc.expr("to_days(d) < to_days('2007-03-08') and to_days(d) >= to_days('2007-03-07')")
queryExpr := tc.expr("d < '2000-03-08 00:00:00'")
succ, err := s.canBePruned(tc.sctx, nil, partitionExpr[0], queryExpr)
c.Assert(err, IsNil)
c.Assert(succ, IsTrue)

queryExpr, err = expression.ParseSimpleExprsWithSchema(ctx, "d > '2018-03-08 00:00:00'", schema)
c.Assert(err, IsNil)
succ, err = s.canBePruned(ctx, nil, partitionExpr[0], queryExpr)
queryExpr = tc.expr("d > '2018-03-08 00:00:00'")
succ, err = s.canBePruned(tc.sctx, nil, partitionExpr[0], queryExpr)
c.Assert(err, IsNil)
c.Assert(succ, IsTrue)

Expand All @@ -72,24 +64,19 @@ func (s *testPartitionPruningSuite) TestCanBePrune(c *C) {
// PARTITION p2 VALUES LESS THAN (UNIX_TIMESTAMP('2010-01-01 00:00:00')),
// PARTITION p3 VALUES LESS THAN (MAXVALUE)
// );
stmt, err = p.ParseOneStmt("create table t (report_updated timestamp)", "", "")
c.Assert(err, IsNil)
tblInfo, err = ddl.BuildTableInfoFromAST(stmt.(*ast.CreateTableStmt))
c.Assert(err, IsNil)
columns = expression.ColumnInfos2ColumnsWithDBName(ctx, model.NewCIStr("t"), tblInfo.Name, tblInfo.Columns)
schema = expression.NewSchema(columns...)
tc = prepareTestCtx(c,
"create table t (report_updated timestamp)",
"unix_timestamp(report_updated)",
)

partitionExpr, err = expression.ParseSimpleExprsWithSchema(ctx, "unix_timestamp(report_updated) < unix_timestamp('2008-04-01') and unix_timestamp(report_updated) >= unix_timestamp('2008-01-01')", schema)
c.Assert(err, IsNil)
queryExpr, err = expression.ParseSimpleExprsWithSchema(ctx, "report_updated > '2008-05-01 00:00:00'", schema)
c.Assert(err, IsNil)
succ, err = s.canBePruned(ctx, nil, partitionExpr[0], queryExpr)
partitionExpr = tc.expr("unix_timestamp(report_updated) < unix_timestamp('2008-04-01') and unix_timestamp(report_updated) >= unix_timestamp('2008-01-01')")
queryExpr = tc.expr("report_updated > '2008-05-01 00:00:00'")
succ, err = s.canBePruned(tc.sctx, nil, partitionExpr[0], queryExpr)
c.Assert(err, IsNil)
c.Assert(succ, IsTrue)

queryExpr, err = expression.ParseSimpleExprsWithSchema(ctx, "report_updated > unix_timestamp('2008-05-01 00:00:00')", schema)
c.Assert(err, IsNil)
succ, err = s.canBePruned(ctx, nil, partitionExpr[0], queryExpr)
queryExpr = tc.expr("report_updated > unix_timestamp('2008-05-01 00:00:00')")
succ, err = s.canBePruned(tc.sctx, nil, partitionExpr[0], queryExpr)
c.Assert(err, IsNil)
_ = succ
// c.Assert(succ, IsTrue)
Expand All @@ -98,3 +85,181 @@ func (s *testPartitionPruningSuite) TestCanBePrune(c *C) {
// Because unix_timestamp('2008-05-01 00:00:00') is fold to constant int 1564761600, and compare it with timestamp (report_updated)
// need to convert 1564761600 to a timestamp, during that step, an error happen and the result is set to <nil>
}

func (s *testPartitionPruningSuite) TestPruneUseBinarySearch(c *C) {
lessThan := lessThanData{data: []int64{4, 7, 11, 14, 17, 0}, maxvalue: true}
cases := []struct {
input dataForPrune
result partitionRange
}{
{dataForPrune{ast.EQ, 66}, partitionRange{5, 6}},
{dataForPrune{ast.EQ, 14}, partitionRange{4, 5}},
{dataForPrune{ast.EQ, 10}, partitionRange{2, 3}},
{dataForPrune{ast.EQ, 3}, partitionRange{0, 1}},
{dataForPrune{ast.LT, 66}, partitionRange{0, 6}},
{dataForPrune{ast.LT, 14}, partitionRange{0, 4}},
{dataForPrune{ast.LT, 10}, partitionRange{0, 3}},
{dataForPrune{ast.LT, 3}, partitionRange{0, 1}},
{dataForPrune{ast.GE, 66}, partitionRange{5, 6}},
{dataForPrune{ast.GE, 14}, partitionRange{4, 6}},
{dataForPrune{ast.GE, 10}, partitionRange{2, 6}},
{dataForPrune{ast.GE, 3}, partitionRange{0, 6}},
{dataForPrune{ast.GT, 66}, partitionRange{5, 6}},
{dataForPrune{ast.GT, 14}, partitionRange{4, 6}},
{dataForPrune{ast.GT, 10}, partitionRange{3, 6}},
{dataForPrune{ast.GT, 3}, partitionRange{1, 6}},
{dataForPrune{ast.GT, 2}, partitionRange{0, 6}},
{dataForPrune{ast.LE, 66}, partitionRange{0, 6}},
{dataForPrune{ast.LE, 14}, partitionRange{0, 5}},
{dataForPrune{ast.LE, 10}, partitionRange{0, 3}},
{dataForPrune{ast.LE, 3}, partitionRange{0, 1}},
{dataForPrune{ast.IsNull, 0}, partitionRange{0, 1}},
{dataForPrune{"illegal", 0}, partitionRange{0, 6}},
}

for i, ca := range cases {
start, end := pruneUseBinarySearch(lessThan, ca.input)
c.Assert(ca.result.start, Equals, start, Commentf("fail = %d", i))
c.Assert(ca.result.end, Equals, end, Commentf("fail = %d", i))
}
}

type testCtx struct {
c *C
sctx sessionctx.Context
schema *expression.Schema
columns []*expression.Column
// names types.NameSlice
lessThan lessThanData
}

func prepareTestCtx(c *C, createTable string, partitionExpr string) *testCtx {
p := parser.New()
stmt, err := p.ParseOneStmt(createTable, "", "")
c.Assert(err, IsNil)
sctx := mock.NewContext()
tblInfo, err := ddl.BuildTableInfoFromAST(stmt.(*ast.CreateTableStmt))
c.Assert(err, IsNil)
columns := expression.ColumnInfos2ColumnsWithDBName(sctx, model.NewCIStr("t"), tblInfo.Name, tblInfo.Columns)
schema := expression.NewSchema(columns...)
return &testCtx{
c: c,
sctx: sctx,
schema: schema,
columns: columns,
// names: names,
}
}

func (tc *testCtx) expr(expr string) []expression.Expression {
res, err := expression.ParseSimpleExprsWithSchema(tc.sctx, expr, tc.schema)
tc.c.Assert(err, IsNil)
return res
}

func (s *testPartitionPruningSuite) TestPartitionRangeForExpr(c *C) {
tc := prepareTestCtx(c,
"create table t (a int)",
"a",
)
lessThan := lessThanData{data: []int64{4, 7, 11, 14, 17, 0}, maxvalue: true}
cases := []struct {
input string
result partitionRangeOR
}{
{"a > 3", partitionRangeOR{{1, 6}}},
{"a < 3", partitionRangeOR{{0, 1}}},
{"a >= 11", partitionRangeOR{{3, 6}}},
{"a > 11", partitionRangeOR{{3, 6}}},
{"a < 11", partitionRangeOR{{0, 3}}},
{"a = 16", partitionRangeOR{{4, 5}}},
{"a > 66", partitionRangeOR{{5, 6}}},
{"a > 2 and a < 10", partitionRangeOR{{0, 3}}},
{"a < 2 or a >= 15", partitionRangeOR{{0, 1}, {4, 6}}},
{"a is null", partitionRangeOR{{0, 1}}},
{"12 > a", partitionRangeOR{{0, 4}}},
{"4 <= a", partitionRangeOR{{1, 6}}},
}

for _, ca := range cases {
expr, err := expression.ParseSimpleExprsWithSchema(tc.sctx, ca.input, tc.schema)
c.Assert(err, IsNil)
result := fullRange(lessThan.length())
result = partitionRangeForExpr(tc.sctx, expr[0], lessThan, tc.columns[0], nil, result)
c.Assert(equalPartitionRangeOR(ca.result, result), IsTrue, Commentf("unexpected:", ca.input))
}
}

func equalPartitionRangeOR(x, y partitionRangeOR) bool {
if len(x) != len(y) {
return false
}
for i := 0; i < len(x); i++ {
if x[i] != y[i] {
return false
}
}
return true
}

func (s *testPartitionPruningSuite) TestPartitionRangeOperation(c *C) {
testIntersectionRange := []struct {
input1 partitionRangeOR
input2 partitionRange
result partitionRangeOR
}{
{input1: partitionRangeOR{{0, 3}, {6, 12}},
input2: partitionRange{4, 7},
result: partitionRangeOR{{6, 7}}},
{input1: partitionRangeOR{{0, 5}},
input2: partitionRange{6, 7},
result: partitionRangeOR{}},
{input1: partitionRangeOR{{0, 4}, {6, 7}, {8, 11}},
input2: partitionRange{3, 9},
result: partitionRangeOR{{3, 4}, {6, 7}, {8, 9}}},
}
for i, ca := range testIntersectionRange {
result := ca.input1.intersectionRange(ca.input2.start, ca.input2.end)
c.Assert(equalPartitionRangeOR(ca.result, result), IsTrue, Commentf("failed %d", i))
}

testIntersection := []struct {
input1 partitionRangeOR
input2 partitionRangeOR
result partitionRangeOR
}{
{input1: partitionRangeOR{{0, 3}, {6, 12}},
input2: partitionRangeOR{{4, 7}},
result: partitionRangeOR{{6, 7}}},
{input1: partitionRangeOR{{4, 7}},
input2: partitionRangeOR{{0, 3}, {6, 12}},
result: partitionRangeOR{{6, 7}}},
{input1: partitionRangeOR{{4, 7}, {8, 10}},
input2: partitionRangeOR{{0, 5}, {6, 12}},
result: partitionRangeOR{{4, 5}, {6, 7}, {8, 10}}},
}
for i, ca := range testIntersection {
result := ca.input1.intersection(ca.input2)
c.Assert(equalPartitionRangeOR(ca.result, result), IsTrue, Commentf("failed %d", i))
}

testUnion := []struct {
input1 partitionRangeOR
input2 partitionRangeOR
result partitionRangeOR
}{
{input1: partitionRangeOR{{0, 1}, {2, 7}},
input2: partitionRangeOR{{3, 5}},
result: partitionRangeOR{{0, 1}, {2, 7}}},
{input1: partitionRangeOR{{2, 7}},
input2: partitionRangeOR{{0, 3}, {4, 12}},
result: partitionRangeOR{{0, 12}}},
{input1: partitionRangeOR{{4, 7}, {8, 10}},
input2: partitionRangeOR{{0, 5}},
result: partitionRangeOR{{0, 7}, {8, 10}}},
}
for i, ca := range testUnion {
result := ca.input1.union(ca.input2)
c.Assert(equalPartitionRangeOR(ca.result, result), IsTrue, Commentf("failed %d", i))
}
}
Loading

0 comments on commit ad569a4

Please sign in to comment.