From 7ecb3159a6370d2c74f32887dc404023ee551812 Mon Sep 17 00:00:00 2001 From: crazycs Date: Mon, 6 May 2019 17:41:50 +0800 Subject: [PATCH] *: add split index region syntax (#10203) --- executor/builder.go | 13 ++++++ executor/executor.go | 4 ++ executor/executor_test.go | 12 ++++++ executor/split.go | 84 ++++++++++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 +- planner/core/common_plans.go | 10 +++++ planner/core/planbuilder.go | 42 ++++++++++++++++++ 8 files changed, 168 insertions(+), 3 deletions(-) create mode 100644 executor/split.go diff --git a/executor/builder.go b/executor/builder.go index 596b64c72b48c..2ccdc3584b865 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -193,6 +193,8 @@ func (b *executorBuilder) build(p plannercore.Plan) Executor { return b.buildWindow(v) case *plannercore.SQLBindPlan: return b.buildSQLBindExec(v) + case *plannercore.SplitIndexRegion: + return b.buildSplitIndexRegion(v) default: if mp, ok := p.(MockPhysicalPlan); ok { return mp.GetExecutor() @@ -1248,6 +1250,17 @@ func (b *executorBuilder) buildUnionAll(v *plannercore.PhysicalUnionAll) Executo return e } +func (b *executorBuilder) buildSplitIndexRegion(v *plannercore.SplitIndexRegion) Executor { + base := newBaseExecutor(b.ctx, nil, v.ExplainID()) + base.initCap = chunk.ZeroCapacity + return &SplitIndexRegionExec{ + baseExecutor: base, + table: v.Table, + indexInfo: v.IndexInfo, + valueLists: v.ValueLists, + } +} + func (b *executorBuilder) buildUpdate(v *plannercore.Update) Executor { tblID2table := make(map[int64]table.Table) for id := range v.SelectPlan.Schema().TblID2Handle { diff --git a/executor/executor.go b/executor/executor.go index eb32d3cc17f2b..afbc20045ba23 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -1379,6 +1379,10 @@ func ResetContextOfStmt(ctx sessionctx.Context, s ast.StmtNode) (err error) { sc.InShowWarning = true sc.SetWarnings(vars.StmtCtx.GetWarnings()) } + case *ast.SplitIndexRegionStmt: + sc.IgnoreTruncate = false + sc.IgnoreZeroInDate = true + sc.AllowInvalidDate = vars.SQLMode.HasAllowInvalidDatesMode() default: sc.IgnoreTruncate = true sc.IgnoreZeroInDate = true diff --git a/executor/executor_test.go b/executor/executor_test.go index cce1ea9b35432..fe2840ec7d852 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -3693,6 +3693,18 @@ func (s *testSuite) TestReadPartitionedTable(c *C) { tk.MustQuery("select a from pt where b = 3").Check(testkit.Rows("3")) } +func (s *testSuite) TestSplitIndexRegion(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a varchar(100),b int, index idx1(b,a))") + tk.MustExec(`split table t index idx1 by (10000,"abcd"),(10000000);`) + _, err := tk.Exec(`split table t index idx1 by ("abcd");`) + c.Assert(err, NotNil) + terr := errors.Cause(err).(*terror.Error) + c.Assert(terr.Code(), Equals, terror.ErrCode(mysql.WarnDataTruncated)) +} + type testOOMSuite struct { store kv.Storage do *domain.Domain diff --git a/executor/split.go b/executor/split.go new file mode 100644 index 0000000000000..dc1b75e8018f3 --- /dev/null +++ b/executor/split.go @@ -0,0 +1,84 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package executor + +import ( + "context" + "math" + + "github.com/pingcap/parser/model" + "github.com/pingcap/tidb/kv" + "github.com/pingcap/tidb/table" + "github.com/pingcap/tidb/table/tables" + "github.com/pingcap/tidb/types" + "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/logutil" + "go.uber.org/zap" +) + +// SplitIndexRegionExec represents a split index regions executor. +type SplitIndexRegionExec struct { + baseExecutor + + table table.Table + indexInfo *model.IndexInfo + valueLists [][]types.Datum +} + +type splitableStore interface { + SplitRegionAndScatter(splitKey kv.Key) (uint64, error) + WaitScatterRegionFinish(regionID uint64) error +} + +// Next implements the Executor Next interface. +func (e *SplitIndexRegionExec) Next(ctx context.Context, _ *chunk.RecordBatch) error { + store := e.ctx.GetStore() + s, ok := store.(splitableStore) + if !ok { + return nil + } + regionIDs := make([]uint64, 0, len(e.valueLists)) + index := tables.NewIndex(e.table.Meta().ID, e.table.Meta(), e.indexInfo) + for _, values := range e.valueLists { + idxKey, _, err := index.GenIndexKey(e.ctx.GetSessionVars().StmtCtx, values, math.MinInt64, nil) + if err != nil { + return err + } + + regionID, err := s.SplitRegionAndScatter(idxKey) + if err != nil { + logutil.Logger(context.Background()).Warn("split table index region failed", + zap.String("table", e.table.Meta().Name.L), + zap.String("index", e.indexInfo.Name.L), + zap.Error(err)) + continue + } + regionIDs = append(regionIDs, regionID) + + } + if !e.ctx.GetSessionVars().WaitTableSplitFinish { + return nil + } + for _, regionID := range regionIDs { + err := s.WaitScatterRegionFinish(regionID) + if err != nil { + logutil.Logger(context.Background()).Warn("wait scatter region failed", + zap.Uint64("regionID", regionID), + zap.String("table", e.table.Meta().Name.L), + zap.String("index", e.indexInfo.Name.L), + zap.Error(err)) + } + } + return nil +} diff --git a/go.mod b/go.mod index 68f788985f536..70a6f40911b88 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e github.com/pingcap/kvproto v0.0.0-20190327032727-3d8cb3a30d5d github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 - github.com/pingcap/parser v0.0.0-20190505092803-4542e963c7f1 + github.com/pingcap/parser v0.0.0-20190506092653-e336082eb825 github.com/pingcap/pd v0.0.0-20190424024702-bd1e2496a669 github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330 diff --git a/go.sum b/go.sum index 827c6392873c9..288f6e6d080d2 100644 --- a/go.sum +++ b/go.sum @@ -160,8 +160,8 @@ github.com/pingcap/kvproto v0.0.0-20190327032727-3d8cb3a30d5d/go.mod h1:QMdbTAXC github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 h1:t2OQTpPJnrPDGlvA+3FwJptMTt6MEPdzK1Wt99oaefQ= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= -github.com/pingcap/parser v0.0.0-20190505092803-4542e963c7f1 h1:YvxFABfyD5Pnp80FUVV4w3zdlmkcwRhQbn7xpTjBwwU= -github.com/pingcap/parser v0.0.0-20190505092803-4542e963c7f1/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= +github.com/pingcap/parser v0.0.0-20190506092653-e336082eb825 h1:U9Kdnknj4n2v76Mg7wazevZ5N9U1OIaMwSNRVLEcLX0= +github.com/pingcap/parser v0.0.0-20190506092653-e336082eb825/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v0.0.0-20190424024702-bd1e2496a669 h1:ZoKjndm/Ig7Ru/wojrQkc/YLUttUdQXoH77gtuWCvL4= github.com/pingcap/pd v0.0.0-20190424024702-bd1e2496a669/go.mod h1:MUCxRzOkYiWZtlyi4MhxjCIj9PgQQ/j+BLNGm7aUsnM= github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible h1:MkWCxgZpJBgY2f4HtwWMMFzSBb3+JPzeJgF3VrXE/bU= diff --git a/planner/core/common_plans.go b/planner/core/common_plans.go index ea467b3ef637d..a918ed17ff218 100644 --- a/planner/core/common_plans.go +++ b/planner/core/common_plans.go @@ -29,6 +29,7 @@ import ( "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/table" + "github.com/pingcap/tidb/types" driver "github.com/pingcap/tidb/types/parser_driver" "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/kvcache" @@ -476,6 +477,15 @@ type LoadStats struct { Path string } +// SplitIndexRegion represents a split index regions plan. +type SplitIndexRegion struct { + baseSchemaProducer + + Table table.Table + IndexInfo *model.IndexInfo + ValueLists [][]types.Datum +} + // DDL represents a DDL statement plan. type DDL struct { baseSchemaProducer diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 1bde08d6f95b5..118b41c8bd9d3 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -279,6 +279,8 @@ func (b *PlanBuilder) Build(node ast.Node) (Plan, error) { return b.buildDropBindPlan(x) case *ast.ChangeStmt: return b.buildChange(x) + case *ast.SplitIndexRegionStmt: + return b.buildSplitIndexRegion(x) } return nil, ErrUnsupportedType.GenWithStack("Unsupported type %T", node) } @@ -1605,6 +1607,46 @@ func (b *PlanBuilder) buildLoadStats(ld *ast.LoadStatsStmt) Plan { return p } +func (b *PlanBuilder) buildSplitIndexRegion(node *ast.SplitIndexRegionStmt) (Plan, error) { + tblInfo := node.Table.TableInfo + indexInfo := tblInfo.FindIndexByName(strings.ToLower(node.IndexName)) + if indexInfo == nil { + return nil, ErrKeyDoesNotExist.GenWithStackByArgs(node.IndexName, tblInfo.Name) + } + + indexValues := make([][]types.Datum, 0, len(node.ValueLists)) + for i, valuesItem := range node.ValueLists { + if len(valuesItem) > len(indexInfo.Columns) { + return nil, ErrWrongValueCountOnRow.GenWithStackByArgs(i + 1) + } + valueList := make([]types.Datum, 0, len(valuesItem)) + for j, valueItem := range valuesItem { + x, ok := valueItem.(*driver.ValueExpr) + if !ok { + return nil, errors.New("expect constant values") + } + colOffset := indexInfo.Columns[j].Offset + value, err := x.Datum.ConvertTo(b.ctx.GetSessionVars().StmtCtx, &tblInfo.Columns[colOffset].FieldType) + if err != nil { + return nil, err + } + + valueList = append(valueList, value) + } + indexValues = append(indexValues, valueList) + } + tableInPlan, ok := b.is.TableByID(tblInfo.ID) + if !ok { + return nil, errors.Errorf("Can't get table %s.", tblInfo.Name.O) + } + return &SplitIndexRegion{ + Table: tableInPlan, + IndexInfo: indexInfo, + ValueLists: indexValues, + }, nil + +} + func (b *PlanBuilder) buildDDL(node ast.DDLNode) (Plan, error) { var authErr error switch v := node.(type) {