From a909fdfc84148c82c3f034bb2061af5a82652297 Mon Sep 17 00:00:00 2001 From: glorv Date: Fri, 9 Oct 2020 19:19:51 +0800 Subject: [PATCH 1/8] support restore view --- lightning/mydump/loader.go | 37 +++++++++++++ lightning/mydump/loader_test.go | 98 ++++++++++++++++++++++++++++++++- lightning/mydump/router.go | 16 ++++-- lightning/mydump/router_test.go | 4 +- lightning/restore/restore.go | 23 +++++++- tests/file_routing/config.toml | 6 ++ tests/file_routing/run.sh | 22 ++++++++ tests/view/config.toml | 1 + tests/view/run.sh | 81 +++++++++++++++++++++++++++ 9 files changed, 280 insertions(+), 8 deletions(-) create mode 100644 tests/view/config.toml create mode 100755 tests/view/run.sh diff --git a/lightning/mydump/loader.go b/lightning/mydump/loader.go index 930cd8b93..cf821f1c1 100644 --- a/lightning/mydump/loader.go +++ b/lightning/mydump/loader.go @@ -32,6 +32,7 @@ type MDDatabaseMeta struct { Name string SchemaFile string Tables []*MDTableMeta + Views []*MDTableMeta charSet string } @@ -80,6 +81,7 @@ type mdLoaderSetup struct { loader *MDLoader dbSchemas []FileInfo tableSchemas []FileInfo + viewSchemas []FileInfo tableDatas []FileInfo dbIndexMap map[string]int tableIndexMap map[filter.Table]int @@ -234,6 +236,17 @@ func (s *mdLoaderSetup) setup(ctx context.Context, store storage.ExternalStorage return errors.Errorf("invalid table schema file, duplicated item - %s", fileInfo.FileMeta.Path) } } + + // setup view schema + for _, fileInfo := range s.viewSchemas { + dbExists, tableExists := s.insertView(fileInfo) + if !dbExists { + return errors.Errorf("invalid table schema file, cannot find db '%s' - %s", fileInfo.TableName.Schema, fileInfo.FileMeta.Path) + } else if !tableExists { + // remove the last `-view.sql` from path as the relate table schema file path + return errors.Errorf("invalid view schema file, miss host table schema for view '%s'", fileInfo.TableName.Name) + } + } } // Sql file for restore data @@ -303,6 +316,8 @@ func (s *mdLoaderSetup) listFiles(ctx context.Context, store storage.ExternalSto s.dbSchemas = append(s.dbSchemas, info) case SourceTypeTableSchema: s.tableSchemas = append(s.tableSchemas, info) + case SourceTypeViewSchema: + s.viewSchemas = append(s.viewSchemas, info) case SourceTypeSQL, SourceTypeCSV, SourceTypeParquet: s.tableDatas = append(s.tableDatas, info) } @@ -346,6 +361,10 @@ func (s *mdLoaderSetup) route() error { dbInfo.count++ knownDBNames[info.TableName.Schema] = dbInfo } + for _, info := range s.viewSchemas { + dbInfo := knownDBNames[info.TableName.Schema] + dbInfo.count++ + } run := func(arr []FileInfo) error { for i, info := range arr { @@ -377,6 +396,9 @@ func (s *mdLoaderSetup) route() error { if err := run(s.tableSchemas); err != nil { return errors.Trace(err) } + if err := run(s.viewSchemas); err != nil { + return errors.Trace(err) + } if err := run(s.tableDatas); err != nil { return errors.Trace(err) } @@ -429,6 +451,21 @@ func (s *mdLoaderSetup) insertTable(fileInfo FileInfo) (*MDTableMeta, bool, bool } } +func (s *mdLoaderSetup) insertView(fileInfo FileInfo) (bool, bool) { + dbMeta, dbExists := s.insertDB(fileInfo.TableName.Schema, "") + _, ok := s.tableIndexMap[fileInfo.TableName] + if ok { + meta := &MDTableMeta{ + DB: fileInfo.TableName.Schema, + Name: fileInfo.TableName.Name, + SchemaFile: fileInfo, + charSet: s.loader.charSet, + } + dbMeta.Views = append(dbMeta.Views, meta) + } + return dbExists, ok +} + func (l *MDLoader) GetDatabases() []*MDDatabaseMeta { return l.dbs } diff --git a/lightning/mydump/loader_test.go b/lightning/mydump/loader_test.go index c782c4663..d66f16417 100644 --- a/lightning/mydump/loader_test.go +++ b/lightning/mydump/loader_test.go @@ -194,6 +194,33 @@ func (s *testMydumpLoaderSuite) TestDataNoHostTable(c *C) { c.Assert(err, ErrorMatches, `invalid data file, miss host table 'tbl' - .*[/\\]?db\.tbl\.sql`) } +func (s *testMydumpLoaderSuite) TestViewNoHostDB(c *C) { + /* + Path/ + notdb-schema-create.sql + db.tbl-schema-view.sql + */ + s.touch(c, "notdb-schema-create.sql") + s.touch(c, "db.tbl-schema-view.sql") + + _, err := md.NewMyDumpLoader(context.Background(), s.cfg) + c.Assert(err, ErrorMatches, `invalid table schema file, cannot find db 'db' - .*[/\\]?db\.tbl-schema-view\.sql`) +} + +func (s *testMydumpLoaderSuite) TestViewNoHostTable(c *C) { + /* + Path/ + db-schema-create.sql + db.tbl-schema-view.sql + */ + + s.touch(c, "db-schema-create.sql") + s.touch(c, "db.tbl-schema-view.sql") + + _, err := md.NewMyDumpLoader(context.Background(), s.cfg) + c.Assert(err, ErrorMatches, `invalid view schema file, miss host table schema for view 'tbl'`) +} + func (s *testMydumpLoaderSuite) TestDataWithoutSchema(c *C) { dir := s.sourceDir p := filepath.Join(dir, "db.tbl.sql") @@ -224,7 +251,6 @@ func (s *testMydumpLoaderSuite) TestTablesWithDots(c *C) { s.touch(c, "db.0002.sql") // insert some tables with file name structures which we're going to ignore. - s.touch(c, "db.v-schema-view.sql") s.touch(c, "db.v-schema-trigger.sql") s.touch(c, "db.v-schema-post.sql") s.touch(c, "db.sql") @@ -264,6 +290,12 @@ func (s *testMydumpLoaderSuite) TestRouter(c *C) { SchemaPattern: "c*", TargetSchema: "c", }, + { + SchemaPattern: "e*", + TablePattern: "f*", + TargetSchema: "v", + TargetTable: "vv", + }, } /* @@ -278,10 +310,15 @@ func (s *testMydumpLoaderSuite) TestRouter(c *C) { a1.s1.1.schema.sql a1.t2-schema.sql a1.t2.1.sql + a1.v1-schema.sql + a1.v1-schema-view.sql c0-schema-create.sql c0.t3-schema.sql c0.t3.1.sql d0-schema-create.sql + e0-schema-create.sql + e0.f0-schema.sql + e0.f0-schema-view.sql */ s.touch(c, "a0-schema-create.sql") @@ -295,6 +332,8 @@ func (s *testMydumpLoaderSuite) TestRouter(c *C) { s.touch(c, "a1.s1.1.sql") s.touch(c, "a1.t2-schema.sql") s.touch(c, "a1.t2.1.sql") + s.touch(c, "a1.v1-schema.sql") + s.touch(c, "a1.v1-schema-view.sql") s.touch(c, "c0-schema-create.sql") s.touch(c, "c0.t3-schema.sql") @@ -302,6 +341,10 @@ func (s *testMydumpLoaderSuite) TestRouter(c *C) { s.touch(c, "d0-schema-create.sql") + s.touch(c, "e0-schema-create.sql") + s.touch(c, "e0.f0-schema.sql") + s.touch(c, "e0.f0-schema-view.sql") + mdl, err := md.NewMyDumpLoader(context.Background(), s.cfg) c.Assert(err, IsNil) c.Assert(mdl.GetDatabases(), DeepEquals, []*md.MDDatabaseMeta{ @@ -315,6 +358,19 @@ func (s *testMydumpLoaderSuite) TestRouter(c *C) { SchemaFile: md.FileInfo{TableName: filter.Table{Schema: "a1", Name: "s1"}, FileMeta: md.SourceFileMeta{Path: "a1.s1-schema.sql", Type: md.SourceTypeTableSchema}}, DataFiles: []md.FileInfo{{TableName: filter.Table{Schema: "a1", Name: "s1"}, FileMeta: md.SourceFileMeta{Path: "a1.s1.1.sql", Type: md.SourceTypeSQL, SortKey: "1"}}}, }, + { + DB: "a1", + Name: "v1", + SchemaFile: md.FileInfo{TableName: filter.Table{Schema: "a1", Name: "v1"}, FileMeta: md.SourceFileMeta{Path: "a1.v1-schema.sql", Type: md.SourceTypeTableSchema}}, + DataFiles: []md.FileInfo{}, + }, + }, + Views: []*md.MDTableMeta{ + { + DB: "a1", + Name: "v1", + SchemaFile: md.FileInfo{TableName: filter.Table{Schema: "a1", Name: "v1"}, FileMeta: md.SourceFileMeta{Path: "a1.v1-schema-view.sql", Type: md.SourceTypeViewSchema}}, + }, }, }, { @@ -349,6 +405,25 @@ func (s *testMydumpLoaderSuite) TestRouter(c *C) { }, }, }, + { + Name: "v", + SchemaFile: "e0-schema-create.sql", + Tables: []*md.MDTableMeta{ + { + DB: "v", + Name: "vv", + SchemaFile: md.FileInfo{TableName: filter.Table{Schema: "v", Name: "vv"}, FileMeta: md.SourceFileMeta{Path: "e0.f0-schema.sql", Type: md.SourceTypeTableSchema}}, + DataFiles: []md.FileInfo{}, + }, + }, + Views: []*md.MDTableMeta{ + { + DB: "v", + Name: "vv", + SchemaFile: md.FileInfo{TableName: filter.Table{Schema: "v", Name: "vv"}, FileMeta: md.SourceFileMeta{Path: "e0.f0-schema-view.sql", Type: md.SourceTypeViewSchema}}, + }, + }, + }, }) } @@ -378,6 +453,12 @@ func (s *testMydumpLoaderSuite) TestFileRouting(c *C) { Table: "$2", Type: "table-schema", }, + { + Pattern: `(?i)^(?:[^./]*/)*([a-z0-9]+)/([a-z0-9_]+)-view\.sql$`, + Schema: "$1", + Table: "$2", + Type: "view-schema", + }, { Pattern: `(?i)^(?:[^./]*/)*([a-z][a-z0-9_]*)/([a-z]+)[0-9]*(?:\.([0-9]+))?\.(sql|csv)$`, Schema: "$1", @@ -399,6 +480,8 @@ func (s *testMydumpLoaderSuite) TestFileRouting(c *C) { s.touch(c, "d1/test0.sql") s.touch(c, "d1/test1.sql") s.touch(c, "d1/test2.001.sql") + s.touch(c, "d1/v1-table.sql") + s.touch(c, "d1/v1-view.sql") _ = s.touch(c, "d1/t1-schema-create.sql") s.touch(c, "d2/schema.sql") s.touch(c, "d2/abc-table.sql") @@ -421,6 +504,19 @@ func (s *testMydumpLoaderSuite) TestFileRouting(c *C) { {TableName: filter.Table{Schema: "d1", Name: "test"}, FileMeta: md.SourceFileMeta{Path: "d1/test2.001.sql", Type: md.SourceTypeSQL}}, }, }, + { + DB: "d1", + Name: "v1", + SchemaFile: md.FileInfo{TableName: filter.Table{Schema: "d1", Name: "v1"}, FileMeta: md.SourceFileMeta{Path: "d1/v1-table.sql", Type: md.SourceTypeTableSchema}}, + DataFiles: []md.FileInfo{}, + }, + }, + Views: []*md.MDTableMeta{ + { + DB: "d1", + Name: "v1", + SchemaFile: md.FileInfo{TableName: filter.Table{Schema: "d1", Name: "v1"}, FileMeta: md.SourceFileMeta{Path: "d1/v1-view.sql", Type: md.SourceTypeViewSchema}}, + }, }, }, { diff --git a/lightning/mydump/router.go b/lightning/mydump/router.go index 93e5b9240..f177d720a 100644 --- a/lightning/mydump/router.go +++ b/lightning/mydump/router.go @@ -19,6 +19,7 @@ const ( SourceTypeIgnore SourceType = iota SourceTypeSchemaSchema SourceTypeTableSchema + SourceTypeViewSchema SourceTypeSQL SourceTypeCSV SourceTypeParquet @@ -27,6 +28,7 @@ const ( const ( SchemaSchema = "schema-schema" TableSchema = "table-schema" + ViewSchema = "view-schema" TypeSQL = "sql" TypeCSV = "csv" TypeParquet = "parquet" @@ -49,6 +51,8 @@ func parseSourceType(t string) (SourceType, error) { return SourceTypeSchemaSchema, nil case TableSchema: return SourceTypeTableSchema, nil + case ViewSchema: + return SourceTypeViewSchema, nil case TypeSQL: return SourceTypeSQL, nil case TypeCSV: @@ -68,6 +72,8 @@ func (s SourceType) String() string { return SchemaSchema case SourceTypeTableSchema: return TableSchema + case SourceTypeViewSchema: + return ViewSchema case SourceTypeCSV: return TypeCSV case SourceTypeSQL: @@ -102,12 +108,14 @@ var ( var ( defaultFileRouteRules = []*config.FileRouteRule{ - // ignore *-schema-view.sql,-schema-trigger.sql,-schema-post.sql files - {Pattern: `(?i).*(-schema-view|-schema-trigger|-schema-post)\.sql`, Type: "ignore"}, + // ignore *-schema-trigger.sql, *-schema-post.sql files + {Pattern: `(?i).*(-schema-trigger|-schema-post)\.sql$`, Type: "ignore"}, // db schema create file pattern, matches files like '{schema}-schema-create.sql' - {Pattern: `(?i)^(?:[^/]*/)*([^/.]+)-schema-create\.sql`, Schema: "$1", Table: "", Type: SchemaSchema}, + {Pattern: `(?i)^(?:[^/]*/)*([^/.]+)-schema-create\.sql$`, Schema: "$1", Table: "", Type: SchemaSchema}, // table schema create file pattern, matches files like '{schema}.{table}-schema.sql' - {Pattern: `(?i)^(?:[^/]*/)*([^/.]+)\.(.*?)-schema\.sql`, Schema: "$1", Table: "$2", Type: TableSchema}, + {Pattern: `(?i)^(?:[^/]*/)*([^/.]+)\.(.*?)-schema\.sql$`, Schema: "$1", Table: "$2", Type: TableSchema}, + // view schema create file pattern, matches files like '{schema}.{table}-schema-view.sql' + {Pattern: `(?i)^(?:[^/]*/)*([^/.]+)\.(.*?)-schema-view\.sql$`, Schema: "$1", Table: "$2", Type: ViewSchema}, // source file pattern, matches files like '{schema}.{table}.0001.{sql|csv}' {Pattern: `(?i)^(?:[^/]*/)*([^/.]+)\.(.*?)(?:\.([0-9]+))?\.(sql|csv|parquet)$`, Schema: "$1", Table: "$2", Type: "$4", Key: "$3"}, } diff --git a/lightning/mydump/router_test.go b/lightning/mydump/router_test.go index 3be8e6e66..df1cce789 100644 --- a/lightning/mydump/router_test.go +++ b/lightning/mydump/router_test.go @@ -98,7 +98,8 @@ func (t *testFileRouterSuite) TestMultiRouteRule(c *C) { // multi rule don't intersect with each other rules := []*config.FileRouteRule{ {Pattern: `(?:[^/]*/)*([^/.]+)-schema-create\.sql`, Schema: "$1", Type: SchemaSchema}, - {Pattern: `(?:[^/]*/)*([^/.]+)\.([^/.]+)-schema\.sql`, Schema: "$1", Table: "$2", Type: TableSchema}, + {Pattern: `(?:[^/]*/)*([^/.]+)\.([^/.]+)-schema\.sql$`, Schema: "$1", Table: "$2", Type: TableSchema}, + {Pattern: `(?:[^/]*/)*([^/.]+)\.([^/.]+)-schema-view\.sql$`, Schema: "$1", Table: "$2", Type: ViewSchema}, {Pattern: `^(?:[^/]*/)*(?P[^/.]+)\.(?P[^./]+)(?:\.(?P[0-9]+))?\.(?Pcsv|sql)(?:\.(?P[A-Za-z0-9]+))?$`, Schema: "$schema", Table: "$table", Type: "$type", Key: "$key", Compression: "$cp"}, } @@ -108,6 +109,7 @@ func (t *testFileRouterSuite) TestMultiRouteRule(c *C) { inputOutputMap := map[string][]string{ "test-schema-create.sql": {"test", "", "", "", SchemaSchema}, "test.t-schema.sql": {"test", "t", "", "", TableSchema}, + "test.v1-schema-view.sql": {"test", "v1", "", "", ViewSchema}, "my_schema.my_table.sql": {"my_schema", "my_table", "", "", "sql"}, "/test/123/my_schema.my_table.sql": {"my_schema", "my_table", "", "", "sql"}, "my_dir/my_schema.my_table.csv": {"my_schema", "my_table", "", "", "csv"}, diff --git a/lightning/restore/restore.go b/lightning/restore/restore.go index c0bad1deb..1ad30f11e 100644 --- a/lightning/restore/restore.go +++ b/lightning/restore/restore.go @@ -339,6 +339,24 @@ func (rc *RestoreController) restoreSchema(ctx context.Context) error { return errors.Annotatef(err, "restore table schema %s failed", dbMeta.Name) } } + + // restore views. Since views can cross database we must restore views after all table schemas are restored. + for _, dbMeta := range rc.dbMetas { + if len(dbMeta.Views) > 0 { + task := log.With(zap.String("db", dbMeta.Name)).Begin(zap.InfoLevel, "restore view schema") + viewsSchema := make(map[string]string) + for _, viewMeta := range dbMeta.Views { + viewsSchema[viewMeta.Name] = viewMeta.GetSchema(ctx, rc.store) + } + err = tidbMgr.InitSchema(ctx, dbMeta.Name, viewsSchema) + + task.End(zap.ErrorLevel, err) + if err != nil { + return errors.Annotatef(err, "restore view schema %s failed", dbMeta.Name) + } + } + + } } dbInfos, err := tidbMgr.LoadSchemaInfo(ctx, rc.dbMetas, rc.backend.FetchRemoteTableModels) if err != nil { @@ -1166,8 +1184,9 @@ func (t *TableRestore) importEngine( } func (t *TableRestore) postProcess(ctx context.Context, rc *RestoreController, cp *TableCheckpoint) error { - if !rc.backend.ShouldPostProcess() { - t.logger.Debug("skip post-processing, not supported by backend") + // if post-process is disabled or the table is empty, just skip + if !rc.backend.ShouldPostProcess() || len(cp.Engines) == 1 { + t.logger.Debug("skip post-processing, not supported by backend or table is empty") rc.saveStatusCheckpoint(t.tableName, WholeTableEngineID, nil, CheckpointStatusAnalyzeSkipped) return nil } diff --git a/tests/file_routing/config.toml b/tests/file_routing/config.toml index daeccb472..b837614aa 100644 --- a/tests/file_routing/config.toml +++ b/tests/file_routing/config.toml @@ -18,6 +18,12 @@ schema = "$1" table = "$2" type = "table-schema" +[[mydumper.files]] +pattern = "(?i)^(?:[^/]*/)*([a-z0-9]+)/([a-z0-9]+)-view\\.sql$" +schema = "$1" +table = "$2" +type = "view-schema" + [[mydumper.files]] path = "ff/test.SQL" schema = "fr" diff --git a/tests/file_routing/run.sh b/tests/file_routing/run.sh index 64bb14ffb..aefd760e0 100755 --- a/tests/file_routing/run.sh +++ b/tests/file_routing/run.sh @@ -29,6 +29,24 @@ echo "INSERT INTO tbl (i, j) VALUES (6, 6), (7, 7), (8, 8), (9, 9);" > "$DBPATH/ echo "INSERT INTO tbl (i, j) VALUES (10, 10);" > "$DBPATH/ff/test.SQL" echo "INSERT INTO tbl (i, j) VALUES (11, 11);" > "$DBPATH/fr/tbl-noused.sql" +# view schema +echo "CREATE TABLE v(i TINYINT);" > "$DBPATH/fr/v-table.sql" +cat > "$DBPATH/fr/v-view.sql" << '_EOF_' +/*!40101 SET NAMES binary*/; +DROP TABLE IF EXISTS `v`; +DROP VIEW IF EXISTS `v`; +SET @PREV_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT; +SET @PREV_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS; +SET @PREV_COLLATION_CONNECTION=@@COLLATION_CONNECTION; +SET character_set_client = utf8; +SET character_set_results = utf8; +SET collation_connection = utf8_general_ci; +CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`192.168.198.178` SQL SECURITY DEFINER VIEW `v` (`i`) AS SELECT `i` FROM `fr`.`tbl` WHERE i <= 5; +SET character_set_client = @PREV_CHARACTER_SET_CLIENT; +SET character_set_results = @PREV_CHARACTER_SET_RESULTS; +SET collation_connection = @PREV_COLLATION_CONNECTION; +_EOF_ + for BACKEND in local importer; do if [ "$BACKEND" = 'local' ]; then check_cluster_version 4 0 0 'local backend' || continue @@ -43,4 +61,8 @@ for BACKEND in local importer; do check_contains "count(*): 10" run_sql 'SELECT sum(j) FROM `fr`.tbl' check_contains "sum(j): 55" + + run_sql 'SELECT sum(i), count(*) FROM `fr`.v' + check_contains "sum(i): 15" + check_contains "count(*): 5" done diff --git a/tests/view/config.toml b/tests/view/config.toml new file mode 100644 index 000000000..629cf31ad --- /dev/null +++ b/tests/view/config.toml @@ -0,0 +1 @@ +[lightning] diff --git a/tests/view/run.sh b/tests/view/run.sh new file mode 100755 index 000000000..fc26934f8 --- /dev/null +++ b/tests/view/run.sh @@ -0,0 +1,81 @@ +#!/bin/sh +# +# Copyright 2019 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euE + +# Populate the mydumper source +DBPATH="$TEST_DIR/fr.mydump" + +echo 'CREATE DATABASE db1;' > "$DBPATH/db1-schema-create.sql" +echo 'CREATE DATABASE db0;' > "$DBPATH/db0-schema-create.sql" +echo "CREATE TABLE tbl(i TINYINT PRIMARY KEY, j INT, s VARCHAR(16));" > "$DBPATH/db1.tbl-schema.sql" +echo "INSERT INTO tbl (i, j, s) VALUES (1, 1, 'test1'),(2, 2, 'test2'), (3, 3, 'test3');" > "$DBPATH/db1.tbl.0.sql" + +# view schema +echo "CREATE TABLE v1(i TINYINT, s VARCHAR(16));" > "$DBPATH/db1.v1-schema.sql" +cat > "$DBPATH/db1.v1-schema-view.sql" << '_EOF_' +/*!40101 SET NAMES binary*/; +DROP TABLE IF EXISTS `v1`; +DROP VIEW IF EXISTS `v1`; +SET @PREV_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT; +SET @PREV_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS; +SET @PREV_COLLATION_CONNECTION=@@COLLATION_CONNECTION; +SET character_set_client = utf8; +SET character_set_results = utf8; +SET collation_connection = utf8_general_ci; +CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`192.168.198.178` SQL SECURITY DEFINER VIEW `v1` (`i`, `s`) AS SELECT `i`,`s` FROM `db1`.`tbl`; +SET character_set_client = @PREV_CHARACTER_SET_CLIENT; +SET character_set_results = @PREV_CHARACTER_SET_RESULTS; +SET collation_connection = @PREV_COLLATION_CONNECTION; +_EOF_ + +# view schema +echo "CREATE TABLE v2(s VARCHAR(16));" > "$DBPATH/db0.v2-schema.sql" +cat > "$DBPATH/db0.v2-schema-view.sql" << '_EOF_' +/*!40101 SET NAMES binary*/; +DROP TABLE IF EXISTS `v2`; +DROP VIEW IF EXISTS `v2`; +SET @PREV_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT; +SET @PREV_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS; +SET @PREV_COLLATION_CONNECTION=@@COLLATION_CONNECTION; +SET character_set_client = utf8; +SET character_set_results = utf8; +SET collation_connection = utf8_general_ci; +CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`192.168.198.178` SQL SECURITY DEFINER VIEW `v2` (`s`) AS SELECT `s` FROM `db1`.`v1` WHERE `i`<2; +SET character_set_client = @PREV_CHARACTER_SET_CLIENT; +SET character_set_results = @PREV_CHARACTER_SET_RESULTS; +SET collation_connection = @PREV_COLLATION_CONNECTION; +_EOF_ + +for BACKEND in local importer tidb; do + if [ "$BACKEND" = 'local' ]; then + check_cluster_version 4 0 0 'local backend' || continue + fi + + run_sql 'DROP DATABASE IF EXISTS db0' + run_sql 'DROP DATABASE IF EXISTS db1' + + # Start importing the tables. + run_lightning -d "$DBPATH" --backend $BACKEND 2> /dev/null + + run_sql 'SELECT count(*), sum(i) FROM `db1`.v1' + check_contains "count(*): 3" + check_contains "sum(i): 6" + + run_sql 'SELECT count(*) FROM `db0`.v2' + check_contains "count(*): 1" + run_sql 'SELECT s FROM `db0`.v2' + check_contains "s: test1" +done From 13bf7ea1ff2707d6240db755a7a5b69d8e341a41 Mon Sep 17 00:00:00 2001 From: glorv Date: Sat, 10 Oct 2020 13:06:44 +0800 Subject: [PATCH 2/8] make router compatible --- lightning/checkpoints/checkpoints_sql_test.go | 2 +- lightning/mydump/router.go | 10 +++++----- tools/go.sum | 1 + 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/lightning/checkpoints/checkpoints_sql_test.go b/lightning/checkpoints/checkpoints_sql_test.go index c79a38f45..a7b15cc5d 100644 --- a/lightning/checkpoints/checkpoints_sql_test.go +++ b/lightning/checkpoints/checkpoints_sql_test.go @@ -431,7 +431,7 @@ func (s *cpSQLSuite) TestDump(c *C) { c.Assert(err, IsNil) c.Assert(csvBuilder.String(), Equals, "table_name,path,offset,type,compression,sort_key,columns,pos,end_offset,prev_rowid_max,rowid_max,kvc_bytes,kvc_kvs,kvc_checksum,create_time,update_time\n"+ - "`db1`.`t2`,/tmp/path/1.sql,0,4,0,,[],55904,102400,681,5000,4491,586,486070148917,2019-04-18 02:45:55 +0000 UTC,2019-04-18 02:45:55 +0000 UTC\n", + "`db1`.`t2`,/tmp/path/1.sql,0,3,0,,[],55904,102400,681,5000,4491,586,486070148917,2019-04-18 02:45:55 +0000 UTC,2019-04-18 02:45:55 +0000 UTC\n", ) s.mock. diff --git a/lightning/mydump/router.go b/lightning/mydump/router.go index f177d720a..d66e973b1 100644 --- a/lightning/mydump/router.go +++ b/lightning/mydump/router.go @@ -28,11 +28,11 @@ const ( const ( SchemaSchema = "schema-schema" TableSchema = "table-schema" - ViewSchema = "view-schema" TypeSQL = "sql" TypeCSV = "csv" TypeParquet = "parquet" TypeIgnore = "ignore" + ViewSchema = "view-schema" ) type Compression int @@ -51,8 +51,6 @@ func parseSourceType(t string) (SourceType, error) { return SourceTypeSchemaSchema, nil case TableSchema: return SourceTypeTableSchema, nil - case ViewSchema: - return SourceTypeViewSchema, nil case TypeSQL: return SourceTypeSQL, nil case TypeCSV: @@ -61,6 +59,8 @@ func parseSourceType(t string) (SourceType, error) { return SourceTypeParquet, nil case TypeIgnore: return SourceTypeIgnore, nil + case ViewSchema: + return SourceTypeViewSchema, nil default: return SourceTypeIgnore, errors.Errorf("unknown source type '%s'", t) } @@ -72,14 +72,14 @@ func (s SourceType) String() string { return SchemaSchema case SourceTypeTableSchema: return TableSchema - case SourceTypeViewSchema: - return ViewSchema case SourceTypeCSV: return TypeCSV case SourceTypeSQL: return TypeSQL case SourceTypeParquet: return TypeParquet + case SourceTypeViewSchema: + return ViewSchema default: return TypeIgnore } diff --git a/tools/go.sum b/tools/go.sum index 835a4676a..420704b64 100644 --- a/tools/go.sum +++ b/tools/go.sum @@ -205,6 +205,7 @@ github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4 github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/failpoint v0.0.0-20200603062251-b230c36c413c h1:cm0zAj+Tab94mp4OH+VoLJiSNQvZO4pWDGJ8KEk2a0c= github.com/pingcap/failpoint v0.0.0-20200603062251-b230c36c413c/go.mod h1:w4PEZ5y16LeofeeGwdgZB4ddv9bLyDuIX+ljstgKZyk= +github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce/go.mod h1:w4PEZ5y16LeofeeGwdgZB4ddv9bLyDuIX+ljstgKZyk= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= From 70020f292f33c6bd14c7339349df1d61dfd3c99f Mon Sep 17 00:00:00 2001 From: glorv Date: Sat, 10 Oct 2020 13:40:52 +0800 Subject: [PATCH 3/8] fix --- lightning/mydump/router.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lightning/mydump/router.go b/lightning/mydump/router.go index d66e973b1..c8ff31e24 100644 --- a/lightning/mydump/router.go +++ b/lightning/mydump/router.go @@ -19,20 +19,20 @@ const ( SourceTypeIgnore SourceType = iota SourceTypeSchemaSchema SourceTypeTableSchema - SourceTypeViewSchema SourceTypeSQL SourceTypeCSV SourceTypeParquet + SourceTypeViewSchema ) const ( SchemaSchema = "schema-schema" TableSchema = "table-schema" + ViewSchema = "view-schema" TypeSQL = "sql" TypeCSV = "csv" TypeParquet = "parquet" TypeIgnore = "ignore" - ViewSchema = "view-schema" ) type Compression int From 2c15563a1288b6840a5a57e4de36309fcc55588d Mon Sep 17 00:00:00 2001 From: glorv Date: Fri, 30 Oct 2020 15:36:00 +0800 Subject: [PATCH 4/8] don't genearte test files in run.sh --- tests/view/data/db0-schema-create.sql | 1 + tests/view/data/db0.v2-schema-view.sql | 13 +++++++ tests/view/data/db0.v2-schema.sql | 1 + tests/view/data/db1-schema-create.sql | 1 + tests/view/data/db1.tbl-schema.sql | 1 + tests/view/data/db1.tbl.0.sql | 5 +++ tests/view/data/db1.v1-schema-view.sql | 13 +++++++ tests/view/data/db1.v1-schema.sql | 1 + tests/view/run.sh | 48 +------------------------- 9 files changed, 37 insertions(+), 47 deletions(-) create mode 100644 tests/view/data/db0-schema-create.sql create mode 100644 tests/view/data/db0.v2-schema-view.sql create mode 100644 tests/view/data/db0.v2-schema.sql create mode 100644 tests/view/data/db1-schema-create.sql create mode 100644 tests/view/data/db1.tbl-schema.sql create mode 100644 tests/view/data/db1.tbl.0.sql create mode 100644 tests/view/data/db1.v1-schema-view.sql create mode 100644 tests/view/data/db1.v1-schema.sql diff --git a/tests/view/data/db0-schema-create.sql b/tests/view/data/db0-schema-create.sql new file mode 100644 index 000000000..1ef9021e4 --- /dev/null +++ b/tests/view/data/db0-schema-create.sql @@ -0,0 +1 @@ +CREATE DATABASE db0; diff --git a/tests/view/data/db0.v2-schema-view.sql b/tests/view/data/db0.v2-schema-view.sql new file mode 100644 index 000000000..887db3745 --- /dev/null +++ b/tests/view/data/db0.v2-schema-view.sql @@ -0,0 +1,13 @@ +/*!40101 SET NAMES binary*/; +DROP TABLE IF EXISTS `v2`; +DROP VIEW IF EXISTS `v2`; +SET @PREV_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT; +SET @PREV_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS; +SET @PREV_COLLATION_CONNECTION=@@COLLATION_CONNECTION; +SET character_set_client = utf8; +SET character_set_results = utf8; +SET collation_connection = utf8_general_ci; +CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`192.168.198.178` SQL SECURITY DEFINER VIEW `v2` (`s`) AS SELECT `s` FROM `db1`.`v1` WHERE `i`<2; +SET character_set_client = @PREV_CHARACTER_SET_CLIENT; +SET character_set_results = @PREV_CHARACTER_SET_RESULTS; +SET collation_connection = @PREV_COLLATION_CONNECTION; diff --git a/tests/view/data/db0.v2-schema.sql b/tests/view/data/db0.v2-schema.sql new file mode 100644 index 000000000..c3245382f --- /dev/null +++ b/tests/view/data/db0.v2-schema.sql @@ -0,0 +1 @@ +CREATE TABLE v2(s VARCHAR(16)); diff --git a/tests/view/data/db1-schema-create.sql b/tests/view/data/db1-schema-create.sql new file mode 100644 index 000000000..3538b4621 --- /dev/null +++ b/tests/view/data/db1-schema-create.sql @@ -0,0 +1 @@ +CREATE DATABASE db1; diff --git a/tests/view/data/db1.tbl-schema.sql b/tests/view/data/db1.tbl-schema.sql new file mode 100644 index 000000000..a6af5d167 --- /dev/null +++ b/tests/view/data/db1.tbl-schema.sql @@ -0,0 +1 @@ +CREATE TABLE tbl(i TINYINT PRIMARY KEY, j INT, s VARCHAR(16)); diff --git a/tests/view/data/db1.tbl.0.sql b/tests/view/data/db1.tbl.0.sql new file mode 100644 index 000000000..6cbe90b91 --- /dev/null +++ b/tests/view/data/db1.tbl.0.sql @@ -0,0 +1,5 @@ +INSERT INTO tbl (i, j, s) +VALUES + (1, 1, 'test1'), + (2, 2, 'test2'), + (3, 3, 'test3'); diff --git a/tests/view/data/db1.v1-schema-view.sql b/tests/view/data/db1.v1-schema-view.sql new file mode 100644 index 000000000..98f4e3f23 --- /dev/null +++ b/tests/view/data/db1.v1-schema-view.sql @@ -0,0 +1,13 @@ +/*!40101 SET NAMES binary*/; +DROP TABLE IF EXISTS `v1`; +DROP VIEW IF EXISTS `v1`; +SET @PREV_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT; +SET @PREV_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS; +SET @PREV_COLLATION_CONNECTION=@@COLLATION_CONNECTION; +SET character_set_client = utf8; +SET character_set_results = utf8; +SET collation_connection = utf8_general_ci; +CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`192.168.198.178` SQL SECURITY DEFINER VIEW `v1` (`i`, `s`) AS SELECT `i`,`s` FROM `db1`.`tbl`; +SET character_set_client = @PREV_CHARACTER_SET_CLIENT; +SET character_set_results = @PREV_CHARACTER_SET_RESULTS; +SET collation_connection = @PREV_COLLATION_CONNECTION; diff --git a/tests/view/data/db1.v1-schema.sql b/tests/view/data/db1.v1-schema.sql new file mode 100644 index 000000000..3a083f542 --- /dev/null +++ b/tests/view/data/db1.v1-schema.sql @@ -0,0 +1 @@ +CREATE TABLE v1(i TINYINT, s VARCHAR(16)); diff --git a/tests/view/run.sh b/tests/view/run.sh index 3726b7083..793e29a02 100755 --- a/tests/view/run.sh +++ b/tests/view/run.sh @@ -14,52 +14,6 @@ # limitations under the License. set -euE - -# Populate the mydumper source -DBPATH="$TEST_DIR/test_view" -mkdir -p $DBPATH - -echo 'CREATE DATABASE db1;' > "$DBPATH/db1-schema-create.sql" -echo 'CREATE DATABASE db0;' > "$DBPATH/db0-schema-create.sql" -echo "CREATE TABLE tbl(i TINYINT PRIMARY KEY, j INT, s VARCHAR(16));" > "$DBPATH/db1.tbl-schema.sql" -echo "INSERT INTO tbl (i, j, s) VALUES (1, 1, 'test1'),(2, 2, 'test2'), (3, 3, 'test3');" > "$DBPATH/db1.tbl.0.sql" - -# view schema -echo "CREATE TABLE v1(i TINYINT, s VARCHAR(16));" > "$DBPATH/db1.v1-schema.sql" -cat > "$DBPATH/db1.v1-schema-view.sql" << '_EOF_' -/*!40101 SET NAMES binary*/; -DROP TABLE IF EXISTS `v1`; -DROP VIEW IF EXISTS `v1`; -SET @PREV_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT; -SET @PREV_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS; -SET @PREV_COLLATION_CONNECTION=@@COLLATION_CONNECTION; -SET character_set_client = utf8; -SET character_set_results = utf8; -SET collation_connection = utf8_general_ci; -CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`192.168.198.178` SQL SECURITY DEFINER VIEW `v1` (`i`, `s`) AS SELECT `i`,`s` FROM `db1`.`tbl`; -SET character_set_client = @PREV_CHARACTER_SET_CLIENT; -SET character_set_results = @PREV_CHARACTER_SET_RESULTS; -SET collation_connection = @PREV_COLLATION_CONNECTION; -_EOF_ - -# view schema -echo "CREATE TABLE v2(s VARCHAR(16));" > "$DBPATH/db0.v2-schema.sql" -cat > "$DBPATH/db0.v2-schema-view.sql" << '_EOF_' -/*!40101 SET NAMES binary*/; -DROP TABLE IF EXISTS `v2`; -DROP VIEW IF EXISTS `v2`; -SET @PREV_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT; -SET @PREV_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS; -SET @PREV_COLLATION_CONNECTION=@@COLLATION_CONNECTION; -SET character_set_client = utf8; -SET character_set_results = utf8; -SET collation_connection = utf8_general_ci; -CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`192.168.198.178` SQL SECURITY DEFINER VIEW `v2` (`s`) AS SELECT `s` FROM `db1`.`v1` WHERE `i`<2; -SET character_set_client = @PREV_CHARACTER_SET_CLIENT; -SET character_set_results = @PREV_CHARACTER_SET_RESULTS; -SET collation_connection = @PREV_COLLATION_CONNECTION; -_EOF_ - for BACKEND in local importer tidb; do if [ "$BACKEND" = 'local' ]; then check_cluster_version 4 0 0 'local backend' || continue @@ -69,7 +23,7 @@ for BACKEND in local importer tidb; do run_sql 'DROP DATABASE IF EXISTS db1' # Start importing the tables. - run_lightning -d "$DBPATH" --backend $BACKEND 2> /dev/null + run_lightning --backend $BACKEND 2> /dev/null run_sql 'SELECT count(*), sum(i) FROM `db1`.v1' check_contains "count(*): 3" From 9b6c77072bdc4cebdef22e44761f8c342dba56ac Mon Sep 17 00:00:00 2001 From: glorv Date: Mon, 2 Nov 2020 14:11:52 +0800 Subject: [PATCH 5/8] execute multi create table stmt in serial --- lightning/restore/tidb.go | 33 +++++++++----- lightning/restore/tidb_test.go | 81 +++++++++++++++++++++++++--------- 2 files changed, 83 insertions(+), 31 deletions(-) diff --git a/lightning/restore/tidb.go b/lightning/restore/tidb.go index 9f1493436..57e4e8ec4 100644 --- a/lightning/restore/tidb.go +++ b/lightning/restore/tidb.go @@ -140,7 +140,7 @@ func (timgr *TiDBManager) InitSchema(ctx context.Context, database string, table for tbl, sqlCreateTable := range tablesSchema { task.Debug("create table", zap.String("schema", sqlCreateTable)) - sqlCreateTable, err = timgr.createTableIfNotExistsStmt(sqlCreateTable, tbl) + sqlCreateStmts, err := timgr.createTableIfNotExistsStmt(sqlCreateTable, tbl) if err != nil { break } @@ -149,9 +149,11 @@ func (timgr *TiDBManager) InitSchema(ctx context.Context, database string, table Logger: sql.Logger.With(zap.String("table", common.UniqueTable(database, tbl))), HideQueryLog: true, } - err = sql2.Exec(ctx, "create table", sqlCreateTable) - if err != nil { - break + for _, s := range sqlCreateStmts { + err = sql2.Exec(ctx, "create table", s) + if err != nil { + break + } } } task.End(zap.ErrorLevel, err) @@ -159,29 +161,40 @@ func (timgr *TiDBManager) InitSchema(ctx context.Context, database string, table return errors.Trace(err) } -func (timgr *TiDBManager) createTableIfNotExistsStmt(createTable, tblName string) (string, error) { +func (timgr *TiDBManager) createTableIfNotExistsStmt(createTable, tblName string) ([]string, error) { stmts, _, err := timgr.parser.Parse(createTable, "", "") if err != nil { - return "", err + return []string{}, err } var res strings.Builder - res.Grow(len(createTable)) ctx := format.NewRestoreCtx(format.DefaultRestoreFlags, &res) + retStmts := make([]string, 0, len(stmts)) for _, stmt := range stmts { - if createTableNode, ok := stmt.(*ast.CreateTableStmt); ok { + switch stmt.(type) { + case *ast.CreateTableStmt: + createTableNode := stmt.(*ast.CreateTableStmt) createTableNode.Table.Schema = model.NewCIStr("") createTableNode.Table.Name = model.NewCIStr(tblName) createTableNode.IfNotExists = true + case *ast.CreateViewStmt: + createViewNode := stmt.(*ast.CreateViewStmt) + createViewNode.ViewName.Name = model.NewCIStr(tblName) + case *ast.DropTableStmt: + dropStmt := stmt.(*ast.DropTableStmt) + dropStmt.Tables[0].Name = model.NewCIStr(tblName) + dropStmt.IfExists = true } if err := stmt.Restore(ctx); err != nil { - return "", err + return []string{}, err } ctx.WritePlain(";") + retStmts = append(retStmts, res.String()) + res.Reset() } - return res.String(), nil + return retStmts, nil } func (timgr *TiDBManager) DropTable(ctx context.Context, tableName string) error { diff --git a/lightning/restore/tidb_test.go b/lightning/restore/tidb_test.go index b1c8a84ef..3d8f61f8f 100644 --- a/lightning/restore/tidb_test.go +++ b/lightning/restore/tidb_test.go @@ -61,7 +61,7 @@ func (s *tidbSuite) TearDownTest(c *C) { } func (s *tidbSuite) TestCreateTableIfNotExistsStmt(c *C) { - createTableIfNotExistsStmt := func(createTable, tableName string) string { + createTableIfNotExistsStmt := func(createTable, tableName string) []string { res, err := s.timgr.createTableIfNotExistsStmt(createTable, tableName) c.Assert(err, IsNil) return res @@ -69,62 +69,62 @@ func (s *tidbSuite) TestCreateTableIfNotExistsStmt(c *C) { c.Assert( createTableIfNotExistsStmt("CREATE TABLE `foo`(`bar` TINYINT(1));", "foo"), - Equals, - "CREATE TABLE IF NOT EXISTS `foo` (`bar` TINYINT(1));", + DeepEquals, + []string{"CREATE TABLE IF NOT EXISTS `foo` (`bar` TINYINT(1));"}, ) c.Assert( createTableIfNotExistsStmt("CREATE TABLE IF NOT EXISTS `foo`(`bar` TINYINT(1));", "foo"), - Equals, - "CREATE TABLE IF NOT EXISTS `foo` (`bar` TINYINT(1));", + DeepEquals, + []string{"CREATE TABLE IF NOT EXISTS `foo` (`bar` TINYINT(1));"}, ) // case insensitive c.Assert( createTableIfNotExistsStmt("/* cOmmEnt */ creAte tablE `fOo`(`bar` TinyinT(1));", "fOo"), - Equals, - "CREATE TABLE IF NOT EXISTS `fOo` (`bar` TINYINT(1));", + DeepEquals, + []string{"CREATE TABLE IF NOT EXISTS `fOo` (`bar` TINYINT(1));"}, ) c.Assert( createTableIfNotExistsStmt("/* coMMenT */ crEatE tAble If not EXISts `FoO`(`bAR` tiNyInT(1));", "FoO"), - Equals, - "CREATE TABLE IF NOT EXISTS `FoO` (`bAR` TINYINT(1));", + DeepEquals, + []string{"CREATE TABLE IF NOT EXISTS `FoO` (`bAR` TINYINT(1));"}, ) // only one "CREATE TABLE" is replaced c.Assert( createTableIfNotExistsStmt("CREATE TABLE `foo`(`bar` INT(1) COMMENT 'CREATE TABLE');", "foo"), - Equals, - "CREATE TABLE IF NOT EXISTS `foo` (`bar` INT(1) COMMENT 'CREATE TABLE');", + DeepEquals, + []string{"CREATE TABLE IF NOT EXISTS `foo` (`bar` INT(1) COMMENT 'CREATE TABLE');"}, ) // upper case becomes shorter c.Assert( createTableIfNotExistsStmt("CREATE TABLE `ſ`(`ı` TINYINT(1));", "ſ"), - Equals, - "CREATE TABLE IF NOT EXISTS `ſ` (`ı` TINYINT(1));", + DeepEquals, + []string{"CREATE TABLE IF NOT EXISTS `ſ` (`ı` TINYINT(1));"}, ) // upper case becomes longer c.Assert( createTableIfNotExistsStmt("CREATE TABLE `ɑ`(`ȿ` TINYINT(1));", "ɑ"), - Equals, - "CREATE TABLE IF NOT EXISTS `ɑ` (`ȿ` TINYINT(1));", + DeepEquals, + []string{"CREATE TABLE IF NOT EXISTS `ɑ` (`ȿ` TINYINT(1));"}, ) // non-utf-8 c.Assert( createTableIfNotExistsStmt("CREATE TABLE `\xcc\xcc\xcc`(`\xdd\xdd\xdd` TINYINT(1));", "\xcc\xcc\xcc"), - Equals, - "CREATE TABLE IF NOT EXISTS `\xcc\xcc\xcc` (`ÝÝÝ` TINYINT(1));", + DeepEquals, + []string{"CREATE TABLE IF NOT EXISTS `\xcc\xcc\xcc` (`ÝÝÝ` TINYINT(1));"}, ) // renaming a table c.Assert( createTableIfNotExistsStmt("create table foo(x int);", "ba`r"), - Equals, - "CREATE TABLE IF NOT EXISTS `ba``r` (`x` INT);", + DeepEquals, + []string{"CREATE TABLE IF NOT EXISTS `ba``r` (`x` INT);"}, ) // conditional comments @@ -134,8 +134,47 @@ func (s *tidbSuite) TestCreateTableIfNotExistsStmt(c *C) { /*!40014 SET FOREIGN_KEY_CHECKS=0*/; CREATE TABLE x.y (z double) ENGINE=InnoDB AUTO_INCREMENT=8343230 DEFAULT CHARSET=utf8; `, "m"), - Equals, - "SET NAMES 'binary';SET @@SESSION.`FOREIGN_KEY_CHECKS`=0;CREATE TABLE IF NOT EXISTS `m` (`z` DOUBLE) ENGINE = InnoDB AUTO_INCREMENT = 8343230 DEFAULT CHARACTER SET = UTF8;", + DeepEquals, + []string{ + "SET NAMES 'binary';", + "SET @@SESSION.`FOREIGN_KEY_CHECKS`=0;", + "CREATE TABLE IF NOT EXISTS `m` (`z` DOUBLE) ENGINE = InnoDB AUTO_INCREMENT = 8343230 DEFAULT CHARACTER SET = UTF8;", + }, + ) + + // create view + c.Assert( + createTableIfNotExistsStmt(` + /*!40101 SET NAMES binary*/; + DROP TABLE IF EXISTS v2; + DROP VIEW IF EXISTS v2; + SET @PREV_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT; + SET @PREV_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS; + SET @PREV_COLLATION_CONNECTION=@@COLLATION_CONNECTION; + SET character_set_client = utf8; + SET character_set_results = utf8; + SET collation_connection = utf8_general_ci; + CREATE ALGORITHM=UNDEFINED DEFINER=root@192.168.198.178 SQL SECURITY DEFINER VIEW v2 (s) AS SELECT s FROM db1.v1 WHERE i<2; + SET character_set_client = @PREV_CHARACTER_SET_CLIENT; + SET character_set_results = @PREV_CHARACTER_SET_RESULTS; + SET collation_connection = @PREV_COLLATION_CONNECTION; + `, "m"), + DeepEquals, + []string{ + "SET NAMES 'binary';", + "DROP TABLE IF EXISTS `m`;", + "DROP VIEW IF EXISTS `m`;", + "SET @`PREV_CHARACTER_SET_CLIENT`=@@`character_set_client`;", + "SET @`PREV_CHARACTER_SET_RESULTS`=@@`character_set_results`;", + "SET @`PREV_COLLATION_CONNECTION`=@@`collation_connection`;", + "SET @@SESSION.`character_set_client`=`utf8`;", + "SET @@SESSION.`character_set_results`=`utf8`;", + "SET @@SESSION.`collation_connection`=`utf8_general_ci`;", + "CREATE ALGORITHM = UNDEFINED DEFINER = `root`@`192.168.198.178` SQL SECURITY DEFINER VIEW `m` (`s`) AS SELECT `s` FROM `db1`.`v1` WHERE `i`<2;", + "SET @@SESSION.`character_set_client`=@`PREV_CHARACTER_SET_CLIENT`;", + "SET @@SESSION.`character_set_results`=@`PREV_CHARACTER_SET_RESULTS`;", + "SET @@SESSION.`collation_connection`=@`PREV_COLLATION_CONNECTION`;", + }, ) } From 9bbc80d83881386994459edb1675c7540fefd10e Mon Sep 17 00:00:00 2001 From: glorv Date: Mon, 2 Nov 2020 14:43:03 +0800 Subject: [PATCH 6/8] fix unit test --- lightning/restore/tidb.go | 3 ++- lightning/restore/tidb_test.go | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lightning/restore/tidb.go b/lightning/restore/tidb.go index 57e4e8ec4..bbdb28ab3 100644 --- a/lightning/restore/tidb.go +++ b/lightning/restore/tidb.go @@ -137,10 +137,11 @@ func (timgr *TiDBManager) InitSchema(ctx context.Context, database string, table } task := sql.Logger.Begin(zap.InfoLevel, "create tables") + var sqlCreateStmts []string for tbl, sqlCreateTable := range tablesSchema { task.Debug("create table", zap.String("schema", sqlCreateTable)) - sqlCreateStmts, err := timgr.createTableIfNotExistsStmt(sqlCreateTable, tbl) + sqlCreateStmts, err = timgr.createTableIfNotExistsStmt(sqlCreateTable, tbl) if err != nil { break } diff --git a/lightning/restore/tidb_test.go b/lightning/restore/tidb_test.go index 3d8f61f8f..64e7c6308 100644 --- a/lightning/restore/tidb_test.go +++ b/lightning/restore/tidb_test.go @@ -191,7 +191,10 @@ func (s *tidbSuite) TestInitSchema(c *C) { ExpectExec("\\QCREATE TABLE IF NOT EXISTS `t1` (`a` INT PRIMARY KEY,`b` VARCHAR(200));\\E"). WillReturnResult(sqlmock.NewResult(2, 1)) s.mockDB. - ExpectExec("\\QSET @@SESSION.`FOREIGN_KEY_CHECKS`=0;CREATE TABLE IF NOT EXISTS `t2` (`xx` TEXT) AUTO_INCREMENT = 11203;\\E"). + ExpectExec("\\QSET @@SESSION.`FOREIGN_KEY_CHECKS`=0;\\E"). + WillReturnResult(sqlmock.NewResult(0, 0)) + s.mockDB. + ExpectExec("\\QCREATE TABLE IF NOT EXISTS `t2` (`xx` TEXT) AUTO_INCREMENT = 11203;\\E"). WillReturnResult(sqlmock.NewResult(2, 1)) s.mockDB. ExpectClose() From b4b1cf4085595fcfa03e95e4c94427a218657322 Mon Sep 17 00:00:00 2001 From: glorv Date: Mon, 23 Nov 2020 14:30:25 +0800 Subject: [PATCH 7/8] fix test --- lightning/restore/tidb_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lightning/restore/tidb_test.go b/lightning/restore/tidb_test.go index 12090f7ed..4e75c1684 100644 --- a/lightning/restore/tidb_test.go +++ b/lightning/restore/tidb_test.go @@ -166,8 +166,8 @@ func (s *tidbSuite) TestCreateTableIfNotExistsStmt(c *C) { DeepEquals, []string{ "SET NAMES 'binary';", - "DROP TABLE IF EXISTS `m`;", - "DROP VIEW IF EXISTS `m`;", + "DROP TABLE IF EXISTS `testdb`.`m`;", + "DROP VIEW IF EXISTS `testdb`.`m`;", "SET @`PREV_CHARACTER_SET_CLIENT`=@@`character_set_client`;", "SET @`PREV_CHARACTER_SET_RESULTS`=@@`character_set_results`;", "SET @`PREV_COLLATION_CONNECTION`=@@`collation_connection`;", From 8fdba8d8716852e4e63a23eca8de934f13716eb7 Mon Sep 17 00:00:00 2001 From: glorv Date: Thu, 26 Nov 2020 11:06:33 +0800 Subject: [PATCH 8/8] resolve comments --- lightning/restore/tidb.go | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/lightning/restore/tidb.go b/lightning/restore/tidb.go index 80530952e..227b8436f 100644 --- a/lightning/restore/tidb.go +++ b/lightning/restore/tidb.go @@ -175,21 +175,18 @@ func createTableIfNotExistsStmt(p *parser.Parser, createTable, dbName, tblName s retStmts := make([]string, 0, len(stmts)) for _, stmt := range stmts { - switch stmt.(type) { + switch node := stmt.(type) { case *ast.CreateTableStmt: - createTableNode := stmt.(*ast.CreateTableStmt) - createTableNode.Table.Schema = model.NewCIStr(dbName) - createTableNode.Table.Name = model.NewCIStr(tblName) - createTableNode.IfNotExists = true + node.Table.Schema = model.NewCIStr(dbName) + node.Table.Name = model.NewCIStr(tblName) + node.IfNotExists = true case *ast.CreateViewStmt: - createViewNode := stmt.(*ast.CreateViewStmt) - createViewNode.ViewName.Schema = model.NewCIStr(dbName) - createViewNode.ViewName.Name = model.NewCIStr(tblName) + node.ViewName.Schema = model.NewCIStr(dbName) + node.ViewName.Name = model.NewCIStr(tblName) case *ast.DropTableStmt: - dropStmt := stmt.(*ast.DropTableStmt) - dropStmt.Tables[0].Schema = model.NewCIStr(dbName) - dropStmt.Tables[0].Name = model.NewCIStr(tblName) - dropStmt.IfExists = true + node.Tables[0].Schema = model.NewCIStr(dbName) + node.Tables[0].Name = model.NewCIStr(tblName) + node.IfExists = true } if err := stmt.Restore(ctx); err != nil { return []string{}, err