From e17a0797478a6521793e5bb258621e4aa803a886 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Sun, 18 Oct 2020 16:09:27 +0800 Subject: [PATCH 1/8] Fix: dump all generated column table --- .../all_generate_column/conf/diff_config.toml | 42 +++++++++++++++++++ tests/all_generate_column/conf/lightning.toml | 20 +++++++++ tests/all_generate_column/run.sh | 37 ++++++++++++++++ v4/export/sql.go | 16 ++++++- 4 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 tests/all_generate_column/conf/diff_config.toml create mode 100644 tests/all_generate_column/conf/lightning.toml create mode 100644 tests/all_generate_column/run.sh diff --git a/tests/all_generate_column/conf/diff_config.toml b/tests/all_generate_column/conf/diff_config.toml new file mode 100644 index 00000000..984ca063 --- /dev/null +++ b/tests/all_generate_column/conf/diff_config.toml @@ -0,0 +1,42 @@ +# diff Configuration. + +log-level = "info" + +chunk-size = 1000 + +check-thread-count = 4 + +sample-percent = 100 + +use-rowid = false + +use-checksum = true + +fix-sql-file = "fix.sql" + +# tables need to check. +[[check-tables]] +schema = "ignore_generate" +tables = ["~t.*"] + +[[table-config]] +schema = "ignore_generate" +table = "t" + +[[table-config.source-tables]] +instance-id = "source-1" +schema = "ignore_generate" +table = "t" + +[[source-db]] +host = "127.0.0.1" +port = 3306 +user = "root" +password = "" +instance-id = "source-1" + +[target-db] +host = "127.0.0.1" +port = 4000 +user = "root" +password = "" diff --git a/tests/all_generate_column/conf/lightning.toml b/tests/all_generate_column/conf/lightning.toml new file mode 100644 index 00000000..ce999426 --- /dev/null +++ b/tests/all_generate_column/conf/lightning.toml @@ -0,0 +1,20 @@ +### tidb-lightning config + +[lightning] +server-mode = false +level = "error" +check-requirements = false + +[tikv-importer] +backend="tidb" +on-duplicate = "error" + +[mydumper] +data-source-dir = "/tmp/dumpling_test_result/sql_res.ignore_generate_column" + +[tidb] +host = "127.0.0.1" +port = 4000 +user = "root" +password = "" +status-port = 10080 diff --git a/tests/all_generate_column/run.sh b/tests/all_generate_column/run.sh new file mode 100644 index 00000000..5f0c1772 --- /dev/null +++ b/tests/all_generate_column/run.sh @@ -0,0 +1,37 @@ +#!/bin/sh + +set -eu +cur=$(cd `dirname $0`; pwd) + +DB_NAME="all_generate" +TABLE_NAME="t" + +# drop database on tidb +export DUMPLING_TEST_PORT=4000 +run_sql "drop database if exists $DB_NAME;" + +# drop database on mysql +export DUMPLING_TEST_PORT=3306 +run_sql "drop database if exists $DB_NAME;" + +# build data on mysql +run_sql "create database $DB_NAME;" + +# build data with generate column full_name +run_sql "create table $DB_NAME.$TABLE_NAME(a int as (1), b int as (2)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;" + +# insert 100 records +run_sql "insert into $DB_NAME.$TABLE_NAME values $(seq -s, 100 | sed 's/,*$//g' | sed "s/[0-9]*/()/g");" + +# dumping +export DUMPLING_TEST_DATABASE=$DB_NAME +run_dumpling + +cat "$cur/conf/lightning.toml" +# use lightning import data to tidb +run_lightning $cur/conf/lightning.toml + +# check mysql and tidb data +check_sync_diff $cur/conf/diff_config.toml + + diff --git a/v4/export/sql.go b/v4/export/sql.go index 3e96d5c0..9f8a61c7 100644 --- a/v4/export/sql.go +++ b/v4/export/sql.go @@ -194,7 +194,13 @@ func SelectAllFromTable(conf *Config, db *sql.Conn, database, table string) (Tab return nil, err } - colTypes, err := GetColumnTypes(db, selectedField, database, table) + var colTypes []*sql.ColumnType + // If all columns are generated + if selectedField == "" { + colTypes, err = GetColumnTypes(db, "*", database, table) + } else { + colTypes, err = GetColumnTypes(db, selectedField, database, table) + } if err != nil { return nil, err } @@ -204,7 +210,13 @@ func SelectAllFromTable(conf *Config, db *sql.Conn, database, table string) (Tab return nil, err } - query := buildSelectQuery(database, table, selectedField, buildWhereCondition(conf, ""), orderByClause) + var query string + // If all columns are generated + if selectedField == "" { + query = fmt.Sprintf("SELECT '' FROM `%s`.`%s`", database, table) + } else { + query = buildSelectQuery(database, table, selectedField, buildWhereCondition(conf, ""), orderByClause) + } return &tableData{ database: database, From 1a28e30d69847b42795568078767954e095f9b17 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Wed, 21 Oct 2020 11:27:34 +0800 Subject: [PATCH 2/8] dump multiple generated columns --- v4/export/ir_impl.go | 2 +- v4/export/writer_util.go | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/v4/export/ir_impl.go b/v4/export/ir_impl.go index 9dbf3a99..459edee7 100644 --- a/v4/export/ir_impl.go +++ b/v4/export/ir_impl.go @@ -138,7 +138,7 @@ func (td *tableData) Rows() SQLRowIter { } func (td *tableData) SelectedField() string { - if td.selectedField == "*" { + if td.selectedField == "*" || td.selectedField == "" { return "" } return fmt.Sprintf("(%s)", td.selectedField) diff --git a/v4/export/writer_util.go b/v4/export/writer_util.go index acfa3e10..c794808e 100644 --- a/v4/export/writer_util.go +++ b/v4/export/writer_util.go @@ -175,13 +175,17 @@ func WriteInsert(pCtx context.Context, tblIR TableDataIR, w storage.Writer, file wp.AddFileSize(insertStatementPrefixLen) for fileRowIter.HasNext() { - if err = fileRowIter.Decode(row); err != nil { - log.Error("scanning from sql.Row failed", zap.Error(err)) - return err - } lastBfSize := bf.Len() - row.WriteToBuffer(bf, escapeBackSlash) + if selectedField != "" { + if err = fileRowIter.Decode(row); err != nil { + log.Error("scanning from sql.Row failed", zap.Error(err)) + return err + } + row.WriteToBuffer(bf, escapeBackSlash) + } else { + bf.WriteString("()") + } counter += 1 wp.AddFileSize(uint64(bf.Len()-lastBfSize) + 2) // 2 is for ",\n" and ";\n" From 5b92d589b2de8fd84fc970e577bb7dc0c624b7d7 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Wed, 21 Oct 2020 11:37:42 +0800 Subject: [PATCH 3/8] fix integration test --- tests/all_generate_column/conf/diff_config.toml | 6 +++--- tests/all_generate_column/conf/lightning.toml | 2 +- tests/all_generate_column/run.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/all_generate_column/conf/diff_config.toml b/tests/all_generate_column/conf/diff_config.toml index 984ca063..1d69e86d 100644 --- a/tests/all_generate_column/conf/diff_config.toml +++ b/tests/all_generate_column/conf/diff_config.toml @@ -16,16 +16,16 @@ fix-sql-file = "fix.sql" # tables need to check. [[check-tables]] -schema = "ignore_generate" +schema = "all_generate_column" tables = ["~t.*"] [[table-config]] -schema = "ignore_generate" +schema = "all_generate_column" table = "t" [[table-config.source-tables]] instance-id = "source-1" -schema = "ignore_generate" +schema = "all_generate_column" table = "t" [[source-db]] diff --git a/tests/all_generate_column/conf/lightning.toml b/tests/all_generate_column/conf/lightning.toml index ce999426..1dada675 100644 --- a/tests/all_generate_column/conf/lightning.toml +++ b/tests/all_generate_column/conf/lightning.toml @@ -10,7 +10,7 @@ backend="tidb" on-duplicate = "error" [mydumper] -data-source-dir = "/tmp/dumpling_test_result/sql_res.ignore_generate_column" +data-source-dir = "/tmp/dumpling_test_result/sql_res.all_generate_column" [tidb] host = "127.0.0.1" diff --git a/tests/all_generate_column/run.sh b/tests/all_generate_column/run.sh index 5f0c1772..a9d7c6a7 100644 --- a/tests/all_generate_column/run.sh +++ b/tests/all_generate_column/run.sh @@ -3,7 +3,7 @@ set -eu cur=$(cd `dirname $0`; pwd) -DB_NAME="all_generate" +DB_NAME="all_generate_column" TABLE_NAME="t" # drop database on tidb From 798d89ada9c2cae0658f5f42d70e46d7b6790ec7 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Sun, 18 Oct 2020 16:09:27 +0800 Subject: [PATCH 4/8] Fix: dump all generated column table --- .../all_generate_column/conf/diff_config.toml | 42 +++++++++++++++++++ tests/all_generate_column/conf/lightning.toml | 20 +++++++++ tests/all_generate_column/run.sh | 37 ++++++++++++++++ v4/export/sql.go | 16 ++++++- 4 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 tests/all_generate_column/conf/diff_config.toml create mode 100644 tests/all_generate_column/conf/lightning.toml create mode 100644 tests/all_generate_column/run.sh diff --git a/tests/all_generate_column/conf/diff_config.toml b/tests/all_generate_column/conf/diff_config.toml new file mode 100644 index 00000000..984ca063 --- /dev/null +++ b/tests/all_generate_column/conf/diff_config.toml @@ -0,0 +1,42 @@ +# diff Configuration. + +log-level = "info" + +chunk-size = 1000 + +check-thread-count = 4 + +sample-percent = 100 + +use-rowid = false + +use-checksum = true + +fix-sql-file = "fix.sql" + +# tables need to check. +[[check-tables]] +schema = "ignore_generate" +tables = ["~t.*"] + +[[table-config]] +schema = "ignore_generate" +table = "t" + +[[table-config.source-tables]] +instance-id = "source-1" +schema = "ignore_generate" +table = "t" + +[[source-db]] +host = "127.0.0.1" +port = 3306 +user = "root" +password = "" +instance-id = "source-1" + +[target-db] +host = "127.0.0.1" +port = 4000 +user = "root" +password = "" diff --git a/tests/all_generate_column/conf/lightning.toml b/tests/all_generate_column/conf/lightning.toml new file mode 100644 index 00000000..ce999426 --- /dev/null +++ b/tests/all_generate_column/conf/lightning.toml @@ -0,0 +1,20 @@ +### tidb-lightning config + +[lightning] +server-mode = false +level = "error" +check-requirements = false + +[tikv-importer] +backend="tidb" +on-duplicate = "error" + +[mydumper] +data-source-dir = "/tmp/dumpling_test_result/sql_res.ignore_generate_column" + +[tidb] +host = "127.0.0.1" +port = 4000 +user = "root" +password = "" +status-port = 10080 diff --git a/tests/all_generate_column/run.sh b/tests/all_generate_column/run.sh new file mode 100644 index 00000000..5f0c1772 --- /dev/null +++ b/tests/all_generate_column/run.sh @@ -0,0 +1,37 @@ +#!/bin/sh + +set -eu +cur=$(cd `dirname $0`; pwd) + +DB_NAME="all_generate" +TABLE_NAME="t" + +# drop database on tidb +export DUMPLING_TEST_PORT=4000 +run_sql "drop database if exists $DB_NAME;" + +# drop database on mysql +export DUMPLING_TEST_PORT=3306 +run_sql "drop database if exists $DB_NAME;" + +# build data on mysql +run_sql "create database $DB_NAME;" + +# build data with generate column full_name +run_sql "create table $DB_NAME.$TABLE_NAME(a int as (1), b int as (2)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;" + +# insert 100 records +run_sql "insert into $DB_NAME.$TABLE_NAME values $(seq -s, 100 | sed 's/,*$//g' | sed "s/[0-9]*/()/g");" + +# dumping +export DUMPLING_TEST_DATABASE=$DB_NAME +run_dumpling + +cat "$cur/conf/lightning.toml" +# use lightning import data to tidb +run_lightning $cur/conf/lightning.toml + +# check mysql and tidb data +check_sync_diff $cur/conf/diff_config.toml + + diff --git a/v4/export/sql.go b/v4/export/sql.go index 6bf10fa1..ad50df3c 100644 --- a/v4/export/sql.go +++ b/v4/export/sql.go @@ -194,7 +194,13 @@ func SelectAllFromTable(conf *Config, db *sql.Conn, database, table string) (Tab return nil, err } - colTypes, err := GetColumnTypes(db, selectedField, database, table) + var colTypes []*sql.ColumnType + // If all columns are generated + if selectedField == "" { + colTypes, err = GetColumnTypes(db, "*", database, table) + } else { + colTypes, err = GetColumnTypes(db, selectedField, database, table) + } if err != nil { return nil, err } @@ -204,7 +210,13 @@ func SelectAllFromTable(conf *Config, db *sql.Conn, database, table string) (Tab return nil, err } - query := buildSelectQuery(database, table, selectedField, buildWhereCondition(conf, ""), orderByClause) + var query string + // If all columns are generated + if selectedField == "" { + query = fmt.Sprintf("SELECT '' FROM `%s`.`%s`", database, table) + } else { + query = buildSelectQuery(database, table, selectedField, buildWhereCondition(conf, ""), orderByClause) + } return &tableData{ database: database, From 41d2fc299c15b98044accecda9278c008c7f3a90 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Wed, 21 Oct 2020 11:27:34 +0800 Subject: [PATCH 5/8] dump multiple generated columns --- v4/export/ir_impl.go | 2 +- v4/export/writer_util.go | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/v4/export/ir_impl.go b/v4/export/ir_impl.go index 9dbf3a99..459edee7 100644 --- a/v4/export/ir_impl.go +++ b/v4/export/ir_impl.go @@ -138,7 +138,7 @@ func (td *tableData) Rows() SQLRowIter { } func (td *tableData) SelectedField() string { - if td.selectedField == "*" { + if td.selectedField == "*" || td.selectedField == "" { return "" } return fmt.Sprintf("(%s)", td.selectedField) diff --git a/v4/export/writer_util.go b/v4/export/writer_util.go index acfa3e10..c794808e 100644 --- a/v4/export/writer_util.go +++ b/v4/export/writer_util.go @@ -175,13 +175,17 @@ func WriteInsert(pCtx context.Context, tblIR TableDataIR, w storage.Writer, file wp.AddFileSize(insertStatementPrefixLen) for fileRowIter.HasNext() { - if err = fileRowIter.Decode(row); err != nil { - log.Error("scanning from sql.Row failed", zap.Error(err)) - return err - } lastBfSize := bf.Len() - row.WriteToBuffer(bf, escapeBackSlash) + if selectedField != "" { + if err = fileRowIter.Decode(row); err != nil { + log.Error("scanning from sql.Row failed", zap.Error(err)) + return err + } + row.WriteToBuffer(bf, escapeBackSlash) + } else { + bf.WriteString("()") + } counter += 1 wp.AddFileSize(uint64(bf.Len()-lastBfSize) + 2) // 2 is for ",\n" and ";\n" From d2cebf0cf5c0936cd1e32ebf1a85a0ab1b8f9d4f Mon Sep 17 00:00:00 2001 From: abingcbc Date: Wed, 21 Oct 2020 11:37:42 +0800 Subject: [PATCH 6/8] fix integration test --- tests/all_generate_column/conf/diff_config.toml | 6 +++--- tests/all_generate_column/conf/lightning.toml | 2 +- tests/all_generate_column/run.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/all_generate_column/conf/diff_config.toml b/tests/all_generate_column/conf/diff_config.toml index 984ca063..1d69e86d 100644 --- a/tests/all_generate_column/conf/diff_config.toml +++ b/tests/all_generate_column/conf/diff_config.toml @@ -16,16 +16,16 @@ fix-sql-file = "fix.sql" # tables need to check. [[check-tables]] -schema = "ignore_generate" +schema = "all_generate_column" tables = ["~t.*"] [[table-config]] -schema = "ignore_generate" +schema = "all_generate_column" table = "t" [[table-config.source-tables]] instance-id = "source-1" -schema = "ignore_generate" +schema = "all_generate_column" table = "t" [[source-db]] diff --git a/tests/all_generate_column/conf/lightning.toml b/tests/all_generate_column/conf/lightning.toml index ce999426..1dada675 100644 --- a/tests/all_generate_column/conf/lightning.toml +++ b/tests/all_generate_column/conf/lightning.toml @@ -10,7 +10,7 @@ backend="tidb" on-duplicate = "error" [mydumper] -data-source-dir = "/tmp/dumpling_test_result/sql_res.ignore_generate_column" +data-source-dir = "/tmp/dumpling_test_result/sql_res.all_generate_column" [tidb] host = "127.0.0.1" diff --git a/tests/all_generate_column/run.sh b/tests/all_generate_column/run.sh index 5f0c1772..a9d7c6a7 100644 --- a/tests/all_generate_column/run.sh +++ b/tests/all_generate_column/run.sh @@ -3,7 +3,7 @@ set -eu cur=$(cd `dirname $0`; pwd) -DB_NAME="all_generate" +DB_NAME="all_generate_column" TABLE_NAME="t" # drop database on tidb From de0553035d046d8a83696ca428f6c037a7c22408 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Wed, 21 Oct 2020 16:42:10 +0800 Subject: [PATCH 7/8] fix mock write insert --- v4/export/test_util.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v4/export/test_util.go b/v4/export/test_util.go index 58813c77..a91b189b 100644 --- a/v4/export/test_util.go +++ b/v4/export/test_util.go @@ -115,8 +115,8 @@ func (m *mockTableIR) ColumnNames() []string { } func (m *mockTableIR) SelectedField() string { - if m.selectedField == "*" { - return "" + if m.selectedField == "*" || m.selectedField == ""{ + return m.selectedField } return m.selectedField } From 2303f4901f58ef2b13a3c657472de1d75d743276 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Thu, 22 Oct 2020 14:31:06 +0800 Subject: [PATCH 8/8] fix write insert into csv --- v4/export/sql.go | 9 ++++----- v4/export/test_util.go | 3 --- v4/export/writer_util.go | 15 ++++++++------- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/v4/export/sql.go b/v4/export/sql.go index ad50df3c..6590344d 100644 --- a/v4/export/sql.go +++ b/v4/export/sql.go @@ -210,13 +210,12 @@ func SelectAllFromTable(conf *Config, db *sql.Conn, database, table string) (Tab return nil, err } - var query string + queryField := selectedField // If all columns are generated - if selectedField == "" { - query = fmt.Sprintf("SELECT '' FROM `%s`.`%s`", database, table) - } else { - query = buildSelectQuery(database, table, selectedField, buildWhereCondition(conf, ""), orderByClause) + if queryField == "" { + queryField = "''" } + query := buildSelectQuery(database, table, queryField, buildWhereCondition(conf, ""), orderByClause) return &tableData{ database: database, diff --git a/v4/export/test_util.go b/v4/export/test_util.go index a91b189b..d7300aa4 100644 --- a/v4/export/test_util.go +++ b/v4/export/test_util.go @@ -115,9 +115,6 @@ func (m *mockTableIR) ColumnNames() []string { } func (m *mockTableIR) SelectedField() string { - if m.selectedField == "*" || m.selectedField == ""{ - return m.selectedField - } return m.selectedField } diff --git a/v4/export/writer_util.go b/v4/export/writer_util.go index 4b6d5b24..131f58f4 100644 --- a/v4/export/writer_util.go +++ b/v4/export/writer_util.go @@ -265,7 +265,7 @@ func WriteInsertInCsv(pCtx context.Context, tblIR TableDataIR, w storage.Writer, err error ) - if !noHeader && len(tblIR.ColumnNames()) != 0 { + if !noHeader && len(tblIR.ColumnNames()) != 0 && tblIR.SelectedField() != "" { for i, col := range tblIR.ColumnNames() { bf.Write(opt.delimiter) escape([]byte(col), bf, getEscapeQuotation(escapeBackSlash, opt.delimiter)) @@ -279,13 +279,14 @@ func WriteInsertInCsv(pCtx context.Context, tblIR TableDataIR, w storage.Writer, wp.currentFileSize += uint64(bf.Len()) for fileRowIter.HasNext() { - if err = fileRowIter.Decode(row); err != nil { - log.Error("scanning from sql.Row failed", zap.Error(err)) - return err - } - lastBfSize := bf.Len() - row.WriteToBufferInCsv(bf, escapeBackSlash, opt) + if tblIR.SelectedField() != "" { + if err = fileRowIter.Decode(row); err != nil { + log.Error("scanning from sql.Row failed", zap.Error(err)) + return err + } + row.WriteToBufferInCsv(bf, escapeBackSlash, opt) + } counter += 1 wp.currentFileSize += uint64(bf.Len()-lastBfSize) + 1 // 1 is for "\n"