From b356941f42554e0751e38e6a047f3763fd291bb6 Mon Sep 17 00:00:00 2001 From: Bingchang Chen <19990626.love@163.com> Date: Thu, 22 Oct 2020 15:17:44 +0800 Subject: [PATCH] Fix: dump all generated column table (#166) * Fix dump all generated column table --- .../all_generate_column/conf/diff_config.toml | 42 +++++++++++++++++++ .../all_generate_column/conf/lightning.toml | 20 +++++++++ dumpling/tests/all_generate_column/run.sh | 37 ++++++++++++++++ dumpling/v4/export/ir_impl.go | 4 +- dumpling/v4/export/sql.go | 15 ++++++- dumpling/v4/export/test_util.go | 3 -- dumpling/v4/export/writer_util.go | 32 ++++++++------ 7 files changed, 133 insertions(+), 20 deletions(-) create mode 100644 dumpling/tests/all_generate_column/conf/diff_config.toml create mode 100644 dumpling/tests/all_generate_column/conf/lightning.toml create mode 100644 dumpling/tests/all_generate_column/run.sh diff --git a/dumpling/tests/all_generate_column/conf/diff_config.toml b/dumpling/tests/all_generate_column/conf/diff_config.toml new file mode 100644 index 00000000..1d69e86d --- /dev/null +++ b/dumpling/tests/all_generate_column/conf/diff_config.toml @@ -0,0 +1,42 @@ +# diff Configuration. + +log-level = "info" + +chunk-size = 1000 + +check-thread-count = 4 + +sample-percent = 100 + +use-rowid = false + +use-checksum = true + +fix-sql-file = "fix.sql" + +# tables need to check. +[[check-tables]] +schema = "all_generate_column" +tables = ["~t.*"] + +[[table-config]] +schema = "all_generate_column" +table = "t" + +[[table-config.source-tables]] +instance-id = "source-1" +schema = "all_generate_column" +table = "t" + +[[source-db]] +host = "127.0.0.1" +port = 3306 +user = "root" +password = "" +instance-id = "source-1" + +[target-db] +host = "127.0.0.1" +port = 4000 +user = "root" +password = "" diff --git a/dumpling/tests/all_generate_column/conf/lightning.toml b/dumpling/tests/all_generate_column/conf/lightning.toml new file mode 100644 index 00000000..1dada675 --- /dev/null +++ b/dumpling/tests/all_generate_column/conf/lightning.toml @@ -0,0 +1,20 @@ +### tidb-lightning config + +[lightning] +server-mode = false +level = "error" +check-requirements = false + +[tikv-importer] +backend="tidb" +on-duplicate = "error" + +[mydumper] +data-source-dir = "/tmp/dumpling_test_result/sql_res.all_generate_column" + +[tidb] +host = "127.0.0.1" +port = 4000 +user = "root" +password = "" +status-port = 10080 diff --git a/dumpling/tests/all_generate_column/run.sh b/dumpling/tests/all_generate_column/run.sh new file mode 100644 index 00000000..a9d7c6a7 --- /dev/null +++ b/dumpling/tests/all_generate_column/run.sh @@ -0,0 +1,37 @@ +#!/bin/sh + +set -eu +cur=$(cd `dirname $0`; pwd) + +DB_NAME="all_generate_column" +TABLE_NAME="t" + +# drop database on tidb +export DUMPLING_TEST_PORT=4000 +run_sql "drop database if exists $DB_NAME;" + +# drop database on mysql +export DUMPLING_TEST_PORT=3306 +run_sql "drop database if exists $DB_NAME;" + +# build data on mysql +run_sql "create database $DB_NAME;" + +# build data with generate column full_name +run_sql "create table $DB_NAME.$TABLE_NAME(a int as (1), b int as (2)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;" + +# insert 100 records +run_sql "insert into $DB_NAME.$TABLE_NAME values $(seq -s, 100 | sed 's/,*$//g' | sed "s/[0-9]*/()/g");" + +# dumping +export DUMPLING_TEST_DATABASE=$DB_NAME +run_dumpling + +cat "$cur/conf/lightning.toml" +# use lightning import data to tidb +run_lightning $cur/conf/lightning.toml + +# check mysql and tidb data +check_sync_diff $cur/conf/diff_config.toml + + diff --git a/dumpling/v4/export/ir_impl.go b/dumpling/v4/export/ir_impl.go index 9dbf3a99..fe896441 100644 --- a/dumpling/v4/export/ir_impl.go +++ b/dumpling/v4/export/ir_impl.go @@ -138,8 +138,8 @@ func (td *tableData) Rows() SQLRowIter { } func (td *tableData) SelectedField() string { - if td.selectedField == "*" { - return "" + if td.selectedField == "*" || td.selectedField == "" { + return td.selectedField } return fmt.Sprintf("(%s)", td.selectedField) } diff --git a/dumpling/v4/export/sql.go b/dumpling/v4/export/sql.go index 6bf10fa1..6590344d 100644 --- a/dumpling/v4/export/sql.go +++ b/dumpling/v4/export/sql.go @@ -194,7 +194,13 @@ func SelectAllFromTable(conf *Config, db *sql.Conn, database, table string) (Tab return nil, err } - colTypes, err := GetColumnTypes(db, selectedField, database, table) + var colTypes []*sql.ColumnType + // If all columns are generated + if selectedField == "" { + colTypes, err = GetColumnTypes(db, "*", database, table) + } else { + colTypes, err = GetColumnTypes(db, selectedField, database, table) + } if err != nil { return nil, err } @@ -204,7 +210,12 @@ func SelectAllFromTable(conf *Config, db *sql.Conn, database, table string) (Tab return nil, err } - query := buildSelectQuery(database, table, selectedField, buildWhereCondition(conf, ""), orderByClause) + queryField := selectedField + // If all columns are generated + if queryField == "" { + queryField = "''" + } + query := buildSelectQuery(database, table, queryField, buildWhereCondition(conf, ""), orderByClause) return &tableData{ database: database, diff --git a/dumpling/v4/export/test_util.go b/dumpling/v4/export/test_util.go index 58813c77..d7300aa4 100644 --- a/dumpling/v4/export/test_util.go +++ b/dumpling/v4/export/test_util.go @@ -115,9 +115,6 @@ func (m *mockTableIR) ColumnNames() []string { } func (m *mockTableIR) SelectedField() string { - if m.selectedField == "*" { - return "" - } return m.selectedField } diff --git a/dumpling/v4/export/writer_util.go b/dumpling/v4/export/writer_util.go index 6edf9fcc..131f58f4 100644 --- a/dumpling/v4/export/writer_util.go +++ b/dumpling/v4/export/writer_util.go @@ -159,8 +159,9 @@ func WriteInsert(pCtx context.Context, tblIR TableDataIR, w storage.Writer, file ) selectedField := tblIR.SelectedField() + // if has generated column - if selectedField != "" { + if selectedField != "" && selectedField != "*" { insertStatementPrefix = fmt.Sprintf("INSERT INTO %s %s VALUES\n", wrapBackTicks(escapeString(tblIR.TableName())), selectedField) } else { @@ -175,13 +176,17 @@ func WriteInsert(pCtx context.Context, tblIR TableDataIR, w storage.Writer, file wp.AddFileSize(insertStatementPrefixLen) for fileRowIter.HasNext() { - if err = fileRowIter.Decode(row); err != nil { - log.Error("scanning from sql.Row failed", zap.Error(err)) - return err - } lastBfSize := bf.Len() - row.WriteToBuffer(bf, escapeBackSlash) + if selectedField != "" { + if err = fileRowIter.Decode(row); err != nil { + log.Error("scanning from sql.Row failed", zap.Error(err)) + return err + } + row.WriteToBuffer(bf, escapeBackSlash) + } else { + bf.WriteString("()") + } counter += 1 wp.AddFileSize(uint64(bf.Len()-lastBfSize) + 2) // 2 is for ",\n" and ";\n" @@ -260,7 +265,7 @@ func WriteInsertInCsv(pCtx context.Context, tblIR TableDataIR, w storage.Writer, err error ) - if !noHeader && len(tblIR.ColumnNames()) != 0 { + if !noHeader && len(tblIR.ColumnNames()) != 0 && tblIR.SelectedField() != "" { for i, col := range tblIR.ColumnNames() { bf.Write(opt.delimiter) escape([]byte(col), bf, getEscapeQuotation(escapeBackSlash, opt.delimiter)) @@ -274,13 +279,14 @@ func WriteInsertInCsv(pCtx context.Context, tblIR TableDataIR, w storage.Writer, wp.currentFileSize += uint64(bf.Len()) for fileRowIter.HasNext() { - if err = fileRowIter.Decode(row); err != nil { - log.Error("scanning from sql.Row failed", zap.Error(err)) - return err - } - lastBfSize := bf.Len() - row.WriteToBufferInCsv(bf, escapeBackSlash, opt) + if tblIR.SelectedField() != "" { + if err = fileRowIter.Decode(row); err != nil { + log.Error("scanning from sql.Row failed", zap.Error(err)) + return err + } + row.WriteToBufferInCsv(bf, escapeBackSlash, opt) + } counter += 1 wp.currentFileSize += uint64(bf.Len()-lastBfSize) + 1 // 1 is for "\n"