Skip to content
This repository has been archived by the owner on Aug 21, 2023. It is now read-only.

Commit

Permalink
make dumpling support dumping view (#158)
Browse files Browse the repository at this point in the history
* support dump view

* add unit tests

* add more comments

* fix integration tests

* address comment

* fix
  • Loading branch information
lichunzhu authored Sep 30, 2020
1 parent a454a3d commit 5129591
Show file tree
Hide file tree
Showing 17 changed files with 192 additions and 40 deletions.
6 changes: 0 additions & 6 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,6 @@ require (
github.com/coreos/go-semver v0.3.0
github.com/docker/go-units v0.4.0
github.com/go-sql-driver/mysql v1.5.0
github.com/grpc-ecosystem/grpc-gateway v1.14.3 // indirect
github.com/json-iterator/go v1.1.9 // indirect
github.com/pingcap/br v0.0.0-20200925095602-bf9cc603382e
github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712
github.com/pingcap/errors v0.11.5-0.20200902104258-eba4f1d8f6de
Expand All @@ -25,9 +23,5 @@ require (
golang.org/x/net v0.0.0-20200822124328-c89045814202 // indirect
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8 // indirect
golang.org/x/text v0.3.3 // indirect
golang.org/x/tools v0.0.0-20200823205832-c024452afbcd // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
gopkg.in/yaml.v2 v2.3.0 // indirect
)
1 change: 1 addition & 0 deletions tests/quote/data/quote-database-schema-create.sql
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
/*!40101 SET NAMES binary*/;
CREATE DATABASE `quo``te/database` /*!40100 DEFAULT CHARACTER SET latin1 */;
1 change: 1 addition & 0 deletions tests/quote/data/quote-database.quote-table-schema.sql
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
/*!40101 SET NAMES binary*/;
CREATE TABLE `quo``te/table` (
`quo``te/col` int(11) NOT NULL,
`a` int(11) DEFAULT NULL,
Expand Down
1 change: 1 addition & 0 deletions tests/views/data/views-schema-create.sql
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
/*!40101 SET NAMES binary*/;
CREATE DATABASE `views` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin */ /*!80016 DEFAULT ENCRYPTION='N' */;
13 changes: 13 additions & 0 deletions tests/views/data/views.v-schema-view.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
/*!40101 SET NAMES binary*/;
DROP TABLE IF EXISTS `v`;
DROP VIEW IF EXISTS `v`;
SET @PREV_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT;
SET @PREV_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS;
SET @PREV_COLLATION_CONNECTION=@@COLLATION_CONNECTION;
SET character_set_client = utf8mb4;
SET character_set_results = utf8mb4;
SET collation_connection = utf8mb4_general_ci;
CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `views`.`v` AS select `views`.`t`.`a` AS `a`,`views`.`t`.`b` AS `b` from `views`.`t`;
SET character_set_client = @PREV_CHARACTER_SET_CLIENT;
SET character_set_results = @PREV_CHARACTER_SET_RESULTS;
SET collation_connection = @PREV_COLLATION_CONNECTION;
6 changes: 5 additions & 1 deletion tests/views/data/views.v-schema.sql
Original file line number Diff line number Diff line change
@@ -1 +1,5 @@
CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `views`.`v` AS select `views`.`t`.`a` AS `a`,`views`.`t`.`b` AS `b` from `views`.`t`;
/*!40101 SET NAMES binary*/;
CREATE TABLE `v`(
`a` int,
`b` int
)ENGINE=MyISAM;
2 changes: 2 additions & 0 deletions tests/views/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@ run_sql "insert into t values $(seq -s, 20 | sed 's/,*$//g' | sed 's/[0-9]*/(\0,

run_dumpling --no-views
file_not_exist "$DUMPLING_OUTPUT_DIR/views.v-schema.sql"
file_not_exist "$DUMPLING_OUTPUT_DIR/views.v-schema-view.sql"

run_dumpling --no-views=false
#diff "$DUMPLING_BASE_NAME/data/views-schema-create.sql" "$DUMPLING_OUTPUT_DIR/views-schema-create.sql"
diff "$DUMPLING_BASE_NAME/data/views.v-schema.sql" "$DUMPLING_OUTPUT_DIR/views.v-schema.sql"
diff "$DUMPLING_BASE_NAME/data/views.v-schema-view.sql" "$DUMPLING_OUTPUT_DIR/views.v-schema-view.sql"
4 changes: 2 additions & 2 deletions v4/export/dump.go
Original file line number Diff line number Diff line change
Expand Up @@ -303,11 +303,11 @@ func dumpTable(ctx context.Context, conf *Config, db *sql.Conn, dbName string, t
if !conf.NoSchemas {
if table.Type == TableTypeView {
viewName := table.Name
createViewSQL, err := ShowCreateView(db, dbName, viewName)
createTableSQL, createViewSQL, err := ShowCreateView(db, dbName, viewName)
if err != nil {
return nil, err
}
return nil, writer.WriteTableMeta(ctx, dbName, viewName, createViewSQL)
return nil, writer.WriteViewMeta(ctx, dbName, viewName, createTableSQL, createViewSQL)
}
createTableSQL, err := ShowCreateTable(db, dbName, tableName)
if err != nil {
Expand Down
8 changes: 8 additions & 0 deletions v4/export/dump_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,15 @@ type testDumpSuite struct{}
type mockWriter struct {
databaseMeta map[string]string
tableMeta map[string]string
viewMeta map[string]string
tableData []TableDataIR
}

func newMockWriter() *mockWriter {
return &mockWriter{
databaseMeta: map[string]string{},
tableMeta: map[string]string{},
viewMeta: map[string]string{},
tableData: nil,
}
}
Expand All @@ -46,6 +48,12 @@ func (m *mockWriter) WriteTableMeta(ctx context.Context, db, table, createSQL st
return nil
}

func (m *mockWriter) WriteViewMeta(ctx context.Context, db, table, createTableSQL, createViewSQL string) error {
m.tableMeta[fmt.Sprintf("%s.%s", db, table)] = createTableSQL
m.viewMeta[fmt.Sprintf("%s.%s", db, table)] = createViewSQL
return nil
}

func (m *mockWriter) WriteTableData(ctx context.Context, ir TableDataIR) error {
m.tableData = append(m.tableData, ir)
return nil
Expand Down
4 changes: 4 additions & 0 deletions v4/export/ir_impl.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"database/sql"
"fmt"
"strconv"
"strings"

"github.com/pkg/errors"
"go.uber.org/zap"
Expand Down Expand Up @@ -285,5 +286,8 @@ func (m *metaData) TargetName() string {
}

func (m *metaData) MetaSQL() string {
if !strings.HasSuffix(m.metaSQL, ";\n") {
m.metaSQL += ";\n"
}
return m.metaSQL
}
1 change: 1 addition & 0 deletions v4/export/prepare.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
const (
outputFileTemplateSchema = "schema"
outputFileTemplateTable = "table"
outputFileTemplateView = "view"
outputFileTemplateData = "data"

defaultOutputFileTemplateBase = `
Expand Down
67 changes: 62 additions & 5 deletions v4/export/sql.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,17 +57,72 @@ func ShowCreateTable(db *sql.Conn, database, table string) (string, error) {
return oneRow[1], nil
}

func ShowCreateView(db *sql.Conn, database, view string) (string, error) {
func ShowCreateView(db *sql.Conn, database, view string) (string, string, error) {
var fieldNames []string
handleFieldRow := func(rows *sql.Rows) error {
var oneRow [6]sql.NullString
err := rows.Scan(&oneRow[0], &oneRow[1], &oneRow[2], &oneRow[3], &oneRow[4], &oneRow[5])
if err != nil {
return err
}
if oneRow[0].Valid {
fieldNames = append(fieldNames, fmt.Sprintf("`%s` int", escapeString(oneRow[0].String)))
}
return nil
}
var oneRow [4]string
handleOneRow := func(rows *sql.Rows) error {
return rows.Scan(&oneRow[0], &oneRow[1], &oneRow[2], &oneRow[3])
}
query := fmt.Sprintf("SHOW CREATE TABLE `%s`.`%s`", escapeString(database), escapeString(view))
err := simpleQuery(db, query, handleOneRow)
var createTableSQL, createViewSQL strings.Builder

// Build createTableSQL
query := fmt.Sprintf("SHOW FIELDS FROM `%s`.`%s`", escapeString(database), escapeString(view))
err := simpleQuery(db, query, handleFieldRow)
if err != nil {
return "", errors.WithMessage(err, query)
return "", "", errors.WithMessage(err, query)
}
return oneRow[1], nil
fmt.Fprintf(&createTableSQL, "CREATE TABLE `%s`(\n", escapeString(view))
createTableSQL.WriteString(strings.Join(fieldNames, ",\n"))
createTableSQL.WriteString("\n)ENGINE=MyISAM;\n")

// Build createViewSQL
fmt.Fprintf(&createViewSQL, "DROP TABLE IF EXISTS `%s`;\n", escapeString(view))
fmt.Fprintf(&createViewSQL, "DROP VIEW IF EXISTS `%s`;\n", escapeString(view))
query = fmt.Sprintf("SHOW CREATE VIEW `%s`.`%s`", escapeString(database), escapeString(view))
err = simpleQuery(db, query, handleOneRow)
if err != nil {
return "", "", errors.WithMessage(err, query)
}
// The result for `show create view` SQL
// mysql> show create view v1;
// +------+-------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------+
// | View | Create View | character_set_client | collation_connection |
// +------+-------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------+
// | v1 | CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v1` (`a`) AS SELECT `t`.`a` AS `a` FROM `test`.`t` | utf8 | utf8_general_ci |
// +------+-------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------+
SetCharset(&createViewSQL, oneRow[2], oneRow[3])
createViewSQL.WriteString(oneRow[1])
createViewSQL.WriteString(";\n")
RestoreCharset(&createViewSQL)

return createTableSQL.String(), createViewSQL.String(), nil
}

func SetCharset(w *strings.Builder, characterSet, collationConnection string) {
w.WriteString("SET @PREV_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT;\n")
w.WriteString("SET @PREV_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS;\n")
w.WriteString("SET @PREV_COLLATION_CONNECTION=@@COLLATION_CONNECTION;\n")

fmt.Fprintf(w, "SET character_set_client = %s;\n", characterSet)
fmt.Fprintf(w, "SET character_set_results = %s;\n", characterSet)
fmt.Fprintf(w, "SET collation_connection = %s;\n", collationConnection)
}

func RestoreCharset(w *strings.Builder) {
w.WriteString("SET character_set_client = @PREV_CHARACTER_SET_CLIENT;\n")
w.WriteString("SET character_set_results = @PREV_CHARACTER_SET_RESULTS;\n")
w.WriteString("SET collation_connection = @PREV_COLLATION_CONNECTION;\n")
}

func ListAllDatabasesTables(db *sql.Conn, databaseNames []string, tableType TableType) (DatabaseTables, error) {
Expand Down Expand Up @@ -515,13 +570,15 @@ func simpleQueryWithArgs(conn *sql.Conn, handleOneRow func(*sql.Rows) error, sql
if err != nil {
return withStack(errors.WithMessage(err, sql))
}
defer rows.Close()

for rows.Next() {
if err := handleOneRow(rows); err != nil {
rows.Close()
return withStack(errors.WithMessage(err, sql))
}
}
rows.Close()
return rows.Err()
}

Expand Down
23 changes: 22 additions & 1 deletion v4/export/sql_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,6 @@ func (s *testDumpSuite) TestBuildSelectField(c *C) {
c.Assert(selectedField, Equals, "`id`,`name`,`quo``te`")
c.Assert(err, IsNil)
c.Assert(mock.ExpectationsWereMet(), IsNil)

}

func (s *testDumpSuite) TestParseSnapshotToTSO(c *C) {
Expand Down Expand Up @@ -232,6 +231,28 @@ func (s *testDumpSuite) TestParseSnapshotToTSO(c *C) {
c.Assert(mock.ExpectationsWereMet(), IsNil)
}

func (s *testDumpSuite) TestShowCreateView(c *C) {
db, mock, err := sqlmock.New()
c.Assert(err, IsNil)
defer db.Close()
conn, err := db.Conn(context.Background())
c.Assert(err, IsNil)

mock.ExpectQuery("SHOW FIELDS FROM `test`.`v`").
WillReturnRows(sqlmock.NewRows([]string{"Field", "Type", "Null", "Key", "Default", "Extra"}).
AddRow("a", "int(11)", "YES", nil, "NULL", nil))

mock.ExpectQuery("SHOW CREATE VIEW `test`.`v`").
WillReturnRows(sqlmock.NewRows([]string{"View", "Create View", "character_set_client", "collation_connection"}).
AddRow("v", "CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v` (`a`) AS SELECT `t`.`a` AS `a` FROM `test`.`t`", "utf8", "utf8_general_ci"))

createTableSQL, createViewSQL, err := ShowCreateView(conn, "test", "v")
c.Assert(err, IsNil)
c.Assert(createTableSQL, Equals, "CREATE TABLE `v`(\n`a` int\n)ENGINE=MyISAM;\n")
c.Assert(createViewSQL, Equals, "DROP TABLE IF EXISTS `v`;\nDROP VIEW IF EXISTS `v`;\nSET @PREV_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT;\nSET @PREV_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS;\nSET @PREV_COLLATION_CONNECTION=@@COLLATION_CONNECTION;\nSET character_set_client = utf8;\nSET character_set_results = utf8;\nSET collation_connection = utf8_general_ci;\nCREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v` (`a`) AS SELECT `t`.`a` AS `a` FROM `test`.`t`;\nSET character_set_client = @PREV_CHARACTER_SET_CLIENT;\nSET character_set_results = @PREV_CHARACTER_SET_RESULTS;\nSET collation_connection = @PREV_COLLATION_CONNECTION;\n")
c.Assert(mock.ExpectationsWereMet(), IsNil)
}

func makeVersion(major, minor, patch int64, preRelease string) *semver.Version {
return &semver.Version{
Major: major,
Expand Down
20 changes: 20 additions & 0 deletions v4/export/writer.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
type Writer interface {
WriteDatabaseMeta(ctx context.Context, db, createSQL string) error
WriteTableMeta(ctx context.Context, db, table, createSQL string) error
WriteViewMeta(ctx context.Context, db, table, createTableSQL, createViewSQL string) error
WriteTableData(ctx context.Context, ir TableDataIR) error
}

Expand Down Expand Up @@ -42,6 +43,22 @@ func (f SimpleWriter) WriteTableMeta(ctx context.Context, db, table, createSQL s
return writeMetaToFile(ctx, db, createSQL, f.cfg.ExternalStorage, fileName+".sql")
}

func (f SimpleWriter) WriteViewMeta(ctx context.Context, db, view, createTableSQL, createViewSQL string) error {
fileNameTable, err := (&outputFileNamer{DB: db, Table: view}).render(f.cfg.OutputFileTemplate, outputFileTemplateTable)
if err != nil {
return err
}
fileNameView, err := (&outputFileNamer{DB: db, Table: view}).render(f.cfg.OutputFileTemplate, outputFileTemplateView)
if err != nil {
return err
}
err = writeMetaToFile(ctx, db, createTableSQL, f.cfg.ExternalStorage, fileNameTable+".sql")
if err != nil {
return err
}
return writeMetaToFile(ctx, db, createViewSQL, f.cfg.ExternalStorage, fileNameView+".sql")
}

type SQLWriter struct{ SimpleWriter }

func (f SQLWriter) WriteTableData(ctx context.Context, ir TableDataIR) error {
Expand Down Expand Up @@ -100,6 +117,9 @@ func writeMetaToFile(ctx context.Context, target, metaSQL string, s storage.Exte
return WriteMeta(ctx, &metaData{
target: target,
metaSQL: metaSQL,
specCmts: []string{
"/*!40101 SET NAMES binary*/;",
},
}, fileWriter)
}

Expand Down
Loading

0 comments on commit 5129591

Please sign in to comment.