From caeba6335f7ad74f35b58139d2ac22f6a55f52eb Mon Sep 17 00:00:00 2001
From: Chunzhu Li <lichunzhu@stu.xjtu.edu.cn>
Date: Fri, 6 Nov 2020 01:13:13 -0600
Subject: [PATCH 1/6] increase maxRetries from 3 to 6 (#583)

---
 pkg/storage/s3.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/storage/s3.go b/pkg/storage/s3.go
index 049971051..f83343185 100644
--- a/pkg/storage/s3.go
+++ b/pkg/storage/s3.go
@@ -37,7 +37,7 @@ const (
 	s3ProviderOption     = "s3.provider"
 	notFound             = "NotFound"
 	// number of retries to make of operations
-	maxRetries = 3
+	maxRetries = 6
 
 	// the maximum number of byte to read for seek
 	maxSkipOffsetByRead = 1 << 16 //64KB

From 412dc71dcb0a9b726e1a926845d0409214bf4bbf Mon Sep 17 00:00:00 2001
From: lvtu <37565148+tongtongyin@users.noreply.github.com>
Date: Fri, 6 Nov 2020 15:23:33 +0800
Subject: [PATCH 2/6] checksum: correct spelling mistake (#587)

---
 pkg/kv/checksum_test.go | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/pkg/kv/checksum_test.go b/pkg/kv/checksum_test.go
index 35d83ef32..d9b91b15a 100644
--- a/pkg/kv/checksum_test.go
+++ b/pkg/kv/checksum_test.go
@@ -22,20 +22,20 @@ import (
 	"github.com/pingcap/br/pkg/kv"
 )
 
-type testKVChcksumSuite struct{}
+type testKVChecksumSuite struct{}
 
-func (s *testKVChcksumSuite) SetUpSuite(c *C)    {}
-func (s *testKVChcksumSuite) TearDownSuite(c *C) {}
+func (s *testKVChecksumSuite) SetUpSuite(c *C)    {}
+func (s *testKVChecksumSuite) TearDownSuite(c *C) {}
 
-var _ = Suite(&testKVChcksumSuite{})
+var _ = Suite(&testKVChecksumSuite{})
 
-func TestKVChcksum(t *testing.T) {
+func TestKVChecksum(t *testing.T) {
 	TestingT(t)
 }
 
 func uint64NotEqual(a uint64, b uint64) bool { return a != b }
 
-func (s *testKVChcksumSuite) TestChcksum(c *C) {
+func (s *testKVChecksumSuite) TestChecksum(c *C) {
 	checksum := kv.NewKVChecksum(0)
 	c.Assert(checksum.Sum(), Equals, uint64(0))
 
@@ -47,7 +47,7 @@ func (s *testKVChcksumSuite) TestChcksum(c *C) {
 	c.Assert(checksum.Sum(), Equals, uint64(0))
 
 	// checksum on real data
-	excpectChecksum := uint64(4850203904608948940)
+	expectChecksum := uint64(4850203904608948940)
 
 	kvs := []kv.Pair{
 		{
@@ -68,16 +68,16 @@ func (s *testKVChcksumSuite) TestChcksum(c *C) {
 	}
 	c.Assert(checksum.SumSize(), Equals, kvBytes)
 	c.Assert(checksum.SumKVS(), Equals, uint64(len(kvs)))
-	c.Assert(checksum.Sum(), Equals, excpectChecksum)
+	c.Assert(checksum.Sum(), Equals, expectChecksum)
 
 	// recompute on same key-value
 	checksum.Update(kvs)
 	c.Assert(checksum.SumSize(), Equals, kvBytes<<1)
 	c.Assert(checksum.SumKVS(), Equals, uint64(len(kvs))<<1)
-	c.Assert(uint64NotEqual(checksum.Sum(), excpectChecksum), IsTrue)
+	c.Assert(uint64NotEqual(checksum.Sum(), expectChecksum), IsTrue)
 }
 
-func (s *testKVChcksumSuite) TestChecksumJSON(c *C) {
+func (s *testKVChecksumSuite) TestChecksumJSON(c *C) {
 	testStruct := &struct {
 		Checksum kv.Checksum
 	}{

From 8d9a51aa282acfae442ab20e7b01669b7f63df7b Mon Sep 17 00:00:00 2001
From: kennytm <kennytm@gmail.com>
Date: Fri, 6 Nov 2020 20:16:09 +0800
Subject: [PATCH 3/6] cmd,task: disable stray command line args, and log the
 command in LogArguments (#579)

* cmd: disallow stray args
* task: in LogArguments, log the command name too
---
 cmd/backup.go      |  6 ++++++
 cmd/debug.go       |  5 +++++
 cmd/restore.go     |  6 ++++++
 pkg/task/common.go | 10 +++++-----
 4 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/cmd/backup.go b/cmd/backup.go
index 18309017f..4e4317cfc 100644
--- a/cmd/backup.go
+++ b/cmd/backup.go
@@ -79,6 +79,9 @@ func newFullBackupCommand() *cobra.Command {
 	command := &cobra.Command{
 		Use:   "full",
 		Short: "backup all database",
+		// prevents incorrect usage like `--checksum false` instead of `--checksum=false`.
+		// the former, according to pflag parsing rules, means `--checksum=true false`.
+		Args: cobra.NoArgs,
 		RunE: func(command *cobra.Command, _ []string) error {
 			// empty db/table means full backup.
 			return runBackupCommand(command, "Full backup")
@@ -93,6 +96,7 @@ func newDBBackupCommand() *cobra.Command {
 	command := &cobra.Command{
 		Use:   "db",
 		Short: "backup a database",
+		Args:  cobra.NoArgs,
 		RunE: func(command *cobra.Command, _ []string) error {
 			return runBackupCommand(command, "Database backup")
 		},
@@ -106,6 +110,7 @@ func newTableBackupCommand() *cobra.Command {
 	command := &cobra.Command{
 		Use:   "table",
 		Short: "backup a table",
+		Args:  cobra.NoArgs,
 		RunE: func(command *cobra.Command, _ []string) error {
 			return runBackupCommand(command, "Table backup")
 		},
@@ -120,6 +125,7 @@ func newRawBackupCommand() *cobra.Command {
 	command := &cobra.Command{
 		Use:   "raw",
 		Short: "(experimental) backup a raw kv range from TiKV cluster",
+		Args:  cobra.NoArgs,
 		RunE: func(command *cobra.Command, _ []string) error {
 			return runBackupRawCommand(command, "Raw backup")
 		},
diff --git a/cmd/debug.go b/cmd/debug.go
index 35f6e502d..65ad9ea6d 100644
--- a/cmd/debug.go
+++ b/cmd/debug.go
@@ -60,6 +60,7 @@ func newCheckSumCommand() *cobra.Command {
 	command := &cobra.Command{
 		Use:   "checksum",
 		Short: "check the backup data",
+		Args:  cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			ctx, cancel := context.WithCancel(GetDefaultContext())
 			defer cancel()
@@ -144,6 +145,7 @@ func newBackupMetaCommand() *cobra.Command {
 	command := &cobra.Command{
 		Use:   "backupmeta",
 		Short: "check the backup meta",
+		Args:  cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			ctx, cancel := context.WithCancel(GetDefaultContext())
 			defer cancel()
@@ -240,6 +242,7 @@ func decodeBackupMetaCommand() *cobra.Command {
 	decodeBackupMetaCmd := &cobra.Command{
 		Use:   "decode",
 		Short: "decode backupmeta to json",
+		Args:  cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ctx, cancel := context.WithCancel(GetDefaultContext())
 			defer cancel()
@@ -300,6 +303,7 @@ func encodeBackupMetaCommand() *cobra.Command {
 	encodeBackupMetaCmd := &cobra.Command{
 		Use:   "encode",
 		Short: "encode backupmeta json file to backupmeta",
+		Args:  cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ctx, cancel := context.WithCancel(GetDefaultContext())
 			defer cancel()
@@ -347,6 +351,7 @@ func setPDConfigCommand() *cobra.Command {
 	pdConfigCmd := &cobra.Command{
 		Use:   "reset-pd-config-as-default",
 		Short: "reset pd config adjusted by BR to default value",
+		Args:  cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ctx, cancel := context.WithCancel(GetDefaultContext())
 			defer cancel()
diff --git a/cmd/restore.go b/cmd/restore.go
index 58cbedb29..cb4d8d786 100644
--- a/cmd/restore.go
+++ b/cmd/restore.go
@@ -106,6 +106,7 @@ func newFullRestoreCommand() *cobra.Command {
 	command := &cobra.Command{
 		Use:   "full",
 		Short: "restore all tables",
+		Args:  cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			return runRestoreCommand(cmd, "Full restore")
 		},
@@ -118,6 +119,7 @@ func newDBRestoreCommand() *cobra.Command {
 	command := &cobra.Command{
 		Use:   "db",
 		Short: "restore tables in a database",
+		Args:  cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			return runRestoreCommand(cmd, "Database restore")
 		},
@@ -130,6 +132,7 @@ func newTableRestoreCommand() *cobra.Command {
 	command := &cobra.Command{
 		Use:   "table",
 		Short: "restore a table",
+		Args:  cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			return runRestoreCommand(cmd, "Table restore")
 		},
@@ -142,6 +145,7 @@ func newLogRestoreCommand() *cobra.Command {
 	command := &cobra.Command{
 		Use:   "cdclog",
 		Short: "(experimental) restore data from cdc log backup",
+		Args:  cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			return runLogRestoreCommand(cmd)
 		},
@@ -155,6 +159,7 @@ func newTiflashReplicaRestoreCommand() *cobra.Command {
 	command := &cobra.Command{
 		Use:   "tiflash-replica",
 		Short: "restore the tiflash replica removed by a failed restore of the older version BR",
+		Args:  cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			return runRestoreTiflashReplicaCommand(cmd, "Restore TiFlash Replica")
 		},
@@ -166,6 +171,7 @@ func newRawRestoreCommand() *cobra.Command {
 	command := &cobra.Command{
 		Use:   "raw",
 		Short: "(experimental) restore a raw kv range to TiKV cluster",
+		Args:  cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			return runRestoreRawCommand(cmd, "Raw restore")
 		},
diff --git a/pkg/task/common.go b/pkg/task/common.go
index 1f5e6aa64..569be489c 100644
--- a/pkg/task/common.go
+++ b/pkg/task/common.go
@@ -420,11 +420,11 @@ func flagToZapField(f *pflag.Flag) zap.Field {
 
 // LogArguments prints origin command arguments.
 func LogArguments(cmd *cobra.Command) {
-	var fields []zap.Field
-	cmd.Flags().VisitAll(func(f *pflag.Flag) {
-		if f.Changed {
-			fields = append(fields, flagToZapField(f))
-		}
+	flags := cmd.Flags()
+	fields := make([]zap.Field, 1, flags.NFlag()+1)
+	fields[0] = zap.String("__command", cmd.CommandPath())
+	flags.Visit(func(f *pflag.Flag) {
+		fields = append(fields, flagToZapField(f))
 	})
 	log.Info("arguments", fields...)
 }

From 11210bd2fa6c48d45a9861f5e407fe9e56438d91 Mon Sep 17 00:00:00 2001
From: 3pointer <luancheng@pingcap.com>
Date: Mon, 9 Nov 2020 18:01:44 +0800
Subject: [PATCH 4/6] cdclog: Fix bugs and add tests for cdclog feature (#573)

* fix restore year column

* cdclog: add unit test for recent bugfix

* cdclog: fix duplicate table bug

* cdclog: fix the bug that drop create table ddl failed

* cdclog: add test for dml&ddl test
---
 pkg/cdclog/buffer.go           |  5 +++
 pkg/cdclog/decoder.go          |  2 +-
 pkg/cdclog/decoder_test.go     | 25 ++++++++++++
 pkg/restore/log_client.go      | 38 ++++++++++++++++--
 pkg/restore/log_client_test.go | 72 ++++++++++++++++++++++++++++++++++
 tests/br_log_restore/run.sh    | 30 +++++++++++++-
 6 files changed, 167 insertions(+), 5 deletions(-)
 create mode 100644 pkg/restore/log_client_test.go

diff --git a/pkg/cdclog/buffer.go b/pkg/cdclog/buffer.go
index f508d5c5c..782253b62 100644
--- a/pkg/cdclog/buffer.go
+++ b/pkg/cdclog/buffer.go
@@ -74,6 +74,11 @@ func NewTableBuffer(tbl table.Table, allocators autoid.Allocators, flushKVPairs
 	return tb
 }
 
+// ResetTableInfo set tableInfo to nil for next reload.
+func (t *TableBuffer) ResetTableInfo() {
+	t.tableInfo = nil
+}
+
 // TableInfo returns the table info of this buffer.
 func (t *TableBuffer) TableInfo() table.Table {
 	return t.tableInfo
diff --git a/pkg/cdclog/decoder.go b/pkg/cdclog/decoder.go
index e68ecf88c..a20da7466 100644
--- a/pkg/cdclog/decoder.go
+++ b/pkg/cdclog/decoder.go
@@ -85,7 +85,7 @@ func (c Column) ToDatum() (types.Datum, error) {
 	)
 
 	switch c.Type {
-	case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong:
+	case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong, mysql.TypeYear:
 		val, err = c.Value.(json.Number).Int64()
 		if err != nil {
 			return types.Datum{}, errors.Trace(err)
diff --git a/pkg/cdclog/decoder_test.go b/pkg/cdclog/decoder_test.go
index 33a0af909..150b9fc88 100644
--- a/pkg/cdclog/decoder_test.go
+++ b/pkg/cdclog/decoder_test.go
@@ -15,9 +15,11 @@ package cdclog
 
 import (
 	"encoding/binary"
+	"encoding/json"
 	"testing"
 
 	"github.com/pingcap/check"
+	"github.com/pingcap/parser/mysql"
 )
 
 func Test(t *testing.T) { check.TestingT(t) }
@@ -141,3 +143,26 @@ func (s *batchSuite) TestDecoder(c *check.C) {
 		index++
 	}
 }
+
+func (s *batchSuite) TestColumn(c *check.C) {
+	// test varbinary columns (same type with varchar 15)
+	col1 := Column{Type: mysql.TypeVarchar, Flag: BinaryFlag, Value: "\\x00\\x01"}
+	col1 = formatColumnVal(col1)
+	dat, err := col1.ToDatum()
+	c.Assert(err, check.IsNil)
+	c.Assert(dat.GetString(), check.Equals, "\x00\x01")
+
+	// test binary columns (same type with varchar 254)
+	col2 := Column{Type: mysql.TypeString, Flag: BinaryFlag, Value: "test\\ttest"}
+	col2 = formatColumnVal(col2)
+	dat, err = col2.ToDatum()
+	c.Assert(err, check.IsNil)
+	c.Assert(dat.GetString(), check.Equals, "test\ttest")
+
+	// test year columns
+	val := json.Number("2020")
+	colYear := Column{Type: mysql.TypeYear, Value: val}
+	dat, err = colYear.ToDatum()
+	c.Assert(err, check.IsNil)
+	c.Assert(dat.GetInt64(), check.Equals, int64(2020))
+}
diff --git a/pkg/restore/log_client.go b/pkg/restore/log_client.go
index c546f85dc..676027d80 100644
--- a/pkg/restore/log_client.go
+++ b/pkg/restore/log_client.go
@@ -137,6 +137,12 @@ func NewLogRestoreClient(
 	return lc, nil
 }
 
+// ResetTSRange used for test.
+func (l *LogClient) ResetTSRange(startTS uint64, endTS uint64) {
+	l.startTS = startTS
+	l.endTS = endTS
+}
+
 func (l *LogClient) maybeTSInRange(ts uint64) bool {
 	// We choose the last event's ts as file name in cdclog when rotate.
 	// so even this file name's ts is larger than l.endTS,
@@ -218,6 +224,10 @@ func (l *LogClient) isDBRelatedDDL(ddl *cdclog.MessageDDL) bool {
 	return false
 }
 
+func (l *LogClient) isDropTable(ddl *cdclog.MessageDDL) bool {
+	return ddl.Type == model.ActionDropTable
+}
+
 func (l *LogClient) doDBDDLJob(ctx context.Context, ddls []string) error {
 	if len(ddls) == 0 {
 		log.Info("no ddls to restore")
@@ -258,7 +268,8 @@ func (l *LogClient) doDBDDLJob(ctx context.Context, ddls []string) error {
 	return nil
 }
 
-func (l *LogClient) needRestoreRowChange(fileName string) (bool, error) {
+// NeedRestoreRowChange determine whether to collect this file by ts range.
+func (l *LogClient) NeedRestoreRowChange(fileName string) (bool, error) {
 	if fileName == logPrefix {
 		// this file name appeared when file sink enabled
 		return true, nil
@@ -290,7 +301,22 @@ func (l *LogClient) collectRowChangeFiles(ctx context.Context) (map[int64][]stri
 
 	// need collect restore tableIDs
 	tableIDs := make([]int64, 0, len(l.meta.Names))
+
+	// we need remove duplicate table name in collection.
+	// when a table create and drop and create again.
+	// then we will have two different table id with same tables.
+	// we should keep the latest table id(larger table id), and filter the old one.
+	nameIDMap := make(map[string]int64)
 	for tableID, name := range l.meta.Names {
+		if tid, ok := nameIDMap[name]; ok {
+			if tid < tableID {
+				nameIDMap[name] = tableID
+			}
+		} else {
+			nameIDMap[name] = tableID
+		}
+	}
+	for name, tableID := range nameIDMap {
 		schema, table := ParseQuoteName(name)
 		if !l.tableFilter.MatchTable(schema, table) {
 			log.Info("filter tables",
@@ -313,7 +339,7 @@ func (l *LogClient) collectRowChangeFiles(ctx context.Context) (map[int64][]stri
 		}
 		err := l.restoreClient.storage.WalkDir(ctx, opt, func(path string, size int64) error {
 			fileName := filepath.Base(path)
-			shouldRestore, err := l.needRestoreRowChange(fileName)
+			shouldRestore, err := l.NeedRestoreRowChange(fileName)
 			if err != nil {
 				return err
 			}
@@ -839,11 +865,17 @@ func (l *LogClient) restoreTableFromPuller(
 			}
 			l.ddlLock.Unlock()
 
+			// if table dropped, we will pull next event to see if this table will create again.
+			// with next create table ddl, we can do reloadTableMeta.
+			if l.isDropTable(ddl) {
+				log.Info("[restoreFromPuller] skip reload because this is a drop table ddl", zap.String("ddl", ddl.Query))
+				l.tableBuffers[tableID].ResetTableInfo()
+				continue
+			}
 			err = l.reloadTableMeta(dom, tableID, item)
 			if err != nil {
 				return errors.Trace(err)
 			}
-
 		case cdclog.RowChanged:
 			if l.tableBuffers[tableID].TableInfo() == nil {
 				err = l.reloadTableMeta(dom, tableID, item)
diff --git a/pkg/restore/log_client_test.go b/pkg/restore/log_client_test.go
new file mode 100644
index 000000000..89752b5c3
--- /dev/null
+++ b/pkg/restore/log_client_test.go
@@ -0,0 +1,72 @@
+// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
+
+package restore_test
+
+import (
+	"context"
+	"math"
+
+	. "github.com/pingcap/check"
+	filter "github.com/pingcap/tidb-tools/pkg/table-filter"
+	"github.com/pingcap/tidb/util/testleak"
+
+	"github.com/pingcap/br/pkg/gluetidb"
+	"github.com/pingcap/br/pkg/mock"
+	"github.com/pingcap/br/pkg/restore"
+)
+
+type testLogRestoreSuite struct {
+	mock *mock.Cluster
+
+	client *restore.LogClient
+}
+
+var _ = Suite(&testLogRestoreSuite{})
+
+func (s *testLogRestoreSuite) SetUpSuite(c *C) {
+	var err error
+	s.mock, err = mock.NewCluster()
+	c.Assert(err, IsNil)
+	restoreClient, err := restore.NewRestoreClient(
+		gluetidb.New(), s.mock.PDClient, s.mock.Storage, nil, defaultKeepaliveCfg)
+	c.Assert(err, IsNil)
+
+	s.client, err = restore.NewLogRestoreClient(
+		context.Background(),
+		restoreClient,
+		0,
+		math.MaxInt64,
+		filter.NewSchemasFilter("test"),
+		8,
+		16,
+		5<<20,
+		16,
+	)
+	c.Assert(err, IsNil)
+}
+
+func (s *testLogRestoreSuite) TearDownSuite(c *C) {
+	testleak.AfterTest(c)()
+}
+
+func (s *testLogRestoreSuite) TestTsInRange(c *C) {
+	fileName1 := "cdclog.1"
+	s.client.ResetTSRange(1, 2)
+	collected, err := s.client.NeedRestoreRowChange(fileName1)
+	c.Assert(err, IsNil)
+	c.Assert(collected, IsTrue)
+
+	// cdclog.3 may have events in [1, 2]
+	// so we should collect it.
+	fileName2 := "cdclog.3"
+	s.client.ResetTSRange(1, 2)
+	collected, err = s.client.NeedRestoreRowChange(fileName2)
+	c.Assert(err, IsNil)
+	c.Assert(collected, IsTrue)
+
+	fileName3 := "cdclog.3"
+	s.client.ResetTSRange(4, 5)
+	collected, err = s.client.NeedRestoreRowChange(fileName3)
+	c.Assert(err, IsNil)
+	c.Assert(collected, IsFalse)
+}
diff --git a/tests/br_log_restore/run.sh b/tests/br_log_restore/run.sh
index 8e10970de..5713c211b 100755
--- a/tests/br_log_restore/run.sh
+++ b/tests/br_log_restore/run.sh
@@ -71,9 +71,26 @@ for i in $(seq $DB_COUNT); do
     row_count_ori[${i}]=$(run_sql "SELECT COUNT(*) FROM $DB${i}.$TABLE;" | awk '/COUNT/{print $2}')
 done
 
+# test drop & create schema/table, finally only db2 has one row
+run_sql "create schema ${DB}_DDL1;"
+run_sql "create table ${DB}_DDL1.t1 (a int primary key, b varchar(10));"
+run_sql "insert into ${DB}_DDL1.t1 values (1, 'x');"
+
+run_sql "drop schema ${DB}_DDL1;"
+run_sql "create schema ${DB}_DDL1;"
+run_sql "create schema ${DB}_DDL2;"
+
+run_sql "create table ${DB}_DDL2.t2 (a int primary key, b varchar(10));"
+run_sql "insert into ${DB}_DDl2.t2 values (2, 'x');"
+
+run_sql "drop table ${DB}_DDL2.t2;"
+run_sql "create table ${DB}_DDL2.t2 (a int primary key, b varchar(10));"
+run_sql "insert into ${DB}_DDL2.t2 values (3, 'x');"
+
 # sleep wait cdc log sync to storage
 # TODO find another way to check cdc log has synced
-sleep 30
+# need wait more time for cdc log synced, because we add some ddl.
+sleep 50
 
 # remove the change feed, because we don't want to record the drop ddl.
 echo "Y" | bin/cdc cli unsafe reset --pd=http://$PD_ADDR
@@ -81,6 +98,8 @@ echo "Y" | bin/cdc cli unsafe reset --pd=http://$PD_ADDR
 for i in $(seq $DB_COUNT); do
     run_sql "DROP DATABASE $DB${i};"
 done
+run_sql "DROP DATABASE ${DB}_DDL1"
+run_sql "DROP DATABASE ${DB}_DDL2"
 
 # restore full
 echo "restore start..."
@@ -92,6 +111,12 @@ for i in $(seq $DB_COUNT); do
 done
 
 fail=false
+row_count=$(run_sql "SELECT COUNT(*) FROM ${DB}_DDL2.t2 WHERE id=3;" | awk '/COUNT/{print $2}')
+if [ "$row_count" -ne 1 ]; then
+    fail=true
+    echo "TEST: [$TEST_NAME] fail on dml&ddl drop test."
+fi
+
 for i in $(seq $DB_COUNT); do
     if [ "${row_count_ori[i]}" != "${row_count_new[i]}" ];then
         fail=true
@@ -110,3 +135,6 @@ fi
 for i in $(seq $DB_COUNT); do
     run_sql "DROP DATABASE $DB${i};"
 done
+
+run_sql "DROP DATABASE ${DB}_DDL1"
+run_sql "DROP DATABASE ${DB}_DDL2"

From 5a4367555bb25cd4f3798b45994a0c20a56d67a0 Mon Sep 17 00:00:00 2001
From: 3pointer <luancheng@pingcap.com>
Date: Tue, 10 Nov 2020 14:07:49 +0800
Subject: [PATCH 5/6] CI: remove unecessary coverage (#589)

* remove uncessary coverage

* add target for codecov

* update codecov

* update codecov

* update codecov
---
 .codecov.yml | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/.codecov.yml b/.codecov.yml
index 674895cd1..9fc8064fa 100644
--- a/.codecov.yml
+++ b/.codecov.yml
@@ -1,10 +1,22 @@
 codecov:
   require_ci_to_pass: yes
 
+comment:
+  layout: "reach, diff, flags, files"
+  behavior: default
+  require_changes: false  # if true: only post the comment if coverage changes
+  require_base: no        # [yes :: must have a base report to post]
+  require_head: yes       # [yes :: must have a head report to post]
+  branches:               # branch names that can post comment
+    - "master"
+
 coverage:
   status:
     project:
       default:
         # Allow the coverage to drop by 3%
+        target: 85%
         threshold: 3%
+        branches:
+          - master
     patch: off

From a3517d674652df83c526b54ff5c90a3b5cdf59a3 Mon Sep 17 00:00:00 2001
From: Lonng <heng@lonng.org>
Date: Tue, 10 Nov 2020 14:50:50 +0800
Subject: [PATCH 6/6] Update the errors/parser dependence to the latest version
 (#593)

Signed-off-by: Lonng <heng@lonng.org>
---
 errors.toml  |  2 +-
 go.mod1      |  6 +++---
 go.sum1      | 12 ++++++------
 tools/go.mod |  2 +-
 tools/go.sum |  2 ++
 5 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/errors.toml b/errors.toml
index 79ac8ce67..ea431923d 100644
--- a/errors.toml
+++ b/errors.toml
@@ -1,5 +1,5 @@
 # AUTOGENERATED BY github.com/pingcap/tiup/components/errdoc/errdoc-gen
-# DO NOT EDIT THIS FILE, PLEASE CHANGE ERROR DEFINITION IF CONTENT IMPROPER.
+# YOU CAN CHANGE THE 'description'/'workaround' FIELDS IF THEM ARE IMPROPER.
 
 ["BR:Backup:ErrBackupChecksumMismatch"]
 error = '''
diff --git a/go.mod1 b/go.mod1
index 3b6738f0e..e03ff54b4 100644
--- a/go.mod1
+++ b/go.mod1
@@ -14,12 +14,12 @@ require (
 	github.com/google/btree v1.0.0
 	github.com/google/uuid v1.1.1
 	github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712
-	github.com/pingcap/errors v0.11.5-0.20201021055732-210aacd3fd99
+	github.com/pingcap/errors v0.11.5-0.20201029093017-5a7df2af2ac7
 	github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce
 	github.com/pingcap/kvproto v0.0.0-20201023092649-e6d6090277c9
 	github.com/pingcap/log v0.0.0-20200828042413-fce0951f1463
-	github.com/pingcap/parser v0.0.0-20201028030005-1328d877c9f3
-	github.com/pingcap/tidb v1.1.0-beta.0.20201031133255-280e8aa35f98
+	github.com/pingcap/parser v0.0.0-20201109022253-d384bee1451e
+	github.com/pingcap/tidb v0.0.0-20201110015039-8d35f17c17f3
 	github.com/pingcap/tidb-tools v4.0.5-0.20200820092506-34ea90c93237+incompatible
 	github.com/pingcap/tipb v0.0.0-20201026044621-45e60c77588f
 	github.com/prometheus/client_golang v1.5.1
diff --git a/go.sum1 b/go.sum1
index 036c5399b..2456b62cc 100644
--- a/go.sum1
+++ b/go.sum1
@@ -513,8 +513,8 @@ github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTw
 github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
 github.com/pingcap/errors v0.11.5-0.20200902104258-eba4f1d8f6de h1:mW8hC2yXTpflfyTeJgcN4aJQfwcYODde8YgjBgAy6do=
 github.com/pingcap/errors v0.11.5-0.20200902104258-eba4f1d8f6de/go.mod h1:g4vx//d6VakjJ0mk7iLBlKA8LFavV/sAVINT/1PFxeQ=
-github.com/pingcap/errors v0.11.5-0.20201021055732-210aacd3fd99 h1:PVuEvTi/LlviMG7X3av44NRwcdPf0tiqL/YdVOIKCpA=
-github.com/pingcap/errors v0.11.5-0.20201021055732-210aacd3fd99/go.mod h1:G7x87le1poQzLB/TqvTJI2ILrSgobnq4Ut7luOwvfvI=
+github.com/pingcap/errors v0.11.5-0.20201029093017-5a7df2af2ac7 h1:wQKuKP2HUtej2gSvx1cZmY4DENUH6tlOxRkfvPT8EBU=
+github.com/pingcap/errors v0.11.5-0.20201029093017-5a7df2af2ac7/go.mod h1:G7x87le1poQzLB/TqvTJI2ILrSgobnq4Ut7luOwvfvI=
 github.com/pingcap/failpoint v0.0.0-20191029060244-12f4ac2fd11d h1:F8vp38kTAckN+v8Jlc98uMBvKIzr1a+UhnLyVYn8Q5Q=
 github.com/pingcap/failpoint v0.0.0-20191029060244-12f4ac2fd11d/go.mod h1:DNS3Qg7bEDhU6EXNHF+XSv/PGznQaMJ5FWvctpm6pQI=
 github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce h1:Y1kCxlCtlPTMtVcOkjUcuQKh+YrluSo7+7YMCQSzy30=
@@ -537,15 +537,15 @@ github.com/pingcap/log v0.0.0-20200511115504-543df19646ad h1:SveG82rmu/GFxYanffx
 github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
 github.com/pingcap/log v0.0.0-20200828042413-fce0951f1463 h1:Jboj+s4jSCp5E1WDgmRUv5rIFKFHaaSWuSZ4wMwXIcc=
 github.com/pingcap/log v0.0.0-20200828042413-fce0951f1463/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
-github.com/pingcap/parser v0.0.0-20201028030005-1328d877c9f3 h1:dfdPB1Ot9cNki/hVUgWFUiM8b05b5JCw7Oq9x6HaDeM=
-github.com/pingcap/parser v0.0.0-20201028030005-1328d877c9f3/go.mod h1:74+OEdwM4B/jMpBRl92ch6CSmSYkQtv2TNxIjFdT/GE=
+github.com/pingcap/parser v0.0.0-20201109022253-d384bee1451e h1:7AXo1anjf9UqG0dvCy9+onp3bQDJjaN8IkSrKErTv3k=
+github.com/pingcap/parser v0.0.0-20201109022253-d384bee1451e/go.mod h1:GbEr2PgY72/4XqPZzmzstlOU/+il/wrjeTNFs6ihsSE=
 github.com/pingcap/sysutil v0.0.0-20200206130906-2bfa6dc40bcd/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI=
 github.com/pingcap/sysutil v0.0.0-20200715082929-4c47bcac246a h1:i2RElJ2aykSqZKeY+3SK18NHhajil8cQdG77wHe+P1Y=
 github.com/pingcap/sysutil v0.0.0-20200715082929-4c47bcac246a/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI=
 github.com/pingcap/sysutil v0.0.0-20201021075216-f93ced2829e2 h1:b2G/eqDeywtdJF3w9nIUdqMmXChsmpLvf4FzUxJ9Vmk=
 github.com/pingcap/sysutil v0.0.0-20201021075216-f93ced2829e2/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI=
-github.com/pingcap/tidb v1.1.0-beta.0.20201031133255-280e8aa35f98 h1:PzWcKNA0+TJjtK88s5vp2PxJNunKhO/LvGwmf49FUUo=
-github.com/pingcap/tidb v1.1.0-beta.0.20201031133255-280e8aa35f98/go.mod h1:gGwitKqKAMNJ+bMNOVF9Ka3EwIJCJIGq7cYCXA0yLxE=
+github.com/pingcap/tidb v0.0.0-20201110015039-8d35f17c17f3 h1:Hh4EYC//LNmCI3VW7uebGu9g1eOLnccgU1o39ELQREE=
+github.com/pingcap/tidb v0.0.0-20201110015039-8d35f17c17f3/go.mod h1:KB3TiWknHfJYMjms4jQdkbVNOZ8gkh3gcWmlZp/97sM=
 github.com/pingcap/tidb-tools v4.0.5-0.20200820092506-34ea90c93237+incompatible h1:qPppnsXVh3KswqRZdSAShGLLPd7dB+5w4lXDnpYn0SQ=
 github.com/pingcap/tidb-tools v4.0.5-0.20200820092506-34ea90c93237+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
 github.com/pingcap/tipb v0.0.0-20201026044621-45e60c77588f h1:J+0TAI+7Hvebz4bM4GnRCRT4MpjYnUxbyi9ky5ZQUsU=
diff --git a/tools/go.mod b/tools/go.mod
index 2c7b80243..fbf708eb8 100644
--- a/tools/go.mod
+++ b/tools/go.mod
@@ -8,6 +8,6 @@ require (
 	github.com/golangci/golangci-lint v1.26.0
 	github.com/mgechev/revive v1.0.2
 	github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce
-	github.com/pingcap/tiup v1.2.1-0.20201020064426-ad64b15d57de
+	github.com/pingcap/tiup v1.2.3
 	golang.org/x/tools v0.0.0-20200527183253-8e7acdbce89d
 )
diff --git a/tools/go.sum b/tools/go.sum
index 7f8117c68..49beec76a 100644
--- a/tools/go.sum
+++ b/tools/go.sum
@@ -673,6 +673,8 @@ github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330/go.mod h1:RtkHW8WbcNx
 github.com/pingcap/tipb v0.0.0-20200417094153-7316d94df1ee/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI=
 github.com/pingcap/tiup v1.2.1-0.20201020064426-ad64b15d57de h1:9Q/AxtNRoA9WmTSnavia3oWrIOSQxDKxXNI0MDbqQ24=
 github.com/pingcap/tiup v1.2.1-0.20201020064426-ad64b15d57de/go.mod h1:q8WzflNHjE1U49k2qstTL0clx2pKh8pkOzUFV4RTvQo=
+github.com/pingcap/tiup v1.2.3 h1:8OCQF7sHhT6VqE8pZU1JTSogPA90OFuWWM/B746x0YY=
+github.com/pingcap/tiup v1.2.3/go.mod h1:q8WzflNHjE1U49k2qstTL0clx2pKh8pkOzUFV4RTvQo=
 github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=