Merge branch 'master' into pd_temp_config

.codecov.yml

@@ -1,10 +1,22 @@
 codecov:
   require_ci_to_pass: yes
 
+comment:
+  layout: "reach, diff, flags, files"
+  behavior: default
+  require_changes: false  # if true: only post the comment if coverage changes
+  require_base: no        # [yes :: must have a base report to post]
+  require_head: yes       # [yes :: must have a head report to post]
+  branches:               # branch names that can post comment
+    - "master"
+
 coverage:
   status:
     project:
       default:
         # Allow the coverage to drop by 3%
+        target: 85%
         threshold: 3%
+        branches:
+          - master
     patch: off

cmd/backup.go

@@ -79,6 +79,9 @@ func newFullBackupCommand() *cobra.Command {
 	command := &cobra.Command{
 		Use:   "full",
 		Short: "backup all database",
+		// prevents incorrect usage like `--checksum false` instead of `--checksum=false`.
+		// the former, according to pflag parsing rules, means `--checksum=true false`.
+		Args: cobra.NoArgs,
 		RunE: func(command *cobra.Command, _ []string) error {
 			// empty db/table means full backup.
 			return runBackupCommand(command, "Full backup")
@@ -93,6 +96,7 @@ func newDBBackupCommand() *cobra.Command {
 	command := &cobra.Command{
 		Use:   "db",
 		Short: "backup a database",
+		Args:  cobra.NoArgs,
 		RunE: func(command *cobra.Command, _ []string) error {
 			return runBackupCommand(command, "Database backup")
 		},
@@ -106,6 +110,7 @@ func newTableBackupCommand() *cobra.Command {
 	command := &cobra.Command{
 		Use:   "table",
 		Short: "backup a table",
+		Args:  cobra.NoArgs,
 		RunE: func(command *cobra.Command, _ []string) error {
 			return runBackupCommand(command, "Table backup")
 		},
@@ -120,6 +125,7 @@ func newRawBackupCommand() *cobra.Command {
 	command := &cobra.Command{
 		Use:   "raw",
 		Short: "(experimental) backup a raw kv range from TiKV cluster",
+		Args:  cobra.NoArgs,
 		RunE: func(command *cobra.Command, _ []string) error {
 			return runBackupRawCommand(command, "Raw backup")
 		},

cmd/debug.go

@@ -60,6 +60,7 @@ func newCheckSumCommand() *cobra.Command {
 	command := &cobra.Command{
 		Use:   "checksum",
 		Short: "check the backup data",
+		Args:  cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			ctx, cancel := context.WithCancel(GetDefaultContext())
 			defer cancel()
@@ -144,6 +145,7 @@ func newBackupMetaCommand() *cobra.Command {
 	command := &cobra.Command{
 		Use:   "backupmeta",
 		Short: "check the backup meta",
+		Args:  cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			ctx, cancel := context.WithCancel(GetDefaultContext())
 			defer cancel()
@@ -240,6 +242,7 @@ func decodeBackupMetaCommand() *cobra.Command {
 	decodeBackupMetaCmd := &cobra.Command{
 		Use:   "decode",
 		Short: "decode backupmeta to json",
+		Args:  cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ctx, cancel := context.WithCancel(GetDefaultContext())
 			defer cancel()
@@ -300,6 +303,7 @@ func encodeBackupMetaCommand() *cobra.Command {
 	encodeBackupMetaCmd := &cobra.Command{
 		Use:   "encode",
 		Short: "encode backupmeta json file to backupmeta",
+		Args:  cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ctx, cancel := context.WithCancel(GetDefaultContext())
 			defer cancel()
@@ -347,6 +351,7 @@ func setPDConfigCommand() *cobra.Command {
 	pdConfigCmd := &cobra.Command{
 		Use:   "reset-pd-config-as-default",
 		Short: "reset pd config adjusted by BR to default value",
+		Args:  cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ctx, cancel := context.WithCancel(GetDefaultContext())
 			defer cancel()

cmd/restore.go

@@ -106,6 +106,7 @@ func newFullRestoreCommand() *cobra.Command {
 	command := &cobra.Command{
 		Use:   "full",
 		Short: "restore all tables",
+		Args:  cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			return runRestoreCommand(cmd, "Full restore")
 		},
@@ -118,6 +119,7 @@ func newDBRestoreCommand() *cobra.Command {
 	command := &cobra.Command{
 		Use:   "db",
 		Short: "restore tables in a database",
+		Args:  cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			return runRestoreCommand(cmd, "Database restore")
 		},
@@ -130,6 +132,7 @@ func newTableRestoreCommand() *cobra.Command {
 	command := &cobra.Command{
 		Use:   "table",
 		Short: "restore a table",
+		Args:  cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			return runRestoreCommand(cmd, "Table restore")
 		},
@@ -142,6 +145,7 @@ func newLogRestoreCommand() *cobra.Command {
 	command := &cobra.Command{
 		Use:   "cdclog",
 		Short: "(experimental) restore data from cdc log backup",
+		Args:  cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			return runLogRestoreCommand(cmd)
 		},
@@ -155,6 +159,7 @@ func newTiflashReplicaRestoreCommand() *cobra.Command {
 	command := &cobra.Command{
 		Use:   "tiflash-replica",
 		Short: "restore the tiflash replica removed by a failed restore of the older version BR",
+		Args:  cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			return runRestoreTiflashReplicaCommand(cmd, "Restore TiFlash Replica")
 		},
@@ -166,6 +171,7 @@ func newRawRestoreCommand() *cobra.Command {
 	command := &cobra.Command{
 		Use:   "raw",
 		Short: "(experimental) restore a raw kv range to TiKV cluster",
+		Args:  cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			return runRestoreRawCommand(cmd, "Raw restore")
 		},

errors.toml

@@ -1,5 +1,5 @@
 # AUTOGENERATED BY github.com/pingcap/tiup/components/errdoc/errdoc-gen
-# DO NOT EDIT THIS FILE, PLEASE CHANGE ERROR DEFINITION IF CONTENT IMPROPER.
+# YOU CAN CHANGE THE 'description'/'workaround' FIELDS IF THEM ARE IMPROPER.
 
 ["BR:Backup:ErrBackupChecksumMismatch"]
 error = '''

go.mod1

@@ -14,12 +14,12 @@ require (
 	github.com/google/btree v1.0.0
 	github.com/google/uuid v1.1.1
 	github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712
-	github.com/pingcap/errors v0.11.5-0.20201021055732-210aacd3fd99
+	github.com/pingcap/errors v0.11.5-0.20201029093017-5a7df2af2ac7
 	github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce
 	github.com/pingcap/kvproto v0.0.0-20201023092649-e6d6090277c9
 	github.com/pingcap/log v0.0.0-20200828042413-fce0951f1463
-	github.com/pingcap/parser v0.0.0-20201028030005-1328d877c9f3
-	github.com/pingcap/tidb v1.1.0-beta.0.20201031133255-280e8aa35f98
+	github.com/pingcap/parser v0.0.0-20201109022253-d384bee1451e
+	github.com/pingcap/tidb v0.0.0-20201110015039-8d35f17c17f3
 	github.com/pingcap/tidb-tools v4.0.5-0.20200820092506-34ea90c93237+incompatible
 	github.com/pingcap/tipb v0.0.0-20201026044621-45e60c77588f
 	github.com/prometheus/client_golang v1.5.1

go.sum1

@@ -513,8 +513,8 @@ github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTw
 github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
 github.com/pingcap/errors v0.11.5-0.20200902104258-eba4f1d8f6de h1:mW8hC2yXTpflfyTeJgcN4aJQfwcYODde8YgjBgAy6do=
 github.com/pingcap/errors v0.11.5-0.20200902104258-eba4f1d8f6de/go.mod h1:g4vx//d6VakjJ0mk7iLBlKA8LFavV/sAVINT/1PFxeQ=
-github.com/pingcap/errors v0.11.5-0.20201021055732-210aacd3fd99 h1:PVuEvTi/LlviMG7X3av44NRwcdPf0tiqL/YdVOIKCpA=
-github.com/pingcap/errors v0.11.5-0.20201021055732-210aacd3fd99/go.mod h1:G7x87le1poQzLB/TqvTJI2ILrSgobnq4Ut7luOwvfvI=
+github.com/pingcap/errors v0.11.5-0.20201029093017-5a7df2af2ac7 h1:wQKuKP2HUtej2gSvx1cZmY4DENUH6tlOxRkfvPT8EBU=
+github.com/pingcap/errors v0.11.5-0.20201029093017-5a7df2af2ac7/go.mod h1:G7x87le1poQzLB/TqvTJI2ILrSgobnq4Ut7luOwvfvI=
 github.com/pingcap/failpoint v0.0.0-20191029060244-12f4ac2fd11d h1:F8vp38kTAckN+v8Jlc98uMBvKIzr1a+UhnLyVYn8Q5Q=
 github.com/pingcap/failpoint v0.0.0-20191029060244-12f4ac2fd11d/go.mod h1:DNS3Qg7bEDhU6EXNHF+XSv/PGznQaMJ5FWvctpm6pQI=
 github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce h1:Y1kCxlCtlPTMtVcOkjUcuQKh+YrluSo7+7YMCQSzy30=
@@ -537,15 +537,15 @@ github.com/pingcap/log v0.0.0-20200511115504-543df19646ad h1:SveG82rmu/GFxYanffx
 github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
 github.com/pingcap/log v0.0.0-20200828042413-fce0951f1463 h1:Jboj+s4jSCp5E1WDgmRUv5rIFKFHaaSWuSZ4wMwXIcc=
 github.com/pingcap/log v0.0.0-20200828042413-fce0951f1463/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
-github.com/pingcap/parser v0.0.0-20201028030005-1328d877c9f3 h1:dfdPB1Ot9cNki/hVUgWFUiM8b05b5JCw7Oq9x6HaDeM=
-github.com/pingcap/parser v0.0.0-20201028030005-1328d877c9f3/go.mod h1:74+OEdwM4B/jMpBRl92ch6CSmSYkQtv2TNxIjFdT/GE=
+github.com/pingcap/parser v0.0.0-20201109022253-d384bee1451e h1:7AXo1anjf9UqG0dvCy9+onp3bQDJjaN8IkSrKErTv3k=
+github.com/pingcap/parser v0.0.0-20201109022253-d384bee1451e/go.mod h1:GbEr2PgY72/4XqPZzmzstlOU/+il/wrjeTNFs6ihsSE=
 github.com/pingcap/sysutil v0.0.0-20200206130906-2bfa6dc40bcd/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI=
 github.com/pingcap/sysutil v0.0.0-20200715082929-4c47bcac246a h1:i2RElJ2aykSqZKeY+3SK18NHhajil8cQdG77wHe+P1Y=
 github.com/pingcap/sysutil v0.0.0-20200715082929-4c47bcac246a/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI=
 github.com/pingcap/sysutil v0.0.0-20201021075216-f93ced2829e2 h1:b2G/eqDeywtdJF3w9nIUdqMmXChsmpLvf4FzUxJ9Vmk=
 github.com/pingcap/sysutil v0.0.0-20201021075216-f93ced2829e2/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI=
-github.com/pingcap/tidb v1.1.0-beta.0.20201031133255-280e8aa35f98 h1:PzWcKNA0+TJjtK88s5vp2PxJNunKhO/LvGwmf49FUUo=
-github.com/pingcap/tidb v1.1.0-beta.0.20201031133255-280e8aa35f98/go.mod h1:gGwitKqKAMNJ+bMNOVF9Ka3EwIJCJIGq7cYCXA0yLxE=
+github.com/pingcap/tidb v0.0.0-20201110015039-8d35f17c17f3 h1:Hh4EYC//LNmCI3VW7uebGu9g1eOLnccgU1o39ELQREE=
+github.com/pingcap/tidb v0.0.0-20201110015039-8d35f17c17f3/go.mod h1:KB3TiWknHfJYMjms4jQdkbVNOZ8gkh3gcWmlZp/97sM=
 github.com/pingcap/tidb-tools v4.0.5-0.20200820092506-34ea90c93237+incompatible h1:qPppnsXVh3KswqRZdSAShGLLPd7dB+5w4lXDnpYn0SQ=
 github.com/pingcap/tidb-tools v4.0.5-0.20200820092506-34ea90c93237+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
 github.com/pingcap/tipb v0.0.0-20201026044621-45e60c77588f h1:J+0TAI+7Hvebz4bM4GnRCRT4MpjYnUxbyi9ky5ZQUsU=

pkg/cdclog/buffer.go

@@ -74,6 +74,11 @@ func NewTableBuffer(tbl table.Table, allocators autoid.Allocators, flushKVPairs
 	return tb
 }
 
+// ResetTableInfo set tableInfo to nil for next reload.
+func (t *TableBuffer) ResetTableInfo() {
+	t.tableInfo = nil
+}
+
 // TableInfo returns the table info of this buffer.
 func (t *TableBuffer) TableInfo() table.Table {
 	return t.tableInfo

pkg/cdclog/decoder.go

@@ -85,7 +85,7 @@ func (c Column) ToDatum() (types.Datum, error) {
 	)
 
 	switch c.Type {
-	case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong:
+	case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong, mysql.TypeYear:
 		val, err = c.Value.(json.Number).Int64()
 		if err != nil {
 			return types.Datum{}, errors.Trace(err)

pkg/cdclog/decoder_test.go

@@ -15,9 +15,11 @@ package cdclog
 
 import (
 	"encoding/binary"
+	"encoding/json"
 	"testing"
 
 	"github.com/pingcap/check"
+	"github.com/pingcap/parser/mysql"
 )
 
 func Test(t *testing.T) { check.TestingT(t) }
@@ -141,3 +143,26 @@ func (s *batchSuite) TestDecoder(c *check.C) {
 		index++
 	}
 }
+
+func (s *batchSuite) TestColumn(c *check.C) {
+	// test varbinary columns (same type with varchar 15)
+	col1 := Column{Type: mysql.TypeVarchar, Flag: BinaryFlag, Value: "\\x00\\x01"}
+	col1 = formatColumnVal(col1)
+	dat, err := col1.ToDatum()
+	c.Assert(err, check.IsNil)
+	c.Assert(dat.GetString(), check.Equals, "\x00\x01")
+
+	// test binary columns (same type with varchar 254)
+	col2 := Column{Type: mysql.TypeString, Flag: BinaryFlag, Value: "test\\ttest"}
+	col2 = formatColumnVal(col2)
+	dat, err = col2.ToDatum()
+	c.Assert(err, check.IsNil)
+	c.Assert(dat.GetString(), check.Equals, "test\ttest")
+
+	// test year columns
+	val := json.Number("2020")
+	colYear := Column{Type: mysql.TypeYear, Value: val}
+	dat, err = colYear.ToDatum()
+	c.Assert(err, check.IsNil)
+	c.Assert(dat.GetInt64(), check.Equals, int64(2020))
+}

pkg/kv/checksum_test.go

@@ -22,20 +22,20 @@ import (
 	"github.com/pingcap/br/pkg/kv"
 )
 
-type testKVChcksumSuite struct{}
+type testKVChecksumSuite struct{}
 
-func (s *testKVChcksumSuite) SetUpSuite(c *C)    {}
-func (s *testKVChcksumSuite) TearDownSuite(c *C) {}
+func (s *testKVChecksumSuite) SetUpSuite(c *C)    {}
+func (s *testKVChecksumSuite) TearDownSuite(c *C) {}
 
-var _ = Suite(&testKVChcksumSuite{})
+var _ = Suite(&testKVChecksumSuite{})
 
-func TestKVChcksum(t *testing.T) {
+func TestKVChecksum(t *testing.T) {
 	TestingT(t)
 }
 
 func uint64NotEqual(a uint64, b uint64) bool { return a != b }
 
-func (s *testKVChcksumSuite) TestChcksum(c *C) {
+func (s *testKVChecksumSuite) TestChecksum(c *C) {
 	checksum := kv.NewKVChecksum(0)
 	c.Assert(checksum.Sum(), Equals, uint64(0))
 
@@ -47,7 +47,7 @@ func (s *testKVChcksumSuite) TestChcksum(c *C) {
 	c.Assert(checksum.Sum(), Equals, uint64(0))
 
 	// checksum on real data
-	excpectChecksum := uint64(4850203904608948940)
+	expectChecksum := uint64(4850203904608948940)
 
 	kvs := []kv.Pair{
 		{
@@ -68,16 +68,16 @@ func (s *testKVChcksumSuite) TestChcksum(c *C) {
 	}
 	c.Assert(checksum.SumSize(), Equals, kvBytes)
 	c.Assert(checksum.SumKVS(), Equals, uint64(len(kvs)))
-	c.Assert(checksum.Sum(), Equals, excpectChecksum)
+	c.Assert(checksum.Sum(), Equals, expectChecksum)
 
 	// recompute on same key-value
 	checksum.Update(kvs)
 	c.Assert(checksum.SumSize(), Equals, kvBytes<<1)
 	c.Assert(checksum.SumKVS(), Equals, uint64(len(kvs))<<1)
-	c.Assert(uint64NotEqual(checksum.Sum(), excpectChecksum), IsTrue)
+	c.Assert(uint64NotEqual(checksum.Sum(), expectChecksum), IsTrue)
 }
 
-func (s *testKVChcksumSuite) TestChecksumJSON(c *C) {
+func (s *testKVChecksumSuite) TestChecksumJSON(c *C) {
 	testStruct := &struct {
 		Checksum kv.Checksum
 	}{

pkg/restore/log_client.go

@@ -137,6 +137,12 @@ func NewLogRestoreClient(
 	return lc, nil
 }
 
+// ResetTSRange used for test.
+func (l *LogClient) ResetTSRange(startTS uint64, endTS uint64) {
+	l.startTS = startTS
+	l.endTS = endTS
+}
+
 func (l *LogClient) maybeTSInRange(ts uint64) bool {
 	// We choose the last event's ts as file name in cdclog when rotate.
 	// so even this file name's ts is larger than l.endTS,
@@ -218,6 +224,10 @@ func (l *LogClient) isDBRelatedDDL(ddl *cdclog.MessageDDL) bool {
 	return false
 }
 
+func (l *LogClient) isDropTable(ddl *cdclog.MessageDDL) bool {
+	return ddl.Type == model.ActionDropTable
+}
+
 func (l *LogClient) doDBDDLJob(ctx context.Context, ddls []string) error {
 	if len(ddls) == 0 {
 		log.Info("no ddls to restore")
@@ -258,7 +268,8 @@ func (l *LogClient) doDBDDLJob(ctx context.Context, ddls []string) error {
 	return nil
 }
 
-func (l *LogClient) needRestoreRowChange(fileName string) (bool, error) {
+// NeedRestoreRowChange determine whether to collect this file by ts range.
+func (l *LogClient) NeedRestoreRowChange(fileName string) (bool, error) {
 	if fileName == logPrefix {
 		// this file name appeared when file sink enabled
 		return true, nil
@@ -290,7 +301,22 @@ func (l *LogClient) collectRowChangeFiles(ctx context.Context) (map[int64][]stri
 
 	// need collect restore tableIDs
 	tableIDs := make([]int64, 0, len(l.meta.Names))
+
+	// we need remove duplicate table name in collection.
+	// when a table create and drop and create again.
+	// then we will have two different table id with same tables.
+	// we should keep the latest table id(larger table id), and filter the old one.
+	nameIDMap := make(map[string]int64)
 	for tableID, name := range l.meta.Names {
+		if tid, ok := nameIDMap[name]; ok {
+			if tid < tableID {
+				nameIDMap[name] = tableID
+			}
+		} else {
+			nameIDMap[name] = tableID
+		}
+	}
+	for name, tableID := range nameIDMap {
 		schema, table := ParseQuoteName(name)
 		if !l.tableFilter.MatchTable(schema, table) {
 			log.Info("filter tables",
@@ -313,7 +339,7 @@ func (l *LogClient) collectRowChangeFiles(ctx context.Context) (map[int64][]stri
 		}
 		err := l.restoreClient.storage.WalkDir(ctx, opt, func(path string, size int64) error {
 			fileName := filepath.Base(path)
-			shouldRestore, err := l.needRestoreRowChange(fileName)
+			shouldRestore, err := l.NeedRestoreRowChange(fileName)
 			if err != nil {
 				return err
 			}
@@ -839,11 +865,17 @@ func (l *LogClient) restoreTableFromPuller(
 			}
 			l.ddlLock.Unlock()
 
+			// if table dropped, we will pull next event to see if this table will create again.
+			// with next create table ddl, we can do reloadTableMeta.
+			if l.isDropTable(ddl) {
+				log.Info("[restoreFromPuller] skip reload because this is a drop table ddl", zap.String("ddl", ddl.Query))
+				l.tableBuffers[tableID].ResetTableInfo()
+				continue
+			}
 			err = l.reloadTableMeta(dom, tableID, item)
 			if err != nil {
 				return errors.Trace(err)
 			}
-
 		case cdclog.RowChanged:
 			if l.tableBuffers[tableID].TableInfo() == nil {
 				err = l.reloadTableMeta(dom, tableID, item)