task: expose table filter for backup full and restore full (pingc…

…ap#313)

* task: expose table filter for `backup full` and `restore full`

* task: move the invalid backup check out

* task: removed a useless `if err != nil` check

Co-authored-by: pingcap-github-bot <sre-bot@pingcap.com>

cmd/backup.go

@@ -84,6 +84,7 @@ func newFullBackupCommand() *cobra.Command {
 			return runBackupCommand(command, "Full backup")
 		},
 	}
+	task.DefineFilterFlags(command)
 	return command
 }
 

cmd/restore.go

@@ -96,6 +96,7 @@ func newFullRestoreCommand() *cobra.Command {
 			return runRestoreCommand(cmd, "Full restore")
 		},
 	}
+	task.DefineFilterFlags(command)
 	return command
 }
 

go.mod

@@ -20,7 +20,7 @@ require (
 	github.com/pingcap/parser v0.0.0-20200603032439-c4ecb4508d2f
 	github.com/pingcap/pd/v4 v4.0.0-rc.2.0.20200520083007-2c251bd8f181
 	github.com/pingcap/tidb v1.1.0-beta.0.20200606093724-b5b4da0e6a90
-	github.com/pingcap/tidb-tools v4.0.0-rc.1.0.20200514040632-f76b3e428e19+incompatible
+	github.com/pingcap/tidb-tools v4.0.0-rc.2.0.20200521050818-6dd445d83fe0+incompatible
 	github.com/pingcap/tipb v0.0.0-20200417094153-7316d94df1ee
 	github.com/prometheus/client_golang v1.5.1
 	github.com/prometheus/common v0.9.1

go.sum

@@ -491,6 +491,8 @@ github.com/pingcap/tidb-tools v4.0.0-rc.1.0.20200421113014-507d2bb3a15e+incompat
 github.com/pingcap/tidb-tools v4.0.0-rc.1.0.20200421113014-507d2bb3a15e+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
 github.com/pingcap/tidb-tools v4.0.0-rc.1.0.20200514040632-f76b3e428e19+incompatible h1:/JKsYjsa5Ug8v5CN4zIbJGIqsvgBUkGwaP/rEScVvWM=
 github.com/pingcap/tidb-tools v4.0.0-rc.1.0.20200514040632-f76b3e428e19+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
+github.com/pingcap/tidb-tools v4.0.0-rc.2.0.20200521050818-6dd445d83fe0+incompatible h1:e+j+rsJYX+J7eTkgjnGBH2/T3NS6GNSPD6nHA5bPdCI=
+github.com/pingcap/tidb-tools v4.0.0-rc.2.0.20200521050818-6dd445d83fe0+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
 github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI=
 github.com/pingcap/tipb v0.0.0-20200417094153-7316d94df1ee h1:XJQ6/LGzOSc/jo33AD8t7jtc4GohxcyODsYnb+kZXJM=
 github.com/pingcap/tipb v0.0.0-20200417094153-7316d94df1ee/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI=

pkg/backup/client.go

@@ -18,7 +18,7 @@ import (
 	"github.com/pingcap/log"
 	"github.com/pingcap/parser/model"
 	pd "github.com/pingcap/pd/v4/client"
-	"github.com/pingcap/tidb-tools/pkg/filter"
+	"github.com/pingcap/tidb-tools/pkg/table-filter"
 	"github.com/pingcap/tidb/distsql"
 	"github.com/pingcap/tidb/domain"
 	"github.com/pingcap/tidb/kv"
@@ -225,7 +225,7 @@ func appendRanges(tbl *model.TableInfo, tblID int64) ([]kv.KeyRange, error) {
 func BuildBackupRangeAndSchema(
 	dom *domain.Domain,
 	storage kv.Storage,
-	tableFilter *filter.Filter,
+	tableFilter filter.Filter,
 	backupTS uint64,
 ) ([]rtree.Range, *Schemas, error) {
 	info, err := dom.GetSnapshotInfoSchema(backupTS)
@@ -247,7 +247,7 @@ func BuildBackupRangeAndSchema(
 		randAlloc := autoid.NewAllocator(storage, dbInfo.ID, false, autoid.AutoRandomType)
 
 		for _, tableInfo := range dbInfo.Tables {
-			if !tableFilter.Match(&filter.Table{Schema: dbInfo.Name.L, Name: tableInfo.Name.L}) {
+			if !tableFilter.MatchTable(dbInfo.Name.O, tableInfo.Name.O) {
 				// Skip tables other than the given table.
 				continue
 			}

pkg/backup/schema_test.go

@@ -8,7 +8,7 @@ import (
 	"sync/atomic"
 
 	. "github.com/pingcap/check"
-	"github.com/pingcap/tidb-tools/pkg/filter"
+	"github.com/pingcap/tidb-tools/pkg/table-filter"
 	"github.com/pingcap/tidb/util/testkit"
 	"github.com/pingcap/tidb/util/testleak"
 
@@ -57,27 +57,23 @@ func (s *testBackupSchemaSuite) TestBuildBackupRangeAndSchema(c *C) {
 	tk := testkit.NewTestKit(c, s.mock.Storage)
 
 	// Table t1 is not exist.
-	testFilter, err := filter.New(false, &filter.Rules{
-		DoTables: []*filter.Table{{Schema: "test", Name: "t1"}},
-	})
+	testFilter, err := filter.Parse([]string{"test.t1"})
 	c.Assert(err, IsNil)
 	_, backupSchemas, err := backup.BuildBackupRangeAndSchema(
 		s.mock.Domain, s.mock.Storage, testFilter, math.MaxUint64)
 	c.Assert(err, IsNil)
 	c.Assert(backupSchemas, IsNil)
 
 	// Database is not exist.
-	fooFilter, err := filter.New(false, &filter.Rules{
-		DoTables: []*filter.Table{{Schema: "foo", Name: "t1"}},
-	})
+	fooFilter, err := filter.Parse([]string{"foo.t1"})
 	c.Assert(err, IsNil)
 	_, backupSchemas, err = backup.BuildBackupRangeAndSchema(
 		s.mock.Domain, s.mock.Storage, fooFilter, math.MaxUint64)
 	c.Assert(err, IsNil)
 	c.Assert(backupSchemas, IsNil)
 
 	// Empty database.
-	noFilter, err := filter.New(false, &filter.Rules{})
+	noFilter, err := filter.Parse([]string{"*.*"})
 	c.Assert(err, IsNil)
 	_, backupSchemas, err = backup.BuildBackupRangeAndSchema(
 		s.mock.Domain, s.mock.Storage, noFilter, math.MaxUint64)

pkg/task/backup.go

@@ -14,7 +14,6 @@ import (
 	"github.com/pingcap/log"
 	"github.com/pingcap/parser/model"
 	"github.com/pingcap/parser/mysql"
-	"github.com/pingcap/tidb-tools/pkg/filter"
 	"github.com/pingcap/tidb/sessionctx/stmtctx"
 	"github.com/pingcap/tidb/sessionctx/variable"
 	"github.com/pingcap/tidb/types"
@@ -113,10 +112,6 @@ func RunBackup(c context.Context, g glue.Glue, cmdName string, cfg *BackupConfig
 	if err != nil {
 		return err
 	}
-	tableFilter, err := filter.New(cfg.CaseSensitive, &cfg.Filter)
-	if err != nil {
-		return err
-	}
 	mgr, err := newMgr(ctx, g, cfg.PD, cfg.TLS, conn.SkipTiFlash, cfg.CheckRequirements)
 	if err != nil {
 		return err
@@ -145,7 +140,7 @@ func RunBackup(c context.Context, g glue.Glue, cmdName string, cfg *BackupConfig
 	isIncrementalBackup := cfg.LastBackupTS > 0
 
 	ranges, backupSchemas, err := backup.BuildBackupRangeAndSchema(
-		mgr.GetDomain(), mgr.GetTiKV(), tableFilter, backupTS)
+		mgr.GetDomain(), mgr.GetTiKV(), cfg.TableFilter, backupTS)
 	if err != nil {
 		return err
 	}

pkg/task/common.go

@@ -7,15 +7,14 @@ import (
 	"crypto/tls"
 	"fmt"
 	"net/url"
-	"regexp"
 	"strings"
 
 	"github.com/gogo/protobuf/proto"
 	"github.com/pingcap/errors"
 	"github.com/pingcap/kvproto/pkg/backup"
 	"github.com/pingcap/log"
 	pd "github.com/pingcap/pd/v4/client"
-	"github.com/pingcap/tidb-tools/pkg/filter"
+	filter "github.com/pingcap/tidb-tools/pkg/table-filter"
 	"github.com/pingcap/tidb/store/tikv"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
@@ -49,6 +48,8 @@ const (
 	flagRateLimitUnit    = "ratelimit-unit"
 	flagConcurrency      = "concurrency"
 	flagChecksum         = "checksum"
+	flagFilter           = "filter"
+	flagCaseSensitive    = "case-sensitive"
 	flagRemoveTiFlash    = "remove-tiflash"
 	flagCheckRequirement = "check-requirements"
 )
@@ -93,10 +94,20 @@ type Config struct {
 	// LogProgress is true means the progress bar is printed to the log instead of stdout.
 	LogProgress bool `json:"log-progress" toml:"log-progress"`
 
-	RemoveTiFlash     bool         `json:"remove-tiflash" toml:"remove-tiflash"`
-	CaseSensitive     bool         `json:"case-sensitive" toml:"case-sensitive"`
-	CheckRequirements bool         `json:"check-requirements" toml:"check-requirements"`
-	Filter            filter.Rules `json:"black-white-list" toml:"black-white-list"`
+	// CaseSensitive should not be used.
+	//
+	// Deprecated: This field is kept only to satisfy the cyclic dependency with TiDB. This field
+	// should be removed after TiDB upgrades the BR dependency.
+	CaseSensitive bool
+	// Filter should not be used, use TableFilter instead.
+	//
+	// Deprecated: This field is kept only to satisfy the cyclic dependency with TiDB. This field
+	// should be removed after TiDB upgrades the BR dependency.
+	Filter filter.MySQLReplicationRules
+
+	TableFilter       filter.Filter `json:"-" toml:"-"`
+	RemoveTiFlash     bool          `json:"remove-tiflash" toml:"remove-tiflash"`
+	CheckRequirements bool          `json:"check-requirements" toml:"check-requirements"`
 }
 
 // DefineCommonFlags defines the flags common to all BRIE commands.
@@ -128,19 +139,26 @@ func DefineCommonFlags(flags *pflag.FlagSet) {
 	storage.DefineFlags(flags)
 }
 
-// DefineDatabaseFlags defines the required --db flag.
+// DefineDatabaseFlags defines the required --db flag for `db` subcommand.
 func DefineDatabaseFlags(command *cobra.Command) {
 	command.Flags().String(flagDatabase, "", "database name")
 	_ = command.MarkFlagRequired(flagDatabase)
 }
 
-// DefineTableFlags defines the required --db and --table flags.
+// DefineTableFlags defines the required --db and --table flags for `table` subcommand.
 func DefineTableFlags(command *cobra.Command) {
 	DefineDatabaseFlags(command)
 	command.Flags().StringP(flagTable, "t", "", "table name")
 	_ = command.MarkFlagRequired(flagTable)
 }
 
+// DefineFilterFlags defines the --filter and --case-sensitive flags for `full` subcommand.
+func DefineFilterFlags(command *cobra.Command) {
+	flags := command.Flags()
+	flags.StringArrayP(flagFilter, "f", []string{"*.*"}, "select tables to process")
+	flags.Bool(flagCaseSensitive, false, "whether the table names used in --filter should be case-sensitive")
+}
+
 // ParseFromFlags parses the TLS config from the flag set.
 func (tls *TLSConfig) ParseFromFlags(flags *pflag.FlagSet) error {
 	var err error
@@ -202,20 +220,39 @@ func (cfg *Config) ParseFromFlags(flags *pflag.FlagSet) error {
 		return errors.Trace(err)
 	}
 
-	if dbFlag := flags.Lookup(flagDatabase); dbFlag != nil {
-		db := escapeFilterName(dbFlag.Value.String())
+	var caseSensitive bool
+	if filterFlag := flags.Lookup(flagFilter); filterFlag != nil {
+		f, err := filter.Parse(filterFlag.Value.(pflag.SliceValue).GetSlice())
+		if err != nil {
+			return err
+		}
+		cfg.TableFilter = f
+		caseSensitive, err = flags.GetBool(flagCaseSensitive)
+		if err != nil {
+			return errors.Trace(err)
+		}
+	} else if dbFlag := flags.Lookup(flagDatabase); dbFlag != nil {
+		db := dbFlag.Value.String()
 		if len(db) == 0 {
 			return errors.New("empty database name is not allowed")
 		}
 		if tblFlag := flags.Lookup(flagTable); tblFlag != nil {
-			tbl := escapeFilterName(tblFlag.Value.String())
+			tbl := tblFlag.Value.String()
 			if len(tbl) == 0 {
 				return errors.New("empty table name is not allowed")
 			}
-			cfg.Filter.DoTables = []*filter.Table{{Schema: db, Name: tbl}}
+			cfg.TableFilter = filter.NewTablesFilter(filter.Table{
+				Schema: db,
+				Name:   tbl,
+			})
 		} else {
-			cfg.Filter.DoDBs = []string{db}
+			cfg.TableFilter = filter.NewSchemasFilter(db)
 		}
+	} else {
+		cfg.TableFilter, _ = filter.Parse([]string{"*.*"})
+	}
+	if !caseSensitive {
+		cfg.TableFilter = filter.CaseInsensitive(cfg.TableFilter)
 	}
 	checkRequirements, err := flags.GetBool(flagCheckRequirement)
 	if err != nil {
@@ -303,13 +340,6 @@ func ReadBackupMeta(
 	return u, s, backupMeta, nil
 }
 
-func escapeFilterName(name string) string {
-	if !strings.HasPrefix(name, "~") {
-		return name
-	}
-	return "~^" + regexp.QuoteMeta(name) + "$"
-}
-
 // flagToZapField checks whether this flag can be logged,
 // if need to log, return its zap field. Or return a field with hidden value.
 func flagToZapField(f *pflag.Flag) zap.Field {

pkg/task/restore.go

@@ -11,7 +11,6 @@ import (
 	"github.com/pingcap/failpoint"
 	"github.com/pingcap/kvproto/pkg/backup"
 	"github.com/pingcap/log"
-	"github.com/pingcap/tidb-tools/pkg/filter"
 	"github.com/pingcap/tidb/config"
 	"github.com/spf13/pflag"
 	"go.uber.org/multierr"
@@ -149,9 +148,9 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
 		return errors.New("cannot do transactional restore from raw kv data")
 	}
 
-	files, tables, dbs, err := filterRestoreFiles(client, cfg)
-	if err != nil {
-		return err
+	files, tables, dbs := filterRestoreFiles(client, cfg)
+	if len(dbs) == 0 && len(tables) != 0 {
+		return errors.New("invalid backup, contain tables but no databases")
 	}
 
 	var newTS uint64
@@ -162,9 +161,6 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
 		}
 	}
 	ddlJobs := restore.FilterDDLJobs(client.GetDDLJobs(), tables)
-	if err != nil {
-		return err
-	}
 
 	// pre-set TiDB config for restore
 	enableTiDBConfig()
@@ -308,16 +304,11 @@ func dropToBlackhole(
 func filterRestoreFiles(
 	client *restore.Client,
 	cfg *RestoreConfig,
-) (files []*backup.File, tables []*utils.Table, dbs []*utils.Database, err error) {
-	tableFilter, err := filter.New(cfg.CaseSensitive, &cfg.Filter)
-	if err != nil {
-		return nil, nil, nil, err
-	}
-
+) (files []*backup.File, tables []*utils.Table, dbs []*utils.Database) {
 	for _, db := range client.GetDatabases() {
 		createdDatabase := false
 		for _, table := range db.Tables {
-			if !tableFilter.Match(&filter.Table{Schema: db.Info.Name.O, Name: table.Info.Name.O}) {
+			if !cfg.TableFilter.MatchTable(db.Info.Name.O, table.Info.Name.O) {
 				continue
 			}
 
@@ -329,9 +320,6 @@ func filterRestoreFiles(
 			tables = append(tables, table)
 		}
 	}
-	if len(dbs) == 0 && len(tables) != 0 {
-		err = errors.New("invalid backup, contain tables but no databases")
-	}
 	return
 }