.*: replace black-white-list by block-allow-list

_utils/terror_gen/errors_release.txt

@@ -140,7 +140,7 @@ ErrConfigMetadataNotSet,[code=20022:class=config:scope=internal:level=medium],"m
 ErrConfigRouteRuleNotFound,[code=20023:class=config:scope=internal:level=medium],"mysql-instance(%d)'s route-rules %s not exist in routes"
 ErrConfigFilterRuleNotFound,[code=20024:class=config:scope=internal:level=medium],"mysql-instance(%d)'s filter-rules %s not exist in filters"
 ErrConfigColumnMappingNotFound,[code=20025:class=config:scope=internal:level=medium],"mysql-instance(%d)'s column-mapping-rules %s not exist in column-mapping"
-ErrConfigBWListNotFound,[code=20026:class=config:scope=internal:level=medium],"mysql-instance(%d)'s list %s not exist in black white list"
+ErrConfigBAListNotFound,[code=20026:class=config:scope=internal:level=medium],"mysql-instance(%d)'s list %s not exist in block allow list"
 ErrConfigMydumperCfgNotFound,[code=20027:class=config:scope=internal:level=medium],"mysql-instance(%d)'s mydumper config %s not exist in mydumpers"
 ErrConfigMydumperPathNotValid,[code=20028:class=config:scope=internal:level=medium],"mysql-instance(%d)'s mydumper-path must specify a valid path to mydumper binary when task-mode is all or full"
 ErrConfigLoaderCfgNotFound,[code=20029:class=config:scope=internal:level=medium],"mysql-instance(%d)'s loader config %s not exist in loaders"
@@ -162,7 +162,7 @@ ErrTaskCheckFailedOpenDB,[code=26002:class=task-check:scope=internal:level=high]
 ErrTaskCheckGenTableRouter,[code=26003:class=task-check:scope=internal:level=medium],"generate table router error"
 ErrTaskCheckGenColumnMapping,[code=26004:class=task-check:scope=internal:level=medium],"generate column mapping error"
 ErrTaskCheckSyncConfigError,[code=26005:class=task-check:scope=internal:level=medium],"%s: %v\n detail: %v"
-ErrTaskCheckGenBWList,[code=26006:class=task-check:scope=internal:level=medium],"generate black white list error"
+ErrTaskCheckGenBAList,[code=26006:class=task-check:scope=internal:level=medium],"generate block allow list error"
 ErrRelayParseUUIDIndex,[code=28001:class=relay-event-lib:scope=internal:level=high],"parse server-uuid.index"
 ErrRelayParseUUIDSuffix,[code=28002:class=relay-event-lib:scope=internal:level=high],"UUID (with suffix) %s not valid"
 ErrRelayUUIDWithSuffixNotFound,[code=28003:class=relay-event-lib:scope=internal:level=high],"no UUID (with suffix) matched %s found in %s, all UUIDs are %v"
@@ -213,7 +213,7 @@ ErrRelayPurgeArgsNotValid,[code=30042:class=relay-unit:scope=internal:level=high
 ErrPreviousGTIDsNotValid,[code=30043:class=relay-unit:scope=internal:level=high],"previousGTIDs %s not valid"
 ErrDumpUnitRuntime,[code=32001:class=dump-unit:scope=internal:level=high],"mydumper runs with error"
 ErrDumpUnitGenTableRouter,[code=32002:class=dump-unit:scope=internal:level=high],"generate table router"
-ErrDumpUnitGenBWList,[code=32003:class=dump-unit:scope=internal:level=high],"generate black white list"
+ErrDumpUnitGenBAList,[code=32003:class=dump-unit:scope=internal:level=high],"generate block allow list"
 ErrLoadUnitCreateSchemaFile,[code=34001:class=load-unit:scope=internal:level=medium],"generate schema file"
 ErrLoadUnitInvalidFileEnding,[code=34002:class=load-unit:scope=internal:level=high],"corresponding ending of sql: ')' not found"
 ErrLoadUnitParseQuoteValues,[code=34003:class=load-unit:scope=internal:level=high],"parse quote values error"
@@ -229,7 +229,7 @@ ErrLoadUnitNoDBFile,[code=34012:class=load-unit:scope=internal:level=high],"inva
 ErrLoadUnitNoTableFile,[code=34013:class=load-unit:scope=internal:level=high],"invalid data sql file, cannot find table - %s"
 ErrLoadUnitDumpDirNotFound,[code=34014:class=load-unit:scope=internal:level=high],"%s does not exist or it's not a dir"
 ErrLoadUnitDuplicateTableFile,[code=34015:class=load-unit:scope=internal:level=high],"invalid table schema file, duplicated item - %s"
-ErrLoadUnitGenBWList,[code=34016:class=load-unit:scope=internal:level=high],"generate black white list"
+ErrLoadUnitGenBAList,[code=34016:class=load-unit:scope=internal:level=high],"generate block allow list"
 ErrSyncerUnitPanic,[code=36001:class=sync-unit:scope=internal:level=high],"panic error: %v"
 ErrSyncUnitInvalidTableName,[code=36002:class=sync-unit:scope=internal:level=high],"extract table name for DML error: %s"
 ErrSyncUnitTableNameQuery,[code=36003:class=sync-unit:scope=internal:level=high],"table name parse error: %s"
@@ -289,7 +289,7 @@ ErrSyncerUnitResolveCasualityFail,[code=36056:class=sync-unit:scope=internal:lev
 ErrSyncerUnitReopenStreamNotSupport,[code=36057:class=sync-unit:scope=internal:level=high],"reopen %T not supported"
 ErrSyncerUnitUpdateConfigInSharding,[code=36058:class=sync-unit:scope=internal:level=high],"try update config when some tables' (%v) sharding DDL not synced not supported"
 ErrSyncerUnitExecWithNoBlockingDDL,[code=36059:class=sync-unit:scope=internal:level=high],"process unit not waiting for sharding DDL to sync"
-ErrSyncerUnitGenBWList,[code=36060:class=sync-unit:scope=internal:level=high],"generate black white list"
+ErrSyncerUnitGenBAList,[code=36060:class=sync-unit:scope=internal:level=high],"generate block allow list"
 ErrSyncerUnitHandleDDLFailed,[code=36061:class=sync-unit:scope=internal:level=high],"fail to handle ddl job for %s"
 ErrSyncerShardDDLConflict,[code=36062:class=sync-unit:scope=internal:level=high],"fail to handle shard ddl %v in optimistic mode, because schema conflict detected"
 ErrSyncerFailpoint,[code=36063:class=sync-unit:scope=internal:level=low],"failpoint specified error"

checker/checker.go

@@ -117,9 +117,9 @@ func (c *Checker) Init(ctx context.Context) (err error) {
 	_, checkSchema := c.checkingItems[config.TableSchemaChecking]
 
 	for _, instance := range c.instances {
-		bw, err := filter.New(instance.cfg.CaseSensitive, instance.cfg.BWList)
+		bw, err := filter.New(instance.cfg.CaseSensitive, instance.cfg.BAList)
 		if err != nil {
-			return terror.ErrTaskCheckGenBWList.Delegate(err)
+			return terror.ErrTaskCheckGenBAList.Delegate(err)
 		}
 		r, err := router.NewTableRouter(instance.cfg.CaseSensitive, instance.cfg.RouteRules)
 		if err != nil {

cmd/dm-syncer/config.go

@@ -338,7 +338,7 @@ func (oc *syncerConfig) convertToNewFormat() (*config.SubTaskConfig, error) {
 			SafeMode:         oc.SafeMode,
 		},
 
-		BWList: &filter.Rules{
+		BAList: &filter.Rules{
 			DoTables:     oc.DoTables,
 			DoDBs:        oc.DoDBs,
 			IgnoreTables: oc.IgnoreTables,

dm/config/subtask.go

@@ -167,7 +167,10 @@ type SubTaskConfig struct {
 	RouteRules         []*router.TableRule   `toml:"route-rules" json:"route-rules"`
 	FilterRules        []*bf.BinlogEventRule `toml:"filter-rules" json:"filter-rules"`
 	ColumnMappingRules []*column.Rule        `toml:"mapping-rule" json:"mapping-rule"`
-	BWList             *filter.Rules         `toml:"black-white-list" json:"black-white-list"`
+
+	// black-white-list is deprecated, use block-allow-list instead
+	BWList *filter.Rules `toml:"black-white-list" json:"black-white-list"`
+	BAList *filter.Rules `toml:"block-allow-list" json:"block-allow-list"`
 
 	MydumperConfig // Mydumper configuration
 	LoaderConfig   // Loader configuration
@@ -304,6 +307,11 @@ func (c *SubTaskConfig) Adjust(verifyDecryptPassword bool) error {
 		}
 	}
 
+	// only when block-allow-list is nil use black-white-list
+	if c.BAList == nil && c.BWList != nil {
+		c.BAList = c.BWList
+	}
+
 	return nil
 }
 

dm/config/subtask_test.go

@@ -15,6 +15,7 @@ package config
 
 import (
 	. "github.com/pingcap/check"
+	"github.com/pingcap/tidb-tools/pkg/filter"
 )
 
 func (t *testConfig) TestSubTask(c *C) {
@@ -148,3 +149,30 @@ func (t *testConfig) TestSubTaskAdjustFail(c *C) {
 		c.Assert(err, ErrorMatches, tc.errorFormat)
 	}
 }
+
+func (t *testConfig) TestSubTaskBlockAllowList(c *C) {
+	filterRules1 := &filter.Rules{
+		DoDBs: []string{"s1"},
+	}
+
+	filterRules2 := &filter.Rules{
+		DoDBs: []string{"s2"},
+	}
+
+	cfg := &SubTaskConfig{
+		Name:     "test",
+		SourceID: "source-1",
+		BWList:   filterRules1,
+	}
+
+	// BAList is nil, will set BAList = BWList
+	err := cfg.Adjust(false)
+	c.Assert(err, IsNil)
+	c.Assert(cfg.BAList, Equals, filterRules1)
+
+	// BAList is not nil, will not update it
+	cfg.BAList = filterRules2
+	err = cfg.Adjust(false)
+	c.Assert(err, IsNil)
+	c.Assert(cfg.BAList, Equals, filterRules2)
+}

dm/config/task.go

@@ -93,7 +93,10 @@ type MySQLInstance struct {
 	FilterRules        []string `yaml:"filter-rules"`
 	ColumnMappingRules []string `yaml:"column-mapping-rules"`
 	RouteRules         []string `yaml:"route-rules"`
-	BWListName         string   `yaml:"black-white-list"`
+
+	// black-white-list is deprecated, use block-allow-list instead
+	BWListName string `yaml:"black-white-list"`
+	BAListName string `yaml:"block-allow-list"`
 
 	MydumperConfigName string          `yaml:"mydumper-config-name"`
 	Mydumper           *MydumperConfig `yaml:"mydumper"`
@@ -111,8 +114,8 @@ type MySQLInstance struct {
 	SyncerThread int `yaml:"syncer-thread"`
 }
 
-// Verify does verification on configs
-func (m *MySQLInstance) Verify() error {
+// VerifyAndAdjust does verification on configs, and adjust some configs
+func (m *MySQLInstance) VerifyAndAdjust() error {
 	if m == nil {
 		return terror.ErrConfigMySQLInstNotFound.Generate()
 	}
@@ -135,6 +138,10 @@ func (m *MySQLInstance) Verify() error {
 		return terror.ErrConfigSyncerCfgConflict.Generate()
 	}
 
+	if len(m.BAListName) == 0 && len(m.BWListName) != 0 {
+		m.BAListName = m.BWListName
+	}
+
 	return nil
 }
 
@@ -275,7 +282,10 @@ type TaskConfig struct {
 	Routes         map[string]*router.TableRule   `yaml:"routes"`
 	Filters        map[string]*bf.BinlogEventRule `yaml:"filters"`
 	ColumnMappings map[string]*column.Rule        `yaml:"column-mappings"`
-	BWList         map[string]*filter.Rules       `yaml:"black-white-list"`
+
+	// black-white-list is deprecated, use block-allow-list instead
+	BWList map[string]*filter.Rules `yaml:"black-white-list"`
+	BAList map[string]*filter.Rules `yaml:"block-allow-list"`
 
 	Mydumpers map[string]*MydumperConfig `yaml:"mydumpers"`
 	Loaders   map[string]*LoaderConfig   `yaml:"loaders"`
@@ -296,6 +306,7 @@ func NewTaskConfig() *TaskConfig {
 		Filters:                 make(map[string]*bf.BinlogEventRule),
 		ColumnMappings:          make(map[string]*column.Rule),
 		BWList:                  make(map[string]*filter.Rules),
+		BAList:                  make(map[string]*filter.Rules),
 		Mydumpers:               make(map[string]*MydumperConfig),
 		Loaders:                 make(map[string]*LoaderConfig),
 		Syncers:                 make(map[string]*SyncerConfig),
@@ -374,7 +385,7 @@ func (c *TaskConfig) adjust() error {
 	iids := make(map[string]int) // source-id -> instance-index
 	duplicateErrorStrings := make([]string, 0)
 	for i, inst := range c.MySQLInstances {
-		if err := inst.Verify(); err != nil {
+		if err := inst.VerifyAndAdjust(); err != nil {
 			return terror.Annotatef(err, "mysql-instance: %s", humanize.Ordinal(i))
 		}
 		if iid, ok := iids[inst.SourceID]; ok {
@@ -412,8 +423,13 @@ func (c *TaskConfig) adjust() error {
 				return terror.ErrConfigColumnMappingNotFound.Generate(i, name)
 			}
 		}
-		if _, ok := c.BWList[inst.BWListName]; len(inst.BWListName) > 0 && !ok {
-			return terror.ErrConfigBWListNotFound.Generate(i, inst.BWListName)
+
+		// only when BAList is empty use BWList
+		if len(c.BAList) == 0 && len(c.BWList) != 0 {
+			c.BAList = c.BWList
+		}
+		if _, ok := c.BAList[inst.BAListName]; len(inst.BAListName) > 0 && !ok {
+			return terror.ErrConfigBAListNotFound.Generate(i, inst.BAListName)
 		}
 
 		if len(inst.MydumperConfigName) > 0 {
@@ -534,7 +550,7 @@ func (c *TaskConfig) SubTaskConfigs(sources map[string]DBConfig) ([]*SubTaskConf
 			cfg.ColumnMappingRules[j] = c.ColumnMappings[name]
 		}
 
-		cfg.BWList = c.BWList[inst.BWListName]
+		cfg.BAList = c.BAList[inst.BAListName]
 
 		cfg.MydumperConfig = *inst.Mydumper
 		cfg.LoaderConfig = *inst.Loader

dm/config/task_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/pingcap/dm/pkg/terror"
 
 	. "github.com/pingcap/check"
+	"github.com/pingcap/tidb-tools/pkg/filter"
 )
 
 func (t *testConfig) TestInvalidTaskConfig(c *C) {
@@ -42,7 +43,7 @@ target-database:
 mysql-instances:
   - source-id: "mysql-replica-01"
     server-id: 101 
-    black-white-list:  "instance"
+    block-allow-list:  "instance"
     route-rules: ["sharding-route-rules-table", "sharding-route-rules-schema"]
     column-mapping-rules: ["instance-1"]
     mydumper-config-name: "global"
@@ -67,7 +68,7 @@ target-database:
 
 mysql-instances:
   - source-id: "mysql-replica-01"
-    black-white-list:  "instance"
+    block-allow-list:  "instance"
     route-rules: ["sharding-route-rules-table", "sharding-route-rules-schema"]
     column-mapping-rules: ["instance-1"]
     mydumper-config-name: "global"
@@ -140,7 +141,7 @@ target-database:
 
 mysql-instances:
   - source-id: "mysql-replica-01"
-    black-white-list:  "instance"
+    block-allow-list:  "instance"
     mydumper-thread: 11
     mydumper-config-name: "global"
     loader-thread: 22
@@ -149,18 +150,18 @@ mysql-instances:
     syncer-config-name: "global"
 
   - source-id: "mysql-replica-02"
-    black-white-list:  "instance"
+    block-allow-list:  "instance"
     mydumper-config-name: "global"
     loader-config-name: "global"
     syncer-config-name: "global"
 
   - source-id: "mysql-replica-03"
-    black-white-list:  "instance"
+    block-allow-list:  "instance"
     mydumper-thread: 44
     loader-thread: 55
     syncer-thread: 66
 
-black-white-list:
+block-allow-list:
   instance:
     do-dbs: ["test"]
 
@@ -221,7 +222,7 @@ mysql-instances:
   - source-id: "mysql-replica-02"
   - source-id: "mysql-replica-03"
 
-black-white-list:
+block-allow-list:
   instance:
     do-dbs: ["test"]
 
@@ -264,3 +265,32 @@ func (t *testConfig) TestCheckDuplicateString(c *C) {
 	sort.Strings(dupeStrings)
 	c.Assert(dupeStrings, DeepEquals, []string{"a", "b", "c"})
 }
+
+func (t *testConfig) TestTaskBlockAllowList(c *C) {
+	filterRules1 := &filter.Rules{
+		DoDBs: []string{"s1"},
+	}
+
+	filterRules2 := &filter.Rules{
+		DoDBs: []string{"s2"},
+	}
+
+	cfg := &TaskConfig{
+		Name:           "test",
+		TaskMode:       "full",
+		TargetDB:       &DBConfig{},
+		MySQLInstances: []*MySQLInstance{{SourceID: "source-1"}},
+		BWList:         map[string]*filter.Rules{"source-1": filterRules1},
+	}
+
+	// BAList is nil, will set BAList = BWList
+	err := cfg.adjust()
+	c.Assert(err, IsNil)
+	c.Assert(cfg.BAList["source-1"], Equals, filterRules1)
+
+	// BAList is not nil, will not update it
+	cfg.BAList = map[string]*filter.Rules{"source-1": filterRules2}
+	err = cfg.adjust()
+	c.Assert(err, IsNil)
+	c.Assert(cfg.BAList["source-1"], Equals, filterRules2)
+}

dm/ctl/master/update_task.go

@@ -28,7 +28,7 @@ import (
 func NewUpdateTaskCmd() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "update-task [-s source ...] <config-file>",
-		Short: "update a task's config for routes, filters, or black-white-list",
+		Short: "update a task's config for routes, filters, or block-allow-list",
 		Run:   updateTaskFunc,
 	}
 	return cmd

dm/dm-ansible/conf/task_advanced.yaml.example

@@ -29,7 +29,7 @@ mysql-instances:             # one or more source database, config more source d
       binlog-pos: 4
     route-rules: ["user-route-rules-schema", "user-route-rules"]
     filter-rules: ["user-filter-1", "user-filter-2"]
-    black-white-list:  "instance"
+    block-allow-list:  "instance"
 
     # `mydumper-config-name` and `mydumper` should only set one
     mydumper-config-name: "global"   # ref `mydumpers` config
@@ -59,7 +59,7 @@ mysql-instances:             # one or more source database, config more source d
       binlog-pos: 320
     route-rules: ["user-route-rules-schema", "user-route-rules"]
     filter-rules: ["user-filter-2"]
-    black-white-list: "instance"
+    block-allow-list: "instance"
 
     mydumper:
       threads: 4
@@ -102,16 +102,16 @@ filters:                     # filter rules, mysql instance can ref rules in it
     events: ["all dml"]             # only do all DML events
     action: Do
 
-black-white-list:
+block-allow-list:
   instance:
-    do-dbs: ["~^test.*", "do"]        # white list of upstream schemas needs to be replicated, regular expression (starts with ~) is supported
-    ignore-dbs: ["mysql", "ignored"]  # black list of upstream schemas needs to be replicated, regular expression (starts with ~) is supported
-    do-tables:                        # white list of upstream tables needs to be replicated, (db-name, tbl-name) regular expression (starts with ~) is supported
+    do-dbs: ["~^test.*", "do"]        # allow list of upstream schemas needs to be replicated, regular expression (starts with ~) is supported
+    ignore-dbs: ["mysql", "ignored"]  # block list of upstream schemas needs to be replicated, regular expression (starts with ~) is supported
+    do-tables:                        # allow list of upstream tables needs to be replicated, (db-name, tbl-name) regular expression (starts with ~) is supported
     - db-name: "~^test.*"
       tbl-name: "~^t.*"
     - db-name: "do"
       tbl-name: "do"
-    ignore-tables:                    # black list of upstream tables needs to be replicated, (db-name, tbl-name) regular expression (starts with ~) is supported
+    ignore-tables:                    # block list of upstream tables needs to be replicated, (db-name, tbl-name) regular expression (starts with ~) is supported
     - db-name: "test"
       tbl-name: "log"
 

dm/dm-ansible/conf/task_basic.yaml.example

@@ -11,7 +11,7 @@ target-database:
 mysql-instances:             # one or more source database, config more source database for sharding merge
   -
     source-id: "instance118-4306" # unique in all instances, used as id when save checkpoints, configs, etc.
-    black-white-list:  "instance"
+    block-allow-list:  "instance"
 
     # number of threads to use for dump data
     mydumper-thread: 4
@@ -24,7 +24,7 @@ mysql-instances:             # one or more source database, config more source d
 
   -
     source-id: "instance118-5306"
-    black-white-list: "instance"
+    block-allow-list: "instance"
 
     # number of threads to use for dump data
     mydumper-thread: 4
@@ -35,10 +35,10 @@ mysql-instances:             # one or more source database, config more source d
     # number of threads to use for replicate binlog
     syncer-thread: 16
 
-black-white-list:
+block-allow-list:
   instance:
-    do-dbs: ["~^test.*", "do"]        # white list of upstream schemas needs to be replicated, regular expression (starts with ~) is supported
-    do-tables:                        # white list of upstream tables needs to be replicated, (db-name, tbl-name) regular expression (starts with ~) is supported
+    do-dbs: ["~^test.*", "do"]        # allow list of upstream schemas needs to be replicated, regular expression (starts with ~) is supported
+    do-tables:                        # allow list of upstream tables needs to be replicated, (db-name, tbl-name) regular expression (starts with ~) is supported
     - db-name: "~^test.*"
       tbl-name: "~^t.*"
     - db-name: "do"