From f9913ed5d7e8e35bbc7375ffb595ffa3a6a5a59e Mon Sep 17 00:00:00 2001 From: kennytm Date: Fri, 29 Mar 2019 00:02:03 +0800 Subject: [PATCH 1/4] config: remove deprecated -compact, -switch-mode flags from tidb-lightning They can still be called from tidb-lightning-ctl. --- cmd/tidb-lightning-ctl/main.go | 3 +- go.sum | 1 + lightning/config/config.go | 23 ++++-------- lightning/lightning.go | 66 ---------------------------------- 4 files changed, 9 insertions(+), 84 deletions(-) diff --git a/cmd/tidb-lightning-ctl/main.go b/cmd/tidb-lightning-ctl/main.go index 6716f3d5b..e3c01bff2 100644 --- a/cmd/tidb-lightning-ctl/main.go +++ b/cmd/tidb-lightning-ctl/main.go @@ -39,8 +39,7 @@ func main() { func run() error { cfg := config.NewConfig() - cfg.FlagSet = flag.NewFlagSet("lightning-ctl", flag.ExitOnError) - fs := cfg.FlagSet + fs := flag.NewFlagSet("lightning-ctl", flag.ExitOnError) fs.StringVar(&cfg.ConfigFile, "config", "tidb-lightning.toml", "tidb-lightning configuration file") diff --git a/go.sum b/go.sum index 56d2a3b17..1462c2b83 100644 --- a/go.sum +++ b/go.sum @@ -89,6 +89,7 @@ github.com/pingcap/check v0.0.0-20171206051426-1c287c953996 h1:ZBdiJCMan6GSo/aPA github.com/pingcap/check v0.0.0-20171206051426-1c287c953996/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= github.com/pingcap/errors v0.11.0 h1:DCJQB8jrHbQ1VVlMFIrbj2ApScNNotVmkSNplu2yUt4= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pingcap/gofail v0.0.0-20181217135706-6a951c1e42c3 h1:04yuCf5NMvLU8rB2m4Qs3rynH7EYpMno3lHkewIOdMo= github.com/pingcap/gofail v0.0.0-20181217135706-6a951c1e42c3/go.mod h1:DazNTg0PTldtpsQiT9I5tVJwV1onHMKBBgXzmJUlMns= github.com/pingcap/goleveldb v0.0.0-20171020084629-8d44bfdf1030 h1:XJLuW0lsP7vAtQ2iPjZwvXZ14m5urp9No+Qr06ZZcTo= github.com/pingcap/goleveldb v0.0.0-20171020084629-8d44bfdf1030/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= diff --git a/lightning/config/config.go b/lightning/config/config.go index d24438fe4..c65759b6a 100644 --- a/lightning/config/config.go +++ b/lightning/config/config.go @@ -51,8 +51,6 @@ type DBStore struct { } type Config struct { - *flag.FlagSet `json:"-"` - App Lightning `toml:"lightning" json:"lightning"` TiDB DBStore `toml:"tidb" json:"tidb"` @@ -66,10 +64,7 @@ type Config struct { Cron Cron `toml:"cron" json:"cron"` // command line flags - ConfigFile string `json:"config-file"` - DoCompact bool `json:"-"` - SwitchMode string `json:"-"` - printVersion bool + ConfigFile string `json:"config-file"` } func (c *Config) String() string { @@ -182,21 +177,22 @@ func NewConfig() *Config { func LoadConfig(args []string) (*Config, error) { cfg := NewConfig() - cfg.FlagSet = flag.NewFlagSet("lightning", flag.ContinueOnError) - fs := cfg.FlagSet + fs := flag.NewFlagSet("lightning", flag.ContinueOnError) // if both `-c` and `-config` are specified, the last one in the command line will take effect. // the default value is assigned immediately after the StringVar() call, // so it is fine to not give any default value for `-c`, to keep the `-h` page clean. fs.StringVar(&cfg.ConfigFile, "c", "", "(deprecated alias of -config)") fs.StringVar(&cfg.ConfigFile, "config", "tidb-lightning.toml", "tidb-lightning configuration file") - fs.BoolVar(&cfg.DoCompact, "compact", false, "do manual compaction on the target cluster, run then exit") - fs.StringVar(&cfg.SwitchMode, "switch-mode", "", "switch tikv into import mode or normal mode, values can be ['import', 'normal'], run then exit") - fs.BoolVar(&cfg.printVersion, "V", false, "print version of lightning") + printVersion := fs.Bool("V", false, "print version of lightning") if err := fs.Parse(args); err != nil { return nil, errors.Trace(err) } + if *printVersion { + fmt.Println(common.GetRawInfo()) + return nil, flag.ErrHelp + } if err := cfg.Load(); err != nil { return nil, errors.Trace(err) @@ -205,11 +201,6 @@ func LoadConfig(args []string) (*Config, error) { } func (cfg *Config) Load() error { - if cfg.printVersion { - fmt.Println(common.GetRawInfo()) - return flag.ErrHelp - } - data, err := ioutil.ReadFile(cfg.ConfigFile) if err != nil { return errors.Trace(err) diff --git a/lightning/lightning.go b/lightning/lightning.go index 26fa255a3..4c4f576e6 100644 --- a/lightning/lightning.go +++ b/lightning/lightning.go @@ -21,12 +21,10 @@ import ( "sync" "github.com/pingcap/errors" - sstpb "github.com/pingcap/kvproto/pkg/import_sstpb" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/pingcap/tidb-lightning/lightning/common" "github.com/pingcap/tidb-lightning/lightning/config" - "github.com/pingcap/tidb-lightning/lightning/kv" "github.com/pingcap/tidb-lightning/lightning/mydump" "github.com/pingcap/tidb-lightning/lightning/restore" ) @@ -72,10 +70,6 @@ func (l *Lightning) Run() error { common.AppLogger.Infof("cfg %s", l.cfg) }) - if l.handleCommandFlagsAndExits() { - return nil - } - l.wg.Add(1) var err error go func() { @@ -86,34 +80,6 @@ func (l *Lightning) Run() error { return errors.Trace(err) } -func (l *Lightning) handleCommandFlagsAndExits() (exits bool) { - if l.cfg.DoCompact { - err := l.doCompact() - if err != nil { - common.AppLogger.Fatalf("compact error %s", errors.ErrorStack(err)) - } - return true - } - - if mode := l.cfg.SwitchMode; mode != "" { - var err error - switch mode { - case config.ImportMode: - err = l.switchMode(sstpb.SwitchMode_Import) - case config.NormalMode: - err = l.switchMode(sstpb.SwitchMode_Normal) - default: - common.AppLogger.Fatalf("invalid mode %s, must use %s or %s", mode, config.ImportMode, config.NormalMode) - } - if err != nil { - common.AppLogger.Fatalf("switch mode error %v", errors.ErrorStack(err)) - } - common.AppLogger.Infof("switch mode to %s", mode) - return true - } - return false -} - func (l *Lightning) run() error { mdl, err := mydump.NewMyDumpLoader(l.cfg) if err != nil { @@ -134,38 +100,6 @@ func (l *Lightning) run() error { return errors.Trace(err) } -func (l *Lightning) doCompact() error { - ctx := context.Background() - - importer, err := kv.NewImporter(ctx, l.cfg.TikvImporter.Addr, l.cfg.TiDB.PdAddr) - if err != nil { - return errors.Trace(err) - } - defer importer.Close() - - if err := importer.Compact(ctx, restore.FullLevelCompact); err != nil { - return errors.Trace(err) - } - - return nil -} - -func (l *Lightning) switchMode(mode sstpb.SwitchMode) error { - ctx := context.Background() - - importer, err := kv.NewImporter(ctx, l.cfg.TikvImporter.Addr, l.cfg.TiDB.PdAddr) - if err != nil { - return errors.Trace(err) - } - defer importer.Close() - - if err := importer.SwitchMode(ctx, mode); err != nil { - return errors.Trace(err) - } - - return nil -} - func (l *Lightning) Stop() { l.shutdown() l.wg.Wait() From 514ee944e75b97baa78e70881f806c7b44a2e67c Mon Sep 17 00:00:00 2001 From: kennytm Date: Fri, 29 Mar 2019 00:08:35 +0800 Subject: [PATCH 2/4] config: search also conf/tidb-lightning.toml for default -config path Fallback to standard default if -config is not supplied --- cmd/tidb-lightning-ctl/main.go | 2 +- lightning/config/config.go | 28 +++++++++++++++++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/cmd/tidb-lightning-ctl/main.go b/cmd/tidb-lightning-ctl/main.go index e3c01bff2..35c25e146 100644 --- a/cmd/tidb-lightning-ctl/main.go +++ b/cmd/tidb-lightning-ctl/main.go @@ -41,7 +41,7 @@ func run() error { cfg := config.NewConfig() fs := flag.NewFlagSet("lightning-ctl", flag.ExitOnError) - fs.StringVar(&cfg.ConfigFile, "config", "tidb-lightning.toml", "tidb-lightning configuration file") + fs.StringVar(&cfg.ConfigFile, "config", "", "tidb-lightning configuration file") compact := fs.Bool("compact", false, "do manual compaction on the target cluster") mode := fs.String("switch-mode", "", "switch tikv into import mode or normal mode, values can be ['import', 'normal']") diff --git a/lightning/config/config.go b/lightning/config/config.go index c65759b6a..8e3f4d9e2 100644 --- a/lightning/config/config.go +++ b/lightning/config/config.go @@ -18,6 +18,7 @@ import ( "flag" "fmt" "io/ioutil" + "os" "runtime" "time" @@ -34,6 +35,8 @@ const ( NormalMode = "normal" ) +var defaultConfigPaths = []string{"tidb-lightning.toml", "conf/tidb-lightning.toml"} + type DBStore struct { Host string `toml:"host" json:"host"` Port int `toml:"port" json:"port"` @@ -156,6 +159,12 @@ func NewConfig() *Config { CheckRequirements: true, }, TiDB: DBStore{ + Host: "127.0.0.1", + Port: 4000, + User: "root", + StatusPort: 10080, + PdAddr: "127.0.0.1:2379", + LogLevel: "error", SQLMode: "STRICT_TRANS_TABLES,NO_ENGINE_SUBSTITUTION", BuildStatsConcurrency: 20, DistSQLScanConcurrency: 100, @@ -171,6 +180,9 @@ func NewConfig() *Config { Separator: ",", }, }, + PostRestore: PostRestore{ + Checksum: true, + }, } } @@ -183,7 +195,7 @@ func LoadConfig(args []string) (*Config, error) { // the default value is assigned immediately after the StringVar() call, // so it is fine to not give any default value for `-c`, to keep the `-h` page clean. fs.StringVar(&cfg.ConfigFile, "c", "", "(deprecated alias of -config)") - fs.StringVar(&cfg.ConfigFile, "config", "tidb-lightning.toml", "tidb-lightning configuration file") + fs.StringVar(&cfg.ConfigFile, "config", "", "tidb-lightning configuration file") printVersion := fs.Bool("V", false, "print version of lightning") if err := fs.Parse(args); err != nil { @@ -201,6 +213,20 @@ func LoadConfig(args []string) (*Config, error) { } func (cfg *Config) Load() error { + // search the default config path if none is specified. + if cfg.ConfigFile == "" { + for _, path := range defaultConfigPaths { + if _, err := os.Stat(path); err == nil { + cfg.ConfigFile = path + break + } + } + // use standard config if unspecified. + if cfg.ConfigFile == "" { + return nil + } + } + data, err := ioutil.ReadFile(cfg.ConfigFile) if err != nil { return errors.Trace(err) From 26b5b99657b4bc9c56da3472782d48c965096b73 Mon Sep 17 00:00:00 2001 From: kennytm Date: Fri, 29 Mar 2019 00:56:29 +0800 Subject: [PATCH 3/4] config: provide command line arguments for some common options --- cmd/tidb-lightning-ctl/main.go | 32 +++++++++++++ lightning/config/config.go | 46 ++++++++++++++++++- tests/_utils/run_lightning | 6 ++- tests/cmdline_override/config.toml | 18 ++++++++ .../data/cmdline_override-schema-create.sql | 1 + .../data/cmdline_override.t-schema.sql | 1 + .../data/cmdline_override.t.sql | 1 + tests/cmdline_override/run.sh | 17 +++++++ 8 files changed, 119 insertions(+), 3 deletions(-) create mode 100644 tests/cmdline_override/config.toml create mode 100644 tests/cmdline_override/data/cmdline_override-schema-create.sql create mode 100644 tests/cmdline_override/data/cmdline_override.t-schema.sql create mode 100644 tests/cmdline_override/data/cmdline_override.t.sql create mode 100755 tests/cmdline_override/run.sh diff --git a/cmd/tidb-lightning-ctl/main.go b/cmd/tidb-lightning-ctl/main.go index 35c25e146..39f94b940 100644 --- a/cmd/tidb-lightning-ctl/main.go +++ b/cmd/tidb-lightning-ctl/main.go @@ -43,6 +43,14 @@ func run() error { fs.StringVar(&cfg.ConfigFile, "config", "", "tidb-lightning configuration file") + logLevel := fs.String("L", "", `log level: info, debug, warn, error, fatal (default "info")`) + logFilePath := fs.String("log-file", "", "log file path") + tidbHost := fs.String("tidb-host", "", "TiDB server host") + tidbPort := fs.Int("tidb-port", 0, "TiDB server port (default 4000)") + tidbUser := fs.String("tidb-user", "", "TiDB user name to connect") + pdAddr := fs.String("pd-urls", "", "PD endpoint address") + importerAddr := fs.String("importer", "", "address (host:port) to connect to tikv-importer") + compact := fs.Bool("compact", false, "do manual compaction on the target cluster") mode := fs.String("switch-mode", "", "switch tikv into import mode or normal mode, values can be ['import', 'normal']") @@ -62,6 +70,30 @@ func run() error { return errors.Trace(err) } + if *logLevel != "" { + cfg.App.LogConfig.Level = *logLevel + } + if *logFilePath != "" { + cfg.App.LogConfig.File = *logFilePath + } + if *tidbHost != "" { + cfg.TiDB.Host = *tidbHost + } + if *tidbPort != 0 { + cfg.TiDB.Port = *tidbPort + } + if *tidbUser != "" { + cfg.TiDB.User = *tidbUser + } + if *pdAddr != "" { + cfg.TiDB.PdAddr = *pdAddr + } + if *importerAddr != "" { + cfg.TikvImporter.Addr = *importerAddr + } + + cfg.Adjust() + ctx := context.Background() if *compact { diff --git a/lightning/config/config.go b/lightning/config/config.go index 8e3f4d9e2..75a723614 100644 --- a/lightning/config/config.go +++ b/lightning/config/config.go @@ -198,6 +198,16 @@ func LoadConfig(args []string) (*Config, error) { fs.StringVar(&cfg.ConfigFile, "config", "", "tidb-lightning configuration file") printVersion := fs.Bool("V", false, "print version of lightning") + logLevel := fs.String("L", "", `log level: info, debug, warn, error, fatal (default "info")`) + logFilePath := fs.String("log-file", "", "log file path") + tidbHost := fs.String("tidb-host", "", "TiDB server host") + tidbPort := fs.Int("tidb-port", 0, "TiDB server port (default 4000)") + tidbUser := fs.String("tidb-user", "", "TiDB user name to connect") + tidbStatusPort := fs.Int("tidb-status", 0, "TiDB server status port (default 10080)") + pdAddr := fs.String("pd-urls", "", "PD endpoint address") + dataSrcPath := fs.String("d", "", "Directory of the dump to import") + importerAddr := fs.String("importer", "", "address (host:port) to connect to tikv-importer") + if err := fs.Parse(args); err != nil { return nil, errors.Trace(err) } @@ -209,6 +219,37 @@ func LoadConfig(args []string) (*Config, error) { if err := cfg.Load(); err != nil { return nil, errors.Trace(err) } + + if *logLevel != "" { + cfg.App.LogConfig.Level = *logLevel + } + if *logFilePath != "" { + cfg.App.LogConfig.File = *logFilePath + } + if *tidbHost != "" { + cfg.TiDB.Host = *tidbHost + } + if *tidbPort != 0 { + cfg.TiDB.Port = *tidbPort + } + if *tidbStatusPort != 0 { + cfg.TiDB.StatusPort = *tidbStatusPort + } + if *tidbUser != "" { + cfg.TiDB.User = *tidbUser + } + if *pdAddr != "" { + cfg.TiDB.PdAddr = *pdAddr + } + if *dataSrcPath != "" { + cfg.Mydumper.SourceDir = *dataSrcPath + } + if *importerAddr != "" { + cfg.TikvImporter.Addr = *importerAddr + } + + cfg.Adjust() + return cfg, nil } @@ -243,6 +284,10 @@ func (cfg *Config) Load() error { return errors.New("invalid config: `mydumper.csv.delimiter` must be one byte long or empty") } + return nil +} + +func (cfg *Config) Adjust() { // handle mydumper if cfg.Mydumper.BatchSize <= 0 { cfg.Mydumper.BatchSize = 100 * _G @@ -271,5 +316,4 @@ func (cfg *Config) Load() error { cfg.Checkpoint.DSN = "/tmp/" + cfg.Checkpoint.Schema + ".pb" } } - return nil } diff --git a/tests/_utils/run_lightning b/tests/_utils/run_lightning index 6b4e37f96..8f93b5931 100755 --- a/tests/_utils/run_lightning +++ b/tests/_utils/run_lightning @@ -15,6 +15,8 @@ set -eu TEST_DIR=/tmp/lightning_test_result +CONFIG="${1-config}" +shift -echo "[$(date)] <<<<<< RUNNING TEST FOR: tests/$TEST_NAME/${1-config}.toml >>>>>>" >> "$TEST_DIR/lightning.log" -bin/tidb-lightning.test -test.coverprofile="$TEST_DIR/cov.$TEST_NAME.$$.out" DEVEL -config "tests/$TEST_NAME/${1-config}.toml" +echo "[$(date)] <<<<<< RUNNING TEST FOR: tests/$TEST_NAME/$CONFIG.toml >>>>>>" >> "$TEST_DIR/lightning.log" +bin/tidb-lightning.test -test.coverprofile="$TEST_DIR/cov.$TEST_NAME.$$.out" DEVEL -config "tests/$TEST_NAME/$CONFIG.toml" "$@" diff --git a/tests/cmdline_override/config.toml b/tests/cmdline_override/config.toml new file mode 100644 index 000000000..aaf4480a9 --- /dev/null +++ b/tests/cmdline_override/config.toml @@ -0,0 +1,18 @@ +[lightning] +check-requirements = false +file = "/tmp/xyzxyzxyz" +level = "xyzxyzxyz" + +[tikv-importer] +addr = "xyzxyzxyz" + +[mydumper] +data-source-dir = "xyzxyzxyz" + +[tidb] +host = "xyzxyzxyz" +port = 12345678 +user = "xyzxyzxyz" +status-port = 12345678 +pd-addr = "xyzxyzxyz" + diff --git a/tests/cmdline_override/data/cmdline_override-schema-create.sql b/tests/cmdline_override/data/cmdline_override-schema-create.sql new file mode 100644 index 000000000..004a73e49 --- /dev/null +++ b/tests/cmdline_override/data/cmdline_override-schema-create.sql @@ -0,0 +1 @@ +create database cmdline_override; \ No newline at end of file diff --git a/tests/cmdline_override/data/cmdline_override.t-schema.sql b/tests/cmdline_override/data/cmdline_override.t-schema.sql new file mode 100644 index 000000000..c74c68204 --- /dev/null +++ b/tests/cmdline_override/data/cmdline_override.t-schema.sql @@ -0,0 +1 @@ +create table t (a int); diff --git a/tests/cmdline_override/data/cmdline_override.t.sql b/tests/cmdline_override/data/cmdline_override.t.sql new file mode 100644 index 000000000..80c7e623e --- /dev/null +++ b/tests/cmdline_override/data/cmdline_override.t.sql @@ -0,0 +1 @@ +insert into t values (15); \ No newline at end of file diff --git a/tests/cmdline_override/run.sh b/tests/cmdline_override/run.sh new file mode 100755 index 000000000..4ceca9a55 --- /dev/null +++ b/tests/cmdline_override/run.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +set -eux + +run_lightning config \ + -L info \ + --log-file "$TEST_DIR/lightning.log" \ + --tidb-host 127.0.0.1 \ + --tidb-port 4000 \ + --tidb-user root \ + --tidb-status 10080 \ + --pd-urls 127.0.0.1:2379 \ + -d tests/cmdline_override/data \ + --importer 127.0.0.1:8808 + +run_sql 'SELECT * FROM cmdline_override.t' +check_contains 'a: 15' From 71f806ee17a899d0e1c3ffb8ab473d8038fd6f9e Mon Sep 17 00:00:00 2001 From: kennytm Date: Thu, 4 Apr 2019 01:27:06 +0800 Subject: [PATCH 4/4] config: stop searching for tidb-lightning.toml if --config is unspecified --- lightning/config/config.go | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/lightning/config/config.go b/lightning/config/config.go index afac7ef73..ee37a6888 100644 --- a/lightning/config/config.go +++ b/lightning/config/config.go @@ -18,7 +18,6 @@ import ( "flag" "fmt" "io/ioutil" - "os" "runtime" "time" @@ -257,18 +256,9 @@ func LoadConfig(args []string) (*Config, error) { } func (cfg *Config) Load() error { - // search the default config path if none is specified. + // use standard config if unspecified. if cfg.ConfigFile == "" { - for _, path := range defaultConfigPaths { - if _, err := os.Stat(path); err == nil { - cfg.ConfigFile = path - break - } - } - // use standard config if unspecified. - if cfg.ConfigFile == "" { - return nil - } + return nil } data, err := ioutil.ReadFile(cfg.ConfigFile)