From 35a532616f06d86ce6e3bf0a3b43d59c360d805e Mon Sep 17 00:00:00 2001 From: Connor1996 Date: Sun, 26 Nov 2017 17:33:21 +0800 Subject: [PATCH 01/14] support ssl/tls --- cmd/pd-ctl/main.go | 15 +++++++-- cmd/pd-tso-bench/main.go | 5 ++- conf/config.toml | 14 +++++++++ pd-client/client.go | 53 +++++++++++++++++++++++++++++-- pdctl/command/global.go | 68 ++++++++++++++++++++++------------------ pdctl/ctl.go | 22 ++++++++----- server/api/redirector.go | 16 ++++++++-- server/config.go | 21 +++++++++++++ server/server.go | 27 +++++++++++++++- 9 files changed, 192 insertions(+), 49 deletions(-) diff --git a/cmd/pd-ctl/main.go b/cmd/pd-ctl/main.go index fab2e1c1da2..f2d404d03e7 100644 --- a/cmd/pd-ctl/main.go +++ b/cmd/pd-ctl/main.go @@ -29,15 +29,21 @@ import ( ) var ( - url string - detach bool - version bool + url string + detach bool + version bool + tlsCAPath string + tlsCertPath string + tlsKeyPath string ) func init() { flag.StringVarP(&url, "pd", "u", "http://127.0.0.1:2379", "The pd address") flag.BoolVarP(&detach, "detach", "d", false, "Run pdctl without readline") flag.BoolVarP(&version, "version", "V", false, "print version information and exit") + flag.StringVar(&tlsCAPath, "ca", "", "path of file that contains list of trusted SSL CAs.") + flag.StringVar(&tlsCertPath, "cert", "", "path of file that contains X509 certificate in PEM format.") + flag.StringVar(&tlsKeyPath, "key", "", "path of file that contains X509 key in PEM format.") } func main() { @@ -115,6 +121,9 @@ func loop() { } args := strings.Split(strings.TrimSpace(line), " ") args = append(args, "-u", url) + args = append(args, "--ca", tlsCAPath) + args = append(args, "--cert", tlsCertPath) + args = append(args, "--key", tlsKeyPath) pdctl.Start(args) } } diff --git a/cmd/pd-tso-bench/main.go b/cmd/pd-tso-bench/main.go index 2512475fee0..2d6c779606e 100644 --- a/cmd/pd-tso-bench/main.go +++ b/cmd/pd-tso-bench/main.go @@ -33,12 +33,15 @@ var ( concurrency = flag.Int("C", 1000, "concurrency") sleep = flag.Duration("sleep", time.Millisecond, "sleep time after a request, used to adjust pressure") interval = flag.Duration("interval", time.Second, "interval to output the statistics") + tlsCAPath = flag.String("ca", "", "path of file that contains list of trusted SSL CAs.") + tlsCertPath = flag.String("cert", "", "path of file that contains X509 certificate in PEM format..") + tlsKeyPath = flag.String("key", "", "path of file that contains X509 key in PEM format.") wg sync.WaitGroup ) func main() { flag.Parse() - pdCli, err := pd.NewClient([]string{*pdAddrs}) + pdCli, err := pd.NewClient([]string{*pdAddrs}, *tlsCAPath, *tlsCertPath, *tlsKeyPath) if err != nil { log.Fatal(err) } diff --git a/conf/config.toml b/conf/config.toml index f7f50cb5b22..548d5ddf423 100644 --- a/conf/config.toml +++ b/conf/config.toml @@ -17,6 +17,19 @@ initial-cluster-state = "new" lease = 3 tso-save-interval = "3s" +# Path of file that contains list of trusted SSL CAs. +tls-ca-path = "" +# Path of file that contains X509 certificate in PEM format. +tls-cert-path = "" +# Path of file that contains X509 key in PEM format. +tls-key-path = "" +# enable client certificate auth, if true following two settings shouldn't be empty +client-cert-auth = false +# Path of file that contains X509 certificate in PEM format for client auth. +# tls-client-cert-path = "" +# Path of file that contains X509 key in PEM format for client auth. +# tls-client-key-path = "" + [log] level = "info" @@ -68,3 +81,4 @@ max-replicas = 3 # For example, ["zone", "rack"] means that we should place replicas to # different zones first, then to different racks if we don't have enough zones. location-labels = [] + diff --git a/pd-client/client.go b/pd-client/client.go index 60be90779d6..9a6a4c34bd4 100644 --- a/pd-client/client.go +++ b/pd-client/client.go @@ -14,6 +14,10 @@ package pd import ( + "crypto/tls" + "crypto/x509" + "io/ioutil" + "net/url" "strings" "sync" "time" @@ -25,6 +29,7 @@ import ( "github.com/pingcap/kvproto/pkg/pdpb" "golang.org/x/net/context" "google.golang.org/grpc" + "google.golang.org/grpc/credentials" ) // Client is a PD (Placement Driver) client. @@ -93,10 +98,14 @@ type client struct { wg sync.WaitGroup ctx context.Context cancel context.CancelFunc + + tlsCAPath string + tlsCertPath string + tlsKeyPath string } // NewClient creates a PD client. -func NewClient(pdAddrs []string) (Client, error) { +func NewClient(pdAddrs []string, tlsCAPath, tlsCertPath, tlsKeyPath string) (Client, error) { log.Infof("[pd] create pd client with endpoints %v", pdAddrs) ctx, cancel := context.WithCancel(context.Background()) c := &client{ @@ -106,6 +115,9 @@ func NewClient(pdAddrs []string) (Client, error) { checkLeaderCh: make(chan struct{}, 1), ctx: ctx, cancel: cancel, + tlsCAPath: tlsCAPath, + tlsCertPath: tlsCertPath, + tlsKeyPath: tlsKeyPath, } c.connMu.clientConns = make(map[string]*grpc.ClientConn) @@ -213,11 +225,46 @@ func (c *client) getOrCreateGRPCConn(addr string) (*grpc.ClientConn, error) { return conn, nil } - cc, err := grpc.Dial(strings.TrimPrefix(addr, "http://"), grpc.WithInsecure()) // TODO: Support HTTPS. + opt := grpc.WithInsecure() + if len(c.tlsCAPath) != 0 { + + certificates := []tls.Certificate{} + if len(c.tlsCertPath) != 0 && len(c.tlsKeyPath) != 0 { + // Load the client certificates from disk + certificate, err := tls.LoadX509KeyPair(c.tlsCertPath, c.tlsKeyPath) + if err != nil { + return nil, errors.Errorf("could not load client key pair: %s", err) + } + certificates = append(certificates, certificate) + } + + // Create a certificate pool from the certificate authority + certPool := x509.NewCertPool() + ca, err := ioutil.ReadFile(c.tlsCAPath) + if err != nil { + return nil, errors.Errorf("could not read ca certificate: %s", err) + } + + // Append the certificates from the CA + if ok := certPool.AppendCertsFromPEM(ca); !ok { + return nil, errors.New("failed to append ca certs") + } + + creds := credentials.NewTLS(&tls.Config{ + Certificates: certificates, + RootCAs: certPool, + }) + + opt = grpc.WithTransportCredentials(creds) + } + u, err := url.Parse(addr) + if err != nil { + return nil, errors.Trace(err) + } + cc, err := grpc.Dial(u.Host, opt) if err != nil { return nil, errors.Trace(err) } - c.connMu.Lock() defer c.connMu.Unlock() if old, ok := c.connMu.clientConns[addr]; ok { diff --git a/pdctl/command/global.go b/pdctl/command/global.go index 5eb06ef06cd..3a719fcd2b3 100644 --- a/pdctl/command/global.go +++ b/pdctl/command/global.go @@ -15,6 +15,8 @@ package command import ( "bytes" + "crypto/tls" + "crypto/x509" "encoding/json" "fmt" "io" @@ -23,20 +25,52 @@ import ( "net/url" "os" - log "github.com/Sirupsen/logrus" "github.com/juju/errors" - "github.com/pingcap/pd/pd-client" "github.com/spf13/cobra" ) var ( - pdClient pd.Client dailClient = &http.Client{} pingPrefix = "pd/ping" errInvalidAddr = errors.New("Invalid pd address, Cannot get connect to it") ) +// InitHttpsClient creates https client with ca file +func InitHttpsClient(tlsCAPath, tlsCertPath, tlsKeyPath string) error { + certificates := []tls.Certificate{} + if len(tlsCertPath) != 0 && len(tlsKeyPath) != 0 { + // Load the client certificates from disk + certificate, err := tls.LoadX509KeyPair(tlsCertPath, tlsKeyPath) + if err != nil { + return errors.Errorf("could not load client key pair: %s", err) + } + certificates = append(certificates, certificate) + } + + // Create a certificate pool from the certificate authority + certPool := x509.NewCertPool() + ca, err := ioutil.ReadFile(tlsCAPath) + if err != nil { + return errors.Errorf("could not read ca certificate: %s", err) + } + + // Append the certificates from the CA + if ok := certPool.AppendCertsFromPEM(ca); !ok { + return errors.New("failed to append ca certs") + } + + tr := &http.Transport{ + TLSClientConfig: &tls.Config{ + Certificates: certificates, + RootCAs: certPool, + }, + } + dailClient = &http.Client{Transport: tr} + + return nil +} + func getRequest(cmd *cobra.Command, prefix string, method string, bodyType string, body io.Reader) (*http.Request, error) { if method == "" { method = http.MethodGet @@ -82,34 +116,6 @@ func genResponseError(r *http.Response) error { return errors.Errorf("[%d] %s", r.StatusCode, res) } -// InitPDClient initialize pd client from cmd -func InitPDClient(cmd *cobra.Command) error { - addr, err := cmd.Flags().GetString("pd") - if err != nil { - return err - } - log.SetOutput(ioutil.Discard) - if pdClient != nil { - return nil - } - err = validPDAddr(addr) - if err != nil { - return err - } - pdClient, err = pd.NewClient([]string{addr}) - if err != nil { - return err - } - return nil -} - -func getClient() (pd.Client, error) { - if pdClient == nil { - return nil, errors.New("Must initialized pdClient firstly") - } - return pdClient, nil -} - func getAddressFromCmd(cmd *cobra.Command, prefix string) string { p, err := cmd.Flags().GetString("pd") if err != nil { diff --git a/pdctl/ctl.go b/pdctl/ctl.go index 3e081fe275f..627c78bf5bf 100644 --- a/pdctl/ctl.go +++ b/pdctl/ctl.go @@ -15,7 +15,6 @@ package pdctl import ( "fmt" - "os" "github.com/pingcap/pd/pdctl/command" "github.com/spf13/cobra" @@ -23,7 +22,10 @@ import ( // CommandFlags are flags that used in all Commands type CommandFlags struct { - URL string + URL string + tlsCAPath string + tlsCertPath string + tlsKeyPath string } var ( @@ -36,6 +38,9 @@ var ( func init() { rootCmd.PersistentFlags().StringVarP(&commandFlags.URL, "pd", "u", "http://127.0.0.1:2379", "pd address") + rootCmd.Flags().StringVar(&commandFlags.tlsCAPath, "ca", "", "path of file that contains list of trusted SSL CAs.") + rootCmd.Flags().StringVar(&commandFlags.tlsCertPath, "cert", "", "path of file that contains X509 certificate in PEM format.") + rootCmd.Flags().StringVar(&commandFlags.tlsKeyPath, "key", "", "path of file that contains X509 key in PEM format.") rootCmd.AddCommand( command.NewConfigCommand(), command.NewRegionCommand(), @@ -60,12 +65,15 @@ func Start(args []string) { rootCmd.SetArgs(args) rootCmd.SilenceErrors = true rootCmd.ParseFlags(args) - err := command.InitPDClient(rootCmd) - if err != nil { - fmt.Println(err) - os.Exit(1) - } rootCmd.SetUsageTemplate(command.UsageTemplate) + + if len(commandFlags.tlsCAPath) != 0 { + if err := command.InitHttpsClient(commandFlags.tlsCAPath, commandFlags.tlsCertPath, commandFlags.tlsKeyPath); err != nil { + fmt.Println(err) + return + } + } + if err := rootCmd.Execute(); err != nil { fmt.Println(rootCmd.UsageString()) } diff --git a/server/api/redirector.go b/server/api/redirector.go index 78a291aaade..42d8465995f 100644 --- a/server/api/redirector.go +++ b/server/api/redirector.go @@ -14,6 +14,7 @@ package api import ( + "crypto/tls" "io/ioutil" "net/http" "net/url" @@ -67,7 +68,12 @@ func (h *redirector) ServeHTTP(w http.ResponseWriter, r *http.Request, next http return } - newCustomReverseProxies(urls).ServeHTTP(w, r) + tlsConfig, err := h.s.GetTLSConfig() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + newCustomReverseProxies(urls, tlsConfig).ServeHTTP(w, r) } type customReverseProxies struct { @@ -75,9 +81,13 @@ type customReverseProxies struct { client *http.Client } -func newCustomReverseProxies(urls []url.URL) *customReverseProxies { +func newCustomReverseProxies(urls []url.URL, tlsConfig *tls.Config) *customReverseProxies { p := &customReverseProxies{ - client: &http.Client{}, + client: &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: tlsConfig, + }, + }, } p.urls = append(p.urls, urls...) diff --git a/server/config.go b/server/config.go index 3a22fab05d4..51cad04bfd7 100644 --- a/server/config.go +++ b/server/config.go @@ -85,6 +85,13 @@ type Config struct { // ElectionInterval is the interval for etcd Raft election. ElectionInterval typeutil.Duration `toml:"election-interval"` + TLSCAPath string `json:"tls-ca-path" toml:"tls-ca-path"` + TLSCertPath string `json:"tls-cert-path" toml:"tls-cert-path"` + TLSKeyPath string `json:"tls-key-path" toml:"tls-key-path"` + ClientCertAuth bool `json:"client-cert-auth" toml:"client-cert-auth"` + TLSClientCertPath string `json:"tls-client-cert-path" toml:"tls-client-cert-path"` + TLSClientKeyPath string `json:"tls-client-key-path" toml:"tls-client-key-path"` + configFile string // For all warnings during parsing. @@ -123,6 +130,14 @@ func NewConfig() *Config { fs.StringVar(&cfg.Log.File.Filename, "log-file", "", "log file path") fs.BoolVar(&cfg.Log.File.LogRotate, "log-rotate", true, "rotate log") fs.StringVar(&cfg.NamespaceClassifier, "namespace-classifier", "default", "namespace classifier (default 'default')") + + fs.StringVar(&cfg.TLSCAPath, "tls-ca", "", "Path of file that contains list of trusted TLS CAs") + fs.StringVar(&cfg.TLSCertPath, "tls-cert", "", "Path of file that contains X509 certificate in PEM format") + fs.StringVar(&cfg.TLSKeyPath, "tls-key", "", "Path of file that contains X509 key in PEM format") + fs.BoolVar(&cfg.ClientCertAuth, "client-cert-auth", false, "whether client authentication is enabled") + fs.StringVar(&cfg.TLSClientCertPath, "tls-client-cert", "", "Path of file that contains X509 certificate in PEM format") + fs.StringVar(&cfg.TLSClientKeyPath, "tls-client-key", "", "Path of file that contains X509 key in PEM format") + cfg.Namespace = make(map[string]NamespaceConfig) return cfg @@ -458,6 +473,12 @@ func (c *Config) genEmbedEtcdConfig() (*embed.Config, error) { cfg.AutoCompactionRetention = c.AutoCompactionRetention cfg.QuotaBackendBytes = int64(c.QuotaBackendBytes) + cfg.ClientTLSInfo.ClientCertAuth = c.ClientCertAuth + cfg.ClientTLSInfo.TrustedCAFile = c.TLSCAPath + cfg.ClientTLSInfo.CertFile = c.TLSCertPath + cfg.ClientTLSInfo.KeyFile = c.TLSKeyPath + cfg.PeerAutoTLS = len(c.TLSCAPath) != 0 + var err error cfg.LPUrls, err = ParseUrls(c.PeerUrls) diff --git a/server/server.go b/server/server.go index eb14c983f17..4f6d6c76107 100644 --- a/server/server.go +++ b/server/server.go @@ -14,6 +14,7 @@ package server import ( + "crypto/tls" "math/rand" "net/http" "path" @@ -26,6 +27,7 @@ import ( log "github.com/Sirupsen/logrus" "github.com/coreos/etcd/clientv3" "github.com/coreos/etcd/embed" + "github.com/coreos/etcd/pkg/transport" "github.com/coreos/etcd/pkg/types" "github.com/juju/errors" "github.com/pingcap/kvproto/pkg/metapb" @@ -120,6 +122,10 @@ func (s *Server) startEtcd() error { if err != nil { return errors.Trace(err) } + + // Wait until etcd is ready for use + <-etcd.Server.ReadyNotify() + // Check cluster ID urlmap, err := types.NewURLsMap(s.cfg.InitialCluster) if err != nil { @@ -131,14 +137,19 @@ func (s *Server) startEtcd() error { endpoints := []string{s.etcdCfg.ACUrls[0].String()} log.Infof("create etcd v3 client with endpoints %v", endpoints) + + tlsConfig, err := s.GetTLSConfig() + if err != nil { + return errors.Trace(err) + } client, err := clientv3.New(clientv3.Config{ Endpoints: endpoints, DialTimeout: etcdTimeout, + TLS: tlsConfig, }) if err != nil { return errors.Trace(err) } - if err = etcdutil.WaitEtcdStart(client, endpoints[0]); err != nil { // See https://github.com/coreos/etcd/issues/6067 // Here may return "not capable" error because we don't start @@ -544,3 +555,17 @@ func (s *Server) GetClusterStatus() (*ClusterStatus, error) { func (s *Server) getAllocIDPath() string { return path.Join(s.rootPath, "alloc_id") } + +// GetTLSConfig gets tls config. +func (s *Server) GetTLSConfig() (*tls.Config, error) { + tlsInfo := transport.TLSInfo{ + CertFile: s.cfg.TLSClientCertPath, + KeyFile: s.cfg.TLSClientKeyPath, + TrustedCAFile: s.cfg.TLSCAPath, + } + tlsConfig, err := tlsInfo.ClientConfig() + if err != nil { + return nil, errors.Trace(err) + } + return tlsConfig, nil +} From 5388ea36954e144d656a6f18ab885e04d0f402ca Mon Sep 17 00:00:00 2001 From: Connor1996 Date: Sun, 26 Nov 2017 18:17:45 +0800 Subject: [PATCH 02/14] fix ci --- pdctl/command/global.go | 4 ++-- pdctl/ctl.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pdctl/command/global.go b/pdctl/command/global.go index 3a719fcd2b3..9ae77058879 100644 --- a/pdctl/command/global.go +++ b/pdctl/command/global.go @@ -36,8 +36,8 @@ var ( errInvalidAddr = errors.New("Invalid pd address, Cannot get connect to it") ) -// InitHttpsClient creates https client with ca file -func InitHttpsClient(tlsCAPath, tlsCertPath, tlsKeyPath string) error { +// InitHTTPSClient creates https client with ca file +func InitHTTPSClient(tlsCAPath, tlsCertPath, tlsKeyPath string) error { certificates := []tls.Certificate{} if len(tlsCertPath) != 0 && len(tlsKeyPath) != 0 { // Load the client certificates from disk diff --git a/pdctl/ctl.go b/pdctl/ctl.go index 627c78bf5bf..339893c29f7 100644 --- a/pdctl/ctl.go +++ b/pdctl/ctl.go @@ -68,7 +68,7 @@ func Start(args []string) { rootCmd.SetUsageTemplate(command.UsageTemplate) if len(commandFlags.tlsCAPath) != 0 { - if err := command.InitHttpsClient(commandFlags.tlsCAPath, commandFlags.tlsCertPath, commandFlags.tlsKeyPath); err != nil { + if err := command.InitHTTPSClient(commandFlags.tlsCAPath, commandFlags.tlsCertPath, commandFlags.tlsKeyPath); err != nil { fmt.Println(err) return } From 9d24250a7e1f335cb5e152fcaafa4ed649093550 Mon Sep 17 00:00:00 2001 From: Connor1996 Date: Sun, 26 Nov 2017 18:29:37 +0800 Subject: [PATCH 03/14] fix test --- pd-client/client_test.go | 2 +- pd-client/leader_change_test.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pd-client/client_test.go b/pd-client/client_test.go index f90f5cca0e9..e3e595d081d 100644 --- a/pd-client/client_test.go +++ b/pd-client/client_test.go @@ -78,7 +78,7 @@ func (s *testClientSuite) SetUpSuite(c *C) { bootstrapServer(c, newHeader(s.srv), s.grpcPDClient) var err error - s.client, err = NewClient(s.srv.GetEndpoints()) + s.client, err = NewClient(s.srv.GetEndpoints(), "", "", "") c.Assert(err, IsNil) s.regionHeartbeat, err = s.grpcPDClient.RegionHeartbeat(context.Background()) c.Assert(err, IsNil) diff --git a/pd-client/leader_change_test.go b/pd-client/leader_change_test.go index bad1cb8784d..447b926794c 100644 --- a/pd-client/leader_change_test.go +++ b/pd-client/leader_change_test.go @@ -78,7 +78,7 @@ func (s *testLeaderChangeSuite) TestLeaderConfigChange(c *C) { svrs, endpoints, closeFunc := s.prepareClusterN(c, 3) defer closeFunc() - cli, err := NewClient(endpoints) + cli, err := NewClient(endpoints, "", "", "") c.Assert(err, IsNil) defer cli.Close() @@ -109,7 +109,7 @@ func (s *testLeaderChangeSuite) TestMemberList(c *C) { _, endpoints, closeFunc := s.prepareClusterN(c, 2) defer closeFunc() - cli, err := NewClient(endpoints[:1]) + cli, err := NewClient(endpoints[:1], "", "", "") c.Assert(err, IsNil) cli.Close() @@ -122,7 +122,7 @@ func (s *testLeaderChangeSuite) TestLeaderChange(c *C) { svrs, endpoints, closeFunc := s.prepareClusterN(c, 3) defer closeFunc() - cli, err := NewClient(endpoints) + cli, err := NewClient(endpoints, "", "", "") c.Assert(err, IsNil) defer cli.Close() @@ -163,7 +163,7 @@ func (s *testLeaderChangeSuite) TestLeaderTransfer(c *C) { servers, endpoints, closeFunc := s.prepareClusterN(c, 2) defer closeFunc() - cli, err := NewClient(endpoints) + cli, err := NewClient(endpoints, "", "", "") c.Assert(err, IsNil) defer cli.Close() From a9f3c1f7d84741ea6a7946acc26be41e7b5c5b9c Mon Sep 17 00:00:00 2001 From: Connor1996 Date: Mon, 27 Nov 2017 14:01:01 +0800 Subject: [PATCH 04/14] use same flag name as etcd --- cmd/pd-ctl/main.go | 4 ++-- cmd/pd-tso-bench/main.go | 2 +- conf/config.toml | 14 ++++++-------- pdctl/ctl.go | 2 +- server/config.go | 14 +++++++------- 5 files changed, 17 insertions(+), 19 deletions(-) diff --git a/cmd/pd-ctl/main.go b/cmd/pd-ctl/main.go index f2d404d03e7..182ce33ffae 100644 --- a/cmd/pd-ctl/main.go +++ b/cmd/pd-ctl/main.go @@ -41,7 +41,7 @@ func init() { flag.StringVarP(&url, "pd", "u", "http://127.0.0.1:2379", "The pd address") flag.BoolVarP(&detach, "detach", "d", false, "Run pdctl without readline") flag.BoolVarP(&version, "version", "V", false, "print version information and exit") - flag.StringVar(&tlsCAPath, "ca", "", "path of file that contains list of trusted SSL CAs.") + flag.StringVar(&tlsCAPath, "cacert", "", "path of file that contains list of trusted SSL CAs.") flag.StringVar(&tlsCertPath, "cert", "", "path of file that contains X509 certificate in PEM format.") flag.StringVar(&tlsKeyPath, "key", "", "path of file that contains X509 key in PEM format.") } @@ -121,7 +121,7 @@ func loop() { } args := strings.Split(strings.TrimSpace(line), " ") args = append(args, "-u", url) - args = append(args, "--ca", tlsCAPath) + args = append(args, "--cacert", tlsCAPath) args = append(args, "--cert", tlsCertPath) args = append(args, "--key", tlsKeyPath) pdctl.Start(args) diff --git a/cmd/pd-tso-bench/main.go b/cmd/pd-tso-bench/main.go index 2d6c779606e..c749d258f90 100644 --- a/cmd/pd-tso-bench/main.go +++ b/cmd/pd-tso-bench/main.go @@ -33,7 +33,7 @@ var ( concurrency = flag.Int("C", 1000, "concurrency") sleep = flag.Duration("sleep", time.Millisecond, "sleep time after a request, used to adjust pressure") interval = flag.Duration("interval", time.Second, "interval to output the statistics") - tlsCAPath = flag.String("ca", "", "path of file that contains list of trusted SSL CAs.") + tlsCAPath = flag.String("cacert", "", "path of file that contains list of trusted SSL CAs.") tlsCertPath = flag.String("cert", "", "path of file that contains X509 certificate in PEM format..") tlsKeyPath = flag.String("key", "", "path of file that contains X509 key in PEM format.") wg sync.WaitGroup diff --git a/conf/config.toml b/conf/config.toml index 548d5ddf423..72129e2e769 100644 --- a/conf/config.toml +++ b/conf/config.toml @@ -17,18 +17,16 @@ initial-cluster-state = "new" lease = 3 tso-save-interval = "3s" -# Path of file that contains list of trusted SSL CAs. -tls-ca-path = "" +# Path of file that contains list of trusted SSL CAs. if set, following four settings shouldn't be empty +tls-cacert-path = "/Users/Connor/cfssl/ca.pem" # Path of file that contains X509 certificate in PEM format. -tls-cert-path = "" +tls-cert-path = "/Users/Connor/cfssl/server.pem" # Path of file that contains X509 key in PEM format. -tls-key-path = "" -# enable client certificate auth, if true following two settings shouldn't be empty -client-cert-auth = false +tls-key-path = "/Users/Connor/cfssl/server-key.pem" # Path of file that contains X509 certificate in PEM format for client auth. -# tls-client-cert-path = "" +tls-client-cert-path = "/Users/Connor/cfssl/client.pem" # Path of file that contains X509 key in PEM format for client auth. -# tls-client-key-path = "" +tls-client-key-path = "/Users/Connor/cfssl/client-key.pem" [log] level = "info" diff --git a/pdctl/ctl.go b/pdctl/ctl.go index 339893c29f7..f555db0a257 100644 --- a/pdctl/ctl.go +++ b/pdctl/ctl.go @@ -38,7 +38,7 @@ var ( func init() { rootCmd.PersistentFlags().StringVarP(&commandFlags.URL, "pd", "u", "http://127.0.0.1:2379", "pd address") - rootCmd.Flags().StringVar(&commandFlags.tlsCAPath, "ca", "", "path of file that contains list of trusted SSL CAs.") + rootCmd.Flags().StringVar(&commandFlags.tlsCAPath, "cacert", "", "path of file that contains list of trusted SSL CAs.") rootCmd.Flags().StringVar(&commandFlags.tlsCertPath, "cert", "", "path of file that contains X509 certificate in PEM format.") rootCmd.Flags().StringVar(&commandFlags.tlsKeyPath, "key", "", "path of file that contains X509 key in PEM format.") rootCmd.AddCommand( diff --git a/server/config.go b/server/config.go index 51cad04bfd7..5bdc7caf1b0 100644 --- a/server/config.go +++ b/server/config.go @@ -85,12 +85,12 @@ type Config struct { // ElectionInterval is the interval for etcd Raft election. ElectionInterval typeutil.Duration `toml:"election-interval"` - TLSCAPath string `json:"tls-ca-path" toml:"tls-ca-path"` - TLSCertPath string `json:"tls-cert-path" toml:"tls-cert-path"` - TLSKeyPath string `json:"tls-key-path" toml:"tls-key-path"` - ClientCertAuth bool `json:"client-cert-auth" toml:"client-cert-auth"` - TLSClientCertPath string `json:"tls-client-cert-path" toml:"tls-client-cert-path"` - TLSClientKeyPath string `json:"tls-client-key-path" toml:"tls-client-key-path"` + TLSCAPath string `toml:"tls-cacert-path" json:"tls-cacert-path"` + TLSCertPath string `toml:"tls-cert-path" json:"tls-cert-path"` + TLSKeyPath string `toml:"tls-key-path" json:"tls-key-path"` + ClientCertAuth bool `toml:"client-cert-auth" json:"client-cert-auth"` + TLSClientCertPath string `toml:"tls-client-cert-path" json:"tls-client-cert-path"` + TLSClientKeyPath string `toml:"tls-client-key-path" json:"tls-client-key-path"` configFile string @@ -131,7 +131,7 @@ func NewConfig() *Config { fs.BoolVar(&cfg.Log.File.LogRotate, "log-rotate", true, "rotate log") fs.StringVar(&cfg.NamespaceClassifier, "namespace-classifier", "default", "namespace classifier (default 'default')") - fs.StringVar(&cfg.TLSCAPath, "tls-ca", "", "Path of file that contains list of trusted TLS CAs") + fs.StringVar(&cfg.TLSCAPath, "tls-cacert", "", "Path of file that contains list of trusted TLS CAs") fs.StringVar(&cfg.TLSCertPath, "tls-cert", "", "Path of file that contains X509 certificate in PEM format") fs.StringVar(&cfg.TLSKeyPath, "tls-key", "", "Path of file that contains X509 key in PEM format") fs.BoolVar(&cfg.ClientCertAuth, "client-cert-auth", false, "whether client authentication is enabled") From c1ae6a0a7b162b525db16bbe5b0cdfe37b2c359a Mon Sep 17 00:00:00 2001 From: Connor1996 Date: Mon, 27 Nov 2017 14:08:27 +0800 Subject: [PATCH 05/14] client auth always enabled when use tls --- server/config.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/server/config.go b/server/config.go index 5bdc7caf1b0..96394ac3d06 100644 --- a/server/config.go +++ b/server/config.go @@ -88,7 +88,6 @@ type Config struct { TLSCAPath string `toml:"tls-cacert-path" json:"tls-cacert-path"` TLSCertPath string `toml:"tls-cert-path" json:"tls-cert-path"` TLSKeyPath string `toml:"tls-key-path" json:"tls-key-path"` - ClientCertAuth bool `toml:"client-cert-auth" json:"client-cert-auth"` TLSClientCertPath string `toml:"tls-client-cert-path" json:"tls-client-cert-path"` TLSClientKeyPath string `toml:"tls-client-key-path" json:"tls-client-key-path"` @@ -134,7 +133,6 @@ func NewConfig() *Config { fs.StringVar(&cfg.TLSCAPath, "tls-cacert", "", "Path of file that contains list of trusted TLS CAs") fs.StringVar(&cfg.TLSCertPath, "tls-cert", "", "Path of file that contains X509 certificate in PEM format") fs.StringVar(&cfg.TLSKeyPath, "tls-key", "", "Path of file that contains X509 key in PEM format") - fs.BoolVar(&cfg.ClientCertAuth, "client-cert-auth", false, "whether client authentication is enabled") fs.StringVar(&cfg.TLSClientCertPath, "tls-client-cert", "", "Path of file that contains X509 certificate in PEM format") fs.StringVar(&cfg.TLSClientKeyPath, "tls-client-key", "", "Path of file that contains X509 key in PEM format") @@ -473,11 +471,10 @@ func (c *Config) genEmbedEtcdConfig() (*embed.Config, error) { cfg.AutoCompactionRetention = c.AutoCompactionRetention cfg.QuotaBackendBytes = int64(c.QuotaBackendBytes) - cfg.ClientTLSInfo.ClientCertAuth = c.ClientCertAuth + cfg.ClientTLSInfo.ClientCertAuth = len(c.TLSCAPath) != 0 cfg.ClientTLSInfo.TrustedCAFile = c.TLSCAPath cfg.ClientTLSInfo.CertFile = c.TLSCertPath cfg.ClientTLSInfo.KeyFile = c.TLSKeyPath - cfg.PeerAutoTLS = len(c.TLSCAPath) != 0 var err error From 9a1dcc32d02c6a0fe6f07f005b4e02a87d4a254f Mon Sep 17 00:00:00 2001 From: Connor1996 Date: Mon, 27 Nov 2017 14:10:20 +0800 Subject: [PATCH 06/14] remove peer auto tls --- server/config.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/config.go b/server/config.go index 96394ac3d06..bf0522784cd 100644 --- a/server/config.go +++ b/server/config.go @@ -475,6 +475,9 @@ func (c *Config) genEmbedEtcdConfig() (*embed.Config, error) { cfg.ClientTLSInfo.TrustedCAFile = c.TLSCAPath cfg.ClientTLSInfo.CertFile = c.TLSCertPath cfg.ClientTLSInfo.KeyFile = c.TLSKeyPath + cfg.PeerTLSInfo.TrustedCAFile = c.TLSCAPath + cfg.PeerTLSInfo.CertFile = c.TLSCertPath + cfg.PeerTLSInfo.KeyFile = c.TLSKeyPath var err error From f9f7091056e2bd3ca376653983364881314cdfd7 Mon Sep 17 00:00:00 2001 From: Connor1996 Date: Mon, 27 Nov 2017 14:46:14 +0800 Subject: [PATCH 07/14] SecurityOption struct --- cmd/pd-tso-bench/main.go | 7 ++++++- conf/config.toml | 10 +++++----- pd-client/client.go | 25 ++++++++++++++----------- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/cmd/pd-tso-bench/main.go b/cmd/pd-tso-bench/main.go index c749d258f90..7952a00bdca 100644 --- a/cmd/pd-tso-bench/main.go +++ b/cmd/pd-tso-bench/main.go @@ -41,7 +41,12 @@ var ( func main() { flag.Parse() - pdCli, err := pd.NewClient([]string{*pdAddrs}, *tlsCAPath, *tlsCertPath, *tlsKeyPath) + + pdCli, err := pd.NewClient([]string{*pdAddrs}, pd.SecurityOption{ + TlsCAPath: *tlsCAPath, + TlsCertPath: *tlsCertPath, + TlsKeyPath: *tlsKeyPath, + }) if err != nil { log.Fatal(err) } diff --git a/conf/config.toml b/conf/config.toml index 72129e2e769..17079271465 100644 --- a/conf/config.toml +++ b/conf/config.toml @@ -18,15 +18,15 @@ lease = 3 tso-save-interval = "3s" # Path of file that contains list of trusted SSL CAs. if set, following four settings shouldn't be empty -tls-cacert-path = "/Users/Connor/cfssl/ca.pem" +tls-cacert-path = "" # Path of file that contains X509 certificate in PEM format. -tls-cert-path = "/Users/Connor/cfssl/server.pem" +tls-cert-path = "" # Path of file that contains X509 key in PEM format. -tls-key-path = "/Users/Connor/cfssl/server-key.pem" +tls-key-path = "" # Path of file that contains X509 certificate in PEM format for client auth. -tls-client-cert-path = "/Users/Connor/cfssl/client.pem" +tls-client-cert-path = "" # Path of file that contains X509 key in PEM format for client auth. -tls-client-key-path = "/Users/Connor/cfssl/client-key.pem" +tls-client-key-path = "" [log] level = "info" diff --git a/pd-client/client.go b/pd-client/client.go index 9a6a4c34bd4..0a1c48adac3 100644 --- a/pd-client/client.go +++ b/pd-client/client.go @@ -99,13 +99,18 @@ type client struct { ctx context.Context cancel context.CancelFunc - tlsCAPath string - tlsCertPath string - tlsKeyPath string + security SecurityOption +} + +// SecurityOption records options about tls +type SecurityOption struct { + TlsCAPath string + TlsCertPath string + TlsKeyPath string } // NewClient creates a PD client. -func NewClient(pdAddrs []string, tlsCAPath, tlsCertPath, tlsKeyPath string) (Client, error) { +func NewClient(pdAddrs []string, security SecurityOption) (Client, error) { log.Infof("[pd] create pd client with endpoints %v", pdAddrs) ctx, cancel := context.WithCancel(context.Background()) c := &client{ @@ -115,9 +120,7 @@ func NewClient(pdAddrs []string, tlsCAPath, tlsCertPath, tlsKeyPath string) (Cli checkLeaderCh: make(chan struct{}, 1), ctx: ctx, cancel: cancel, - tlsCAPath: tlsCAPath, - tlsCertPath: tlsCertPath, - tlsKeyPath: tlsKeyPath, + security: security, } c.connMu.clientConns = make(map[string]*grpc.ClientConn) @@ -226,12 +229,12 @@ func (c *client) getOrCreateGRPCConn(addr string) (*grpc.ClientConn, error) { } opt := grpc.WithInsecure() - if len(c.tlsCAPath) != 0 { + if len(c.security.TlsCAPath) != 0 { certificates := []tls.Certificate{} - if len(c.tlsCertPath) != 0 && len(c.tlsKeyPath) != 0 { + if len(c.security.TlsCertPath) != 0 && len(c.security.TlsKeyPath) != 0 { // Load the client certificates from disk - certificate, err := tls.LoadX509KeyPair(c.tlsCertPath, c.tlsKeyPath) + certificate, err := tls.LoadX509KeyPair(c.security.TlsCertPath, c.security.TlsKeyPath) if err != nil { return nil, errors.Errorf("could not load client key pair: %s", err) } @@ -240,7 +243,7 @@ func (c *client) getOrCreateGRPCConn(addr string) (*grpc.ClientConn, error) { // Create a certificate pool from the certificate authority certPool := x509.NewCertPool() - ca, err := ioutil.ReadFile(c.tlsCAPath) + ca, err := ioutil.ReadFile(c.security.TlsCAPath) if err != nil { return nil, errors.Errorf("could not read ca certificate: %s", err) } From 2f89728fa563c1647f4e65283c83ec6da6840e8c Mon Sep 17 00:00:00 2001 From: Connor1996 Date: Mon, 27 Nov 2017 14:54:58 +0800 Subject: [PATCH 08/14] fix ci --- cmd/pd-tso-bench/main.go | 6 +++--- pd-client/client.go | 14 +++++++------- pd-client/client_test.go | 2 +- pd-client/leader_change_test.go | 8 ++++---- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/cmd/pd-tso-bench/main.go b/cmd/pd-tso-bench/main.go index 7952a00bdca..ea59f823cc1 100644 --- a/cmd/pd-tso-bench/main.go +++ b/cmd/pd-tso-bench/main.go @@ -43,9 +43,9 @@ func main() { flag.Parse() pdCli, err := pd.NewClient([]string{*pdAddrs}, pd.SecurityOption{ - TlsCAPath: *tlsCAPath, - TlsCertPath: *tlsCertPath, - TlsKeyPath: *tlsKeyPath, + TLSCAPath: *tlsCAPath, + TLSCertPath: *tlsCertPath, + TLSKeyPath: *tlsKeyPath, }) if err != nil { log.Fatal(err) diff --git a/pd-client/client.go b/pd-client/client.go index 0a1c48adac3..423bb1dfa24 100644 --- a/pd-client/client.go +++ b/pd-client/client.go @@ -104,9 +104,9 @@ type client struct { // SecurityOption records options about tls type SecurityOption struct { - TlsCAPath string - TlsCertPath string - TlsKeyPath string + TLSCAPath string + TLSCertPath string + TLSKeyPath string } // NewClient creates a PD client. @@ -229,12 +229,12 @@ func (c *client) getOrCreateGRPCConn(addr string) (*grpc.ClientConn, error) { } opt := grpc.WithInsecure() - if len(c.security.TlsCAPath) != 0 { + if len(c.security.TLSCAPath) != 0 { certificates := []tls.Certificate{} - if len(c.security.TlsCertPath) != 0 && len(c.security.TlsKeyPath) != 0 { + if len(c.security.TLSCertPath) != 0 && len(c.security.TLSKeyPath) != 0 { // Load the client certificates from disk - certificate, err := tls.LoadX509KeyPair(c.security.TlsCertPath, c.security.TlsKeyPath) + certificate, err := tls.LoadX509KeyPair(c.security.TLSCertPath, c.security.TLSKeyPath) if err != nil { return nil, errors.Errorf("could not load client key pair: %s", err) } @@ -243,7 +243,7 @@ func (c *client) getOrCreateGRPCConn(addr string) (*grpc.ClientConn, error) { // Create a certificate pool from the certificate authority certPool := x509.NewCertPool() - ca, err := ioutil.ReadFile(c.security.TlsCAPath) + ca, err := ioutil.ReadFile(c.security.TLSCAPath) if err != nil { return nil, errors.Errorf("could not read ca certificate: %s", err) } diff --git a/pd-client/client_test.go b/pd-client/client_test.go index e3e595d081d..f49c2573333 100644 --- a/pd-client/client_test.go +++ b/pd-client/client_test.go @@ -78,7 +78,7 @@ func (s *testClientSuite) SetUpSuite(c *C) { bootstrapServer(c, newHeader(s.srv), s.grpcPDClient) var err error - s.client, err = NewClient(s.srv.GetEndpoints(), "", "", "") + s.client, err = NewClient(s.srv.GetEndpoints(), SecurityOption{}) c.Assert(err, IsNil) s.regionHeartbeat, err = s.grpcPDClient.RegionHeartbeat(context.Background()) c.Assert(err, IsNil) diff --git a/pd-client/leader_change_test.go b/pd-client/leader_change_test.go index 447b926794c..1ef9843de1b 100644 --- a/pd-client/leader_change_test.go +++ b/pd-client/leader_change_test.go @@ -78,7 +78,7 @@ func (s *testLeaderChangeSuite) TestLeaderConfigChange(c *C) { svrs, endpoints, closeFunc := s.prepareClusterN(c, 3) defer closeFunc() - cli, err := NewClient(endpoints, "", "", "") + cli, err := NewClient(endpoints, SecurityOption{}) c.Assert(err, IsNil) defer cli.Close() @@ -109,7 +109,7 @@ func (s *testLeaderChangeSuite) TestMemberList(c *C) { _, endpoints, closeFunc := s.prepareClusterN(c, 2) defer closeFunc() - cli, err := NewClient(endpoints[:1], "", "", "") + cli, err := NewClient(endpoints[:1], SecurityOption{}) c.Assert(err, IsNil) cli.Close() @@ -122,7 +122,7 @@ func (s *testLeaderChangeSuite) TestLeaderChange(c *C) { svrs, endpoints, closeFunc := s.prepareClusterN(c, 3) defer closeFunc() - cli, err := NewClient(endpoints, "", "", "") + cli, err := NewClient(endpoints, SecurityOption{}) c.Assert(err, IsNil) defer cli.Close() @@ -163,7 +163,7 @@ func (s *testLeaderChangeSuite) TestLeaderTransfer(c *C) { servers, endpoints, closeFunc := s.prepareClusterN(c, 2) defer closeFunc() - cli, err := NewClient(endpoints, "", "", "") + cli, err := NewClient(endpoints, SecurityOption{}) c.Assert(err, IsNil) defer cli.Close() From 942da1520876cb2b05b772c4e90539dc73ba31ed Mon Sep 17 00:00:00 2001 From: Connor1996 Date: Tue, 28 Nov 2017 14:25:12 +0800 Subject: [PATCH 09/14] address comment --- pd-client/client.go | 2 +- pdctl/command/global.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pd-client/client.go b/pd-client/client.go index 423bb1dfa24..c8304bbc572 100644 --- a/pd-client/client.go +++ b/pd-client/client.go @@ -249,7 +249,7 @@ func (c *client) getOrCreateGRPCConn(addr string) (*grpc.ClientConn, error) { } // Append the certificates from the CA - if ok := certPool.AppendCertsFromPEM(ca); !ok { + if !certPool.AppendCertsFromPEM(ca) { return nil, errors.New("failed to append ca certs") } diff --git a/pdctl/command/global.go b/pdctl/command/global.go index 9ae77058879..6b33da4e429 100644 --- a/pdctl/command/global.go +++ b/pdctl/command/global.go @@ -56,7 +56,7 @@ func InitHTTPSClient(tlsCAPath, tlsCertPath, tlsKeyPath string) error { } // Append the certificates from the CA - if ok := certPool.AppendCertsFromPEM(ca); !ok { + if !certPool.AppendCertsFromPEM(ca) { return errors.New("failed to append ca certs") } From 7cc7bfbe3abc1efcd9ad6694667684051068014a Mon Sep 17 00:00:00 2001 From: Connor1996 Date: Wed, 29 Nov 2017 15:49:11 +0800 Subject: [PATCH 10/14] pd server don't need client cert --- conf/config.toml | 4 ---- pkg/etcdutil/etcdutil.go | 7 +++++-- server/config.go | 10 +++------- server/server.go | 15 ++++++++------- 4 files changed, 16 insertions(+), 20 deletions(-) diff --git a/conf/config.toml b/conf/config.toml index 17079271465..6941283bcae 100644 --- a/conf/config.toml +++ b/conf/config.toml @@ -23,10 +23,6 @@ tls-cacert-path = "" tls-cert-path = "" # Path of file that contains X509 key in PEM format. tls-key-path = "" -# Path of file that contains X509 certificate in PEM format for client auth. -tls-client-cert-path = "" -# Path of file that contains X509 key in PEM format for client auth. -tls-client-key-path = "" [log] level = "info" diff --git a/pkg/etcdutil/etcdutil.go b/pkg/etcdutil/etcdutil.go index e30604e631a..24017d2229c 100644 --- a/pkg/etcdutil/etcdutil.go +++ b/pkg/etcdutil/etcdutil.go @@ -15,6 +15,7 @@ package etcdutil import ( "context" + "crypto/tls" "net/http" "time" @@ -43,7 +44,7 @@ const ( // CheckClusterID checks Etcd's cluster ID, returns an error if mismatch. // This function will never block even quorum is not satisfied. -func CheckClusterID(localClusterID types.ID, um types.URLsMap) error { +func CheckClusterID(localClusterID types.ID, um types.URLsMap, tlsConfig *tls.Config) error { if len(um) == 0 { return nil } @@ -54,7 +55,9 @@ func CheckClusterID(localClusterID types.ID, um types.URLsMap) error { } for _, u := range peerURLs { - trp := &http.Transport{} + trp := &http.Transport{ + TLSClientConfig: tlsConfig, + } remoteCluster, gerr := etcdserver.GetClusterFromRemotePeers([]string{u}, trp) trp.CloseIdleConnections() if gerr != nil { diff --git a/server/config.go b/server/config.go index bf0522784cd..8daa52b5c9c 100644 --- a/server/config.go +++ b/server/config.go @@ -85,11 +85,9 @@ type Config struct { // ElectionInterval is the interval for etcd Raft election. ElectionInterval typeutil.Duration `toml:"election-interval"` - TLSCAPath string `toml:"tls-cacert-path" json:"tls-cacert-path"` - TLSCertPath string `toml:"tls-cert-path" json:"tls-cert-path"` - TLSKeyPath string `toml:"tls-key-path" json:"tls-key-path"` - TLSClientCertPath string `toml:"tls-client-cert-path" json:"tls-client-cert-path"` - TLSClientKeyPath string `toml:"tls-client-key-path" json:"tls-client-key-path"` + TLSCAPath string `toml:"tls-cacert-path" json:"tls-cacert-path"` + TLSCertPath string `toml:"tls-cert-path" json:"tls-cert-path"` + TLSKeyPath string `toml:"tls-key-path" json:"tls-key-path"` configFile string @@ -133,8 +131,6 @@ func NewConfig() *Config { fs.StringVar(&cfg.TLSCAPath, "tls-cacert", "", "Path of file that contains list of trusted TLS CAs") fs.StringVar(&cfg.TLSCertPath, "tls-cert", "", "Path of file that contains X509 certificate in PEM format") fs.StringVar(&cfg.TLSKeyPath, "tls-key", "", "Path of file that contains X509 key in PEM format") - fs.StringVar(&cfg.TLSClientCertPath, "tls-client-cert", "", "Path of file that contains X509 certificate in PEM format") - fs.StringVar(&cfg.TLSClientKeyPath, "tls-client-key", "", "Path of file that contains X509 key in PEM format") cfg.Namespace = make(map[string]NamespaceConfig) diff --git a/server/server.go b/server/server.go index f1217f3010d..da24be9fc36 100644 --- a/server/server.go +++ b/server/server.go @@ -128,19 +128,20 @@ func (s *Server) startEtcd() error { if err != nil { return errors.Trace(err) } - if err = etcdutil.CheckClusterID(etcd.Server.Cluster().ID(), urlmap); err != nil { + tlsConfig, err := s.GetTLSConfig() + if err != nil { return errors.Trace(err) } + if err = etcdutil.CheckClusterID(etcd.Server.Cluster().ID(), urlmap, tlsConfig); err != nil { + return errors.Trace(err) + } + // Wait etcd until it is ready to use <-etcd.Server.ReadyNotify() endpoints := []string{s.etcdCfg.ACUrls[0].String()} log.Infof("create etcd v3 client with endpoints %v", endpoints) - tlsConfig, err := s.GetTLSConfig() - if err != nil { - return errors.Trace(err) - } client, err := clientv3.New(clientv3.Config{ Endpoints: endpoints, DialTimeout: etcdTimeout, @@ -550,8 +551,8 @@ func (s *Server) getAllocIDPath() string { // GetTLSConfig gets tls config. func (s *Server) GetTLSConfig() (*tls.Config, error) { tlsInfo := transport.TLSInfo{ - CertFile: s.cfg.TLSClientCertPath, - KeyFile: s.cfg.TLSClientKeyPath, + CertFile: s.cfg.TLSCertPath, + KeyFile: s.cfg.TLSKeyPath, TrustedCAFile: s.cfg.TLSCAPath, } tlsConfig, err := tlsInfo.ClientConfig() From 261e6c50ac0ff04a125d0a4c56b9b48cefb7e678 Mon Sep 17 00:00:00 2001 From: Connor1996 Date: Wed, 29 Nov 2017 15:50:11 +0800 Subject: [PATCH 11/14] support abort when wait for etcd to be ready --- server/server.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/server/server.go b/server/server.go index da24be9fc36..1a40d6935d5 100644 --- a/server/server.go +++ b/server/server.go @@ -17,11 +17,14 @@ import ( "crypto/tls" "math/rand" "net/http" + "os" + "os/signal" "path" "strconv" "strings" "sync" "sync/atomic" + "syscall" "time" log "github.com/Sirupsen/logrus" @@ -136,8 +139,19 @@ func (s *Server) startEtcd() error { return errors.Trace(err) } + sc := make(chan os.Signal, 1) + signal.Notify(sc, + syscall.SIGHUP, + syscall.SIGINT, + syscall.SIGTERM, + syscall.SIGQUIT) + + select { // Wait etcd until it is ready to use - <-etcd.Server.ReadyNotify() + case <-etcd.Server.ReadyNotify(): + case sig := <-sc: + return errors.Errorf("receive signal %v when waiting embed etcd to be ready", sig) + } endpoints := []string{s.etcdCfg.ACUrls[0].String()} log.Infof("create etcd v3 client with endpoints %v", endpoints) From 047530235d1833210435480f61a2cd808b15c6f7 Mon Sep 17 00:00:00 2001 From: Connor1996 Date: Wed, 29 Nov 2017 16:56:58 +0800 Subject: [PATCH 12/14] fix test --- pkg/etcdutil/etcdutil_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/etcdutil/etcdutil_test.go b/pkg/etcdutil/etcdutil_test.go index 29eefe5c1c2..c84566e68c9 100644 --- a/pkg/etcdutil/etcdutil_test.go +++ b/pkg/etcdutil/etcdutil_test.go @@ -14,6 +14,7 @@ package etcdutil import ( + "crypto/tls" "fmt" "io/ioutil" "net/url" @@ -120,7 +121,7 @@ func (s *testEtcdutilSuite) TestMemberHelpers(c *C) { // Test CheckClusterID urlmap, err := types.NewURLsMap(cfg2.InitialCluster) c.Assert(err, IsNil) - err = CheckClusterID(etcd1.Server.Cluster().ID(), urlmap) + err = CheckClusterID(etcd1.Server.Cluster().ID(), urlmap, &tls.Config{}) c.Assert(err, IsNil) // Test RemoveEtcdMember From 66ad5dae5ecaafa6fb8e9fc69be5c162125b9111 Mon Sep 17 00:00:00 2001 From: Connor1996 Date: Fri, 1 Dec 2017 16:03:33 +0800 Subject: [PATCH 13/14] use security section --- cmd/pd-ctl/main.go | 24 ++++++++++++------------ cmd/pd-tso-bench/main.go | 12 ++++++------ conf/config.toml | 7 ++++--- pd-client/client.go | 14 +++++++------- pdctl/command/global.go | 8 ++++---- pdctl/ctl.go | 18 +++++++++--------- server/config.go | 34 +++++++++++++++++++++------------- server/server.go | 6 +++--- 8 files changed, 66 insertions(+), 57 deletions(-) diff --git a/cmd/pd-ctl/main.go b/cmd/pd-ctl/main.go index 182ce33ffae..035fcb0a744 100644 --- a/cmd/pd-ctl/main.go +++ b/cmd/pd-ctl/main.go @@ -29,21 +29,21 @@ import ( ) var ( - url string - detach bool - version bool - tlsCAPath string - tlsCertPath string - tlsKeyPath string + url string + detach bool + version bool + CAPath string + CertPath string + KeyPath string ) func init() { flag.StringVarP(&url, "pd", "u", "http://127.0.0.1:2379", "The pd address") flag.BoolVarP(&detach, "detach", "d", false, "Run pdctl without readline") flag.BoolVarP(&version, "version", "V", false, "print version information and exit") - flag.StringVar(&tlsCAPath, "cacert", "", "path of file that contains list of trusted SSL CAs.") - flag.StringVar(&tlsCertPath, "cert", "", "path of file that contains X509 certificate in PEM format.") - flag.StringVar(&tlsKeyPath, "key", "", "path of file that contains X509 key in PEM format.") + flag.StringVar(&CAPath, "cacert", "", "path of file that contains list of trusted SSL CAs.") + flag.StringVar(&CertPath, "cert", "", "path of file that contains X509 certificate in PEM format.") + flag.StringVar(&KeyPath, "key", "", "path of file that contains X509 key in PEM format.") } func main() { @@ -121,9 +121,9 @@ func loop() { } args := strings.Split(strings.TrimSpace(line), " ") args = append(args, "-u", url) - args = append(args, "--cacert", tlsCAPath) - args = append(args, "--cert", tlsCertPath) - args = append(args, "--key", tlsKeyPath) + args = append(args, "--cacert", CAPath) + args = append(args, "--cert", CertPath) + args = append(args, "--key", KeyPath) pdctl.Start(args) } } diff --git a/cmd/pd-tso-bench/main.go b/cmd/pd-tso-bench/main.go index ea59f823cc1..a9401d1d8fd 100644 --- a/cmd/pd-tso-bench/main.go +++ b/cmd/pd-tso-bench/main.go @@ -33,9 +33,9 @@ var ( concurrency = flag.Int("C", 1000, "concurrency") sleep = flag.Duration("sleep", time.Millisecond, "sleep time after a request, used to adjust pressure") interval = flag.Duration("interval", time.Second, "interval to output the statistics") - tlsCAPath = flag.String("cacert", "", "path of file that contains list of trusted SSL CAs.") - tlsCertPath = flag.String("cert", "", "path of file that contains X509 certificate in PEM format..") - tlsKeyPath = flag.String("key", "", "path of file that contains X509 key in PEM format.") + CAPath = flag.String("cacert", "", "path of file that contains list of trusted SSL CAs.") + CertPath = flag.String("cert", "", "path of file that contains X509 certificate in PEM format..") + KeyPath = flag.String("key", "", "path of file that contains X509 key in PEM format.") wg sync.WaitGroup ) @@ -43,9 +43,9 @@ func main() { flag.Parse() pdCli, err := pd.NewClient([]string{*pdAddrs}, pd.SecurityOption{ - TLSCAPath: *tlsCAPath, - TLSCertPath: *tlsCertPath, - TLSKeyPath: *tlsKeyPath, + CAPath: *CAPath, + CertPath: *CertPath, + KeyPath: *KeyPath, }) if err != nil { log.Fatal(err) diff --git a/conf/config.toml b/conf/config.toml index 6941283bcae..1c8568f7963 100644 --- a/conf/config.toml +++ b/conf/config.toml @@ -17,12 +17,13 @@ initial-cluster-state = "new" lease = 3 tso-save-interval = "3s" +[security] # Path of file that contains list of trusted SSL CAs. if set, following four settings shouldn't be empty -tls-cacert-path = "" +cacert-path = "" # Path of file that contains X509 certificate in PEM format. -tls-cert-path = "" +cert-path = "" # Path of file that contains X509 key in PEM format. -tls-key-path = "" +key-path = "" [log] level = "info" diff --git a/pd-client/client.go b/pd-client/client.go index c8304bbc572..4eae2e1c2b0 100644 --- a/pd-client/client.go +++ b/pd-client/client.go @@ -104,9 +104,9 @@ type client struct { // SecurityOption records options about tls type SecurityOption struct { - TLSCAPath string - TLSCertPath string - TLSKeyPath string + CAPath string + CertPath string + KeyPath string } // NewClient creates a PD client. @@ -229,12 +229,12 @@ func (c *client) getOrCreateGRPCConn(addr string) (*grpc.ClientConn, error) { } opt := grpc.WithInsecure() - if len(c.security.TLSCAPath) != 0 { + if len(c.security.CAPath) != 0 { certificates := []tls.Certificate{} - if len(c.security.TLSCertPath) != 0 && len(c.security.TLSKeyPath) != 0 { + if len(c.security.CertPath) != 0 && len(c.security.KeyPath) != 0 { // Load the client certificates from disk - certificate, err := tls.LoadX509KeyPair(c.security.TLSCertPath, c.security.TLSKeyPath) + certificate, err := tls.LoadX509KeyPair(c.security.CertPath, c.security.KeyPath) if err != nil { return nil, errors.Errorf("could not load client key pair: %s", err) } @@ -243,7 +243,7 @@ func (c *client) getOrCreateGRPCConn(addr string) (*grpc.ClientConn, error) { // Create a certificate pool from the certificate authority certPool := x509.NewCertPool() - ca, err := ioutil.ReadFile(c.security.TLSCAPath) + ca, err := ioutil.ReadFile(c.security.CAPath) if err != nil { return nil, errors.Errorf("could not read ca certificate: %s", err) } diff --git a/pdctl/command/global.go b/pdctl/command/global.go index 6b33da4e429..dc966d84758 100644 --- a/pdctl/command/global.go +++ b/pdctl/command/global.go @@ -37,11 +37,11 @@ var ( ) // InitHTTPSClient creates https client with ca file -func InitHTTPSClient(tlsCAPath, tlsCertPath, tlsKeyPath string) error { +func InitHTTPSClient(CAPath, CertPath, KeyPath string) error { certificates := []tls.Certificate{} - if len(tlsCertPath) != 0 && len(tlsKeyPath) != 0 { + if len(CertPath) != 0 && len(KeyPath) != 0 { // Load the client certificates from disk - certificate, err := tls.LoadX509KeyPair(tlsCertPath, tlsKeyPath) + certificate, err := tls.LoadX509KeyPair(CertPath, KeyPath) if err != nil { return errors.Errorf("could not load client key pair: %s", err) } @@ -50,7 +50,7 @@ func InitHTTPSClient(tlsCAPath, tlsCertPath, tlsKeyPath string) error { // Create a certificate pool from the certificate authority certPool := x509.NewCertPool() - ca, err := ioutil.ReadFile(tlsCAPath) + ca, err := ioutil.ReadFile(CAPath) if err != nil { return errors.Errorf("could not read ca certificate: %s", err) } diff --git a/pdctl/ctl.go b/pdctl/ctl.go index f555db0a257..fd0a24d707a 100644 --- a/pdctl/ctl.go +++ b/pdctl/ctl.go @@ -22,10 +22,10 @@ import ( // CommandFlags are flags that used in all Commands type CommandFlags struct { - URL string - tlsCAPath string - tlsCertPath string - tlsKeyPath string + URL string + CAPath string + CertPath string + KeyPath string } var ( @@ -38,9 +38,9 @@ var ( func init() { rootCmd.PersistentFlags().StringVarP(&commandFlags.URL, "pd", "u", "http://127.0.0.1:2379", "pd address") - rootCmd.Flags().StringVar(&commandFlags.tlsCAPath, "cacert", "", "path of file that contains list of trusted SSL CAs.") - rootCmd.Flags().StringVar(&commandFlags.tlsCertPath, "cert", "", "path of file that contains X509 certificate in PEM format.") - rootCmd.Flags().StringVar(&commandFlags.tlsKeyPath, "key", "", "path of file that contains X509 key in PEM format.") + rootCmd.Flags().StringVar(&commandFlags.CAPath, "cacert", "", "path of file that contains list of trusted SSL CAs.") + rootCmd.Flags().StringVar(&commandFlags.CertPath, "cert", "", "path of file that contains X509 certificate in PEM format.") + rootCmd.Flags().StringVar(&commandFlags.KeyPath, "key", "", "path of file that contains X509 key in PEM format.") rootCmd.AddCommand( command.NewConfigCommand(), command.NewRegionCommand(), @@ -67,8 +67,8 @@ func Start(args []string) { rootCmd.ParseFlags(args) rootCmd.SetUsageTemplate(command.UsageTemplate) - if len(commandFlags.tlsCAPath) != 0 { - if err := command.InitHTTPSClient(commandFlags.tlsCAPath, commandFlags.tlsCertPath, commandFlags.tlsKeyPath); err != nil { + if len(commandFlags.CAPath) != 0 { + if err := command.InitHTTPSClient(commandFlags.CAPath, commandFlags.CertPath, commandFlags.KeyPath); err != nil { fmt.Println(err) return } diff --git a/server/config.go b/server/config.go index 8daa52b5c9c..373997889ce 100644 --- a/server/config.go +++ b/server/config.go @@ -85,9 +85,7 @@ type Config struct { // ElectionInterval is the interval for etcd Raft election. ElectionInterval typeutil.Duration `toml:"election-interval"` - TLSCAPath string `toml:"tls-cacert-path" json:"tls-cacert-path"` - TLSCertPath string `toml:"tls-cert-path" json:"tls-cert-path"` - TLSKeyPath string `toml:"tls-key-path" json:"tls-key-path"` + Security SecurityConfig `toml:"security" json:"security"` configFile string @@ -128,9 +126,9 @@ func NewConfig() *Config { fs.BoolVar(&cfg.Log.File.LogRotate, "log-rotate", true, "rotate log") fs.StringVar(&cfg.NamespaceClassifier, "namespace-classifier", "default", "namespace classifier (default 'default')") - fs.StringVar(&cfg.TLSCAPath, "tls-cacert", "", "Path of file that contains list of trusted TLS CAs") - fs.StringVar(&cfg.TLSCertPath, "tls-cert", "", "Path of file that contains X509 certificate in PEM format") - fs.StringVar(&cfg.TLSKeyPath, "tls-key", "", "Path of file that contains X509 key in PEM format") + fs.StringVar(&cfg.Security.CAPath, "cacert", "", "Path of file that contains list of trusted TLS CAs") + fs.StringVar(&cfg.Security.CertPath, "cert", "", "Path of file that contains X509 certificate in PEM format") + fs.StringVar(&cfg.Security.KeyPath, "key", "", "Path of file that contains X509 key in PEM format") cfg.Namespace = make(map[string]NamespaceConfig) @@ -435,6 +433,16 @@ func (c *NamespaceConfig) adjust(opt *scheduleOption) { adjustUint64(&c.MaxReplicas, uint64(opt.GetMaxReplicas(namespace.DefaultNamespace))) } +// SecurityConfig is the configuration for supporting tls. +type SecurityConfig struct { + // CAPath is the path of file that contains list of trusted SSL CAs. if set, following four settings shouldn't be empty + CAPath string `toml:"cacert-path" json:"cacert-path"` + // CertPath is the path of file that contains X509 certificate in PEM format. + CertPath string `toml:"cert-path" json:"cert-path"` + // KeyPath is the path of file that contains X509 key in PEM format. + KeyPath string `toml:"key-path" json:"key-path"` +} + // ParseUrls parse a string into multiple urls. // Export for api. func ParseUrls(s string) ([]url.URL, error) { @@ -467,13 +475,13 @@ func (c *Config) genEmbedEtcdConfig() (*embed.Config, error) { cfg.AutoCompactionRetention = c.AutoCompactionRetention cfg.QuotaBackendBytes = int64(c.QuotaBackendBytes) - cfg.ClientTLSInfo.ClientCertAuth = len(c.TLSCAPath) != 0 - cfg.ClientTLSInfo.TrustedCAFile = c.TLSCAPath - cfg.ClientTLSInfo.CertFile = c.TLSCertPath - cfg.ClientTLSInfo.KeyFile = c.TLSKeyPath - cfg.PeerTLSInfo.TrustedCAFile = c.TLSCAPath - cfg.PeerTLSInfo.CertFile = c.TLSCertPath - cfg.PeerTLSInfo.KeyFile = c.TLSKeyPath + cfg.ClientTLSInfo.ClientCertAuth = len(c.Security.CAPath) != 0 + cfg.ClientTLSInfo.TrustedCAFile = c.Security.CAPath + cfg.ClientTLSInfo.CertFile = c.Security.CertPath + cfg.ClientTLSInfo.KeyFile = c.Security.KeyPath + cfg.PeerTLSInfo.TrustedCAFile = c.Security.CAPath + cfg.PeerTLSInfo.CertFile = c.Security.CertPath + cfg.PeerTLSInfo.KeyFile = c.Security.KeyPath var err error diff --git a/server/server.go b/server/server.go index 1a40d6935d5..755ffbc9236 100644 --- a/server/server.go +++ b/server/server.go @@ -565,9 +565,9 @@ func (s *Server) getAllocIDPath() string { // GetTLSConfig gets tls config. func (s *Server) GetTLSConfig() (*tls.Config, error) { tlsInfo := transport.TLSInfo{ - CertFile: s.cfg.TLSCertPath, - KeyFile: s.cfg.TLSKeyPath, - TrustedCAFile: s.cfg.TLSCAPath, + CertFile: s.cfg.Security.CertPath, + KeyFile: s.cfg.Security.KeyPath, + TrustedCAFile: s.cfg.Security.CAPath, } tlsConfig, err := tlsInfo.ClientConfig() if err != nil { From ca1380149afc419cad84e18f2fb5c4079b5d77f4 Mon Sep 17 00:00:00 2001 From: Connor1996 Date: Fri, 1 Dec 2017 16:13:15 +0800 Subject: [PATCH 14/14] fix typo --- cmd/pd-ctl/main.go | 18 +++++++++--------- cmd/pd-tso-bench/main.go | 12 ++++++------ 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/cmd/pd-ctl/main.go b/cmd/pd-ctl/main.go index 035fcb0a744..32c39f33b1c 100644 --- a/cmd/pd-ctl/main.go +++ b/cmd/pd-ctl/main.go @@ -32,18 +32,18 @@ var ( url string detach bool version bool - CAPath string - CertPath string - KeyPath string + caPath string + certPath string + keyPath string ) func init() { flag.StringVarP(&url, "pd", "u", "http://127.0.0.1:2379", "The pd address") flag.BoolVarP(&detach, "detach", "d", false, "Run pdctl without readline") flag.BoolVarP(&version, "version", "V", false, "print version information and exit") - flag.StringVar(&CAPath, "cacert", "", "path of file that contains list of trusted SSL CAs.") - flag.StringVar(&CertPath, "cert", "", "path of file that contains X509 certificate in PEM format.") - flag.StringVar(&KeyPath, "key", "", "path of file that contains X509 key in PEM format.") + flag.StringVar(&caPath, "cacert", "", "path of file that contains list of trusted SSL CAs.") + flag.StringVar(&certPath, "cert", "", "path of file that contains X509 certificate in PEM format.") + flag.StringVar(&keyPath, "key", "", "path of file that contains X509 key in PEM format.") } func main() { @@ -121,9 +121,9 @@ func loop() { } args := strings.Split(strings.TrimSpace(line), " ") args = append(args, "-u", url) - args = append(args, "--cacert", CAPath) - args = append(args, "--cert", CertPath) - args = append(args, "--key", KeyPath) + args = append(args, "--cacert", caPath) + args = append(args, "--cert", certPath) + args = append(args, "--key", keyPath) pdctl.Start(args) } } diff --git a/cmd/pd-tso-bench/main.go b/cmd/pd-tso-bench/main.go index a9401d1d8fd..d8e10ee6cc4 100644 --- a/cmd/pd-tso-bench/main.go +++ b/cmd/pd-tso-bench/main.go @@ -33,9 +33,9 @@ var ( concurrency = flag.Int("C", 1000, "concurrency") sleep = flag.Duration("sleep", time.Millisecond, "sleep time after a request, used to adjust pressure") interval = flag.Duration("interval", time.Second, "interval to output the statistics") - CAPath = flag.String("cacert", "", "path of file that contains list of trusted SSL CAs.") - CertPath = flag.String("cert", "", "path of file that contains X509 certificate in PEM format..") - KeyPath = flag.String("key", "", "path of file that contains X509 key in PEM format.") + caPath = flag.String("cacert", "", "path of file that contains list of trusted SSL CAs.") + certPath = flag.String("cert", "", "path of file that contains X509 certificate in PEM format..") + keyPath = flag.String("key", "", "path of file that contains X509 key in PEM format.") wg sync.WaitGroup ) @@ -43,9 +43,9 @@ func main() { flag.Parse() pdCli, err := pd.NewClient([]string{*pdAddrs}, pd.SecurityOption{ - CAPath: *CAPath, - CertPath: *CertPath, - KeyPath: *KeyPath, + CAPath: *caPath, + CertPath: *certPath, + KeyPath: *keyPath, }) if err != nil { log.Fatal(err)