add integration tests

client/pkg/transport/listener.go

@@ -178,6 +178,7 @@ type TLSInfo struct {
 	tlsConfig       atomic.Value // *tls.Config
 	refreshOnce     sync.Once
 	RefreshDuration time.Duration
+	refreshDone     chan struct{}
 }
 
 func (info *TLSInfo) String() string {
@@ -330,6 +331,7 @@ func SelfCert(lg *zap.Logger, dirpath string, hosts []string, selfSignedCertVali
 func (info *TLSInfo) startRefresh() {
 	info.refreshOnce.Do(
 		func() {
+			info.refreshDone = make(chan struct{})
 			info.loadTLSConfig()
 			go info.tlsConfigRefreshLoop()
 		},
@@ -359,8 +361,13 @@ func (info *TLSInfo) tlsConfigRefreshLoop() {
 	}
 	ticker := time.NewTicker(refreshDuration)
 	defer ticker.Stop()
-	for range ticker.C {
-		info.loadTLSConfig()
+	for {
+		select {
+		case <-ticker.C:
+			info.loadTLSConfig()
+		case <-info.refreshDone:
+			return
+		}
 	}
 }
 
@@ -620,6 +627,10 @@ func (info *TLSInfo) ClientConfig() (*tls.Config, error) {
 	return cfg, nil
 }
 
+func (info *TLSInfo) Close() {
+	close(info.refreshDone)
+}
+
 // IsClosedConnError returns true if the error is from closing listener, cmux.
 // copied from golang.org/x/net/http2/http2.go
 func IsClosedConnError(err error) bool {

go.mod

@@ -42,6 +42,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.1.1 // indirect
 	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
+	github.com/creack/pty v1.1.11 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect

go.sum

@@ -65,6 +65,7 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw=
 github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=

tests/integration/root_ca_rotation_test.go

@@ -83,6 +83,7 @@ func generateCerts(privKey *ecdsa.PrivateKey, rootCA *x509.Certificate, dir, suf
 		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageContentCommitment,
 		BasicConstraintsValid: true,
 		IPAddresses:           []net.IP{net.ParseIP("127.0.0.1")},
+		DNSNames:              []string{"localhost"},
 	}
 	caBytes, err := x509.CreateCertificate(rand.Reader, &tmpl, rootCA, &priv.PublicKey, privKey)
 	if err != nil {
@@ -134,17 +135,18 @@ func TestRootCARotation(t *testing.T) {
 	rootCAPath := path.Join(tmpdir, "ca-cert.pem")
 	logger := zap.NewExample()
 	rootCA, caBytes, privKey := createRootCertificateAuthority(rootCAPath, []byte{}, t)
-	generateCerts(privKey, rootCA, tmpdir, "_itest", t)
+	generateCerts(privKey, rootCA, tmpdir, "_itest_old", t)
 
 	tlsInfo := &transport.TLSInfo{
 		TrustedCAFile:   rootCAPath,
-		CertFile:        path.Join(tmpdir, "cert_itest.pem"),
-		KeyFile:         path.Join(tmpdir, "key_itest.pem"),
-		ClientCertFile:  path.Join(tmpdir, "cert_itest.pem"),
-		ClientKeyFile:   path.Join(tmpdir, "key_itest.pem"),
+		CertFile:        path.Join(tmpdir, "cert_itest_old.pem"),
+		KeyFile:         path.Join(tmpdir, "key_itest_old.pem"),
+		ClientCertFile:  path.Join(tmpdir, "cert_itest_old.pem"),
+		ClientKeyFile:   path.Join(tmpdir, "key_itest_old.pem"),
 		Logger:          logger,
 		RefreshDuration: 100 * time.Millisecond,
 	}
+	defer tlsInfo.Close()
 
 	cluster := integration.NewClusterV3(
 		t,
@@ -159,6 +161,7 @@ func TestRootCARotation(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+
 	cli, cerr := integration.NewClient(t, clientv3.Config{
 		Endpoints:   []string{cluster.Members[0].GRPCURL()},
 		DialTimeout: time.Second,
@@ -173,4 +176,57 @@ func TestRootCARotation(t *testing.T) {
 	if cerr != nil {
 		t.Fatalf("expected TLS handshake success, got %v", cerr)
 	}
+
+	// regenerate rootCA and sign new certs
+	rootCA, _, privKey = createRootCertificateAuthority(rootCAPath, caBytes, t)
+	generateCerts(privKey, rootCA, tmpdir, "_itest_new", t)
+
+	// give server some time to reload new CA
+	time.Sleep(time.Second)
+
+	newClientTlsinfo := &transport.TLSInfo{
+		TrustedCAFile:  rootCAPath,
+		CertFile:       path.Join(tmpdir, "cert_itest_new.pem"),
+		KeyFile:        path.Join(tmpdir, "key_itest_new.pem"),
+		ClientCertFile: path.Join(tmpdir, "cert_itest_new.pem"),
+		ClientKeyFile:  path.Join(tmpdir, "key_itest_new.pem"),
+		Logger:         logger,
+	}
+
+	// old rootCA certs
+	cli, cerr = integration.NewClient(t, clientv3.Config{
+		Endpoints:   []string{cluster.Members[0].GRPCURL()},
+		DialTimeout: time.Second,
+		DialOptions: []grpc.DialOption{grpc.WithBlock()},
+		TLS:         cc,
+	})
+
+	if cli != nil {
+		cli.Close()
+	}
+
+	if cerr != nil {
+		t.Fatalf("expected TLS handshake success, got %v", cerr)
+	}
+
+	// new rootCA certs
+	cc, err = newClientTlsinfo.ClientConfig()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	cli, cerr = integration.NewClient(t, clientv3.Config{
+		Endpoints:   []string{cluster.Members[0].GRPCURL()},
+		DialTimeout: time.Second,
+		DialOptions: []grpc.DialOption{grpc.WithBlock()},
+		TLS:         cc,
+	})
+
+	if cli != nil {
+		cli.Close()
+	}
+
+	if cerr != nil {
+		t.Fatalf("expected TLS handshake success, got %v", cerr)
+	}
 }