From 270b5781d5fbd6ef89c1ce57f67b58ffe5c9ced0 Mon Sep 17 00:00:00 2001 From: Yi-Shu Tai Date: Sun, 31 Oct 2021 17:58:46 -0700 Subject: [PATCH] add integration tests --- client/pkg/transport/listener.go | 15 ++++- go.mod | 1 + go.sum | 1 + tests/integration/root_ca_rotation_test.go | 66 ++++++++++++++++++++-- 4 files changed, 76 insertions(+), 7 deletions(-) diff --git a/client/pkg/transport/listener.go b/client/pkg/transport/listener.go index a8971e69c0c4..cbb8fabab7a2 100644 --- a/client/pkg/transport/listener.go +++ b/client/pkg/transport/listener.go @@ -178,6 +178,7 @@ type TLSInfo struct { tlsConfig atomic.Value // *tls.Config refreshOnce sync.Once RefreshDuration time.Duration + refreshDone chan struct{} } func (info *TLSInfo) String() string { @@ -330,6 +331,7 @@ func SelfCert(lg *zap.Logger, dirpath string, hosts []string, selfSignedCertVali func (info *TLSInfo) startRefresh() { info.refreshOnce.Do( func() { + info.refreshDone = make(chan struct{}) info.loadTLSConfig() go info.tlsConfigRefreshLoop() }, @@ -359,8 +361,13 @@ func (info *TLSInfo) tlsConfigRefreshLoop() { } ticker := time.NewTicker(refreshDuration) defer ticker.Stop() - for range ticker.C { - info.loadTLSConfig() + for { + select { + case <-ticker.C: + info.loadTLSConfig() + case <-info.refreshDone: + return + } } } @@ -620,6 +627,10 @@ func (info *TLSInfo) ClientConfig() (*tls.Config, error) { return cfg, nil } +func (info *TLSInfo) Close() { + close(info.refreshDone) +} + // IsClosedConnError returns true if the error is from closing listener, cmux. // copied from golang.org/x/net/http2/http2.go func IsClosedConnError(err error) bool { diff --git a/go.mod b/go.mod index dbdef2f5a058..6d3234b46dc0 100644 --- a/go.mod +++ b/go.mod @@ -42,6 +42,7 @@ require ( github.com/cespare/xxhash/v2 v2.1.1 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect + github.com/creack/pty v1.1.11 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect diff --git a/go.sum b/go.sum index 78811a2aba06..067e5e051d32 100644 --- a/go.sum +++ b/go.sum @@ -65,6 +65,7 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/tests/integration/root_ca_rotation_test.go b/tests/integration/root_ca_rotation_test.go index 7531b250d3ca..da751bce675c 100644 --- a/tests/integration/root_ca_rotation_test.go +++ b/tests/integration/root_ca_rotation_test.go @@ -83,6 +83,7 @@ func generateCerts(privKey *ecdsa.PrivateKey, rootCA *x509.Certificate, dir, suf KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageContentCommitment, BasicConstraintsValid: true, IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + DNSNames: []string{"localhost"}, } caBytes, err := x509.CreateCertificate(rand.Reader, &tmpl, rootCA, &priv.PublicKey, privKey) if err != nil { @@ -134,17 +135,18 @@ func TestRootCARotation(t *testing.T) { rootCAPath := path.Join(tmpdir, "ca-cert.pem") logger := zap.NewExample() rootCA, caBytes, privKey := createRootCertificateAuthority(rootCAPath, []byte{}, t) - generateCerts(privKey, rootCA, tmpdir, "_itest", t) + generateCerts(privKey, rootCA, tmpdir, "_itest_old", t) tlsInfo := &transport.TLSInfo{ TrustedCAFile: rootCAPath, - CertFile: path.Join(tmpdir, "cert_itest.pem"), - KeyFile: path.Join(tmpdir, "key_itest.pem"), - ClientCertFile: path.Join(tmpdir, "cert_itest.pem"), - ClientKeyFile: path.Join(tmpdir, "key_itest.pem"), + CertFile: path.Join(tmpdir, "cert_itest_old.pem"), + KeyFile: path.Join(tmpdir, "key_itest_old.pem"), + ClientCertFile: path.Join(tmpdir, "cert_itest_old.pem"), + ClientKeyFile: path.Join(tmpdir, "key_itest_old.pem"), Logger: logger, RefreshDuration: 100 * time.Millisecond, } + defer tlsInfo.Close() cluster := integration.NewClusterV3( t, @@ -159,6 +161,7 @@ func TestRootCARotation(t *testing.T) { if err != nil { t.Fatal(err) } + cli, cerr := integration.NewClient(t, clientv3.Config{ Endpoints: []string{cluster.Members[0].GRPCURL()}, DialTimeout: time.Second, @@ -173,4 +176,57 @@ func TestRootCARotation(t *testing.T) { if cerr != nil { t.Fatalf("expected TLS handshake success, got %v", cerr) } + + // regenerate rootCA and sign new certs + rootCA, _, privKey = createRootCertificateAuthority(rootCAPath, caBytes, t) + generateCerts(privKey, rootCA, tmpdir, "_itest_new", t) + + // give server some time to reload new CA + time.Sleep(time.Second) + + newClientTlsinfo := &transport.TLSInfo{ + TrustedCAFile: rootCAPath, + CertFile: path.Join(tmpdir, "cert_itest_new.pem"), + KeyFile: path.Join(tmpdir, "key_itest_new.pem"), + ClientCertFile: path.Join(tmpdir, "cert_itest_new.pem"), + ClientKeyFile: path.Join(tmpdir, "key_itest_new.pem"), + Logger: logger, + } + + // old rootCA certs + cli, cerr = integration.NewClient(t, clientv3.Config{ + Endpoints: []string{cluster.Members[0].GRPCURL()}, + DialTimeout: time.Second, + DialOptions: []grpc.DialOption{grpc.WithBlock()}, + TLS: cc, + }) + + if cli != nil { + cli.Close() + } + + if cerr != nil { + t.Fatalf("expected TLS handshake success, got %v", cerr) + } + + // new rootCA certs + cc, err = newClientTlsinfo.ClientConfig() + if err != nil { + t.Fatal(err) + } + + cli, cerr = integration.NewClient(t, clientv3.Config{ + Endpoints: []string{cluster.Members[0].GRPCURL()}, + DialTimeout: time.Second, + DialOptions: []grpc.DialOption{grpc.WithBlock()}, + TLS: cc, + }) + + if cli != nil { + cli.Close() + } + + if cerr != nil { + t.Fatalf("expected TLS handshake success, got %v", cerr) + } }