diff --git a/README.md b/README.md index d13fd8a..5c06808 100644 --- a/README.md +++ b/README.md @@ -48,21 +48,20 @@ Let's go through an example leveraging the `go test` flow: 4. Use `e2emon.AsInstrumented` if you want to be able to query your service for metrics, which is a great way to assess it's internal state in tests! For example see following Etcd definition: - ```go mdox-exec="sed -n '228,243p' db/db.go" + ```go mdox-exec="sed -n '351,363p' db/db.go" return e2emon.AsInstrumented(env.Runnable(name).WithPorts(map[string]int{AccessPortName: 2379, "metrics": 9000}).Init( - e2e.StartOptions{ - Image: o.image, - Command: e2e.NewCommand( - "/usr/local/bin/etcd", - "--listen-client-urls=http://0.0.0.0:2379", - "--advertise-client-urls=http://0.0.0.0:2379", - "--listen-metrics-urls=http://0.0.0.0:9000", - "--log-level=error", - ), - Readiness: e2e.NewHTTPReadinessProbe("metrics", "/health", 200, 204), - }, + e2e.StartOptions{ + Image: o.image, + Command: e2e.NewCommand( + "/usr/local/bin/etcd", + "--listen-client-urls=http://0.0.0.0:2379", + "--advertise-client-urls=http://0.0.0.0:2379", + "--listen-metrics-urls=http://0.0.0.0:9000", + "--log-level=error", + ), + Readiness: e2e.NewHTTPReadinessProbe("metrics", "/health", 200, 204), + }, ), "metrics") - } ``` 5. Program your scenario as you want. You can start, wait for their readiness, stop, check their metrics and use their network endpoints from both unit test (`Endpoint`) as well as within each workload (`InternalEndpoint`). You can also access workload directory. There is a shared directory across all workloads. Check `Dir` and `InternalDir` runnable methods. diff --git a/db/db.go b/db/db.go index 398d76f..9aad647 100644 --- a/db/db.go +++ b/db/db.go @@ -6,12 +6,20 @@ package e2edb import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" "fmt" + "math/big" + "net" "os" "path/filepath" "strconv" "strings" + "time" + "github.com/efficientgo/core/errors" "github.com/efficientgo/e2e" e2emon "github.com/efficientgo/e2e/monitoring" e2eprof "github.com/efficientgo/e2e/profiling" @@ -32,6 +40,7 @@ type options struct { type minioOptions struct { enableSSE bool + enableTLS bool } func WithImage(image string) Option { @@ -52,6 +61,12 @@ func WithMinioSSE() Option { } } +func WithMinioTLS() Option { + return func(o *options) { + o.minioOptions.enableTLS = true + } +} + const AccessPortName = "http" func NewPrometheus(env e2e.Environment, name string, opts ...Option) *e2emon.Prometheus { @@ -85,7 +100,6 @@ func NewMinio(env e2e.Environment, name, bktName string, opts ...Option) *e2emon "MINIO_ROOT_USER=" + MinioAccessKey, "MINIO_ROOT_PASSWORD=" + MinioSecretKey, "MINIO_BROWSER=" + "off", - "ENABLE_HTTPS=" + "0", } f := env.Runnable(name).WithPorts(ports).Future() @@ -105,6 +119,56 @@ func NewMinio(env e2e.Environment, name, bktName string, opts ...Option) *e2emon command += "curl -sSL --tlsv1.3 -O 'https://raw.githubusercontent.com/minio/kes/master/root.key' -O 'https://raw.githubusercontent.com/minio/kes/master/root.cert' && cp root.* /home/me/ && " } + var readiness e2e.ReadinessProbe + + if o.minioOptions.enableTLS { + if err := os.MkdirAll(filepath.Join(f.Dir(), "certs", "CAs"), 0750); err != nil { + return &e2emon.InstrumentedRunnable{Runnable: e2e.NewFailedRunnable(name, errors.Wrap(err, "create certs dir"))} + } + + if err := genCerts( + filepath.Join(f.Dir(), "certs", "public.crt"), + filepath.Join(f.Dir(), "certs", "private.key"), + filepath.Join(f.Dir(), "certs", "CAs", "ca.crt"), + fmt.Sprintf("%s-%s", env.Name(), name), + ); err != nil { + return &e2emon.InstrumentedRunnable{Runnable: e2e.NewFailedRunnable(name, errors.Wrap(err, "fail to generate certs"))} + } + + envVars = append(envVars, "ENABLE_HTTPS="+"1") + command = command + fmt.Sprintf( + "su - me -s /bin/sh -c 'mkdir -p %s && %s /opt/bin/minio server --certs-dir %s/certs --address :%v --quiet %v'", + filepath.Join(f.Dir(), bktName), + strings.Join(envVars, " "), + f.Dir(), + ports[AccessPortName], + f.Dir(), + ) + + readiness = e2e.NewHTTPSReadinessProbe( + AccessPortName, + "/minio/health/cluster", + 200, + 200, + ) + } else { + envVars = append(envVars, "ENABLE_HTTPS="+"0") + command = command + fmt.Sprintf( + "su - me -s /bin/sh -c 'mkdir -p %s && %s /opt/bin/minio server --address :%v --quiet %v'", + filepath.Join(f.Dir(), bktName), + strings.Join(envVars, " "), + ports[AccessPortName], + f.Dir(), + ) + + readiness = e2e.NewHTTPReadinessProbe( + AccessPortName, + "/minio/health/cluster", + 200, + 200, + ) + } + return e2emon.AsInstrumented(f.Init( e2e.StartOptions{ Image: o.image, @@ -112,24 +176,83 @@ func NewMinio(env e2e.Environment, name, bktName string, opts ...Option) *e2emon Command: e2e.NewCommandWithoutEntrypoint( "sh", "-c", - command+fmt.Sprintf( - "su - me -s /bin/sh -c 'mkdir -p %s && %s /opt/bin/minio server --address :%v --quiet %v'", - filepath.Join(f.Dir(), bktName), - strings.Join(envVars, " "), - ports[AccessPortName], - f.Dir(), - ), - ), - Readiness: e2e.NewHTTPReadinessProbe( - AccessPortName, - "/minio/health/cluster", - 200, - 200, + command, ), + Readiness: readiness, }, ), AccessPortName) } +// genCerts generates certificates and writes those to the provided paths. +func genCerts(certPath, privkeyPath, caPath, serverName string) error { + var caRoot = &x509.Certificate{ + SerialNumber: big.NewInt(2019), + NotAfter: time.Now().AddDate(10, 0, 0), + IsCA: true, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + + var cert = &x509.Certificate{ + SerialNumber: big.NewInt(1658), + DNSNames: []string{serverName}, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")}, + NotAfter: time.Now().AddDate(10, 0, 0), + SubjectKeyId: []byte{1, 2, 3}, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature, + } + + caPrivKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return err + } + + certPrivKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return err + } + // Generate CA cert. + caBytes, err := x509.CreateCertificate(rand.Reader, caRoot, caRoot, &caPrivKey.PublicKey, caPrivKey) + if err != nil { + return err + } + caPEM := pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: caBytes, + }) + err = os.WriteFile(caPath, caPEM, 0644) + if err != nil { + return err + } + + // Sign the cert with the CA private key. + certBytes, err := x509.CreateCertificate(rand.Reader, cert, caRoot, &certPrivKey.PublicKey, caPrivKey) + if err != nil { + return err + } + certPEM := pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: certBytes, + }) + err = os.WriteFile(certPath, certPEM, 0644) + if err != nil { + return err + } + + certPrivKeyPEM := pem.EncodeToMemory(&pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(certPrivKey), + }) + err = os.WriteFile(privkeyPath, certPrivKeyPEM, 0644) + if err != nil { + return err + } + + return nil +} + func NewConsul(env e2e.Environment, name string, opts ...Option) *e2emon.InstrumentedRunnable { o := options{image: "consul:1.8.4"} for _, opt := range opts {