diff --git a/.github/workflows/functional.yml b/.github/workflows/functional.yml index 0163227394..4cec276d40 100644 --- a/.github/workflows/functional.yml +++ b/.github/workflows/functional.yml @@ -119,3 +119,10 @@ jobs: ngf_tag=${{ steps.ngf-meta.outputs.version }} make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=telemetry working-directory: ./tests + + - name: Run functional tests + run: | + ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric + ngf_tag=${{ steps.ngf-meta.outputs.version }} + make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} + working-directory: ./tests diff --git a/tests/framework/request.go b/tests/framework/request.go index f5a03c32a6..ae82307905 100644 --- a/tests/framework/request.go +++ b/tests/framework/request.go @@ -5,16 +5,19 @@ import ( "context" "crypto/tls" "fmt" + "log" "net" "net/http" "strings" "time" + + "k8s.io/apimachinery/pkg/util/wait" ) // Get sends a GET request to the specified url. // It resolves to the specified address instead of using DNS. // The status and body of the response is returned, or an error. -func Get(url, address string, timeout time.Duration) (int, string, error) { +func Get(url, address string, requestTimeout time.Duration) (int, string, error) { dialer := &net.Dialer{} http.DefaultTransport.(*http.Transport).DialContext = func( @@ -27,7 +30,7 @@ func Get(url, address string, timeout time.Duration) (int, string, error) { return dialer.DialContext(ctx, network, fmt.Sprintf("%s:%s", address, port)) } - ctx, cancel := context.WithTimeout(context.Background(), timeout) + ctx, cancel := context.WithTimeout(context.Background(), requestTimeout) defer cancel() req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) @@ -60,3 +63,36 @@ func Get(url, address string, timeout time.Duration) (int, string, error) { return resp.StatusCode, body.String(), nil } + +// GetWithRetry retries the Get function until it succeeds or the context times out. +func GetWithRetry( + ctx context.Context, + url, + address string, + requestTimeout time.Duration, +) (int, string, error) { + var statusCode int + var body string + + err := wait.PollUntilContextCancel( + ctx, + 500*time.Millisecond, + true, /* poll immediately */ + func(ctx context.Context) (bool, error) { + var getErr error + statusCode, body, getErr = Get(url, address, requestTimeout) + if getErr != nil { + return false, getErr + } + + if statusCode != 200 { + log.Printf("got %d code instead of expected 200\n", statusCode) + return false, nil + } + + return true, nil + }, + ) + + return statusCode, body, err +} diff --git a/tests/go.mod b/tests/go.mod index 183f0f4cd6..fe909c6657 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -3,6 +3,7 @@ module github.com/nginxinc/nginx-gateway-fabric/tests go 1.22.2 require ( + github.com/nginxinc/nginx-gateway-fabric v0.1.0-rc.1.0.20240520213534-10bae0bc9f00 github.com/onsi/ginkgo/v2 v2.17.3 github.com/onsi/gomega v1.33.1 github.com/prometheus/client_golang v1.19.1 @@ -67,7 +68,7 @@ require ( golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.20.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect google.golang.org/grpc v1.63.2 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/tests/go.sum b/tests/go.sum index e51df13e53..bcea894f8c 100644 --- a/tests/go.sum +++ b/tests/go.sum @@ -86,6 +86,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/nginxinc/nginx-gateway-fabric v0.1.0-rc.1.0.20240520213534-10bae0bc9f00 h1:Q8HhAjmq6KADTmXHAhh5p1pg+QXrca9ayjVvxQpTYng= +github.com/nginxinc/nginx-gateway-fabric v0.1.0-rc.1.0.20240520213534-10bae0bc9f00/go.mod h1:eG/LWjOU50RDukuBhRMGihLgpW/vcOLnhwEZql+31d4= github.com/onsi/ginkgo/v2 v2.17.3 h1:oJcvKpIb7/8uLpDDtnQuf18xVnwKp8DTD7DQ6gTd/MU= github.com/onsi/ginkgo/v2 v2.17.3/go.mod h1:nP2DPOQoNsQmsVyv5rDA8JkXQoCs6goXIvr/PRJ1eCc= github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= @@ -122,8 +124,8 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= -go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -176,8 +178,8 @@ gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuB gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca h1:PupagGYwj8+I4ubCxcmcBRk3VlUWtTg5huQpZR9flmE= gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de h1:cZGRis4/ot9uVm639a+rHCUaG0JJHEsdyzSQTMX+suY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:H4O17MA/PE9BsGx3w+a+W2VOLLD1Qf7oJneAoU6WktY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda h1:LI5DOvAxUPMv/50agcLLoo+AdWc1irS9Rzz4vPuD1V4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= diff --git a/tests/suite/dataplane_perf_test.go b/tests/suite/dataplane_perf_test.go index 3860133e79..a1c3a6fb4f 100644 --- a/tests/suite/dataplane_perf_test.go +++ b/tests/suite/dataplane_perf_test.go @@ -23,11 +23,8 @@ var _ = Describe("Dataplane performance", Ordered, Label("nfr", "performance"), "dp-perf/gateway.yaml", "dp-perf/cafe-routes.yaml", } - ns := &core.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: "dp-perf", - }, - } + + var ns core.Namespace var addr string targetURL := "http://cafe.example.com" @@ -56,7 +53,13 @@ var _ = Describe("Dataplane performance", Ordered, Label("nfr", "performance"), } BeforeAll(func() { - Expect(resourceManager.Apply([]client.Object{ns})).To(Succeed()) + ns = core.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dp-perf", + }, + } + + Expect(resourceManager.Apply([]client.Object{&ns})).To(Succeed()) Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(ns.Name)).To(Succeed()) diff --git a/tests/suite/graceful_recovery_test.go b/tests/suite/graceful_recovery_test.go index b4148039e4..b9cfe5d131 100644 --- a/tests/suite/graceful_recovery_test.go +++ b/tests/suite/graceful_recovery_test.go @@ -36,11 +36,7 @@ var _ = Describe("Graceful Recovery test", Ordered, Label("nfr", "graceful-recov "graceful-recovery/cafe-routes.yaml", } - ns := &core.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: "graceful-recovery", - }, - } + var ns core.Namespace teaURL := "https://cafe.example.com/tea" coffeeURL := "http://cafe.example.com/coffee" @@ -63,7 +59,13 @@ var _ = Describe("Graceful Recovery test", Ordered, Label("nfr", "graceful-recov }) BeforeEach(func() { - Expect(resourceManager.Apply([]client.Object{ns})).To(Succeed()) + ns = core.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "graceful-recovery", + }, + } + + Expect(resourceManager.Apply([]client.Object{&ns})).To(Succeed()) Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(ns.Name)).To(Succeed()) @@ -82,13 +84,13 @@ var _ = Describe("Graceful Recovery test", Ordered, Label("nfr", "graceful-recov }) It("recovers when NGF container is restarted", func() { - runRecoveryTest(teaURL, coffeeURL, ngfPodName, ngfContainerName, files, ns) + runRecoveryTest(teaURL, coffeeURL, ngfPodName, ngfContainerName, files, &ns) }) It("recovers when nginx container is restarted", func() { // FIXME(bjee19) remove Skip() when https://github.com/nginxinc/nginx-gateway-fabric/issues/1108 is completed. Skip("Test currently fails due to this issue: https://github.com/nginxinc/nginx-gateway-fabric/issues/1108") - runRecoveryTest(teaURL, coffeeURL, ngfPodName, nginxContainerName, files, ns) + runRecoveryTest(teaURL, coffeeURL, ngfPodName, nginxContainerName, files, &ns) }) }) @@ -206,10 +208,10 @@ func checkForWorkingTraffic(teaURL, coffeeURL string) error { } func checkForFailingTraffic(teaURL, coffeeURL string) error { - if err := expectRequestToFail(teaURL, address, "URI: /tea"); err != nil { + if err := expectRequestToFail(teaURL, address); err != nil { return err } - if err := expectRequestToFail(coffeeURL, address, "URI: /coffee"); err != nil { + if err := expectRequestToFail(coffeeURL, address); err != nil { return err } return nil @@ -228,7 +230,7 @@ func expectRequestToSucceed(appURL, address string, responseBodyMessage string) return err } -func expectRequestToFail(appURL, address string, responseBodyMessage string) error { +func expectRequestToFail(appURL, address string) error { status, body, err := framework.Get(appURL, address, timeoutConfig.RequestTimeout) if status != 0 { return errors.New("expected http status to be 0") diff --git a/tests/suite/longevity_test.go b/tests/suite/longevity_test.go index 2ddf66c442..d42ebc8b26 100644 --- a/tests/suite/longevity_test.go +++ b/tests/suite/longevity_test.go @@ -31,16 +31,18 @@ var _ = Describe("Longevity", Label("longevity-setup", "longevity-teardown"), fu "longevity/prom.yaml", } - ns = &core.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: "longevity", - }, - } + ns core.Namespace labelFilter = GinkgoLabelFilter() ) BeforeEach(func() { + ns = core.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "longevity", + }, + } + if !strings.Contains(labelFilter, "longevity") { Skip("skipping longevity test unless 'longevity' label is explicitly defined when running") } @@ -51,7 +53,7 @@ var _ = Describe("Longevity", Label("longevity-setup", "longevity-teardown"), fu Skip("'longevity-setup' label not specified; skipping...") } - Expect(resourceManager.Apply([]client.Object{ns})).To(Succeed()) + Expect(resourceManager.Apply([]client.Object{&ns})).To(Succeed()) Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed()) Expect(resourceManager.ApplyFromFiles(promFile, ngfNamespace)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(ns.Name)).To(Succeed()) diff --git a/tests/suite/manifests/hello-world/apps.yaml b/tests/suite/manifests/hello-world/apps.yaml new file mode 100644 index 0000000000..9e0df181c8 --- /dev/null +++ b/tests/suite/manifests/hello-world/apps.yaml @@ -0,0 +1,98 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hello +spec: + replicas: 1 + selector: + matchLabels: + app: hello + template: + metadata: + labels: + app: hello + spec: + containers: + - name: hello + image: nginxdemos/nginx-hello:plain-text + ports: + - containerPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: hello +spec: + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + name: http + selector: + app: hello +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: world +spec: + replicas: 1 + selector: + matchLabels: + app: world + template: + metadata: + labels: + app: world + spec: + containers: + - name: world + image: nginxdemos/nginx-hello:plain-text + ports: + - containerPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: world +spec: + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + name: http + selector: + app: world +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hello-world +spec: + replicas: 1 + selector: + matchLabels: + app: hello-world + template: + metadata: + labels: + app: hello-world + spec: + containers: + - name: hello-world + image: nginxdemos/nginx-hello:plain-text + ports: + - containerPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: hello-world +spec: + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + name: http + selector: + app: hello-world diff --git a/tests/suite/manifests/hello/gateway.yaml b/tests/suite/manifests/hello-world/gateway.yaml similarity index 85% rename from tests/suite/manifests/hello/gateway.yaml rename to tests/suite/manifests/hello-world/gateway.yaml index e6507f613b..81234f9500 100644 --- a/tests/suite/manifests/hello/gateway.yaml +++ b/tests/suite/manifests/hello-world/gateway.yaml @@ -8,4 +8,4 @@ spec: - name: http port: 80 protocol: HTTP - hostname: "*.example.com" + hostname: foo.example.com diff --git a/tests/suite/manifests/hello-world/routes.yaml b/tests/suite/manifests/hello-world/routes.yaml new file mode 100644 index 0000000000..9b7b456e98 --- /dev/null +++ b/tests/suite/manifests/hello-world/routes.yaml @@ -0,0 +1,50 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: hello +spec: + parentRefs: + - name: gateway + sectionName: http + rules: + - matches: + - path: + type: Exact + value: /hello + backendRefs: + - name: hello + port: 80 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: world +spec: + parentRefs: + - name: gateway + sectionName: http + rules: + - matches: + - path: + type: Exact + value: /world + backendRefs: + - name: world + port: 80 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: hello-world +spec: + parentRefs: + - name: gateway + sectionName: http + rules: + - matches: + - path: + type: Exact + value: /helloworld + backendRefs: + - name: hello-world + port: 80 diff --git a/tests/suite/manifests/hello/hello.yaml b/tests/suite/manifests/hello/hello.yaml deleted file mode 100644 index 1d566b2159..0000000000 --- a/tests/suite/manifests/hello/hello.yaml +++ /dev/null @@ -1,32 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: hello -spec: - replicas: 1 - selector: - matchLabels: - app: hello - template: - metadata: - labels: - app: hello - spec: - containers: - - name: hello - image: nginxdemos/nginx-hello:plain-text - ports: - - containerPort: 8080 ---- -apiVersion: v1 -kind: Service -metadata: - name: hello -spec: - ports: - - port: 80 - targetPort: 8080 - protocol: TCP - name: http - selector: - app: hello diff --git a/tests/suite/manifests/hello/route.yaml b/tests/suite/manifests/hello/route.yaml deleted file mode 100644 index e70cce260f..0000000000 --- a/tests/suite/manifests/hello/route.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: gateway.networking.k8s.io/v1 -kind: HTTPRoute -metadata: - name: hello -spec: - parentRefs: - - name: gateway - sectionName: http - hostnames: - - "hello.example.com" - rules: - - matches: - - path: - type: Exact - value: /hello - backendRefs: - - name: hello - port: 80 diff --git a/tests/suite/manifests/tracing/nginxproxy.yaml b/tests/suite/manifests/tracing/nginxproxy.yaml new file mode 100644 index 0000000000..ed1f621047 --- /dev/null +++ b/tests/suite/manifests/tracing/nginxproxy.yaml @@ -0,0 +1,12 @@ +apiVersion: gateway.nginx.org/v1alpha1 +kind: NginxProxy +metadata: + name: nginx-proxy +spec: + telemetry: + exporter: + endpoint: otel-collector-opentelemetry-collector.collector.svc:4317 + serviceName: my-test-svc + spanAttributes: + - key: testkey1 + value: testval1 diff --git a/tests/suite/manifests/tracing/policy-multiple.yaml b/tests/suite/manifests/tracing/policy-multiple.yaml new file mode 100644 index 0000000000..851f573fdd --- /dev/null +++ b/tests/suite/manifests/tracing/policy-multiple.yaml @@ -0,0 +1,17 @@ +apiVersion: gateway.nginx.org/v1alpha1 +kind: ObservabilityPolicy +metadata: + name: test-observability-policy +spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: hello + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: world + tracing: + strategy: ratio + spanAttributes: + - key: testkey2 + value: testval2 diff --git a/tests/suite/manifests/tracing/policy-single.yaml b/tests/suite/manifests/tracing/policy-single.yaml new file mode 100644 index 0000000000..6d8da1581c --- /dev/null +++ b/tests/suite/manifests/tracing/policy-single.yaml @@ -0,0 +1,14 @@ +apiVersion: gateway.nginx.org/v1alpha1 +kind: ObservabilityPolicy +metadata: + name: test-observability-policy +spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: hello + tracing: + strategy: ratio + spanAttributes: + - key: testkey2 + value: testval2 diff --git a/tests/suite/sample_test.go b/tests/suite/sample_test.go index 56d08713a6..a2f28c2410 100644 --- a/tests/suite/sample_test.go +++ b/tests/suite/sample_test.go @@ -1,9 +1,11 @@ package suite import ( + "context" "fmt" "net/http" "strconv" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -16,18 +18,21 @@ import ( var _ = Describe("Basic test example", Label("functional"), func() { files := []string{ - "hello/hello.yaml", - "hello/gateway.yaml", - "hello/route.yaml", - } - ns := &core.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: "hello", - }, + "hello-world/apps.yaml", + "hello-world/gateway.yaml", + "hello-world/routes.yaml", } + var ns core.Namespace + BeforeEach(func() { - Expect(resourceManager.Apply([]client.Object{ns})).To(Succeed()) + ns = core.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "helloworld", + }, + } + + Expect(resourceManager.Apply([]client.Object{&ns})).To(Succeed()) Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(ns.Name)).To(Succeed()) }) @@ -38,11 +43,15 @@ var _ = Describe("Basic test example", Label("functional"), func() { }) It("sends traffic", func() { - url := "http://hello.example.com/hello" + url := "http://foo.example.com/hello" if portFwdPort != 0 { - url = fmt.Sprintf("http://hello.example.com:%s/hello", strconv.Itoa(portFwdPort)) + url = fmt.Sprintf("http://foo.example.com:%s/hello", strconv.Itoa(portFwdPort)) } - status, body, err := framework.Get(url, address, timeoutConfig.RequestTimeout) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + status, body, err := framework.GetWithRetry(ctx, url, address, timeoutConfig.RequestTimeout) Expect(err).ToNot(HaveOccurred()) Expect(status).To(Equal(http.StatusOK)) Expect(body).To(ContainSubstring("URI: /hello")) diff --git a/tests/suite/system_suite_test.go b/tests/suite/system_suite_test.go index 0dd0b3f4bb..a5d017e2cb 100644 --- a/tests/suite/system_suite_test.go +++ b/tests/suite/system_suite_test.go @@ -27,8 +27,10 @@ import ( ctlr "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" v1 "sigs.k8s.io/gateway-api/apis/v1" + ngfAPI "github.com/nginxinc/nginx-gateway-fabric/apis/v1alpha1" "github.com/nginxinc/nginx-gateway-fabric/tests/framework" ) @@ -79,6 +81,7 @@ var ( const ( releaseName = "ngf-test" ngfNamespace = "nginx-gateway" + gatewayClassName = "nginx" ngfHTTPForwardedPort = 10080 ngfHTTPSForwardedPort = 10443 ) @@ -102,6 +105,8 @@ func setup(cfg setupConfig, extraInstallArgs ...string) { Expect(coordination.AddToScheme(scheme)).To(Succeed()) Expect(v1.AddToScheme(scheme)).To(Succeed()) Expect(batchv1.AddToScheme(scheme)).To(Succeed()) + Expect(gatewayv1.AddToScheme(scheme)).To(Succeed()) + Expect(ngfAPI.AddToScheme(scheme)).To(Succeed()) options := client.Options{ Scheme: scheme, diff --git a/tests/suite/telemetry_test.go b/tests/suite/telemetry_test.go index ec6cbf0999..46a9947a0b 100644 --- a/tests/suite/telemetry_test.go +++ b/tests/suite/telemetry_test.go @@ -1,13 +1,17 @@ package suite import ( + "context" "fmt" "os/exec" "strings" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" core "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" crClient "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -135,7 +139,20 @@ func uninstallCollector() ([]byte, error) { "--namespace", collectorNamespace, } - return exec.Command("helm", args...).CombinedOutput() + output, err := exec.Command("helm", args...).CombinedOutput() + if err != nil { + return output, err + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + err = k8sClient.Delete(ctx, &core.Namespace{ObjectMeta: metav1.ObjectMeta{Name: collectorNamespace}}) + if err != nil && !apierrors.IsNotFound(err) { + return nil, err + } + + return nil, resourceManager.DeleteNamespace(collectorNamespace) } func assertConsecutiveLinesInLogs(logs string, expectedLines []string) { diff --git a/tests/suite/tracing_test.go b/tests/suite/tracing_test.go new file mode 100644 index 0000000000..507885513d --- /dev/null +++ b/tests/suite/tracing_test.go @@ -0,0 +1,269 @@ +package suite + +import ( + "context" + "errors" + "fmt" + "net/http" + "strconv" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + core "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + v1 "sigs.k8s.io/gateway-api/apis/v1" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + + ngfAPI "github.com/nginxinc/nginx-gateway-fabric/apis/v1alpha1" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/conditions" + "github.com/nginxinc/nginx-gateway-fabric/tests/framework" +) + +var _ = Describe("Tracing", Label("functional"), func() { + files := []string{ + "hello-world/apps.yaml", + "hello-world/gateway.yaml", + "hello-world/routes.yaml", + } + var ns core.Namespace + + var collectorPodName, helloURL, worldURL, helloworldURL string + + BeforeEach(func() { + ns = core.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "helloworld", + }, + } + + output, err := installCollector() + Expect(err).ToNot(HaveOccurred(), string(output)) + + collectorPodNames, err := resourceManager.GetPodNames( + collectorNamespace, + client.MatchingLabels{ + "app.kubernetes.io/name": "opentelemetry-collector", + }, + ) + + Expect(err).ToNot(HaveOccurred()) + Expect(collectorPodNames).To(HaveLen(1)) + + collectorPodName = collectorPodNames[0] + + Expect(resourceManager.Apply([]client.Object{&ns})).To(Succeed()) + Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed()) + Expect(resourceManager.WaitForAppsToBeReady(ns.Name)).To(Succeed()) + + url := "http://foo.example.com" + helloURL = url + "/hello" + worldURL = url + "/world" + helloworldURL = url + "/helloworld" + if portFwdPort != 0 { + helloURL = fmt.Sprintf("%s:%s/hello", url, strconv.Itoa(portFwdPort)) + worldURL = fmt.Sprintf("%s:%s/world", url, strconv.Itoa(portFwdPort)) + helloworldURL = fmt.Sprintf("%s:%s/helloworld", url, strconv.Itoa(portFwdPort)) + } + }) + + AfterEach(func() { + output, err := uninstallCollector() + Expect(err).ToNot(HaveOccurred(), string(output)) + + Expect(resourceManager.DeleteFromFiles(files, ns.Name)).To(Succeed()) + Expect(resourceManager.DeleteNamespace(ns.Name)).To(Succeed()) + }) + + updateGatewayClass := func() error { + ctx, cancel := context.WithTimeout(context.Background(), timeoutConfig.CreateTimeout) + defer cancel() + + key := types.NamespacedName{Name: "nginx"} + var gwClass gatewayv1.GatewayClass + if err := k8sClient.Get(ctx, key, &gwClass); err != nil { + return err + } + + gwClass.Spec.ParametersRef = &gatewayv1.ParametersReference{ + Group: ngfAPI.GroupName, + Kind: gatewayv1.Kind("NginxProxy"), + Name: "nginx-proxy", + } + + return k8sClient.Update(ctx, &gwClass) + } + + sendTraceRequests := func(ctx context.Context, url string, count int) { + for range count { + status, _, err := framework.GetWithRetry(ctx, url, address, timeoutConfig.RequestTimeout) + Expect(err).ToNot(HaveOccurred()) + Expect(status).To(Equal(http.StatusOK)) + } + } + + It("sends tracing spans for one policy attached to one route", func() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + sendTraceRequests(ctx, helloURL, 5) + + // verify that no traces exist yet + logs, err := resourceManager.GetPodLogs(collectorNamespace, collectorPodName, &core.PodLogOptions{}) + Expect(err).ToNot(HaveOccurred()) + Expect(logs).ToNot(ContainSubstring("service.name: Str(ngf:helloworld:gateway:my-test-svc)")) + + // install tracing configuration + traceFiles := []string{ + "tracing/nginxproxy.yaml", + "tracing/policy-single.yaml", + } + Expect(resourceManager.ApplyFromFiles(traceFiles, ns.Name)).To(Succeed()) + Expect(updateGatewayClass()).To(Succeed()) + + Eventually( + func() error { + return verifyGatewayClassResolvedRefs() + }). + WithTimeout(timeoutConfig.GetTimeout). + WithPolling(500 * time.Millisecond). + Should(Succeed()) + + Eventually( + func() error { + return verifyPolicyStatus() + }). + WithTimeout(timeoutConfig.GetTimeout). + WithPolling(500 * time.Millisecond). + Should(Succeed()) + + // send traffic and verify that traces exist for hello app + findTraces := func() bool { + sendTraceRequests(ctx, helloURL, 25) + sendTraceRequests(ctx, worldURL, 25) + sendTraceRequests(ctx, helloworldURL, 25) + + logs, err := resourceManager.GetPodLogs(collectorNamespace, collectorPodName, &core.PodLogOptions{}) + Expect(err).ToNot(HaveOccurred()) + return strings.Contains(logs, "service.name: Str(ngf:helloworld:gateway:my-test-svc)") + } + + // wait for expected first line to show up + Eventually(findTraces, "1m", "5s").Should(BeTrue()) + + logs, err = resourceManager.GetPodLogs(collectorNamespace, collectorPodName, &core.PodLogOptions{}) + Expect(err).ToNot(HaveOccurred()) + + Expect(logs).To(ContainSubstring("http.method: Str(GET)")) + Expect(logs).To(ContainSubstring("http.target: Str(/hello)")) + Expect(logs).To(ContainSubstring("testkey1: Str(testval1)")) + Expect(logs).To(ContainSubstring("testkey2: Str(testval2)")) + + // verify traces don't exist for other apps + Expect(logs).ToNot(ContainSubstring("http.target: Str(/world)")) + Expect(logs).ToNot(ContainSubstring("http.target: Str(/helloworld)")) + }) + + It("sends tracing spans for one policy attached to multiple routes", func() { + // install tracing configuration + traceFiles := []string{ + "tracing/nginxproxy.yaml", + "tracing/policy-multiple.yaml", + } + Expect(resourceManager.ApplyFromFiles(traceFiles, ns.Name)).To(Succeed()) + Expect(updateGatewayClass()).To(Succeed()) + + Eventually( + func() error { + return verifyGatewayClassResolvedRefs() + }). + WithTimeout(timeoutConfig.GetTimeout). + WithPolling(500 * time.Millisecond). + Should(Succeed()) + + Eventually( + func() error { + return verifyPolicyStatus() + }). + WithTimeout(timeoutConfig.GetTimeout). + WithPolling(500 * time.Millisecond). + Should(Succeed()) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // send traffic and verify that traces exist for hello app + findTraces := func() bool { + sendTraceRequests(ctx, helloURL, 25) + sendTraceRequests(ctx, worldURL, 25) + sendTraceRequests(ctx, helloworldURL, 25) + + logs, err := resourceManager.GetPodLogs(collectorNamespace, collectorPodName, &core.PodLogOptions{}) + Expect(err).ToNot(HaveOccurred()) + return strings.Contains(logs, "service.name: Str(ngf:helloworld:gateway:my-test-svc)") + } + + // wait for expected first line to show up + Eventually(findTraces, "1m", "5s").Should(BeTrue()) + + logs, err := resourceManager.GetPodLogs(collectorNamespace, collectorPodName, &core.PodLogOptions{}) + Expect(err).ToNot(HaveOccurred()) + + Expect(logs).To(ContainSubstring("http.method: Str(GET)")) + Expect(logs).To(ContainSubstring("http.target: Str(/hello)")) + Expect(logs).To(ContainSubstring("http.target: Str(/world)")) + Expect(logs).To(ContainSubstring("testkey1: Str(testval1)")) + Expect(logs).To(ContainSubstring("testkey2: Str(testval2)")) + + // verify traces don't exist for helloworld apps + Expect(logs).ToNot(ContainSubstring("http.target: Str(/helloworld)")) + }) +}) + +func verifyGatewayClassResolvedRefs() error { + ctx, cancel := context.WithTimeout(context.Background(), timeoutConfig.GetTimeout) + defer cancel() + + var gc v1.GatewayClass + if err := k8sClient.Get(ctx, types.NamespacedName{Name: gatewayClassName}, &gc); err != nil { + return err + } + + for _, cond := range gc.Status.Conditions { + if cond.Type == string(conditions.GatewayClassResolvedRefs) && cond.Status == metav1.ConditionTrue { + return nil + } + } + + return errors.New("ResolvedRefs status not set to true on GatewayClass") +} + +func verifyPolicyStatus() error { + ctx, cancel := context.WithTimeout(context.Background(), timeoutConfig.GetTimeout) + defer cancel() + + var pol ngfAPI.ObservabilityPolicy + key := types.NamespacedName{Name: "test-observability-policy", Namespace: "helloworld"} + if err := k8sClient.Get(ctx, key, &pol); err != nil { + return err + } + + var count int + for _, ancestor := range pol.Status.Ancestors { + for _, cond := range ancestor.Conditions { + if cond.Type == string(gatewayv1alpha2.PolicyConditionAccepted) && cond.Status == metav1.ConditionTrue { + count++ + } + } + } + + if count != len(pol.Status.Ancestors) { + return errors.New("Policy not accepted") + } + + return nil +} diff --git a/tests/suite/upgrade_test.go b/tests/suite/upgrade_test.go index e749292078..52eb32f435 100644 --- a/tests/suite/upgrade_test.go +++ b/tests/suite/upgrade_test.go @@ -35,11 +35,7 @@ var _ = Describe("Upgrade testing", Label("nfr", "upgrade"), func() { "ngf-upgrade/cafe-routes.yaml", } - ns = &core.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: "ngf-upgrade", - }, - } + ns core.Namespace valuesFile = "manifests/ngf-upgrade/values.yaml" resultsFile *os.File @@ -60,7 +56,13 @@ var _ = Describe("Upgrade testing", Label("nfr", "upgrade"), func() { } setup(cfg, "--values", valuesFile) - Expect(resourceManager.Apply([]client.Object{ns})).To(Succeed()) + ns = core.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ngf-upgrade", + }, + } + + Expect(resourceManager.Apply([]client.Object{&ns})).To(Succeed()) Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(ns.Name)).To(Succeed())