Add functional tests for tracing

Verify end to end tracing by creating a collector and sending trace data to it. Test cases: 1. one policy attached to one route 2. one policy attached to multiple routes
nginxinc · May 21, 2024 · 3b3687d · 3b3687d
1 parent 6a9290a
commit 3b3687d
Show file tree

Hide file tree

Showing 20 changed files with 596 additions and 100 deletions.
diff --git a/.github/workflows/functional.yml b/.github/workflows/functional.yml
@@ -119,3 +119,10 @@ jobs:
           ngf_tag=${{ steps.ngf-meta.outputs.version }}
           make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=telemetry
         working-directory: ./tests
+
+      - name: Run functional tests
+        run: |
+          ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric
+          ngf_tag=${{ steps.ngf-meta.outputs.version }}
+          make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag}
+        working-directory: ./tests
diff --git a/tests/framework/request.go b/tests/framework/request.go
@@ -5,16 +5,19 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
+	"log"
 	"net"
 	"net/http"
 	"strings"
 	"time"
+
+	"k8s.io/apimachinery/pkg/util/wait"
 )
 
 // Get sends a GET request to the specified url.
 // It resolves to the specified address instead of using DNS.
 // The status and body of the response is returned, or an error.
-func Get(url, address string, timeout time.Duration) (int, string, error) {
+func Get(url, address string, requestTimeout time.Duration) (int, string, error) {
 	dialer := &net.Dialer{}
 
 	http.DefaultTransport.(*http.Transport).DialContext = func(
@@ -27,7 +30,7 @@ func Get(url, address string, timeout time.Duration) (int, string, error) {
 		return dialer.DialContext(ctx, network, fmt.Sprintf("%s:%s", address, port))
 	}
 
-	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	ctx, cancel := context.WithTimeout(context.Background(), requestTimeout)
 	defer cancel()
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
@@ -60,3 +63,36 @@ func Get(url, address string, timeout time.Duration) (int, string, error) {
 
 	return resp.StatusCode, body.String(), nil
 }
+
+// GetWithRetry retries the Get function until it succeeds or the context times out.
+func GetWithRetry(
+	ctx context.Context,
+	url,
+	address string,
+	requestTimeout time.Duration,
+) (int, string, error) {
+	var statusCode int
+	var body string
+
+	err := wait.PollUntilContextCancel(
+		ctx,
+		500*time.Millisecond,
+		true, /* poll immediately */
+		func(ctx context.Context) (bool, error) {
+			var getErr error
+			statusCode, body, getErr = Get(url, address, requestTimeout)
+			if getErr != nil {
+				return false, getErr
+			}
+
+			if statusCode != 200 {
+				log.Printf("got %d code instead of expected 200\n", statusCode)
+				return false, nil
+			}
+
+			return true, nil
+		},
+	)
+
+	return statusCode, body, err
+}
diff --git a/tests/go.mod b/tests/go.mod
@@ -3,6 +3,7 @@ module github.com/nginxinc/nginx-gateway-fabric/tests
 go 1.22.2
 
 require (
+	github.com/nginxinc/nginx-gateway-fabric v0.1.0-rc.1.0.20240520213534-10bae0bc9f00
 	github.com/onsi/ginkgo/v2 v2.17.3
 	github.com/onsi/gomega v1.33.1
 	github.com/prometheus/client_golang v1.19.1
@@ -67,7 +68,7 @@ require (
 	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/tools v0.20.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect
 	google.golang.org/grpc v1.63.2 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect

diff --git a/tests/go.sum b/tests/go.sum
@@ -86,6 +86,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/nginxinc/nginx-gateway-fabric v0.1.0-rc.1.0.20240520213534-10bae0bc9f00 h1:Q8HhAjmq6KADTmXHAhh5p1pg+QXrca9ayjVvxQpTYng=
+github.com/nginxinc/nginx-gateway-fabric v0.1.0-rc.1.0.20240520213534-10bae0bc9f00/go.mod h1:eG/LWjOU50RDukuBhRMGihLgpW/vcOLnhwEZql+31d4=
 github.com/onsi/ginkgo/v2 v2.17.3 h1:oJcvKpIb7/8uLpDDtnQuf18xVnwKp8DTD7DQ6gTd/MU=
 github.com/onsi/ginkgo/v2 v2.17.3/go.mod h1:nP2DPOQoNsQmsVyv5rDA8JkXQoCs6goXIvr/PRJ1eCc=
 github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk=
@@ -122,8 +124,8 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
-go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
-go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
+go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
+go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -176,8 +178,8 @@ gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuB
 gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca h1:PupagGYwj8+I4ubCxcmcBRk3VlUWtTg5huQpZR9flmE=
 gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
 gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de h1:cZGRis4/ot9uVm639a+rHCUaG0JJHEsdyzSQTMX+suY=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:H4O17MA/PE9BsGx3w+a+W2VOLLD1Qf7oJneAoU6WktY=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda h1:LI5DOvAxUPMv/50agcLLoo+AdWc1irS9Rzz4vPuD1V4=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY=
 google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM=
 google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA=
 google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=

diff --git a/tests/suite/dataplane_perf_test.go b/tests/suite/dataplane_perf_test.go
@@ -23,11 +23,8 @@ var _ = Describe("Dataplane performance", Ordered, Label("nfr", "performance"),
 		"dp-perf/gateway.yaml",
 		"dp-perf/cafe-routes.yaml",
 	}
-	ns := &core.Namespace{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: "dp-perf",
-		},
-	}
+
+	var ns core.Namespace
 
 	var addr string
 	targetURL := "http://cafe.example.com"
@@ -56,7 +53,13 @@ var _ = Describe("Dataplane performance", Ordered, Label("nfr", "performance"),
 	}
 
 	BeforeAll(func() {
-		Expect(resourceManager.Apply([]client.Object{ns})).To(Succeed())
+		ns = core.Namespace{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "dp-perf",
+			},
+		}
+
+		Expect(resourceManager.Apply([]client.Object{&ns})).To(Succeed())
 		Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed())
 		Expect(resourceManager.WaitForAppsToBeReady(ns.Name)).To(Succeed())
 

diff --git a/tests/suite/graceful_recovery_test.go b/tests/suite/graceful_recovery_test.go
@@ -36,11 +36,7 @@ var _ = Describe("Graceful Recovery test", Ordered, Label("nfr", "graceful-recov
 		"graceful-recovery/cafe-routes.yaml",
 	}
 
-	ns := &core.Namespace{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: "graceful-recovery",
-		},
-	}
+	var ns core.Namespace
 
 	teaURL := "https://cafe.example.com/tea"
 	coffeeURL := "http://cafe.example.com/coffee"
@@ -63,7 +59,13 @@ var _ = Describe("Graceful Recovery test", Ordered, Label("nfr", "graceful-recov
 	})
 
 	BeforeEach(func() {
-		Expect(resourceManager.Apply([]client.Object{ns})).To(Succeed())
+		ns = core.Namespace{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "graceful-recovery",
+			},
+		}
+
+		Expect(resourceManager.Apply([]client.Object{&ns})).To(Succeed())
 		Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed())
 		Expect(resourceManager.WaitForAppsToBeReady(ns.Name)).To(Succeed())
 
@@ -82,13 +84,13 @@ var _ = Describe("Graceful Recovery test", Ordered, Label("nfr", "graceful-recov
 	})
 
 	It("recovers when NGF container is restarted", func() {
-		runRecoveryTest(teaURL, coffeeURL, ngfPodName, ngfContainerName, files, ns)
+		runRecoveryTest(teaURL, coffeeURL, ngfPodName, ngfContainerName, files, &ns)
 	})
 
 	It("recovers when nginx container is restarted", func() {
 		// FIXME(bjee19) remove Skip() when https://github.com/nginxinc/nginx-gateway-fabric/issues/1108 is completed.
 		Skip("Test currently fails due to this issue: https://github.com/nginxinc/nginx-gateway-fabric/issues/1108")
-		runRecoveryTest(teaURL, coffeeURL, ngfPodName, nginxContainerName, files, ns)
+		runRecoveryTest(teaURL, coffeeURL, ngfPodName, nginxContainerName, files, &ns)
 	})
 })
 
@@ -206,10 +208,10 @@ func checkForWorkingTraffic(teaURL, coffeeURL string) error {
 }
 
 func checkForFailingTraffic(teaURL, coffeeURL string) error {
-	if err := expectRequestToFail(teaURL, address, "URI: /tea"); err != nil {
+	if err := expectRequestToFail(teaURL, address); err != nil {
 		return err
 	}
-	if err := expectRequestToFail(coffeeURL, address, "URI: /coffee"); err != nil {
+	if err := expectRequestToFail(coffeeURL, address); err != nil {
 		return err
 	}
 	return nil
@@ -228,7 +230,7 @@ func expectRequestToSucceed(appURL, address string, responseBodyMessage string)
 	return err
 }
 
-func expectRequestToFail(appURL, address string, responseBodyMessage string) error {
+func expectRequestToFail(appURL, address string) error {
 	status, body, err := framework.Get(appURL, address, timeoutConfig.RequestTimeout)
 	if status != 0 {
 		return errors.New("expected http status to be 0")

diff --git a/tests/suite/longevity_test.go b/tests/suite/longevity_test.go
@@ -31,16 +31,18 @@ var _ = Describe("Longevity", Label("longevity-setup", "longevity-teardown"), fu
 			"longevity/prom.yaml",
 		}
 
-		ns = &core.Namespace{
-			ObjectMeta: metav1.ObjectMeta{
-				Name: "longevity",
-			},
-		}
+		ns core.Namespace
 
 		labelFilter = GinkgoLabelFilter()
 	)
 
 	BeforeEach(func() {
+		ns = core.Namespace{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "longevity",
+			},
+		}
+
 		if !strings.Contains(labelFilter, "longevity") {
 			Skip("skipping longevity test unless 'longevity' label is explicitly defined when running")
 		}
@@ -51,7 +53,7 @@ var _ = Describe("Longevity", Label("longevity-setup", "longevity-teardown"), fu
 			Skip("'longevity-setup' label not specified; skipping...")
 		}
 
-		Expect(resourceManager.Apply([]client.Object{ns})).To(Succeed())
+		Expect(resourceManager.Apply([]client.Object{&ns})).To(Succeed())
 		Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed())
 		Expect(resourceManager.ApplyFromFiles(promFile, ngfNamespace)).To(Succeed())
 		Expect(resourceManager.WaitForAppsToBeReady(ns.Name)).To(Succeed())

diff --git a/tests/suite/manifests/hello-world/apps.yaml b/tests/suite/manifests/hello-world/apps.yaml
@@ -0,0 +1,98 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hello
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: hello
+  template:
+    metadata:
+      labels:
+        app: hello
+    spec:
+      containers:
+      - name: hello
+        image: nginxdemos/nginx-hello:plain-text
+        ports:
+        - containerPort: 8080
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: hello
+spec:
+  ports:
+  - port: 80
+    targetPort: 8080
+    protocol: TCP
+    name: http
+  selector:
+    app: hello
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: world
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: world
+  template:
+    metadata:
+      labels:
+        app: world
+    spec:
+      containers:
+      - name: world
+        image: nginxdemos/nginx-hello:plain-text
+        ports:
+        - containerPort: 8080
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: world
+spec:
+  ports:
+  - port: 80
+    targetPort: 8080
+    protocol: TCP
+    name: http
+  selector:
+    app: world
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hello-world
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: hello-world
+  template:
+    metadata:
+      labels:
+        app: hello-world
+    spec:
+      containers:
+      - name: hello-world
+        image: nginxdemos/nginx-hello:plain-text
+        ports:
+        - containerPort: 8080
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: hello-world
+spec:
+  ports:
+  - port: 80
+    targetPort: 8080
+    protocol: TCP
+    name: http
+  selector:
+    app: hello-world
diff --git a/tests/suite/manifests/hello/gateway.yaml → .../suite/manifests/hello-world/gateway.yaml b/tests/suite/manifests/hello/gateway.yaml → .../suite/manifests/hello-world/gateway.yaml
@@ -8,4 +8,4 @@ spec:
   - name: http
     port: 80
     protocol: HTTP
-    hostname: "*.example.com"
+    hostname: foo.example.com