Performance Test changes for EE

changelog/v1.15.0-beta18/performance-tests-additions-for-ee.yaml

@@ -0,0 +1,9 @@
+changelog:
+  - type: NON_USER_FACING
+    issueLink: https://github.com/solo-io/solo-projects/issues/5038
+    resolvesIssue: false
+    description: >-
+      Add WithHealthChecks method to UpstreamBuilder and export other performance test helpers for use in EE
+
+      skipCI-kube-tests:true
+      skipCI-docs-build:true

projects/gloo/pkg/translator/performance_test.go

@@ -3,9 +3,9 @@ package translator_test
 import (
 	"context"
 	"fmt"
-	"strings"
 
 	"github.com/solo-io/gloo/test/ginkgo/decorators"
+	"go.uber.org/zap"
 
 	"github.com/solo-io/go-utils/contextutils"
 
@@ -39,14 +39,6 @@ import (
 	"github.com/solo-io/gloo/projects/gloo/pkg/plugins"
 )
 
-// benchmarkConfig allows configuration for benchmarking tests to be reused for similar cases
-// This struct can be factored out to an accessible location should additional benchmarking suites be added
-type benchmarkConfig struct {
-	iterations        int                   // the number of iterations to attempt for a particular entry
-	maxDur            time.Duration         // the maximum time to spend on a particular entry even if not all iterations are complete
-	benchmarkMatchers []types.GomegaMatcher // matchers representing the assertions we wish to make for a particular entry
-}
-
 // Tests are run as part of the "Nightly" action in a GHA using the default Linux runner
 // More info on that machine can be found here: https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources
 // When developing new tests, users should manually run that action in order to test performance under the same parameters
@@ -83,7 +75,7 @@ var _ = Describe("Translation - Benchmarking Tests", decorators.Performance, Lab
 	// We measure the duration of the translation of the snapshot, benchmarking according to the benchmarkConfig
 	// Labels are used to add context to the entry description
 	DescribeTable("Benchmark table",
-		func(snapBuilder *gloohelpers.ScaledSnapshotBuilder, config benchmarkConfig, labels ...string) {
+		func(snapBuilder *gloohelpers.ScaledSnapshotBuilder, config *gloohelpers.BenchmarkConfig, labels ...string) {
 			var (
 				apiSnap *v1snap.ApiSnapshot
 				proxy   *v1.Proxy
@@ -95,6 +87,11 @@ var _ = Describe("Translation - Benchmarking Tests", decorators.Performance, Lab
 				tooFastWarningCount int
 			)
 
+			// Translating logs at info level, which is very noisy when running repeatedly
+			originalLogLevel := contextutils.GetLogLevel()
+			contextutils.SetLogLevel(zap.ErrorLevel)
+			defer contextutils.SetLogLevel(originalLogLevel)
+
 			apiSnap = snapBuilder.Build()
 
 			params := plugins.Params{
@@ -105,7 +102,7 @@ var _ = Describe("Translation - Benchmarking Tests", decorators.Performance, Lab
 			Expect(apiSnap.Proxies).NotTo(BeEmpty())
 			proxy = apiSnap.Proxies[0]
 
-			desc := generateDesc(snapBuilder, config, labels...)
+			desc := gloohelpers.GenerateBenchmarkDesc(snapBuilder, config, labels...)
 
 			experiment := gmeasure.NewExperiment(fmt.Sprintf("Experiment - %s", desc))
 
@@ -135,7 +132,7 @@ var _ = Describe("Translation - Benchmarking Tests", decorators.Performance, Lab
 				// If desired, a field can be added to benchmarkConfig to allow benchmarking according to User and/or
 				// System time
 				experiment.RecordDuration(desc, res.Total)
-			}, gmeasure.SamplingConfig{N: config.iterations, Duration: config.maxDur})
+			}, gmeasure.SamplingConfig{N: config.Iterations, Duration: config.MaxDur})
 
 			if tooFastWarningCount > 0 {
 				logger := contextutils.LoggerFrom(params.Ctx)
@@ -144,9 +141,9 @@ var _ = Describe("Translation - Benchmarking Tests", decorators.Performance, Lab
 
 			durations := experiment.Get(desc).Durations
 
-			Expect(durations).Should(And(config.benchmarkMatchers...))
+			Expect(durations).Should(And(config.GetMatchers()...))
 		},
-		generateDesc, // generate descriptions for table entries with nil descriptions
+		gloohelpers.GenerateBenchmarkDesc, // generate descriptions for table entries with nil descriptions
 		Entry("basic", gloohelpers.NewInjectedSnapshotBuilder(basicSnap), basicConfig),
 		Entry(nil, gloohelpers.NewScaledSnapshotBuilder().WithUpstreamCount(10).WithEndpointCount(1), basicConfig, "upstream scale"),
 		Entry(nil, gloohelpers.NewScaledSnapshotBuilder().WithUpstreamCount(1000).WithEndpointCount(1), oneKUpstreamsConfig, "upstream scale"),
@@ -160,20 +157,6 @@ var _ = Describe("Translation - Benchmarking Tests", decorators.Performance, Lab
 	)
 })
 
-func generateDesc(b *gloohelpers.ScaledSnapshotBuilder, _ benchmarkConfig, labels ...string) string {
-	labelPrefix := ""
-	if len(labels) > 0 {
-		labelPrefix = fmt.Sprintf("(%s) ", strings.Join(labels, ", "))
-	}
-
-	if b.HasInjectedSnapshot() {
-		return fmt.Sprintf("%sinjected snapshot", labelPrefix)
-	}
-
-	// If/when additional Snapshot fields are included in testing, the description should be updated accordingly
-	return fmt.Sprintf("%s%d endpoint(s), %d upstream(s)", labelPrefix, b.EndpointCount(), b.UpstreamCount())
-}
-
 // Test assets: Add blocks for logical groupings of tests, including:
 // - in-line snapshot definitions for tests that require granularly-configured/heterogeneous resources (ie testing a particular field or feature)
 // - benchmarkConfigs for particular groups of use cases depending on processing time requirements/expectations
@@ -191,23 +174,31 @@ var basicSnap = &v1snap.ApiSnapshot{
 	Upstreams: []*v1.Upstream{gloohelpers.Upstream(0)},
 }
 
-var basicConfig = benchmarkConfig{
-	iterations: 1000,
-	maxDur:     10 * time.Second,
-	benchmarkMatchers: []types.GomegaMatcher{
+var basicConfig = &gloohelpers.BenchmarkConfig{
+	Iterations: 1000,
+	MaxDur:     10 * time.Second,
+	GhaMatchers: []types.GomegaMatcher{
 		matchers.HaveMedianLessThan(50 * time.Millisecond),
 		matchers.HavePercentileLessThan(90, 100*time.Millisecond),
 	},
+	LocalMatchers: []types.GomegaMatcher{
+		matchers.HaveMedianLessThan(2 * time.Millisecond),
+		matchers.HavePercentileLessThan(90, 4*time.Millisecond),
+	},
 }
 
 /* 1k Upstreams Scale Test */
-var oneKUpstreamsConfig = benchmarkConfig{
-	iterations: 100,
-	maxDur:     30 * time.Second,
-	benchmarkMatchers: []types.GomegaMatcher{
+var oneKUpstreamsConfig = &gloohelpers.BenchmarkConfig{
+	Iterations: 100,
+	MaxDur:     30 * time.Second,
+	GhaMatchers: []types.GomegaMatcher{
 		matchers.HaveMedianLessThan(time.Second),
 		matchers.HavePercentileLessThan(90, 2*time.Second),
 	},
+	LocalMatchers: []types.GomegaMatcher{
+		matchers.HaveMedianLessThan(40 * time.Millisecond),
+		matchers.HavePercentileLessThan(90, 80*time.Millisecond),
+	},
 }
 
 /* Upstream SNI Test */

test/helpers/benchmark.go

@@ -1,6 +1,15 @@
 package helpers
 
-import "time"
+import (
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/solo-io/gloo/test/testutils"
+
+	"github.com/onsi/gomega/types"
+)
 
 // Result represents the result of measuring a function's execution time.
 type Result struct {
@@ -11,3 +20,28 @@ type Result struct {
 	// Time spent in user mode + kernel mode
 	Total time.Duration
 }
+
+// BenchmarkConfig allows configuration for benchmarking tests to be reused for similar cases
+// This struct can be factored out to an accessible location should additional benchmarking suites be added
+type BenchmarkConfig struct {
+	Iterations    int                   // the number of iterations to attempt for a particular entry
+	MaxDur        time.Duration         // the maximum time to spend on a particular entry even if not all iterations are complete
+	LocalMatchers []types.GomegaMatcher // matchers representing the assertions we wish to make for a particular entry when running locally
+	GhaMatchers   []types.GomegaMatcher // matchers representing the assertions we wish to make for a particular entry when running in a GHA
+}
+
+func (bc *BenchmarkConfig) GetMatchers() []types.GomegaMatcher {
+	if os.Getenv(testutils.GithubAction) != "" {
+		return bc.GhaMatchers
+	}
+	return bc.LocalMatchers
+}
+
+func GenerateBenchmarkDesc(b *ScaledSnapshotBuilder, _ *BenchmarkConfig, labels ...string) string {
+	labelPrefix := ""
+	if len(labels) > 0 {
+		labelPrefix = fmt.Sprintf("(%s) ", strings.Join(labels, ", "))
+	}
+
+	return fmt.Sprintf("%s%s", labelPrefix, b.description())
+}

test/helpers/scaled_snapshots.go

@@ -104,6 +104,15 @@ func (b *ScaledSnapshotBuilder) Build() *gloosnapshot.ApiSnapshot {
 	}
 }
 
+func (b *ScaledSnapshotBuilder) description() string {
+	if b.HasInjectedSnapshot() {
+		return "injected snapshot"
+	}
+
+	// If/when additional Snapshot fields are included in testing, the description should be updated accordingly
+	return fmt.Sprintf("%d endpoint(s), %d upstream(s)", b.EndpointCount(), b.UpstreamCount())
+}
+
 func upMeta(i int) *core.Metadata {
 	return &core.Metadata{
 		Name:      fmt.Sprintf("test-%06d", i),

test/helpers/upstreams.go

@@ -3,13 +3,16 @@ package helpers
 import (
 	"fmt"
 
+	"github.com/solo-io/gloo/projects/gloo/pkg/api/external/envoy/api/v2/core"
 	v1 "github.com/solo-io/gloo/projects/gloo/pkg/api/v1"
 	"github.com/solo-io/gloo/projects/gloo/pkg/api/v1/ssl"
 )
 
 // UpstreamBuilder contains options for building Upstreams to be included in scaled Snapshots
 type UpstreamBuilder struct {
 	sniPattern sniPattern
+
+	healthChecks []*core.HealthCheck
 }
 
 type sniPattern int
@@ -34,9 +37,16 @@ func (b *UpstreamBuilder) WithConsistentSni() *UpstreamBuilder {
 	return b
 }
 
+func (b *UpstreamBuilder) WithHealthChecks(healthChecks []*core.HealthCheck) *UpstreamBuilder {
+	b.healthChecks = healthChecks
+	return b
+}
+
 func (b *UpstreamBuilder) Build(i int) *v1.Upstream {
 	up := Upstream(i)
 
+	up.HealthChecks = b.healthChecks
+
 	switch b.sniPattern {
 	case uniqueSni:
 		up.SslConfig = &ssl.UpstreamSslConfig{

test/testutils/env.go

@@ -54,6 +54,10 @@ const (
 	// ServiceLogLevel is used to set the log level for the test services. See services/logging.go for more details
 	ServiceLogLevel = "SERVICE_LOG_LEVEL"
 
+	// GithubAction is used by Github Actions and is the name of the currently running action or ID of a step
+	// https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables
+	GithubAction = "GITHUB_ACITON"
+
 	// GcloudBuildId is used by Cloudbuild to identify the build id
 	// This is set when running tests in Cloudbuild
 	GcloudBuildId = "GCLOUD_BUILD_ID"