[ADXT-199] Early start long e2e tests environment deployment on the CI (priority on EKS)

.gitlab/e2e/e2e.yml

@@ -167,10 +167,23 @@ new-e2e-containers:
       - EXTRA_PARAMS: "--run TestKindSuite -c ddinfra:osDescriptor=ubuntu:20.04"
       - EXTRA_PARAMS: "--run TestKindSuite -c ddinfra:osDescriptor=ubuntu:22.04"
       - EXTRA_PARAMS: --run TestEKSSuite
+        E2E_PRE_INITIALIZED: "true"
       - EXTRA_PARAMS: --run TestECSSuite
       - EXTRA_PARAMS: --run TestDockerSuite
       - EXTRA_PARAMS: --skip "Test(Kind|EKS|ECS|Docker)Suite"
 
+new-e2e-containers-eks-init:
+  extends: .new_e2e_template
+  needs: []
+  rules:
+    - !reference [.on_container_or_e2e_changes]
+    - !reference [.manual]
+  variables:
+    TARGETS: ./tests/containers
+    TEAM: container-integrations
+    EXTRA_PARAMS: --run TestEKSSuite
+    E2E_INIT_ONLY: "true"
+
 new-e2e-remote-config:
   extends: .new_e2e_template_needs_deb_x64
   rules:
@@ -252,7 +265,33 @@ new-e2e-npm-docker:
   variables:
     TARGETS: ./tests/npm
     TEAM: network-performance-monitoring
-    EXTRA_PARAMS: --run "Test(ECSVM|EC2VMContainerized|EKSVM)Suite"
+    EXTRA_PARAMS: --run "Test(ECSVM|EC2VMContainerized)Suite"
+
+
+new-e2e-npm-eks-init:
+  extends: .new_e2e_template
+  needs: []
+  rules:
+    - !reference [.on_npm_or_e2e_changes]
+    - !reference [.manual]
+  variables:
+    TARGETS: ./tests/npm
+    TEAM: network-performance-monitoring
+    EXTRA_PARAMS: --run "TestEKSVMSuite"
+    E2E_INIT_ONLY: "true"
+
+new-e2e-npm-eks:
+  extends: .new_e2e_template
+  rules:
+    - !reference [.on_npm_or_e2e_changes]
+    - !reference [.manual]
+  needs:
+    - qa_agent
+  variables:
+    TARGETS: ./tests/npm
+    TEAM: network-performance-monitoring
+    EXTRA_PARAMS: --run "TestEKSVMSuite"
+    E2E_PRE_INITIALIZED: "true"
 
 new-e2e-aml:
   extends: .new_e2e_template

test/new-e2e/go.mod

@@ -29,7 +29,7 @@ require (
 	// `TEST_INFRA_DEFINITIONS_BUILDIMAGES` matches the commit sha in the module version
 	// Example: 	github.com/DataDog/test-infra-definitions v0.0.0-YYYYMMDDHHmmSS-0123456789AB
 	// => TEST_INFRA_DEFINITIONS_BUILDIMAGES: 0123456789AB
-	github.com/DataDog/test-infra-definitions v0.0.0-20240702121350-81094fa60efe
+	github.com/DataDog/test-infra-definitions v0.0.0-20240704155019-dc1a8c1eaa1f
 	github.com/aws/aws-sdk-go-v2 v1.30.0
 	github.com/aws/aws-sdk-go-v2/config v1.27.19
 	github.com/aws/aws-sdk-go-v2/service/ec2 v1.164.2

test/new-e2e/pkg/e2e/suite.go

@@ -189,6 +189,7 @@ type BaseSuite[Env any] struct {
 	currentProvisioners  ProvisionerMap
 
 	firstFailTest string
+	initOnly      bool
 }
 
 //
@@ -306,6 +307,10 @@ func (bs *BaseSuite[Env]) reconcileEnv(targetProvisioners ProvisionerMap) error
 		resources.Merge(provisionerResources)
 	}
 
+	if bs.initOnly {
+		return nil
+	}
+
 	// Env is taken as parameter as some fields may have keys set by Env pulumi program.
 	err = bs.buildEnvFromResources(resources, newEnvFields, newEnvValues)
 	if err != nil {
@@ -328,6 +333,11 @@ func (bs *BaseSuite[Env]) reconcileEnv(targetProvisioners ProvisionerMap) error
 
 func (bs *BaseSuite[Env]) createEnv() (*Env, []reflect.StructField, []reflect.Value, error) {
 	var env Env
+	initOnly, err := runner.GetProfile().ParamStore().GetBoolWithDefault(parameters.InitOnly, false)
+	if err == nil {
+		bs.initOnly = initOnly
+	}
+
 	envFields := reflect.VisibleFields(reflect.TypeOf(&env).Elem())
 	envValue := reflect.ValueOf(&env)
 
@@ -467,6 +477,9 @@ func (bs *BaseSuite[Env]) SetupSuite() {
 		// `panic()` is required to stop the execution of the test suite. Otherwise `testify.Suite` will keep on running suite tests.
 		panic(err)
 	}
+	if bs.initOnly {
+		bs.T().Skip("INIT_ONLY is set, skipping tests")
+	}
 }
 
 // BeforeTest is executed right before the test starts and receives the suite and test names as input.
@@ -513,6 +526,11 @@ func (bs *BaseSuite[Env]) TearDownSuite() {
 		return
 	}
 
+	if bs.initOnly {
+		bs.T().Logf("INIT_ONLY is set, skipping deletion")
+		return
+	}
+
 	if bs.firstFailTest != "" && bs.params.skipDeleteOnFailure {
 		bs.Require().FailNow(fmt.Sprintf("%v failed. As SkipDeleteOnFailure feature is enabled the tests after %v were skipped. "+
 			"The environment of %v was kept.", bs.firstFailTest, bs.firstFailTest, bs.firstFailTest))

test/new-e2e/pkg/environments/aws/kubernetes/eks.go

@@ -172,6 +172,10 @@ func EKSRunFunc(ctx *pulumi.Context, env *environments.AwsKubernetes, params *Pr
 			return err
 		}
 
+		if params.eksInitOnly {
+			return nil
+		}
+
 		// Building Kubernetes provider
 		eksKubeProvider, err := kubernetes.NewProvider(awsEnv.Ctx(), awsEnv.Namer.ResourceName("k8s-provider"), &kubernetes.ProviderArgs{
 			Kubeconfig:            cluster.KubeconfigJson,

test/new-e2e/pkg/environments/aws/kubernetes/params.go

@@ -34,6 +34,7 @@ type ProvisionerParams struct {
 	eksLinuxARMNodeGroup     bool
 	eksBottlerocketNodeGroup bool
 	eksWindowsNodeGroup      bool
+	eksInitOnly              bool
 	deployDogstatsd          bool
 }
 
@@ -131,6 +132,14 @@ func WithEKSWindowsNodeGroup() ProvisionerOption {
 	}
 }
 
+// WithEKSInitOnly enable EKS init only
+func WithEKSInitOnly() ProvisionerOption {
+	return func(params *ProvisionerParams) error {
+		params.eksInitOnly = true
+		return nil
+	}
+}
+
 // WithDeployDogstatsd deploy standalone dogstatd
 func WithDeployDogstatsd() ProvisionerOption {
 	return func(params *ProvisionerParams) error {

test/new-e2e/pkg/runner/ci_profile.go

@@ -41,15 +41,31 @@ func NewCIProfile() (Profile, error) {
 	// TODO move to job script
 	os.Setenv("PULUMI_CONFIG_PASSPHRASE", passVal)
 
+	// Test early create resources
+
 	// Building name prefix
 	jobID := os.Getenv("CI_JOB_ID")
 	projectID := os.Getenv("CI_PROJECT_ID")
 	if jobID == "" || projectID == "" {
 		return nil, fmt.Errorf("unable to compute name prefix, missing variables job id: %s, project id: %s", jobID, projectID)
 	}
-
+	uniqueID := jobID
 	store := parameters.NewEnvStore(EnvPrefix)
 
+	initOnly, err := store.GetBoolWithDefault(parameters.InitOnly, false)
+	if err != nil {
+		return nil, err
+	}
+
+	preinitialized, err := store.GetBoolWithDefault(parameters.PreInitialized, false)
+	if err != nil {
+		return nil, err
+	}
+
+	if initOnly || preinitialized {
+		uniqueID = os.Getenv("CI_PIPELINE_ID") // We use pipeline ID for init only and pre-initialized jobs, to be able to share state
+	}
+
 	// get environments from store
 	environmentsStr, err := store.GetWithDefault(parameters.Environments, defaultCIEnvironments)
 	if err != nil {
@@ -69,7 +85,7 @@ func NewCIProfile() (Profile, error) {
 
 	return ciProfile{
 		baseProfile: newProfile("e2eci", ciEnvironments, store, &secretStore, outputRoot),
-		ciUniqueID:  "ci-" + jobID + "-" + projectID,
+		ciUniqueID:  "ci-" + uniqueID + "-" + projectID,
 	}, nil
 }
 

test/new-e2e/pkg/runner/parameters/const.go

@@ -49,4 +49,8 @@ const (
 	PulumiVerboseProgressStreams StoreKey = "pulumi_verbose_progress_streams"
 	// DevMode config flag parameter name
 	DevMode StoreKey = "dev_mode"
+	// InitOnly config flag parameter name
+	InitOnly StoreKey = "init_only"
+	// PreInitialized config flag parameter name
+	PreInitialized StoreKey = "pre_initialized"
 )

test/new-e2e/tests/containers/eks_test.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/components"
 	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner"
+	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner/parameters"
 	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/infra"
 	"github.com/DataDog/test-infra-definitions/scenarios/aws/eks"
 
@@ -22,10 +23,17 @@ import (
 
 type eksSuite struct {
 	k8sSuite
+
+	initOnly bool
 }
 
 func TestEKSSuite(t *testing.T) {
-	suite.Run(t, &eksSuite{})
+	var initOnly bool
+	initOnlyParam, err := runner.GetProfile().ParamStore().GetBoolWithDefault(parameters.InitOnly, false)
+	if err == nil {
+		initOnly = initOnlyParam
+	}
+	suite.Run(t, &eksSuite{initOnly: initOnly})
 }
 
 func (suite *eksSuite) SetupSuite() {
@@ -37,6 +45,9 @@ func (suite *eksSuite) SetupSuite() {
 		"ddtestworkload:deploy": auto.ConfigValue{Value: "true"},
 		"dddogstatsd:deploy":    auto.ConfigValue{Value: "true"},
 	}
+	if suite.initOnly {
+		stackConfig["ddinfra:initOnly"] = auto.ConfigValue{Value: "true"}
+	}
 
 	_, stackOutput, err := infra.GetStackManager().GetStackNoDeleteOnFailure(
 		ctx,
@@ -55,6 +66,10 @@ func (suite *eksSuite) SetupSuite() {
 		suite.T().FailNow()
 	}
 
+	if suite.initOnly {
+		suite.T().Skip("E2E_INIT_ONLY is set, skipping tests")
+	}
+
 	fakeintake := &components.FakeIntake{}
 	fiSerialized, err := json.Marshal(stackOutput.Outputs["dd-Fakeintake-aws-ecs"].Value)
 	suite.Require().NoError(err)
@@ -79,6 +94,11 @@ func (suite *eksSuite) SetupSuite() {
 }
 
 func (suite *eksSuite) TearDownSuite() {
+	if suite.initOnly {
+		suite.T().Logf("E2E_INIT_ONLY is set, skipping deletion")
+		return
+	}
+
 	suite.k8sSuite.TearDownSuite()
 
 	ctx := context.Background()

test/new-e2e/tests/containers/main_test.go

@@ -13,14 +13,21 @@ import (
 	"testing"
 
 	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner"
+	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner/parameters"
 	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/infra"
 )
 
 var keepStacks = flag.Bool("keep-stacks", false, "Do not destroy the Pulumi stacks at the end of the tests")
 
 func TestMain(m *testing.M) {
 	code := m.Run()
-	if runner.GetProfile().AllowDevMode() && *keepStacks {
+	var initOnly bool
+	initOnlyParameter, err := runner.GetProfile().ParamStore().GetBoolWithDefault(parameters.InitOnly, false)
+	if err == nil {
+		initOnly = initOnlyParameter
+	}
+
+	if runner.GetProfile().AllowDevMode() && *keepStacks || initOnly {
 		fmt.Fprintln(os.Stderr, "Keeping stacks")
 	} else {
 		fmt.Fprintln(os.Stderr, "Cleaning up stacks")

test/new-e2e/tests/npm/eks_1host_test.go

@@ -22,6 +22,8 @@ import (
 	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e"
 	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments"
 	envkube "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes"
+	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner"
+	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner/parameters"
 
 	"github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes"
 )
@@ -37,7 +39,7 @@ type eksVMSuite struct {
 	e2e.BaseSuite[eksHttpbinEnv]
 }
 
-func eksHttpbinEnvProvisioner() e2e.PulumiEnvRunFunc[eksHttpbinEnv] {
+func eksHttpbinEnvProvisioner(opts ...envkube.ProvisionerOption) e2e.PulumiEnvRunFunc[eksHttpbinEnv] {
 	return func(ctx *pulumi.Context, env *eksHttpbinEnv) error {
 		awsEnv, err := aws.NewEnvironment(ctx)
 		if err != nil {
@@ -73,10 +75,15 @@ func eksHttpbinEnvProvisioner() e2e.PulumiEnvRunFunc[eksHttpbinEnv] {
 			return npmtools.K8sAppDefinition(&awsEnv, kubeProvider, "npmtools", testURL)
 		}
 
-		params := envkube.GetProvisionerParams(
+		provisionerOpts := []envkube.ProvisionerOption{
 			envkube.WithEKSLinuxNodeGroup(),
 			envkube.WithAgentOptions(kubernetesagentparams.WithHelmValues(systemProbeConfigNPMHelmValues)),
 			envkube.WithWorkloadApp(npmToolsWorkload),
+		}
+		provisionerOpts = append(provisionerOpts, opts...)
+
+		params := envkube.GetProvisionerParams(
+			provisionerOpts...,
 		)
 		envkube.EKSRunFunc(ctx, &env.AwsKubernetes, params)
 
@@ -87,17 +94,22 @@ func eksHttpbinEnvProvisioner() e2e.PulumiEnvRunFunc[eksHttpbinEnv] {
 // TestEKSVMSuite will validate running the agent
 func TestEKSVMSuite(t *testing.T) {
 	t.Parallel()
+	provisionerOpts := []envkube.ProvisionerOption{}
+
+	initOnly, err := runner.GetProfile().ParamStore().GetBoolWithDefault(parameters.InitOnly, false)
+	if err == nil && initOnly {
+		provisionerOpts = append(provisionerOpts, envkube.WithEKSInitOnly())
+	}
 
 	s := &eksVMSuite{}
-	e2eParams := []e2e.SuiteOption{e2e.WithProvisioner(e2e.NewTypedPulumiProvisioner("eksHttpbin", eksHttpbinEnvProvisioner(), nil))}
+	e2eParams := []e2e.SuiteOption{e2e.WithProvisioner(e2e.NewTypedPulumiProvisioner("eksHttpbin", eksHttpbinEnvProvisioner(provisionerOpts...), nil))}
 
 	e2e.Run(t, s, e2eParams...)
 }
 
 // BeforeTest will be called before each test
 func (v *eksVMSuite) BeforeTest(suiteName, testName string) {
 	v.BaseSuite.BeforeTest(suiteName, testName)
-
 	// default is to reset the current state of the fakeintake aggregators
 	if !v.BaseSuite.IsDevMode() {
 		v.Env().FakeIntake.Client().FlushServerAndResetAggregators()