[ADXT-199] Early start long e2e tests environment deployment on the CI (priority on EKS)

.gitlab/e2e/e2e.yml

@@ -103,11 +103,53 @@ new-e2e-containers:
       - EXTRA_PARAMS: "--run TestKindSuite -c ddinfra:kubernetesVersion=1.29"
       - EXTRA_PARAMS: "--run TestKindSuite -c ddinfra:osDescriptor=ubuntu:20.04"
       - EXTRA_PARAMS: "--run TestKindSuite -c ddinfra:osDescriptor=ubuntu:22.04"
-      - EXTRA_PARAMS: --run TestEKSSuite
       - EXTRA_PARAMS: --run TestECSSuite
       - EXTRA_PARAMS: --run TestDockerSuite
       - EXTRA_PARAMS: --skip "Test(Kind|EKS|ECS|Docker)Suite"
 
+new-e2e-containers-eks-init:
+  extends: .new_e2e_template
+  needs:
+    - !reference [.needs_new_e2e_template]
+  rules:
+    - !reference [.on_container_or_e2e_changes]
+    - !reference [.manual]
+  variables:
+    TARGETS: ./tests/containers
+    TEAM: container-integrations
+    EXTRA_PARAMS: --run TestEKSSuite
+    E2E_INIT_ONLY: "true"
+
+# This job does not do anything. This is a workaround to be able to run new-e2e-containers-eks even if the init job fails. Any better solution would be appreciated
+new-e2e-containers-eks-sync:
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/test-infra-definitions/runner$TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX:$TEST_INFRA_DEFINITIONS_BUILDIMAGES
+  stage: e2e
+  tags: ["arch:amd64"]
+  script:
+    - exit 0
+  needs:
+    - new-e2e-containers-eks-init
+  rules:
+    - !reference [.on_container_or_e2e_changes]
+    - !reference [.manual]
+  when: always
+
+new-e2e-containers-eks:
+  extends: .new_e2e_template
+  rules:
+    - !reference [.on_container_or_e2e_changes]
+    - !reference [.manual]
+  needs:
+    - !reference [.needs_new_e2e_template]
+    - new-e2e-containers-eks-sync
+    - qa_agent
+    - qa_dca
+  variables:
+    TARGETS: ./tests/containers
+    TEAM: container-integrations
+    EXTRA_PARAMS: --run TestEKSSuite
+    E2E_PRE_INITIALIZED: "true"
+
 new-e2e-remote-config:
   extends: .new_e2e_template_needs_deb_x64
   rules:
@@ -210,7 +252,50 @@ new-e2e-npm-docker:
   variables:
     TARGETS: ./tests/npm
     TEAM: network-performance-monitoring
-    EXTRA_PARAMS: --run "Test(ECSVM|EC2VMContainerized|EKSVM)Suite"
+    EXTRA_PARAMS: --run "Test(ECSVM|EC2VMContainerized)Suite"
+
+
+new-e2e-npm-eks-init:
+  extends: .new_e2e_template
+  needs:
+    - !reference [.needs_new_e2e_template]
+  rules:
+    - !reference [.on_npm_or_e2e_changes]
+    - !reference [.manual]
+  variables:
+    TARGETS: ./tests/npm
+    TEAM: network-performance-monitoring
+    EXTRA_PARAMS: --run "TestEKSVMSuite"
+    E2E_INIT_ONLY: "true"
+
+new-e2e-npm-eks-sync:
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/test-infra-definitions/runner$TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX:$TEST_INFRA_DEFINITIONS_BUILDIMAGES
+  tags: ["arch:amd64"]
+  stage: e2e
+  script:
+    - exit 0
+  needs:
+    - new-e2e-npm-eks-init
+  rules:
+    - !reference [.on_npm_or_e2e_changes]
+    - !reference [.manual]
+  when: always
+
+new-e2e-npm-eks:
+  extends: .new_e2e_template
+  rules:
+    - !reference [.on_npm_or_e2e_changes]
+    - !reference [.manual]
+  needs:
+    - !reference [.needs_new_e2e_template]
+    - new-e2e-npm-eks-sync
+    - qa_agent
+    - qa_dca
+  variables:
+    TARGETS: ./tests/npm
+    TEAM: network-performance-monitoring
+    EXTRA_PARAMS: --run "TestEKSVMSuite"
+    E2E_PRE_INITIALIZED: "true"
 
 new-e2e-aml:
   extends: .new_e2e_template
@@ -501,3 +586,15 @@ trigger-flakes-finder:
       - artifact: flake-finder-gitlab-ci.yml
         job: generate-flakes-finder-pipeline
   allow_failure: true
+
+new-e2e-eks-cleanup-on-failure:
+  extends:
+    - .new_e2e_template_needs_container_deploy
+  needs:
+    - new-e2e-containers-eks-sync
+    - new-e2e-npm-eks-sync
+  script:
+    - inv new-e2e-tests.cleanup-remote-stacks --pipeline-id=$E2E_PIPELINE_ID --pulumi-backend=dd-pulumi-state
+  when: on_failure
+  allow_failure: true
+

.gitlab/package_build/linux.yml

@@ -88,6 +88,8 @@ datadog-agent-6-x64:
 # build Agent 7 binaries for x86_64
 datadog-agent-7-x64:
   extends: [.agent_build_common, .agent_build_x86, .agent_7_build]
+  script:
+    - exit 1 
 
 # build Agent 6 binaries for arm64
 datadog-agent-6-arm64:

tasks/new_e2e_tests.py

@@ -5,6 +5,7 @@
 from __future__ import annotations
 
 import json
+import multiprocessing
 import os
 import os.path
 import shutil
@@ -22,6 +23,7 @@
 from tasks.libs.common.go import download_go_dependencies
 from tasks.libs.common.utils import REPO_PATH, color_message, running_in_ci
 from tasks.modules import DEFAULT_MODULES
+from tools.e2e_stacks import destroy_remote_stack
 
 
 @task(
@@ -194,6 +196,49 @@ def clean(ctx, locks=True, stacks=False, output=False, skip_destroy=False):
         _clean_output()
 
 
+@task
+def cleanup_remote_stacks(ctx, pipeline_id, pulumi_backend):
+    """
+    Clean up remote stacks created by the pipeline
+    """
+    # if not running_in_ci():
+    #     raise Exit("This task should be run in CI only", 1)
+    print(f"Running aws s3api list-objects --bucket {pulumi_backend} --prefix .pulumi/stacks/e2eci/ci-{pipeline_id}")
+    res = ctx.run(
+        f"aws s3api list-objects --bucket {pulumi_backend} --prefix .pulumi/stacks/e2eci/ci-{pipeline_id}",
+        hide=True,
+        warn=True,
+    )
+    if res.exited != 0:
+        print(f"Failed to list stacks in bucket {pulumi_backend}:", res.stdout, res.stderr)
+        return
+    eks_stacks = set()
+    stacks = json.loads(res.stdout)
+    if "Contents" in stacks:
+        for stack in stacks["Contents"]:
+            if "Key" in stack:
+                stack_id = stack["Key"].split("/")[-1].replace(".json.bak", "").replace(".json", "")
+                if "eks" in stack_id:
+                    eks_stacks.add(f"organization/e2eci/{stack_id}")
+
+    pool = multiprocessing.Pool(len(eks_stacks))
+    res = pool.map(destroy_remote_stack, eks_stacks)
+    destroyed_stack = set()
+    failed_stack = set()
+    for r, stack in res:
+        if r.returncode != 0:
+            failed_stack.add(stack)
+        else:
+            destroyed_stack.add(stack)
+        print(f"Stack {stack}: {r.stdout} {r.stderr}")
+    pool.close()
+
+    for stack in destroyed_stack:
+        print(f"Stack {stack} destroyed successfully")
+    for stack in failed_stack:
+        print(f"Failed to destroy stack {stack}")
+
+
 @task
 def deps(ctx, verbose=False):
     """

tasks/tools/e2e_stacks.py

@@ -0,0 +1,6 @@
+import subprocess
+
+
+# This function cannot be defined in a file that imports invoke.tasks. Otherwise it fails when called with multiprocessing.
+def destroy_remote_stack(stack: str):
+    return subprocess.run(["pulumi", "destroy", "--yes", "--stack", stack], capture_output=True, text=True), stack

test/new-e2e/pkg/e2e/suite.go

@@ -189,6 +189,7 @@ type BaseSuite[Env any] struct {
 	currentProvisioners  ProvisionerMap
 
 	firstFailTest string
+	initOnly      bool
 }
 
 //
@@ -306,6 +307,10 @@ func (bs *BaseSuite[Env]) reconcileEnv(targetProvisioners ProvisionerMap) error
 		resources.Merge(provisionerResources)
 	}
 
+	if bs.initOnly {
+		return nil
+	}
+
 	// Env is taken as parameter as some fields may have keys set by Env pulumi program.
 	err = bs.buildEnvFromResources(resources, newEnvFields, newEnvValues)
 	if err != nil {
@@ -328,6 +333,11 @@ func (bs *BaseSuite[Env]) reconcileEnv(targetProvisioners ProvisionerMap) error
 
 func (bs *BaseSuite[Env]) createEnv() (*Env, []reflect.StructField, []reflect.Value, error) {
 	var env Env
+	initOnly, err := runner.GetProfile().ParamStore().GetBoolWithDefault(parameters.InitOnly, false)
+	if err == nil {
+		bs.initOnly = initOnly
+	}
+
 	envFields := reflect.VisibleFields(reflect.TypeOf(&env).Elem())
 	envValue := reflect.ValueOf(&env)
 
@@ -467,6 +477,9 @@ func (bs *BaseSuite[Env]) SetupSuite() {
 		// `panic()` is required to stop the execution of the test suite. Otherwise `testify.Suite` will keep on running suite tests.
 		panic(err)
 	}
+	if bs.initOnly {
+		bs.T().Skip("INIT_ONLY is set, skipping tests")
+	}
 }
 
 // BeforeTest is executed right before the test starts and receives the suite and test names as input.
@@ -513,6 +526,11 @@ func (bs *BaseSuite[Env]) TearDownSuite() {
 		return
 	}
 
+	if bs.initOnly {
+		bs.T().Logf("INIT_ONLY is set, skipping deletion")
+		return
+	}
+
 	if bs.firstFailTest != "" && bs.params.skipDeleteOnFailure {
 		bs.Require().FailNow(fmt.Sprintf("%v failed. As SkipDeleteOnFailure feature is enabled the tests after %v were skipped. "+
 			"The environment of %v was kept.", bs.firstFailTest, bs.firstFailTest, bs.firstFailTest))

test/new-e2e/pkg/environments/aws/kubernetes/eks.go

@@ -175,9 +175,14 @@ func EKSRunFunc(ctx *pulumi.Context, env *environments.Kubernetes, params *Provi
 			return err
 		}
 
+		if params.eksInitOnly {
+			return nil
+		}
+
 		kubeConfig, err := cluster.GetKubeconfig(ctx, &eks.ClusterGetKubeconfigArgs{
 			ProfileName: pulumi.String(awsEnv.Profile()),
 		})
+
 		if err != nil {
 			return err
 		}

test/new-e2e/pkg/environments/aws/kubernetes/params.go

@@ -35,6 +35,7 @@ type ProvisionerParams struct {
 	eksLinuxARMNodeGroup     bool
 	eksBottlerocketNodeGroup bool
 	eksWindowsNodeGroup      bool
+	eksInitOnly              bool
 	awsEnv                   *aws.Environment
 	deployDogstatsd          bool
 }
@@ -133,6 +134,14 @@ func WithEKSWindowsNodeGroup() ProvisionerOption {
 	}
 }
 
+// WithEKSInitOnly enable EKS init only
+func WithEKSInitOnly() ProvisionerOption {
+	return func(params *ProvisionerParams) error {
+		params.eksInitOnly = true
+		return nil
+	}
+}
+
 // WithDeployDogstatsd deploy standalone dogstatd
 func WithDeployDogstatsd() ProvisionerOption {
 	return func(params *ProvisionerParams) error {

test/new-e2e/pkg/runner/ci_profile.go

@@ -51,9 +51,23 @@ func NewCIProfile() (Profile, error) {
 	if jobID == "" || projectID == "" {
 		return nil, fmt.Errorf("unable to compute name prefix, missing variables job id: %s, project id: %s", jobID, projectID)
 	}
-
+	uniqueID := jobID
 	store := parameters.NewEnvStore(EnvPrefix)
 
+	initOnly, err := store.GetBoolWithDefault(parameters.InitOnly, false)
+	if err != nil {
+		return nil, err
+	}
+
+	preinitialized, err := store.GetBoolWithDefault(parameters.PreInitialized, false)
+	if err != nil {
+		return nil, err
+	}
+
+	if initOnly || preinitialized {
+		uniqueID = os.Getenv("CI_PIPELINE_ID") // We use pipeline ID for init only and pre-initialized jobs, to be able to share state
+	}
+
 	// get environments from store
 	environmentsStr, err := store.GetWithDefault(parameters.Environments, "")
 	if err != nil {
@@ -74,7 +88,7 @@ func NewCIProfile() (Profile, error) {
 
 	return ciProfile{
 		baseProfile: newProfile("e2eci", ciEnvironments, store, &secretStore, outputRoot),
-		ciUniqueID:  "ci-" + jobID + "-" + projectID,
+		ciUniqueID:  "ci-" + uniqueID + "-" + projectID,
 	}, nil
 }
 

test/new-e2e/pkg/runner/parameters/const.go

@@ -49,4 +49,8 @@ const (
 	PulumiVerboseProgressStreams StoreKey = "pulumi_verbose_progress_streams"
 	// DevMode allows to keep the stack after the test completes
 	DevMode StoreKey = "dev_mode"
+	// InitOnly config flag parameter name
+	InitOnly StoreKey = "init_only"
+	// PreInitialized config flag parameter name
+	PreInitialized StoreKey = "pre_initialized"
 )

test/new-e2e/tests/containers/eks_test.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/components"
 	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner"
+	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner/parameters"
 	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/infra"
 
 	"github.com/pulumi/pulumi/sdk/v3/go/auto"
@@ -23,10 +24,17 @@ import (
 
 type eksSuite struct {
 	k8sSuite
+
+	initOnly bool
 }
 
 func TestEKSSuite(t *testing.T) {
-	suite.Run(t, &eksSuite{})
+	var initOnly bool
+	initOnlyParam, err := runner.GetProfile().ParamStore().GetBoolWithDefault(parameters.InitOnly, false)
+	if err == nil {
+		initOnly = initOnlyParam
+	}
+	suite.Run(t, &eksSuite{initOnly: initOnly})
 }
 
 func (suite *eksSuite) SetupSuite() {
@@ -38,6 +46,9 @@ func (suite *eksSuite) SetupSuite() {
 		"ddtestworkload:deploy": auto.ConfigValue{Value: "true"},
 		"dddogstatsd:deploy":    auto.ConfigValue{Value: "true"},
 	}
+	if suite.initOnly {
+		stackConfig["ddinfra:initOnly"] = auto.ConfigValue{Value: "true"}
+	}
 
 	_, stackOutput, err := infra.GetStackManager().GetStackNoDeleteOnFailure(
 		ctx,
@@ -56,6 +67,10 @@ func (suite *eksSuite) SetupSuite() {
 		suite.T().FailNow()
 	}
 
+	if suite.initOnly {
+		suite.T().Skip("E2E_INIT_ONLY is set, skipping tests")
+	}
+
 	fakeintake := &components.FakeIntake{}
 	fiSerialized, err := json.Marshal(stackOutput.Outputs["dd-Fakeintake-aws-ecs"].Value)
 	suite.Require().NoError(err)
@@ -84,6 +99,11 @@ func (suite *eksSuite) SetupSuite() {
 }
 
 func (suite *eksSuite) TearDownSuite() {
+	if suite.initOnly {
+		suite.T().Logf("E2E_INIT_ONLY is set, skipping deletion")
+		return
+	}
+
 	suite.k8sSuite.TearDownSuite()
 
 	ctx := context.Background()