[ADXT-199] Early start long e2e tests environment deployment on the CI (priority on EKS)

.gitlab-ci.yml

@@ -98,7 +98,9 @@ stages:
   - internal_image_deploy
   - install_script_testing
   - e2e_pre_test
+  - e2e_init
   - e2e
+  - e2e_cleanup
   - e2e_k8s
   - e2e_install_packages
   - kitchen_cleanup
@@ -220,6 +222,7 @@ variables:
   E2E_TESTS_AZURE_TENANT_ID: ci.datadog-agent.e2e_tests_azure_tenant_id  # agent-devx-loops
   E2E_TESTS_AZURE_SUBSCRIPTION_ID: ci.datadog-agent.e2e_tests_azure_subscription_id  # agent-devx-loops
   E2E_TESTS_GCP_CREDENTIALS: ci.datadog-agent.e2e_tests_gcp_credentials  # agent-devx-loops
+  E2E_PULUMI_CONFIG_PASSPHRASE: ci.datadog-agent.pulumi_password  # agent-devx-loops
   KITCHEN_EC2_SSH_KEY: ci.datadog-agent.aws_ec2_kitchen_ssh_key  # agent-devx-loops
   KITCHEN_AZURE_CLIENT_ID: ci.datadog-agent.azure_kitchen_client_id  # agent-devx-loops
   KITCHEN_AZURE_CLIENT_SECRET: ci.datadog-agent.azure_kitchen_client_secret  # agent-devx-loops
@@ -892,6 +895,7 @@ workflow:
   - changes:
       paths:
         # TODO: Add paths that should trigger tests for npm
+        - pkg/network/**/*
         - test/new-e2e/tests/npm/**/*
       compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
 

.gitlab/e2e/e2e.yml

@@ -103,11 +103,41 @@ new-e2e-containers:
       - EXTRA_PARAMS: "--run TestKindSuite -c ddinfra:kubernetesVersion=1.29"
       - EXTRA_PARAMS: "--run TestKindSuite -c ddinfra:osDescriptor=ubuntu:20.04"
       - EXTRA_PARAMS: "--run TestKindSuite -c ddinfra:osDescriptor=ubuntu:22.04"
-      - EXTRA_PARAMS: --run TestEKSSuite
       - EXTRA_PARAMS: --run TestECSSuite
       - EXTRA_PARAMS: --run TestDockerSuite
       - EXTRA_PARAMS: --skip "Test(Kind|EKS|ECS|Docker)Suite"
 
+new-e2e-containers-eks-init:
+  stage: e2e_init
+  extends: .new_e2e_template
+  needs:
+    - !reference [.needs_new_e2e_template]
+  rules:
+    - !reference [.on_container_or_e2e_changes]
+    - !reference [.manual]
+  variables:
+    TARGETS: ./tests/containers
+    TEAM: container-integrations
+    EXTRA_PARAMS: --run TestEKSSuite
+    E2E_INIT_ONLY: "true"
+  allow_failure: true
+
+new-e2e-containers-eks:
+  extends: .new_e2e_template
+  rules:
+    - !reference [.on_container_or_e2e_changes]
+    - !reference [.manual]
+  needs:
+    - !reference [.needs_new_e2e_template]
+    - new-e2e-containers-eks-init
+    - qa_agent
+    - qa_dca
+  variables:
+    TARGETS: ./tests/containers
+    TEAM: container-integrations
+    EXTRA_PARAMS: --run TestEKSSuite
+    E2E_PRE_INITIALIZED: "true"
+
 new-e2e-remote-config:
   extends: .new_e2e_template_needs_deb_x64
   rules:
@@ -210,7 +240,39 @@ new-e2e-npm-docker:
   variables:
     TARGETS: ./tests/npm
     TEAM: network-performance-monitoring
-    EXTRA_PARAMS: --run "Test(ECSVM|EC2VMContainerized|EKSVM)Suite"
+    EXTRA_PARAMS: --run "Test(ECSVM|EC2VMContainerized)Suite"
+
+
+new-e2e-npm-eks-init:
+  stage: e2e_init
+  extends: .new_e2e_template
+  needs:
+    - !reference [.needs_new_e2e_template]
+  rules:
+    - !reference [.on_npm_or_e2e_changes]
+    - !reference [.manual]
+  variables:
+    TARGETS: ./tests/npm
+    TEAM: network-performance-monitoring
+    EXTRA_PARAMS: --run "TestEKSVMSuite"
+    E2E_INIT_ONLY: "true"
+  allow_failure: true
+
+new-e2e-npm-eks:
+  extends: .new_e2e_template
+  rules:
+    - !reference [.on_npm_or_e2e_changes]
+    - !reference [.manual]
+  needs:
+    - !reference [.needs_new_e2e_template]
+    - new-e2e-npm-eks-init
+    - qa_agent
+    - qa_dca
+  variables:
+    TARGETS: ./tests/npm
+    TEAM: network-performance-monitoring
+    EXTRA_PARAMS: --run "TestEKSVMSuite"
+    E2E_PRE_INITIALIZED: "true"
 
 new-e2e-aml:
   extends: .new_e2e_template
@@ -516,3 +578,28 @@ trigger-flakes-finder:
       - artifact: flake-finder-gitlab-ci.yml
         job: generate-flakes-finder-pipeline
   allow_failure: true
+
+new-e2e-eks-cleanup-on-failure:
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/test-infra-definitions/runner$TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX:$TEST_INFRA_DEFINITIONS_BUILDIMAGES
+  tags: ["arch:amd64"]
+  stage: e2e_cleanup
+  dependencies: [go_e2e_deps]
+  script:
+    # Setup AWS Credentials
+    - mkdir -p ~/.aws
+    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_PROFILE >> ~/.aws/config || exit $?
+    - export AWS_PROFILE=agent-qa-ci
+    # Now all `aws` commands target the agent-qa profile
+    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SSH_PUBLIC_KEY_RSA > $E2E_PUBLIC_KEY_PATH || exit $?
+    - touch $E2E_PRIVATE_KEY_PATH && chmod 600 $E2E_PRIVATE_KEY_PATH && $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SSH_KEY_RSA > $E2E_PRIVATE_KEY_PATH || exit $?
+    # Use S3 backend
+    - PULUMI_CONFIG_PASSPHRASE=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $E2E_PULUMI_CONFIG_PASSPHRASE) || exit $?; export PULUMI_CONFIG_PASSPHRASE
+    - pulumi login "s3://dd-pulumi-state?region=us-east-1&awssdk=v2&profile=$AWS_PROFILE"
+    - inv -e new-e2e-tests.cleanup-remote-stacks --stack-regex "ci-$CI_PIPELINE_ID.*eks.*" --pulumi-backend=dd-pulumi-state
+  variables:
+    E2E_PUBLIC_KEY_PATH: /tmp/agent-qa-ssh-key.pub
+    E2E_PRIVATE_KEY_PATH: /tmp/agent-qa-ssh-key
+    E2E_PIPELINE_ID: $CI_PIPELINE_ID
+  when: always
+  allow_failure: true
+

tasks/linter.py

@@ -562,6 +562,7 @@ def job_change_path(ctx, job_files=None):
         'new-e2e-cws',
         'new-e2e-language-detection',
         'new-e2e-npm-docker',
+        'new-e2e-eks-cleanup',
         'new-e2e-npm-packages',
         'new-e2e-orchestrator',
         'new-e2e-package-signing-amazonlinux-a6-x86_64',

tasks/new_e2e_tests.py

@@ -5,8 +5,10 @@
 from __future__ import annotations
 
 import json
+import multiprocessing
 import os
 import os.path
+import re
 import shutil
 import tempfile
 from pathlib import Path
@@ -22,6 +24,7 @@
 from tasks.libs.common.go import download_go_dependencies
 from tasks.libs.common.utils import REPO_PATH, color_message, running_in_ci
 from tasks.modules import DEFAULT_MODULES
+from tools.e2e_stacks import destroy_remote_stack
 
 
 @task(
@@ -194,6 +197,58 @@ def clean(ctx, locks=True, stacks=False, output=False, skip_destroy=False):
         _clean_output()
 
 
+@task
+def cleanup_remote_stacks(ctx, stack_regex, pulumi_backend):
+    """
+    Clean up remote stacks created by the pipeline
+    """
+    if not running_in_ci():
+        raise Exit("This task should be run in CI only", 1)
+
+    stack_regex = re.compile(stack_regex)
+
+    # Ideally we'd use the pulumi CLI to list all the stacks. However we have way too much stacks in the bucket so the commands hang forever.
+    # Once the bucket is cleaned up we can switch to the pulumi CLI
+    res = ctx.run(
+        "pulumi stack ls --all --json",
+        hide=True,
+        warn=True,
+    )
+    if res.exited != 0:
+        print(f"Failed to list stacks in {pulumi_backend}:", res.stdout, res.stderr)
+        return
+    to_delete_stacks = set()
+    stacks = json.loads(res.stdout)
+    print(stacks)
+    for stack in stacks:
+        stack_id = (
+            stack.get("name", "")
+            .split("/")[-1]
+            .replace(".json.bak", "")
+            .replace(".json", "")
+            .replace(".pulumi/stacks/e2eci", "")
+        )
+        if stack_regex.match(stack_id):
+            to_delete_stacks.add(f"organization/e2eci/{stack_id}")
+
+    print("About to delete the following stacks:", to_delete_stacks)
+    with multiprocessing.Pool(len(to_delete_stacks)) as pool:
+        res = pool.map(destroy_remote_stack, to_delete_stacks)
+        destroyed_stack = set()
+        failed_stack = set()
+        for r, stack in res:
+            if r.returncode != 0:
+                failed_stack.add(stack)
+            else:
+                destroyed_stack.add(stack)
+            print(f"Stack {stack}: {r.stdout} {r.stderr}")
+
+    for stack in destroyed_stack:
+        print(f"Stack {stack} destroyed successfully")
+    for stack in failed_stack:
+        print(f"Failed to destroy stack {stack}")
+
+
 @task
 def deps(ctx, verbose=False):
     """

tasks/tools/e2e_stacks.py

@@ -0,0 +1,8 @@
+import subprocess
+
+
+# This function cannot be defined in a file that imports invoke.tasks. Otherwise it fails when called with multiprocessing.
+def destroy_remote_stack(stack: str):
+    return subprocess.run(
+        ["pulumi", "destroy", "--remove", "--yes", "--stack", stack], capture_output=True, text=True
+    ), stack

test/new-e2e/pkg/e2e/suite.go

@@ -149,11 +149,12 @@ import (
 	"testing"
 	"time"
 
+	"github.com/DataDog/test-infra-definitions/common/utils"
+	"github.com/DataDog/test-infra-definitions/components"
+
 	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner"
 	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner/parameters"
 	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/infra"
-	"github.com/DataDog/test-infra-definitions/common/utils"
-	"github.com/DataDog/test-infra-definitions/components"
 
 	"github.com/stretchr/testify/suite"
 )
@@ -189,6 +190,7 @@ type BaseSuite[Env any] struct {
 	currentProvisioners  ProvisionerMap
 
 	firstFailTest string
+	initOnly      bool
 }
 
 //
@@ -212,7 +214,6 @@ func (bs *BaseSuite[Env]) UpdateEnv(newProvisioners ...Provisioner) {
 		uniqueIDs[provisioner.ID()] = struct{}{}
 		targetProvisioners[provisioner.ID()] = provisioner
 	}
-
 	if err := bs.reconcileEnv(targetProvisioners); err != nil {
 		panic(err)
 	}
@@ -229,6 +230,11 @@ func (bs *BaseSuite[Env]) init(options []SuiteOption, self Suite[Env]) {
 		o(&bs.params)
 	}
 
+	initOnly, err := runner.GetProfile().ParamStore().GetBoolWithDefault(parameters.InitOnly, false)
+	if err == nil {
+		bs.initOnly = initOnly
+	}
+
 	if !runner.GetProfile().AllowDevMode() {
 		bs.params.devMode = false
 	}
@@ -306,6 +312,11 @@ func (bs *BaseSuite[Env]) reconcileEnv(targetProvisioners ProvisionerMap) error
 		resources.Merge(provisionerResources)
 	}
 
+	// When INIT_ONLY is set, we only partially provision the environment so we do not want initialize the environment
+	if bs.initOnly {
+		return nil
+	}
+
 	// Env is taken as parameter as some fields may have keys set by Env pulumi program.
 	err = bs.buildEnvFromResources(resources, newEnvFields, newEnvValues)
 	if err != nil {
@@ -328,6 +339,7 @@ func (bs *BaseSuite[Env]) reconcileEnv(targetProvisioners ProvisionerMap) error
 
 func (bs *BaseSuite[Env]) createEnv() (*Env, []reflect.StructField, []reflect.Value, error) {
 	var env Env
+
 	envFields := reflect.VisibleFields(reflect.TypeOf(&env).Elem())
 	envValue := reflect.ValueOf(&env)
 
@@ -467,6 +479,10 @@ func (bs *BaseSuite[Env]) SetupSuite() {
 		// `panic()` is required to stop the execution of the test suite. Otherwise `testify.Suite` will keep on running suite tests.
 		panic(err)
 	}
+
+	if bs.initOnly {
+		bs.T().Skip("INIT_ONLY is set, skipping tests")
+	}
 }
 
 // BeforeTest is executed right before the test starts and receives the suite and test names as input.
@@ -513,6 +529,11 @@ func (bs *BaseSuite[Env]) TearDownSuite() {
 		return
 	}
 
+	if bs.initOnly {
+		bs.T().Logf("INIT_ONLY is set, skipping deletion")
+		return
+	}
+
 	if bs.firstFailTest != "" && bs.params.skipDeleteOnFailure {
 		bs.Require().FailNow(fmt.Sprintf("%v failed. As SkipDeleteOnFailure feature is enabled the tests after %v were skipped. "+
 			"The environment of %v was kept.", bs.firstFailTest, bs.firstFailTest, bs.firstFailTest))

test/new-e2e/pkg/environments/aws/kubernetes/eks.go

@@ -175,9 +175,14 @@ func EKSRunFunc(ctx *pulumi.Context, env *environments.Kubernetes, params *Provi
 			return err
 		}
 
+		if awsEnv.InitOnly() {
+			return nil
+		}
+
 		kubeConfig, err := cluster.GetKubeconfig(ctx, &eks.ClusterGetKubeconfigArgs{
 			ProfileName: pulumi.String(awsEnv.Profile()),
 		})
+
 		if err != nil {
 			return err
 		}

test/new-e2e/pkg/runner/ci_profile.go

@@ -51,9 +51,23 @@ func NewCIProfile() (Profile, error) {
 	if jobID == "" || projectID == "" {
 		return nil, fmt.Errorf("unable to compute name prefix, missing variables job id: %s, project id: %s", jobID, projectID)
 	}
-
+	uniqueID := jobID
 	store := parameters.NewEnvStore(EnvPrefix)
 
+	initOnly, err := store.GetBoolWithDefault(parameters.InitOnly, false)
+	if err != nil {
+		return nil, err
+	}
+
+	preInitialized, err := store.GetBoolWithDefault(parameters.PreInitialized, false)
+	if err != nil {
+		return nil, err
+	}
+
+	if initOnly || preInitialized {
+		uniqueID = fmt.Sprintf("init-%s", os.Getenv("CI_PIPELINE_ID")) // We use pipeline ID for init only and pre-initialized jobs, to be able to share state
+	}
+
 	// get environments from store
 	environmentsStr, err := store.GetWithDefault(parameters.Environments, "")
 	if err != nil {
@@ -74,7 +88,7 @@ func NewCIProfile() (Profile, error) {
 
 	return ciProfile{
 		baseProfile: newProfile("e2eci", ciEnvironments, store, &secretStore, outputRoot),
-		ciUniqueID:  "ci-" + jobID + "-" + projectID,
+		ciUniqueID:  "ci-" + uniqueID + "-" + projectID,
 	}, nil
 }
 

test/new-e2e/pkg/runner/configmap.go

@@ -35,6 +35,9 @@ const (
 	// InfraExtraResourcesTags pulumi config parameter name
 	InfraExtraResourcesTags = commonconfig.DDInfraConfigNamespace + ":" + commonconfig.DDInfraExtraResourcesTags
 
+	//InfraInitOnly pulumi config parameter name
+	InfraInitOnly = commonconfig.DDInfraConfigNamespace + ":" + commonconfig.DDInfraInitOnly
+
 	// AWSKeyPairName pulumi config parameter name
 	AWSKeyPairName = commonconfig.DDInfraConfigNamespace + ":" + infraaws.DDInfraDefaultKeyPairParamName
 	// AWSPublicKeyPath pulumi config parameter name
@@ -124,6 +127,7 @@ func BuildStackParameters(profile Profile, scenarioConfig ConfigMap) (ConfigMap,
 		parameters.ExtraResourcesTags: {InfraExtraResourcesTags},
 		parameters.PipelineID:         {AgentPipelineID},
 		parameters.CommitSHA:          {AgentCommitSHA},
+		parameters.InitOnly:           {InfraInitOnly},
 	}
 
 	for storeKey, configMapKeys := range params {

test/new-e2e/pkg/runner/configmap_test.go

@@ -36,6 +36,7 @@ func Test_BuildStackParameters(t *testing.T) {
 		"ddinfra:aws/defaultKeyPairName":        auto.ConfigValue{Value: "key_pair_name", Secret: false},
 		"ddinfra:env":                           auto.ConfigValue{Value: "", Secret: false},
 		"ddinfra:extraResourcesTags":            auto.ConfigValue{Value: "extra_resources_tags", Secret: false},
+		"ddinfra:initOnly":                      auto.ConfigValue{Value: "init_only", Secret: false},
 		"ddinfra:aws/defaultPublicKeyPath":      auto.ConfigValue{Value: "public_key_path", Secret: false},
 		"ddinfra:aws/defaultPrivateKeyPath":     auto.ConfigValue{Value: "private_key_path", Secret: false},
 		"ddinfra:aws/defaultPrivateKeyPassword": auto.ConfigValue{Value: "private_key_password", Secret: true},