oci/int: Add separate resource cleanup step

Introduce a destroy-only mode in the test runner to run terraform destroy for the respective cloud provider configurations. This can be used to destroy cloud resources without going through the whole provision-test process. Add a new step in github actions workflow to run the test binary in destoy-only mode at the very end irrespective of the result of the previous steps. This ensures that the infrastructure is always destroyed, even if the CI job is cancelled. This is added to solve a recent CI failure due to a failure in GCP which resulted in the cluster provisioning to take more than 30 minutes, which is the test timeout duration. After the timeout, the test binary got terminated and couldn't perform graceful stop and cleanup. To work around such scenarios, the cleanup can be run separately at the end with its own timeout to not affect the test runtime. Signed-off-by: Sunny <darkowlzz@protonmail.com>
fluxcd · Jan 8, 2024 · 62c1416 · 62c1416
1 parent 1876c5e
commit 62c1416
Show file tree

Hide file tree

Showing 6 changed files with 66 additions and 13 deletions.
diff --git a/.github/workflows/integration-azure.yaml b/.github/workflows/integration-azure.yaml
@@ -57,3 +57,15 @@ jobs:
           TF_VAR_enable_wi: ${{ matrix.enable-workload-id }}
           TF_VAR_wi_k8s_sa_name: test-workload-id
           TF_VAR_wi_k8s_sa_ns: default
+      - name: Ensure resource cleanup
+        if: ${{ always() }}
+        run: . .env && make destroy-azure
+        env:
+          ARM_CLIENT_ID: ${{ secrets.OCI_E2E_AZ_ARM_CLIENT_ID }}
+          ARM_CLIENT_SECRET: ${{ secrets.OCI_E2E_AZ_ARM_CLIENT_SECRET }}
+          ARM_SUBSCRIPTION_ID: ${{ secrets.OCI_E2E_AZ_ARM_SUBSCRIPTION_ID }}
+          ARM_TENANT_ID: ${{ secrets.OCI_E2E_AZ_ARM_TENANT_ID }}
+          TF_VAR_azure_location: ${{ vars.TF_VAR_azure_location }}
+          TF_VAR_enable_wi: ${{ matrix.enable-workload-id }}
+          TF_VAR_wi_k8s_sa_name: test-workload-id
+          TF_VAR_wi_k8s_sa_ns: default
diff --git a/.github/workflows/integration-gcp.yaml b/.github/workflows/integration-gcp.yaml
@@ -71,3 +71,13 @@ jobs:
           TF_VAR_enable_wi: ${{ matrix.enable-workload-id }}
           TF_VAR_wi_k8s_sa_name: test-workload-id
           TF_VAR_wi_k8s_sa_ns: default
+      - name: Ensure resource cleanup
+        if: ${{ always() }}
+        run: . .env && make destroy-gcp
+        env:
+          TF_VAR_gcp_project_id: ${{ vars.TF_VAR_gcp_project_id }}
+          TF_VAR_gcp_region: ${{ vars.TF_VAR_gcp_region }}
+          TF_VAR_gcp_zone: ${{ vars.TF_VAR_gcp_zone }}
+          TF_VAR_enable_wi: ${{ matrix.enable-workload-id }}
+          TF_VAR_wi_k8s_sa_name: test-workload-id
+          TF_VAR_wi_k8s_sa_ns: default
diff --git a/oci/tests/integration/Makefile b/oci/tests/integration/Makefile
@@ -25,3 +25,15 @@ test-azure:
 
 test-gcp:
 	$(MAKE) test PROVIDER_ARG="-provider gcp"
+
+destroy:
+	go test -timeout $(TEST_TIMEOUT) -v ./ $(GO_TEST_ARGS) $(PROVIDER_ARG) -destroy-only --tags=integration
+
+destroy-aws:
+	$(MAKE) destroy PROVIDER_ARG="-provider aws"
+
+destroy-azure:
+	$(MAKE) destroy PROVIDER_ARG="-provider azure"
+
+destroy-gcp:
+	$(MAKE) destroy PROVIDER_ARG="-provider gcp"
diff --git a/oci/tests/integration/go.mod b/oci/tests/integration/go.mod
@@ -6,8 +6,9 @@ replace github.com/fluxcd/pkg/oci => ../../
 
 require (
 	github.com/fluxcd/pkg/oci v0.32.0
-	github.com/fluxcd/test-infra/tftestenv v0.0.0-20230720084205-d40ee5473f22
+	github.com/fluxcd/test-infra/tftestenv v0.0.0-20240108135005-b58e0c4e0cfa
 	github.com/google/go-containerregistry v0.17.0
+	github.com/hashicorp/terraform-exec v0.18.1
 	github.com/hashicorp/terraform-json v0.17.1
 	github.com/onsi/gomega v1.30.0
 	k8s.io/api v0.28.4
@@ -65,7 +66,6 @@ require (
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-version v1.6.0 // indirect
 	github.com/hashicorp/hc-install v0.5.1 // indirect
-	github.com/hashicorp/terraform-exec v0.18.1 // indirect
 	github.com/imdario/mergo v0.3.15 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect

diff --git a/oci/tests/integration/go.sum b/oci/tests/integration/go.sum
@@ -71,8 +71,8 @@ github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc
 github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI=
 github.com/evanphx/json-patch/v5 v5.7.0 h1:nJqP7uwL84RJInrohHfW0Fx3awjbm8qZeFv0nW9SYGc=
 github.com/evanphx/json-patch/v5 v5.7.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
-github.com/fluxcd/test-infra/tftestenv v0.0.0-20230720084205-d40ee5473f22 h1:1f0EQM0kPX2px9FanVMyD/UrHFQ9zqFg58M42Y8bBts=
-github.com/fluxcd/test-infra/tftestenv v0.0.0-20230720084205-d40ee5473f22/go.mod h1:liFlLEXgambGVdWSJ4JzbIHf1Vjpp1HwUyPazPIVZug=
+github.com/fluxcd/test-infra/tftestenv v0.0.0-20240108135005-b58e0c4e0cfa h1:JdI+rVwGF5gBYt+UBijOVzXtq7aAU80vgksMNXSCCfU=
+github.com/fluxcd/test-infra/tftestenv v0.0.0-20240108135005-b58e0c4e0cfa/go.mod h1:liFlLEXgambGVdWSJ4JzbIHf1Vjpp1HwUyPazPIVZug=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=

diff --git a/oci/tests/integration/suite_test.go b/oci/tests/integration/suite_test.go
@@ -29,6 +29,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/hashicorp/terraform-exec/tfexec"
 	tfjson "github.com/hashicorp/terraform-json"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -80,6 +81,9 @@ var (
 	// verbose flag to enable output of terraform execution.
 	verbose = flag.Bool("verbose", false, "verbose output of the environment setup")
 
+	// destroyOnly flag to destroy any provisioned infrastructure.
+	destroyOnly = flag.Bool("destroy-only", false, "run in destroy-only mode and delete any existing infrastructure")
+
 	// testRepos is a map of registry common name and URL of the test
 	// repositories. This is used as the test cases to run the tests against.
 	// The registry common name need not be the actual registry address but an
@@ -153,15 +157,6 @@ func TestMain(m *testing.M) {
 	flag.Parse()
 	ctx := context.TODO()
 
-	appImg := os.Getenv("TEST_IMG")
-	if appImg == "" {
-		log.Fatal("TEST_IMG must be set to the test application image, cannot be empty")
-	}
-
-	localImgs := map[string]string{
-		"app": appImg,
-	}
-
 	// Validate the provider.
 	if *targetProvider == "" {
 		log.Fatalf("-provider flag must be set to one of %v", supportedProviders)
@@ -181,6 +176,30 @@ func TestMain(m *testing.M) {
 		log.Fatalf("Failed to get provider config for %q", *targetProvider)
 	}
 
+	// Run destroy-only mode if enabled.
+	if *destroyOnly {
+		log.Println("Running in destroy-only mode...")
+		envOpts := []tftestenv.EnvironmentOption{
+			tftestenv.WithVerbose(*verbose),
+			// Ignore any state lock in destroy-only mode.
+			tftestenv.WithTfDestroyOptions(tfexec.Lock(false)),
+		}
+		if err := tftestenv.Destroy(ctx, providerCfg.terraformPath, envOpts...); err != nil {
+			panic(err)
+		}
+		os.Exit(0)
+	}
+
+	// Check the test app image.
+	appImg := os.Getenv("TEST_IMG")
+	if appImg == "" {
+		log.Fatal("TEST_IMG must be set to the test application image, cannot be empty")
+	}
+
+	localImgs := map[string]string{
+		"app": appImg,
+	}
+
 	// Construct scheme to be added to the kubeclient.
 	scheme := runtime.NewScheme()