From ac7d391c04fa701ad5a05b56b2dbaaf1954ed112 Mon Sep 17 00:00:00 2001 From: bbensky Date: Wed, 10 Jul 2024 14:02:14 -0600 Subject: [PATCH 01/12] integration testing setup Co-authored-by: Ryan --- .circleci/config.yml | 32 ++++++++++++++++++++++++++++---- e2e/pre.sh | 6 ++++++ e2e/test.sh | 22 ++++++++++++++++++++++ e2e/testsuite.yaml | 12 ++++++++++++ 4 files changed, 68 insertions(+), 4 deletions(-) create mode 100644 e2e/pre.sh create mode 100644 e2e/test.sh create mode 100644 e2e/testsuite.yaml diff --git a/.circleci/config.yml b/.circleci/config.yml index 0b6c275..2870117 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,10 +1,24 @@ version: 2.1 orbs: - rok8s: fairwinds/rok8s-scripts@11 + rok8s: fairwinds/rok8s-scripts@14 oss-docs: fairwinds/oss-docs@0 references: + e2e_config: &e2e_config + command_runner_image: quay.io/reactiveops/ci-images:v14-bullseye + kind_node_image: "kindest/node:v1.29.4@sha256:3abb816a5b1061fb15c6e9e60856ec40d56b7b52bcea5f5f1350bc6e2320b6f8" + executor: golang-exec + pre_script: e2e/pre.sh + store-test-results: /tmp/test-results/ + script: e2e/test.sh + requires: + - test + filters: + branches: + only: /.*/ + tags: + ignore: /.*/ install_vault: &install_vault run: name: install hashicorp vault @@ -18,7 +32,7 @@ references: jobs: test: docker: - - image: cimg/go:1.21 + - image: cimg/go:1.22 steps: - checkout - run: @@ -43,7 +57,7 @@ jobs: working_directory: /go/src/github.com/fairwindsops/nova resource_class: large docker: - - image: goreleaser/goreleaser:v1.22.1 + - image: goreleaser/goreleaser:v2.0.1 steps: - checkout - setup_remote_docker @@ -56,7 +70,7 @@ jobs: resource_class: large shell: /bin/bash docker: - - image: goreleaser/goreleaser:v1.22.1 + - image: goreleaser/goreleaser:v2.0.1 steps: - checkout - setup_remote_docker @@ -82,6 +96,16 @@ workflows: only: /.*/ tags: ignore: /.*/ + - rok8s/kubernetes_e2e_tests: + requires: + - test + filters: + branches: + only: /.*/ + tags: + ignore: /.*/ + name: functional tests + <<: *e2e_config release: jobs: - oss-docs/publish-docs: diff --git a/e2e/pre.sh b/e2e/pre.sh new file mode 100644 index 0000000..f54c0f1 --- /dev/null +++ b/e2e/pre.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +set -e + +make build +docker cp ./ e2e-command-runner:/pluto \ No newline at end of file diff --git a/e2e/test.sh b/e2e/test.sh new file mode 100644 index 0000000..1b097df --- /dev/null +++ b/e2e/test.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -e + + +printf "\n\n" +echo "***************************" +echo "** Install and Run Venom **" +echo "***************************" +printf "\n\n" + +curl -LO https://github.com/ovh/venom/releases/download/v1.1.0/venom.linux-amd64 +mv venom.linux-amd64 /usr/local/bin/venom +chmod +x /usr/local/bin/venom + +cp /nova/nova /usr/local/bin/nova + +cd /nova/e2e +mkdir -p /tmp/test-results +helm delete -n kube-system hostpath-provisioner || true +venom run testsuite.yaml --output-dir=/tmp/test-results +exit $? \ No newline at end of file diff --git a/e2e/testsuite.yaml b/e2e/testsuite.yaml new file mode 100644 index 0000000..b325f65 --- /dev/null +++ b/e2e/testsuite.yaml @@ -0,0 +1,12 @@ +name: Nova Tests +description: Collection of e2e tests for Nova. +testcases: +- name: Title of First TestCase + steps: + - script: echo 'foo' + assertions: + - result.code ShouldEqual 0 + - script: echo 'bar' + assertions: + - result.systemout ShouldNotContainSubstring foo + - result.timeseconds ShouldBeLessThan 1 \ No newline at end of file From a337a4fda6bec76c8c577a3b69a45c36c81081ab Mon Sep 17 00:00:00 2001 From: bbensky Date: Wed, 10 Jul 2024 14:10:58 -0600 Subject: [PATCH 02/12] adding golang-exec --- .circleci/config.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2870117..804c774 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,6 +4,11 @@ orbs: rok8s: fairwinds/rok8s-scripts@14 oss-docs: fairwinds/oss-docs@0 +executors: + golang-exec: + docker: + - image: cimg/go:1.21 + references: e2e_config: &e2e_config command_runner_image: quay.io/reactiveops/ci-images:v14-bullseye From bdcafe37478597f990597afeebfea96f4ad143ba Mon Sep 17 00:00:00 2001 From: bbensky Date: Wed, 10 Jul 2024 14:20:12 -0600 Subject: [PATCH 03/12] adding goreleaser version --- .circleci/config.yml | 2 +- .goreleaser.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 804c774..86ab76c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -66,7 +66,7 @@ jobs: steps: - checkout - setup_remote_docker - - run: goreleaser --snapshot + - run: goreleaser --snapshot --skip sign - store_artifacts: path: dist destination: snapshot diff --git a/.goreleaser.yml b/.goreleaser.yml index f742ad4..92d00ca 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -1,3 +1,4 @@ +version: 2 project_name: nova release: prerelease: auto From e552fb4479e402419d2a55481ec2104ea3ce6ce8 Mon Sep 17 00:00:00 2001 From: bbensky Date: Wed, 10 Jul 2024 14:30:28 -0600 Subject: [PATCH 04/12] updating goreleaser config --- .goreleaser.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.goreleaser.yml b/.goreleaser.yml index 92d00ca..095fd30 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -35,10 +35,10 @@ changelog: - '^test:' brews: - name: nova - tap: + repository: owner: FairwindsOps name: homebrew-tap - folder: Formula + directory: Formula description: Check installed Helm charts for updates test: | system "#{bin}/nova version" From 39bc8c03bb31ff151f644ef294f4420f4cdfed7e Mon Sep 17 00:00:00 2001 From: bbensky Date: Wed, 10 Jul 2024 14:44:43 -0600 Subject: [PATCH 05/12] fix pre.sh --- e2e/pre.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/pre.sh b/e2e/pre.sh index f54c0f1..4ea6f87 100644 --- a/e2e/pre.sh +++ b/e2e/pre.sh @@ -3,4 +3,4 @@ set -e make build -docker cp ./ e2e-command-runner:/pluto \ No newline at end of file +docker cp ./ e2e-command-runner:/nova \ No newline at end of file From a7da90ebe7bb04b2be39611d822cd95bb1ea2ff1 Mon Sep 17 00:00:00 2001 From: bbensky Date: Wed, 17 Jul 2024 15:03:55 -0600 Subject: [PATCH 06/12] added first test --- e2e/testsuite.yaml | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/e2e/testsuite.yaml b/e2e/testsuite.yaml index b325f65..81d2671 100644 --- a/e2e/testsuite.yaml +++ b/e2e/testsuite.yaml @@ -1,12 +1,26 @@ name: Nova Tests description: Collection of e2e tests for Nova. testcases: -- name: Title of First TestCase +- name: install helm charts steps: - - script: echo 'foo' + - script: | + helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx + helm repo update + # helm install ingress-nginx-current ingress-nginx/ingress-nginx + helm install --version 4.6.0 ingress-nginx-old ingress-nginx/ingress-nginx assertions: - result.code ShouldEqual 0 - - script: echo 'bar' + +- name: nova_run + steps: + - type: exec + script: nova find --output-file output.json + +- name: validate outdated chart version + steps: + - type: exec + script: | + jq '.helm[] | select(.release == "ingress-nginx-old" and .outdated == true)' ./output.json assertions: - - result.systemout ShouldNotContainSubstring foo - - result.timeseconds ShouldBeLessThan 1 \ No newline at end of file + - result.code ShouldEqual 0 + - result.systemoutjson ShouldNotBeEmpty From 9a65d9e114ba9364a3835a4e92f21f8da4031b6b Mon Sep 17 00:00:00 2001 From: Ryan Miron Date: Wed, 18 Sep 2024 13:04:39 -0400 Subject: [PATCH 07/12] cat file to stdout for sanity check --- e2e/testsuite.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/e2e/testsuite.yaml b/e2e/testsuite.yaml index 81d2671..df4a870 100644 --- a/e2e/testsuite.yaml +++ b/e2e/testsuite.yaml @@ -14,7 +14,9 @@ testcases: - name: nova_run steps: - type: exec - script: nova find --output-file output.json + script: | + nova find --output-file output.json + cat ./output.json - name: validate outdated chart version steps: From 7205557a2941e2bbc0b183e8b3b59bfaeef1beac Mon Sep 17 00:00:00 2001 From: Ryan Miron Date: Wed, 18 Sep 2024 13:24:06 -0400 Subject: [PATCH 08/12] catting venom logs --- e2e/test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/e2e/test.sh b/e2e/test.sh index 1b097df..7436463 100644 --- a/e2e/test.sh +++ b/e2e/test.sh @@ -18,5 +18,6 @@ cp /nova/nova /usr/local/bin/nova cd /nova/e2e mkdir -p /tmp/test-results helm delete -n kube-system hostpath-provisioner || true -venom run testsuite.yaml --output-dir=/tmp/test-results +venom run -v testsuite.yaml --output-dir=/tmp/test-results +cat /tmp/test-results/venom.log exit $? \ No newline at end of file From e6ce04af203b2f6e18f4c0ed7aca32246017a9a7 Mon Sep 17 00:00:00 2001 From: Ryan Miron Date: Wed, 18 Sep 2024 13:39:52 -0400 Subject: [PATCH 09/12] store artifacts --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 86ab76c..21b912c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,6 +16,7 @@ references: executor: golang-exec pre_script: e2e/pre.sh store-test-results: /tmp/test-results/ + store-artifacts: /tmp/output/ script: e2e/test.sh requires: - test From 0dda70b1442dec656a9b4c31cc0d335c4a6dfd02 Mon Sep 17 00:00:00 2001 From: Ryan Miron Date: Wed, 18 Sep 2024 13:47:05 -0400 Subject: [PATCH 10/12] changed artifacts path --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 21b912c..d088620 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,7 +16,7 @@ references: executor: golang-exec pre_script: e2e/pre.sh store-test-results: /tmp/test-results/ - store-artifacts: /tmp/output/ + store-artifacts: /tmp/test-results/ script: e2e/test.sh requires: - test From 261de616f919512937f4cd7b076ed06bbb5dd91c Mon Sep 17 00:00:00 2001 From: Ryan Miron Date: Fri, 20 Sep 2024 13:41:33 -0400 Subject: [PATCH 11/12] use build-linux --- e2e/pre.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/pre.sh b/e2e/pre.sh index 4ea6f87..0268410 100644 --- a/e2e/pre.sh +++ b/e2e/pre.sh @@ -2,5 +2,5 @@ set -e -make build +make build-linux docker cp ./ e2e-command-runner:/nova \ No newline at end of file From 5ed16e0f5c70d01d2773ff8ed2abfa9adecbdd0b Mon Sep 17 00:00:00 2001 From: bbensky Date: Fri, 20 Sep 2024 12:01:12 -0600 Subject: [PATCH 12/12] latest release testcase --- e2e/testsuite.yaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/e2e/testsuite.yaml b/e2e/testsuite.yaml index df4a870..bd32f30 100644 --- a/e2e/testsuite.yaml +++ b/e2e/testsuite.yaml @@ -6,7 +6,7 @@ testcases: - script: | helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx helm repo update - # helm install ingress-nginx-current ingress-nginx/ingress-nginx + helm install ingress-nginx-latest ingress-nginx/ingress-nginx --set controller.ingressClassResource.name=nginx-latest helm install --version 4.6.0 ingress-nginx-old ingress-nginx/ingress-nginx assertions: - result.code ShouldEqual 0 @@ -26,3 +26,12 @@ testcases: assertions: - result.code ShouldEqual 0 - result.systemoutjson ShouldNotBeEmpty + +- name: validate latest chart version + steps: + - type: exec + script: | + jq '.helm[] | select(.release == "ingress-nginx-latest" and .outdated == false)' ./output.json + assertions: + - result.code ShouldEqual 0 + - result.systemoutjson ShouldNotBeEmpty