show e2e test results in the github workflow (vmware-tanzu#288)

Co-authored-by: Chandra Pamuluri <cpamuluri@cpamuluri-a01.vmware.com>

.github/workflows/cli-coexistence_tests.yaml

@@ -62,5 +62,5 @@ jobs:
       - name: Tests Results Summary
         if: always()
         run: |
-          TEST_RESULTS_MD=$(./hack/scripts/process-ginkgo-test-results.sh test/e2e/coexistence/coexistence-tests.json)
+          TEST_RESULTS_MD=$(./hack/scripts/process-ginkgo-test-results.sh test/e2e/coexistence)
           echo "$TEST_RESULTS_MD" >> $GITHUB_STEP_SUMMARY

.github/workflows/cli_core_e2e_test.yaml

@@ -1,4 +1,4 @@
-name: Tanzu CLI Core Tests
+name: Tanzu CLI Core E2E Tests
 
 on:
   pull_request:
@@ -44,9 +44,15 @@ jobs:
         run: |
           make start-test-central-repo
 
-      - name: E2E Tests
+      - name: Run CLI E2E Tests
         env:
           TANZU_API_TOKEN: ${{ secrets.TANZU_API_TOKEN }}
           TANZU_CLI_TMC_UNSTABLE_URL: ${{ secrets.TANZU_CLI_TMC_UNSTABLE_URL }}
         run: |
           make e2e-cli-core
+
+      - name: Tests Results Summary
+        if: always()
+        run: |
+          TEST_RESULTS_MD=$(./hack/scripts/process-ginkgo-test-results.sh ./test/e2e/testresults)
+          echo "$TEST_RESULTS_MD" >> $GITHUB_STEP_SUMMARY

.gitignore

@@ -26,5 +26,5 @@ _output/
 # Output of the go coverage tool, specifically when used with LiteIDE
 coverage.txt
 *.out
-test/e2e/coexistence/coexistence-tests.json
-e2e-test-output.txt
+test/e2e/coexistence/
+test/e2e/testresults/

hack/scripts/process-ginkgo-test-results.sh

@@ -3,18 +3,18 @@
 # Copyright 2023 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-# This script accepts a ginkgo generated test reports json file as an argument and processes the json and generates a github flavored markdown table of Test results summary
+# This script accepts a file directory path which has ginkgo generated test reports json files as an argument and processes the json files and generates a github flavored markdown table of Test results summary
 
 if [ $# -eq 0 ]
 then
     echo "Usage: $0 ginkgo generated test report json is required"
     exit 1
 fi
 
-# Accepts file as an argument
-# Usage:  sh process-ginkgo-test-results.sh results.json
+# Accepts file directory as an argument, which has ginkgo test results files in json format
+# Usage:  sh process-ginkgo-test-results.sh ./testresults
+
 
-json=$(cat "$1")
 
 # Print the table header
 echo "| :memo: Test Suite Description | Total Tests | Passed | Failed |"
@@ -25,45 +25,48 @@ total_tests=0
 total_passed=0
 total_failed=0
 
-# Loop through each suite
-for suite in $(echo "$json" | jq -r '.[] | @base64'); do
-  suite_json=$(echo "$suite" | base64 --decode | jq -r '.')
-
-  # Get suite description
-  suite_description=$(echo "$suite_json" | jq -r '.SuiteDescription')
-
-  # Counters for suite tests
-  suite_tests=0
-  suite_passed=0
-  suite_failed=0
-
-  # Loop through each spec in the suite
-  for spec in $(echo "$suite_json" | jq -r '.SpecReports[] | @base64'); do
-    spec_json=$(echo "$spec" | base64 --decode | jq -r '.')
-    state=$(echo "$spec_json" | jq -r '.State')
-
-    # Increment counters
-    ((suite_tests++))
-    ((total_tests++))
-
-    if [ "$state" == "passed" ]; then
-      ((suite_passed++))
-      ((total_passed++))
+# Loop through each file in the given directory
+for file in `ls $1`; do
+  json=$(cat "$1/$file")
+  # Loop through each suite
+  for suite in $(echo "$json" | jq -r '.[] | @base64'); do
+    suite_json=$(echo "$suite" | base64 --decode | jq -r '.')
+
+    # Get suite description
+    suite_description=$(echo "$suite_json" | jq -r '.SuiteDescription')
+
+    # Counters for suite tests
+    suite_tests=0
+    suite_passed=0
+    suite_failed=0
+
+    # Loop through each spec in the suite
+    for spec in $(echo "$suite_json" | jq -r '.SpecReports[] | @base64'); do
+      spec_json=$(echo "$spec" | base64 --decode | jq -r '.')
+      state=$(echo "$spec_json" | jq -r '.State')
+
+      # Increment counters
+      ((suite_tests++))
+      ((total_tests++))
+
+      if [ "$state" == "passed" ]; then
+        ((suite_passed++))
+        ((total_passed++))
+      else
+        ((suite_failed++))
+        ((total_failed++))
+      fi
+    done
+
+    # Print the suite row with color and icon depending on the result
+    if [ "$suite_failed" -eq 0 ]; then
+      echo "| $suite_description | $suite_tests | $suite_passed | $suite_failed |"
     else
-      ((suite_failed++))
-      ((total_failed++))
+      echo "| $suite_description | $suite_tests | $suite_passed | :x: $suite_failed |"
     fi
-  done
-
-  # Print the suite row with color and icon depending on the result
-  if [ "$suite_failed" -eq 0 ]; then
-    echo "| $suite_description | $suite_tests | $suite_passed | $suite_failed |"
-  else
-    echo "| $suite_description | $suite_tests | $suite_passed | :x: $suite_failed |"
-  fi
 
+  done
 done
-
 # Print the total line with color and icon depending on the result
 if [ "$total_failed" -eq 0 ]; then
   echo "| **Total** | **$total_tests** | **$total_passed** | **$total_failed** |"

test/e2e/Makefile

@@ -6,6 +6,7 @@ E2E_TEST_TIMEOUT ?= 60m
 GOTEST_VERBOSE ?= -v
 ROOT_DIR := $(shell git rev-parse --show-toplevel)
 GO := go
+GINKGO := ginkgo
 
 ifndef TANZU_API_TOKEN
 TANZU_API_TOKEN = ""
@@ -54,21 +55,21 @@ e2e-cli-core-all: e2e-cli-lifecycle e2e-cli-config e2e-plugin-compatibility-test
 e2e-cli-lifecycle:
 	export TANZU_CLI_CEIP_OPT_IN_PROMPT_ANSWER="No" ; \
 	export TANZU_CLI_EULA_PROMPT_ANSWER="Yes" ; \
-	${GO} test ${ROOT_DIR}/test/e2e/cli_lifecycle -timeout ${E2E_TEST_TIMEOUT} -race -coverprofile ${E2E_TEST_OUTPUT} ${GOTEST_VERBOSE} ; \
+	${GINKGO} --keep-going --output-dir ${ROOT_DIR}/test/e2e/testresults --json-report=results.json --keep-separate-reports --race --nodes=1 ${GOTEST_VERBOSE} -r ${ROOT_DIR}/test/e2e/cli_lifecycle  --trace > /tmp/out && { cat /tmp/out | grep -Ev 'STEP:|seconds|.go:'; rm /tmp/out; } || { exit_code=$$?; cat /tmp/out | grep -Ev 'STEP:|seconds|.go:'; rm /tmp/out; exit $$exit_code; } \
 
 .PHONY: e2e-cli-config ## Execute CLI config life cycle specific e2e tests
 e2e-cli-config:
 	export TANZU_CLI_CEIP_OPT_IN_PROMPT_ANSWER="No" ; \
 	export TANZU_CLI_EULA_PROMPT_ANSWER="Yes" ; \
-	${GO} test ${ROOT_DIR}/test/e2e/config -timeout ${E2E_TEST_TIMEOUT} -race -coverprofile ${E2E_TEST_OUTPUT} ${GOTEST_VERBOSE} ; \
+	${GINKGO} --keep-going --output-dir ${ROOT_DIR}/test/e2e/testresults --json-report=results.json --keep-separate-reports --race --nodes=1 ${GOTEST_VERBOSE} -r ${ROOT_DIR}/test/e2e/config  --trace > /tmp/out && { cat /tmp/out | grep -Ev 'STEP:|seconds|.go:'; rm /tmp/out; } || { exit_code=$$?; cat /tmp/out | grep -Ev 'STEP:|seconds|.go:'; rm /tmp/out; exit $$exit_code; } \
 
 .PHONY: e2e-plugin-compatibility-tests ## Execute CLI Core Plugin Compatibility E2E test cases
 e2e-plugin-compatibility-tests:
 	export TANZU_CLI_E2E_TEST_CENTRAL_REPO_URL=$(TANZU_CLI_E2E_TEST_CENTRAL_REPO_URL) ; \
 	export TANZU_CLI_PLUGIN_DISCOVERY_IMAGE_SIGNATURE_VERIFICATION_SKIP_LIST=$(TANZU_CLI_E2E_TEST_CENTRAL_REPO_URL) ; \
 	export TANZU_CLI_CEIP_OPT_IN_PROMPT_ANSWER="No" ; \
 	export TANZU_CLI_EULA_PROMPT_ANSWER="Yes" ; \
-	${GO} test ${ROOT_DIR}/test/e2e/plugins_compatibility -timeout ${E2E_TEST_TIMEOUT} -race -coverprofile ${E2E_TEST_OUTPUT} ${GOTEST_VERBOSE} ; \
+	${GINKGO} --keep-going --output-dir ${ROOT_DIR}/test/e2e/testresults --json-report=results.json --keep-separate-reports --race --nodes=1 ${GOTEST_VERBOSE} -r ${ROOT_DIR}/test/e2e/plugins_compatibility  --trace > /tmp/out && { cat /tmp/out | grep -Ev 'STEP:|seconds|.go:'; rm /tmp/out; } || { exit_code=$$?; cat /tmp/out | grep -Ev 'STEP:|seconds|.go:'; rm /tmp/out; exit $$exit_code; } \
 
 .PHONY: e2e-plugin-lifecycle-tests ## Execute CLI Core Plugin life cycle E2E test cases
 e2e-plugin-lifecycle-tests:
@@ -78,7 +79,8 @@ e2e-plugin-lifecycle-tests:
 	export TANZU_CLI_E2E_TEST_LOCAL_CENTRAL_REPO_CA_CERT_PATH=${TANZU_CLI_E2E_TEST_LOCAL_CENTRAL_REPO_CA_CERT_PATH} ; \
 	export TANZU_CLI_CEIP_OPT_IN_PROMPT_ANSWER="No" ; \
 	export TANZU_CLI_EULA_PROMPT_ANSWER="Yes" ; \
-	${GO} test ${ROOT_DIR}/test/e2e/plugin_lifecycle -timeout ${E2E_TEST_TIMEOUT} -race -coverprofile ${E2E_TEST_OUTPUT} ${GOTEST_VERBOSE} ; \
+	${GINKGO} --keep-going --output-dir ${ROOT_DIR}/test/e2e/testresults --json-report=results.json --keep-separate-reports --race --nodes=1 ${GOTEST_VERBOSE} -r ${ROOT_DIR}/test/e2e/plugin_lifecycle  --trace > /tmp/out && { cat /tmp/out | grep -Ev 'STEP:|seconds|.go:'; rm /tmp/out; } || { exit_code=$$?; cat /tmp/out | grep -Ev 'STEP:|seconds|.go:'; rm /tmp/out; exit $$exit_code; } \
+
 
 .PHONY: e2e-plugin-sync-k8s ## Execute CLI Core Plugin sync E2E test cases for k8s target
 e2e-plugin-sync-k8s:
@@ -88,7 +90,7 @@ e2e-plugin-sync-k8s:
 	export TANZU_CLI_E2E_TEST_LOCAL_CENTRAL_REPO_CA_CERT_PATH=${TANZU_CLI_E2E_TEST_LOCAL_CENTRAL_REPO_CA_CERT_PATH} ; \
 	export TANZU_CLI_CEIP_OPT_IN_PROMPT_ANSWER="No" ; \
 	export TANZU_CLI_EULA_PROMPT_ANSWER="Yes" ; \
-	${GO} test ${ROOT_DIR}/test/e2e/plugin_sync/k8s -timeout ${E2E_TEST_TIMEOUT} -race -coverprofile ${E2E_TEST_OUTPUT} ${GOTEST_VERBOSE} ; \
+	${GINKGO} --keep-going --output-dir ${ROOT_DIR}/test/e2e/testresults --json-report=results.json --keep-separate-reports --race --nodes=1 ${GOTEST_VERBOSE} -r ${ROOT_DIR}/test/e2e/plugin_sync/k8s  --trace > /tmp/out && { cat /tmp/out | grep -Ev 'STEP:|seconds|.go:'; rm /tmp/out; } || { exit_code=$$?; cat /tmp/out | grep -Ev 'STEP:|seconds|.go:'; rm /tmp/out; exit $$exit_code; } \
 
 
 ## To run TMC tests, we need to set environment variable TANZU_API_TOKEN, in case of github workflow, its been set as environment variable
@@ -105,7 +107,7 @@ e2e-plugin-sync-tmc:
 		export TANZU_CLI_EULA_PROMPT_ANSWER="Yes" ; \
 		export TANZU_CLI_E2E_TEST_ENVIRONMENT="true" ; \
 		export TANZU_API_TOKEN=$(TANZU_API_TOKEN) ; \
-		${GO} test -p 1 ${ROOT_DIR}/test/e2e/plugin_sync/tmc -timeout ${E2E_TEST_TIMEOUT} -race -coverprofile ${E2E_TEST_OUTPUT} ${GOTEST_VERBOSE} ; \
+		${GINKGO} --keep-going --output-dir ${ROOT_DIR}/test/e2e/testresults --json-report=results.json --keep-separate-reports --race --nodes=1 ${GOTEST_VERBOSE} -r ${ROOT_DIR}/test/e2e/plugin_sync/tmc  --trace > /tmp/out && { cat /tmp/out | grep -Ev 'STEP:|seconds|.go:'; rm /tmp/out; } || { exit_code=$$?; cat /tmp/out | grep -Ev 'STEP:|seconds|.go:'; rm /tmp/out; exit $$exit_code; } \
 	fi
 
 
@@ -117,7 +119,7 @@ e2e-context-tmc-tests:
 	else \
 	    export TANZU_CLI_CEIP_OPT_IN_PROMPT_ANSWER="No" ; \
 		export TANZU_CLI_EULA_PROMPT_ANSWER="Yes" ; \
-		${GO} test ${ROOT_DIR}/test/e2e/context/tmc -timeout ${E2E_TEST_TIMEOUT} -race -coverprofile ${E2E_TEST_OUTPUT} ${GOTEST_VERBOSE} ; \
+		${GINKGO} --keep-going --output-dir ${ROOT_DIR}/test/e2e/testresults --json-report=results.json --keep-separate-reports --race --nodes=1 ${GOTEST_VERBOSE} -r ${ROOT_DIR}/test/e2e/context/tmc  --trace > /tmp/out && { cat /tmp/out | grep -Ev 'STEP:|seconds|.go:'; rm /tmp/out; } || { exit_code=$$?; cat /tmp/out | grep -Ev 'STEP:|seconds|.go:'; rm /tmp/out; exit $$exit_code; } \
 	fi
 
 .PHONY: e2e-context-k8s-tests ## Execute CLI context life cycle e2e tests for k8s target
@@ -128,7 +130,7 @@ e2e-context-k8s-tests:
 	export TANZU_CLI_E2E_TEST_LOCAL_CENTRAL_REPO_CA_CERT_PATH=${TANZU_CLI_E2E_TEST_LOCAL_CENTRAL_REPO_CA_CERT_PATH} ; \
 	export TANZU_CLI_CEIP_OPT_IN_PROMPT_ANSWER="No" ; \
 	export TANZU_CLI_EULA_PROMPT_ANSWER="Yes" ; \
-	${GO} test `go list ${ROOT_DIR}/test/e2e/context/... | grep -v test/e2e/context/tmc` -timeout ${E2E_TEST_TIMEOUT} -race -coverprofile ${E2E_TEST_OUTPUT} ${GOTEST_VERBOSE}
+	${GINKGO} --keep-going --output-dir ${ROOT_DIR}/test/e2e/testresults --json-report=results.json --keep-separate-reports --race --nodes=1 ${GOTEST_VERBOSE} -r ${ROOT_DIR}/test/e2e/context/k8s  --trace > /tmp/out && { cat /tmp/out | grep -Ev 'STEP:|seconds|.go:'; rm /tmp/out; } || { exit_code=$$?; cat /tmp/out | grep -Ev 'STEP:|seconds|.go:'; rm /tmp/out; exit $$exit_code; } \
 
 .PHONY: e2e-airgapped-tests ## Execute CLI airgapped tests
 e2e-airgapped-tests:
@@ -140,4 +142,4 @@ e2e-airgapped-tests:
 	export TANZU_CLI_E2E_AIRGAPPED_REPO=$(TANZU_CLI_E2E_AIRGAPPED_REPO) ; \
 	export TANZU_CLI_CEIP_OPT_IN_PROMPT_ANSWER="Yes" ; \
 	export TANZU_CLI_EULA_PROMPT_ANSWER="Yes" ; \
-	${GO} test ${ROOT_DIR}/test/e2e/airgapped -timeout ${E2E_TEST_TIMEOUT} -race -coverprofile ${E2E_TEST_OUTPUT} ${GOTEST_VERBOSE} ;
+	${GINKGO} --keep-going --output-dir ${ROOT_DIR}/test/e2e/testresults --json-report=results.json --keep-separate-reports --race --nodes=1 ${GOTEST_VERBOSE} -r ${ROOT_DIR}/test/e2e/airgapped  --trace > /tmp/out && { cat /tmp/out | grep -Ev 'STEP:|seconds|.go:'; rm /tmp/out; } || { exit_code=$$?; cat /tmp/out | grep -Ev 'STEP:|seconds|.go:'; rm /tmp/out; exit $$exit_code; } \

test/e2e/cli_lifecycle/cli_lifecycle_suite_test.go

@@ -12,5 +12,5 @@ import (
 
 func TestCliLifecycle(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "CliLifecycle Suite")
+	RunSpecs(t, "CLI lifecycle E2E Test Suite")
 }

test/e2e/coexistence/cli_coexistence_suite_test.go

@@ -16,7 +16,7 @@ import (
 
 func TestCliCoexistence(t *testing.T) {
 	gomega.RegisterFailHandler(ginkgo.Fail)
-	ginkgo.RunSpecs(t, "CLI Coexistence Tests involving legacy Tanzu CLI and new Tanzu CLI")
+	ginkgo.RunSpecs(t, "CLI Coexistence E2E Test Suite")
 }
 
 var (

test/e2e/context/k8s/context_suite_test.go

@@ -15,7 +15,7 @@ import (
 
 func TestContext(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "Context-K8S Suite")
+	RunSpecs(t, "Context K8S E2E Test Suite")
 }
 
 var (

test/e2e/context/tmc/tmc_suite_test.go

@@ -17,7 +17,7 @@ import (
 
 func TestTmc(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "Context-TMC Suite")
+	RunSpecs(t, "Context TMC E2E Test Suite")
 }
 
 var (

test/e2e/plugin_lifecycle/plugin_lifecycle_suite_test.go

@@ -17,7 +17,7 @@ import (
 
 func TestPluginLifecycle(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "Plugin-Lifecycle E2E Test Suite")
+	RunSpecs(t, "Plugin Lifecycle E2E Test Suite")
 }
 
 var (

test/e2e/plugin_sync/k8s/plugin_sync_k8s_lifecycle_suite_test.go

@@ -19,7 +19,7 @@ import (
 
 func TestPluginSyncLifecycle(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "Plugin-Sync-k8s-Lifecycle E2E Test Suite")
+	RunSpecs(t, "Plugin-Sync K8S Lifecycle E2E Test Suite")
 }
 
 var (

test/e2e/plugin_sync/tmc/plugin_sync_tmc_lifecycle_suite_test.go

@@ -20,7 +20,7 @@ import (
 
 func TestPluginSyncLifecycle(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "Plugin-Sync-TMC-Lifecycle E2E Test Suite")
+	RunSpecs(t, "Plugin-Sync K8S Lifecycle E2E Test Suite")
 }
 
 var (

test/e2e/plugins_compatibility/plugins_compatibility_suite_test.go

@@ -19,7 +19,7 @@ import (
 
 func TestPluginsCompatibility(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "PluginsCompatibility Suite")
+	RunSpecs(t, "Plugins Compatibility E2E Test Suite")
 }
 
 var (