From 4b2bd90f74ce4a229d0da27122cdbc86216a288f Mon Sep 17 00:00:00 2001 From: Jintao Zhang Date: Fri, 26 Jan 2024 14:36:49 +0800 Subject: [PATCH] Independent trigger performance test Signed-off-by: Jintao Zhang --- .github/workflows/_performance_tests.yaml | 113 ++++++++++++++++++ .github/workflows/e2e_trigger_via_label.yaml | 1 - .github/workflows/performance_targeted.yaml | 92 ++++++++++++++ .../performance_trigger_via_label.yaml | 27 +++++ 4 files changed, 232 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/_performance_tests.yaml create mode 100644 .github/workflows/performance_targeted.yaml create mode 100644 .github/workflows/performance_trigger_via_label.yaml diff --git a/.github/workflows/_performance_tests.yaml b/.github/workflows/_performance_tests.yaml new file mode 100644 index 0000000000..57a38c1e17 --- /dev/null +++ b/.github/workflows/_performance_tests.yaml @@ -0,0 +1,113 @@ +name: performance tests + +on: + workflow_call: + inputs: + kic-image: + description: KIC image to be tested (e.g. `kong/kubernetes-ingress-controller:v2.9.0`). + type: string + required: true + kong-image: + description: Kong image to be tested (e.g. `kong:3.1.0`). + type: string + required: false + kong-effective-version: + description: Effective version of Kong (e.g. `3.4`). Used when testing against nightly Kong images. + type: string + required: false + load-local-image: + description: Whether to load the local built image into the Docker daemon (from artifact). + type: boolean + default: false + run-performance: + description: Run performance E2E tests + type: boolean + default: false + res-number-for-perf: + description: The number of resource for performance tests + type: string + default: 10000 + +jobs: + performance: + if: ${{ inputs.run-performance }} + runs-on: ubuntu-latest + needs: dependencies-versions + strategy: + fail-fast: false + steps: + - name: Download built image artifact + if: ${{ inputs.load-local-image }} + uses: actions/download-artifact@v3 + with: + name: kic-image + path: /tmp + + - name: Load built image + if: ${{ inputs.load-local-image }} + run: | + docker load --input /tmp/image.tar + docker image ls -a + + - name: checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: setup golang + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - uses: Kong/kong-license@master + id: license + with: + op-token: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} + + - name: split image and tag + id: split + env: + KONG: ${{ inputs.kong-image }} + CONTROLLER: ${{ inputs.controller-image }} + run: | + if [ "${{ inputs.kong-image }}" != "" ]; then + export kong_image=$(echo ${{ inputs.kong-image }} | awk '{split($0,a,":"); print a[1]}') + export kong_tag=$(echo ${{ inputs.kong-image }} | awk '{split($0,a,":"); print a[2]}') + echo "kong-image=$kong_image" >> $GITHUB_OUTPUT + echo "kong-tag=$kong_tag" >> $GITHUB_OUTPUT + fi + if [ "${{ inputs.kic-image }}" != "" ]; then + export kic_image=$(echo ${{ inputs.kic-image }} | awk '{split($0,a,":"); print a[1]}') + export kic_tag=$(echo ${{ inputs.kic-image }} | awk '{split($0,a,":"); print a[2]}') + echo "kic-image=$kic_image" >> $GITHUB_OUTPUT + echo "kic-tag=$kic_tag" >> $GITHUB_OUTPUT + fi + + + - name: run performance tests + run: make test.performance + env: + TEST_CONTROLLER_IMAGE: ${{ steps.split.outputs.kic-image }} + TEST_CONTROLLER_TAG: ${{ steps.split.outputs.kic-tag }} + TEST_KONG_LOAD_IMAGES: ${{ inputs.load-local-image }} + TEST_KONG_IMAGE: ${{ steps.split.outputs.kong-image }} + TEST_KONG_TAG: ${{ steps.split.outputs.kong-tag }} + KONG_LICENSE_DATA: ${{ steps.license.outputs.license }} + PERF_RES_NUMBER: ${{ inputs.res-number-for-perf }} + NCPU: 1 # it was found that github actions (specifically) did not seem to perform well when spawning + # multiple kind clusters within a single job, so only 1 is allowed at a time. + GOTESTSUM_JUNITFILE: "performance-tests.xml" + + - name: upload diagnostics + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: diagnostics-e2e-tests + path: /tmp/ktf-diag* + if-no-files-found: ignore + + - name: collect test report + uses: actions/upload-artifact@v3 + with: + name: tests-report + path: "*-tests.xml" diff --git a/.github/workflows/e2e_trigger_via_label.yaml b/.github/workflows/e2e_trigger_via_label.yaml index a3d437e92c..ab67518dc2 100644 --- a/.github/workflows/e2e_trigger_via_label.yaml +++ b/.github/workflows/e2e_trigger_via_label.yaml @@ -25,6 +25,5 @@ jobs: gh workflow run ${WORKFLOW} --ref ${BRANCH} \ -f run-gke=false \ -f run-istio=true \ - -f run-performance=true \ -f all-supported-k8s-versions=true \ -f pr-number=${PR_NUMBER} diff --git a/.github/workflows/performance_targeted.yaml b/.github/workflows/performance_targeted.yaml new file mode 100644 index 0000000000..aacb774aa4 --- /dev/null +++ b/.github/workflows/performance_targeted.yaml @@ -0,0 +1,92 @@ +name: performance tests (targeted) +run-name: performance tests (targeted), branch:${{ github.ref_name }}, triggered by @${{ github.actor }} + +concurrency: + # Limit the concurrency of e2e tests to run only 1 workflow for ref (branch). + # Ref: https://docs.github.com/en/actions/using-jobs/using-concurrency + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + workflow_dispatch: + inputs: + controller-image: + description: KIC Docker image to test with. If empty, builds an image from the dispatch branch. + type: string + required: false + run-performance: + description: Run performance E2E tests + type: boolean + default: false + res-number-for-perf: + description: The number of resource for performance tests + type: string + default: 10000 + pr-number: + description: PR number to post a comment in. If empty, no comment is posted. + type: string + required: false + +jobs: + post-comment-in-pr: + if: ${{ github.event.inputs.pr-number != '' }} + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ secrets.K8S_TEAM_BOT_GH_PAT }} + # URL is the current workflow's run URL. + # Sadly this is not readily available in github's context. + URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + PR_NUMBER: ${{ github.event.inputs.pr-number }} + RUN_GKE: ${{ inputs.run-gke }} + steps: + - uses: actions/checkout@v4 + - run: | + MSG="performance (targeted) tests with KIND-based clusters were started at ${URL}" + gh pr comment ${PR_NUMBER} --body "${MSG}" + # Remove the 'ci/run-performance' label from the PR to prevent the `performance_trigger_via_label.yaml` + # workflow from running the `performance_targeted.yaml` again. + - run: gh pr edit ${PR_NUMBER} --remove-label ci/run-performance + + build-image: + if: ${{ inputs.controller-image == '' }} + uses: ./.github/workflows/_docker_build.yaml + secrets: inherit + + # We need to pick an image to use for the tests. If the input specified one, we use that. Otherwise, we use the one + # built by the previous job. + choose-image: + runs-on: ubuntu-latest + if: ${{ !cancelled() }} + needs: build-image + outputs: + image: ${{ steps.choose.outputs.image }} + steps: + - name: Choose image + id: choose + run: | + if [ "${{ inputs.controller-image }}" == "" ]; then + echo "image=${{ needs.build-image.outputs.image }}" >> $GITHUB_OUTPUT + else + echo "image=${{ inputs.controller-image }}" >> $GITHUB_OUTPUT + fi + + - name: Print image + run: echo "Using ${{ steps.choose.outputs.image }}" + + run: + needs: choose-image + if: ${{ !cancelled() }} + uses: ./.github/workflows/_performance_tests.yaml + secrets: inherit + with: + kic-image: ${{ needs.choose-image.outputs.image }} + load-local-image: ${{ inputs.controller-image == '' }} + run-performance: ${{ inputs.run-performance }} + res-number-for-perf: ${{ inputs.res-number-for-perf }} + + test-reports: + needs: run + uses: ./.github/workflows/_test_reports.yaml + secrets: inherit + with: + coverage: false diff --git a/.github/workflows/performance_trigger_via_label.yaml b/.github/workflows/performance_trigger_via_label.yaml new file mode 100644 index 0000000000..f090561a17 --- /dev/null +++ b/.github/workflows/performance_trigger_via_label.yaml @@ -0,0 +1,27 @@ +name: trigger perf test (targeted) on label + +on: + pull_request: + types: + - labeled + +jobs: + trigger-performance-tests-targeted: + if: contains(github.event.*.labels.*.name, 'ci/run-performance') + + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ secrets.K8S_TEAM_BOT_GH_PAT }} + WORKFLOW: .github/workflows/performance_targeted.yaml + BRANCH: ${{ github.event.pull_request.head.ref }} + PR_NUMBER: ${{ github.event.pull_request.number }} + + steps: + - uses: actions/checkout@v4 + # Do not run e2e tests on GKE-based clusters for specific PR, because currently + # there is no way to use an image built from PR's code for those tests. + # https://github.com/Kong/kubernetes-testing-framework/issues/587 + - run: | + gh workflow run ${WORKFLOW} --ref ${BRANCH} \ + -f run-performance=true \ + -f pr-number=${PR_NUMBER}