feat: scheduled tests for installer Action

.github/workflows/schedule.installer.yml

@@ -0,0 +1,211 @@
+name: verifier action
+on:
+  # Daily run.
+  schedule:
+    - cron: "0 4 * * *"
+  workflow_dispatch:
+    inputs:
+      version:
+        type: string
+        description: The version to to test for pre-release.
+        required: true
+
+permissions: read-all
+
+env:
+  GH_TOKEN: ${{ secrets.CREATE_ISSUES }}
+  ISSUE_REPOSITORY: ${{ github.repository }}
+  MINIMUM_INSTALLER_VERSION: v2.0.1
+
+jobs:
+  list-verifiers:
+    runs-on: ubuntu-latest
+    outputs:
+      # https://github.blog/changelog/2020-04-15-github-actions-new-workflow-features/#new-fromjson-method-in-expressions
+      # https://docs.github.com/en/actions/learn-github-actions/expressions#fromjson
+      version: ${{ steps.generate-versions.outputs.version }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 # tag=v3.1.2
+        with:
+          # Note: the example-package need to be checkout'ed in the default workspace.
+          repository: slsa-framework/example-package
+          ref: main
+
+      - name: Checkout
+        uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 # tag=v3.1.2
+        with:
+          path: __THIS_REPO__
+
+      - name: Generate verifier list
+        if: inputs.version == ''
+        id: generate-list
+        run: ./__THIS_REPO__/.github/workflows/scripts/schedule.actions/verifier-installer.sh
+
+      - name: Generate pre-release list
+        if: inputs.version != ''
+        id: generate-prerelease
+        env:
+          PRE_RELEASE_VERSION: ${{ inputs.version }}
+        run: echo "version=[\"$PRE_RELEASE_VERSION\"]" >> "$GITHUB_OUTPUT"
+
+      - name: Generate pre-release list
+        id: generate-versions
+        env:
+          PRE_RELEASE_VERSION: ${{ steps.generate-prerelease.outputs.version }}
+          LIST_VERSION: ${{ steps.generate-list.outputs.version }}
+        run: |
+          if [[ -n $PRE_RELEASE_VERSION ]]; then
+            echo "version=$PRE_RELEASE_VERSION" >> "$GITHUB_OUTPUT"
+          else
+            echo "version=$LIST_VERSION" >> "$GITHUB_OUTPUT"
+          fi
+
+  verifier-run:
+    needs: list-verifiers
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        version: ${{ fromJson(needs.list-verifiers.outputs.version) }}
+    steps:
+      - name: Debug
+        env:
+          VERSION: ${{ matrix.version }}
+        run: echo "version is '$VERSION'"
+
+      - name: Checkout this repository
+        if: ${{ inputs.version != '' || ! contains(matrix.version, '-rc' ) }}
+        uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 # tag=v3.1.2
+        with:
+          repository: ${{ github.repository }}
+          ref: ${{ matrix.version }}
+
+      # Install at tag.
+      # ==============
+      - name: Run the Action at tag
+        if: ${{ inputs.version != '' || ! contains(matrix.version, '-rc' ) }}
+        env:
+          SLSA_VERIFIER_CI_ACTION_REF: ${{ matrix.version }}
+        uses: ./actions/installer
+
+      - name: Verify the version
+        env:
+          VERSION: ${{ matrix.version }}
+        if: ${{ inputs.version != '' || ! contains(matrix.version, '-rc' ) }}
+        run: |
+          version=$(slsa-verifier version 2>&1 | grep GitVersion | cut -d ':' -f2 | tr -d "[:space:]")
+          slsa-verifier version
+          echo "version: $version"
+          echo "VERSION: $VERSION"
+          # Note: the version reported by the slsa-verifier does not contain the leading `v`.
+          [ "$version" == "${VERSION:1}" ]
+
+      - name: Delete the binary
+        if: ${{ inputs.version != '' || ! contains(matrix.version, '-rc' ) }}
+        run: |
+          binary_path=$(which slsa-verifier)
+          echo "binary_path: $binary_path"
+          rm -rf "$binary_path"
+
+      # Install at commit sha.
+      # =====================
+      - name: Get sha1
+        if: ${{ inputs.version != '' || ! contains(matrix.version, '-rc' ) }}
+        id: commit
+        env:
+          VERSION: ${{ matrix.version }}
+        run: |
+          # TODO: use $GITHUB_REPOSITORY
+          commit_sha=$(gh api -H "Accept: application/vnd.github+json" /repos/slsa-framework/slsa-verifier/git/ref/tags/"$VERSION" | jq -r '.object.sha')
+          echo "commit_sha=$commit_sha" >> "$GITHUB_OUTPUT"
+
+      - name: Run the Action at commit
+        if: ${{ inputs.version != '' || ! contains(matrix.version, '-rc' ) }}
+        env:
+          SLSA_VERIFIER_CI_ACTION_REF: ${{ steps.commit.outputs.commit_sha }}
+        uses: ./actions/installer
+
+      - name: Verify the version
+        env:
+          VERSION: ${{ matrix.version }}
+        if: ${{ inputs.version != '' || ! contains(matrix.version, '-rc' ) }}
+        run: |
+          version=$(slsa-verifier version 2>&1 | grep GitVersion | cut -d ':' -f2 | tr -d "[:space:]")
+          slsa-verifier version
+          echo "version: $version"
+          echo "VERSION: $VERSION"
+          # Note: the version reported by the slsa-verifier does not contain the leading `v`.
+          [ "$version" == "${VERSION:1}" ]
+
+      - name: Delete the binary
+        if: ${{ inputs.version != '' || ! contains(matrix.version, '-rc' ) }}
+        run: |
+          binary_path=$(which slsa-verifier)
+          echo "binary_path: $binary_path"
+          rm -rf "$binary_path"
+
+      # Install at invalid commit.
+      # =========================
+      - name: Install invalid commit
+        id: invalid-commit
+        if: ${{ inputs.version != '' || ! contains(matrix.version, '-rc' ) }}
+        continue-on-error: true
+        env:
+          SLSA_VERIFIER_CI_ACTION_REF: 55ca6286e3e4f4fba5d0448333fa99fc5a404a73
+        uses: ./actions/installer
+      - env:
+          SUCCESS: ${{ steps.invalid-commit.outcome == 'failure' }}
+        run: |
+          [ "$SUCCESS" == "true" ]
+
+      # Install at inexistent tag.
+      # =========================
+      - name: Install non-existent tag
+        id: inexistent-tag
+        if: ${{ inputs.version != '' || ! contains(matrix.version, '-rc' ) }}
+        continue-on-error: true
+        env:
+          SLSA_VERIFIER_CI_ACTION_REF: v100.3.5
+        uses: ./actions/installer
+      - env:
+          SUCCESS: ${{ steps.inexistent-tag.outcome == 'failure' }}
+        run: |
+          [ "$SUCCESS" == "true" ]
+
+      # Install at empty tag.
+      # =====================
+      - name: Install empty tag
+        id: empty-tag
+        if: ${{ inputs.version != '' || ! contains(matrix.version, '-rc' ) }}
+        continue-on-error: true
+        env:
+          SLSA_VERIFIER_CI_ACTION_REF:
+        uses: ./actions/installer
+      - env:
+          SUCCESS: ${{ steps.inexistent-tag.outcome == 'failure' }}
+        run: |
+          [ "$SUCCESS" == "true" ]
+
+  if-succeed:
+    needs: [verifier-run]
+    runs-on: ubuntu-latest
+    # We use `== 'failure'` instead of ` != 'success'` because we want to ignore skipped jobs, if there are any.
+    if: always() && inputs.version == '' && needs.verifier-run.result != 'failure'
+    steps:
+      - uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # tag=v2.4.0
+        with:
+          repository: slsa-framework/example-package
+          ref: main
+      - run: ./.github/workflows/scripts/e2e-report-success.sh
+
+  if-failed:
+
+    needs: [verifier-run]
+    runs-on: ubuntu-latest
+    if: always() && inputs.version == '' && needs.verifier-run.result == 'failure'
+    steps:
+      - uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # tag=v2.4.0
+        with:
+          repository: slsa-framework/example-package
+          ref: main
+      - run: ./.github/workflows/scripts/e2e-report-failure.sh

.github/workflows/scripts/schedule.actions/verifier-installer.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# shellcheck source=/dev/null
+source "./.github/workflows/scripts/e2e-utils.sh"
+
+minimum_version="$MINIMUM_INSTALLER_VERSION"
+list="\"$minimum_version\""
+# Check the releases.
+echo "Listing releases"
+# Note: can remove -R option.
+release_list=$(gh -R slsa-framework/slsa-verifier release list)
+while read -r line; do
+    tag=$(echo "$line" | cut -f1)
+    if version_ge "$tag" "$minimum_version"; then
+        echo " INFO: found version to test: $tag"
+        list="$list, \"$tag\""
+    fi
+done <<<"$release_list"
+
+versions="[$list]"
+echo "version=$versions" >> "$GITHUB_OUTPUT"

RELEASE.md

@@ -41,8 +41,10 @@ Check the following:
 3. Ensure that the latest release can be installed via a `go install`.
 4. Verify that the version reported by the `version` command is correct:
 ```shell
-$ ./slsa-verifier version 2>&1 | grep GitVersion 
+$ ./slsa-verifier version 2>&1 | grep GitVersion
 ```
+5. Ensure the installer Action works by manually running the [schedule.installer.yml](https://github.com/slsa-framework/slsa-verifier/actions/workflows/pre-submit.actions.yml). 
+
 
 If both of these steps succeed, then move on to the [Final Release](#final-release).
 
@@ -117,6 +119,8 @@ $ sed -i "s/v1.0.0/v1.1.1/g" ./README.md
 4. Send a pull request with the changes. In the description, explain the steps to verify the hash update, i.e., reviewers shoud LGTM only if the provenance verification succeeds
 and the hash in the pull request matches the one computed on the binary. You can use [#slsa-framework/slsa-github-generator#113](https://github.com/slsa-framework/slsa-github-generator/pull/113) as example.
 
+5. Update the e2e.installer-action.yml [here](https://github.com/slsa-framework/example-package/blob/main/.github/workflows/e2e.installer-action.yml#L9) and [here](https://github.com/slsa-framework/example-package/blob/main/.github/workflows/e2e.installer-action.yml#L17) with the newly released version; and [here](https://github.com/slsa-framework/example-package/blob/main/.github/workflows/e2e.installer-action.yml#L27) with its corresponding commit sha.
+
 ## Update builders
 
 Send a similar pull request to update the hash and version of the verifier for the workflow [slsa-framework/slsa-github-generator/blob/main/.github/workflows/builder_go_slsa3.yml#L30-L31](https://github.com/slsa-framework/slsa-github-generator/blob/main/.github/workflows/builder_go_slsa3.yml#L30-L31). Explain the steps to verify the hash. If the pull request for the verifier is already merged, you can simply point to it instead.

actions/installer/dist/index.js

@@ -179,10 +179,14 @@ function run() {
         // Validate binary provenance
         try {
             const { exitCode, stdout, stderr } = yield exec.getExecOutput(`${bootstrapVerifierPath}`, [
-                "verify-artifact", downloadedBinaryPath,
-                "--provenance-path", downloadedProvenancePath,
-                "--source-uri", "github.com/slsa-framework/slsa-verifier",
-                "--source-tag", version,
+                "verify-artifact",
+                downloadedBinaryPath,
+                "--provenance-path",
+                downloadedProvenancePath,
+                "--source-uri",
+                "github.com/slsa-framework/slsa-verifier",
+                "--source-tag",
+                version,
             ]);
             if (exitCode !== 0) {
                 throw new Error(`Unable to verify binary provenance. Aborting installation. stdout: ${stdout}; stderr: ${stderr}`);