feat: Support large subjects file in generic generator

.github/workflows/generator_generic_slsa3.yml

@@ -21,6 +21,7 @@ env:
   # Generator
   BUILDER_BINARY: slsa-generator-generic-linux-amd64 # Name of the binary in the release assets.
   BUILDER_DIR: internal/builders/generic # Source directory if we compile the builder.
+  SUBJECTS_FILENAME: "subjects.sha256sum.base64.231b023dad77ada71cb6fb328090d00a" # Filename containing the subjects. A random value is appended to avoid name collisions.
 
 defaults:
   run:
@@ -31,7 +32,13 @@ on:
     inputs:
       base64-subjects:
         description: "Artifacts for which to generate provenance, formatted the same as the output of sha256sum (SHA256 NAME\\n[...]) and base64 encoded."
-        required: true
+        required: false
+        type: string
+      base64-subjects-as-file:
+        description: >
+          The file 'handle' representing the filename containing the artifacts for which to generate provenance, formatted the same as the output of sha256sum (SHA256 NAME\\n[...]) and base64 encoded. 'actions/generator/generic/create-base64-subjects-from-file'.
+          The handle must be created using Action 'slsa-framework/slsa-github-generator/actions/generator/generic/create-base64-subjects-from-file'.
+        required: false
         type: string
       upload-assets:
         description: >
@@ -145,6 +152,7 @@ jobs:
       outcome: ${{ steps.final.outputs.outcome }}
       provenance-sha256: ${{ steps.sign-prov.outputs.provenance-sha256 }}
       provenance-name: ${{ steps.sign-prov.outputs.provenance-name }}
+      subject-artifact-name: ${{ steps.metadata.outputs.artifact_name }}
     runs-on: ubuntu-latest
     needs: [detect-env]
     permissions:
@@ -165,6 +173,48 @@ jobs:
           directory: "${{ env.BUILDER_DIR }}"
           allow-private-repository: ${{ inputs.private-repository }}
 
+      - name: Extract subjects file metadata
+        id: metadata
+        continue-on-error: true
+        if: inputs.base64-subjects-as-file != ''
+        env:
+          UNTRUSTED_SUBJECTS_AS_FILE: "${{ inputs.base64-subjects-as-file }}"
+        run: |
+          set -euo pipefail
+          obj=$(echo "${UNTRUSTED_SUBJECTS_AS_FILE}" | base64 -d | jq)
+          echo "UNTRUSTED_SUBJECTS_AS_FILE: ${obj}"
+          artifact_name=$(echo "${obj}" | jq -r '.artifact_name')
+          filename=$(echo "${obj}" | jq -r '.filename')
+          sha256=$(echo "${obj}" | jq -r '.sha256')
+
+          # shellcheck disable=SC2129
+          echo "artifact_name=${artifact_name}" >> "$GITHUB_OUTPUT"
+          echo "filename=${filename}" >> "$GITHUB_OUTPUT"
+          echo "sha256=${sha256}" >> "$GITHUB_OUTPUT"
+
+      - name: Download subjects file
+        continue-on-error: true
+        if: inputs.base64-subjects-as-file != ''
+        uses: slsa-framework/slsa-github-generator/.github/actions/secure-download-artifact@main
+        with:
+          name: "${{ steps.metadata.outputs.artifact_name }}"
+          path: "${{ steps.metadata.outputs.filename }}"
+          sha256: "${{ steps.metadata.outputs.sha256 }}"
+
+      - name: Create subject file
+        continue-on-error: true
+        env:
+          UNTRUSTED_SUBJECTS: "${{ inputs.base64-subjects }}"
+          UNTRUSTED_SUBJECTS_FILENAME: "${{ steps.metadata.outputs.filename }}"
+        run: |
+          set -euo pipefail
+          # NOTE: SUBJECTS_FILE is trusted and declared at the top of the file.
+          if [[ -n "${UNTRUSTED_SUBJECTS_FILENAME}" ]]; then
+            mv "${UNTRUSTED_SUBJECTS_FILENAME}" "${SUBJECTS_FILENAME}"
+          else
+            echo "${UNTRUSTED_SUBJECTS}" > "${SUBJECTS_FILENAME}"
+          fi
+
       - name: Create and sign provenance
         id: sign-prov
         continue-on-error: true
@@ -173,7 +223,6 @@ jobs:
         # See: https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#understanding-the-risk-of-script-injections
         env:
           GITHUB_CONTEXT: "${{ toJSON(github) }}"
-          UNTRUSTED_SUBJECTS: "${{ inputs.base64-subjects }}"
           UNTRUSTED_PROVENANCE_NAME: "${{ inputs.provenance-name }}"
           UNTRUSTED_DEPRECATED_ATTESTATION_NAME: "${{ inputs.attestation-name }}"
         run: |
@@ -195,7 +244,7 @@ jobs:
           # number of subjects based on in-toto attestation bundle file naming conventions.
           # See: https://github.com/in-toto/attestation/blob/main/spec/bundle.md#file-naming-convention
           # NOTE: The attest commmand outputs the provenance-name and provenance-sha256
-          "$GITHUB_WORKSPACE/$BUILDER_BINARY" attest --subjects "${UNTRUSTED_SUBJECTS}" -g "$untrusted_prov_name"
+          "$GITHUB_WORKSPACE/$BUILDER_BINARY" attest --subjects-filename "${SUBJECTS_FILENAME}" -g "$untrusted_prov_name"
 
       - name: Upload the signed provenance
         id: upload-prov
@@ -281,3 +330,6 @@ jobs:
           set -euo pipefail
           echo "outcome=$([ "$SUCCESS" == "true" ] && echo "success" || echo "failure")" >> "$GITHUB_OUTPUT"
           [ "$CONTINUE" == "true" ] || [ "$SUCCESS" == "true" ] || exit 27
+
+  # cleanup deletes internal artifacts used by the generator workflow
+  # TODO(#2382): Delete artifacts ${{ needs.generator.outputs.subject-artifact-name }}

actions/generator/generic/create-base64-subjects-from-file/action.yml

@@ -0,0 +1,71 @@
+# Copyright 2023 SLSA Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "Secure subjects file sharing generic generator"
+description: "Create a list of subjects from a file"
+inputs:
+  path:
+    description: 'A path to a file containing the base64-subjects.'
+    required: true
+outputs:
+  handle:
+    description: "Object handle representing the file."
+    value: "${{ steps.object.outputs.base64 }}"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Generate random value
+      id: rng
+      uses: slsa-framework/slsa-github-generator/.github/actions/rng@main
+
+    - name: Generate random name
+      id: name
+      shell: bash
+      env:
+        UNTRUSTED_PATH: "${{ inputs.path }}"
+        RNG: "${{ steps.rng.outputs.random }}"
+      run: |
+        set -euo pipefail
+
+        name=$(basename "${UNTRUSTED_PATH}")
+        if [[ -z "${UNTRUSTED_PATH}" ]]; then
+          echo "error: empty path"
+          exit 1
+        fi
+        echo "artifact_name=${name}-${RNG}" >> "$GITHUB_OUTPUT"
+        echo "filename=${name}" >> "$GITHUB_OUTPUT"
+
+    - name: Upload file
+      id: upload
+      uses: slsa-framework/slsa-github-generator/.github/actions/secure-upload-artifact@main
+      with:
+        name: "${{ steps.name.outputs.artifact_name }}"
+        path: "${{ inputs.path }}"
+
+    - name: Create object
+      id: object
+      shell: bash
+      env:
+        UNTRUSTED_ARTIFACT_NAME: "${{ steps.name.outputs.artifact_name }}"
+        SHA256: "${{ steps.upload.outputs.sha256 }}"
+        UNTRUSTED_FILENAME: "${{ steps.name.outputs.filename }}"
+      run: |
+        set -euo pipefail
+
+        object="{\"artifact_name\": \"${UNTRUSTED_ARTIFACT_NAME}\", \"sha256\": \"${SHA256}\", \"filename\": \"${UNTRUSTED_FILENAME}\"}"
+        echo "$object" | jq
+        base64_object=$(echo "$object" | base64 -w0)
+
+        echo "base64=${base64_object}" >> "$GITHUB_OUTPUT"

internal/builders/generic/README.md

@@ -98,6 +98,33 @@ provenance:
     base64-subjects: "${{ needs.build.outputs.hashes }}"
 ```
 
+The `base64-subjects` input has a maximum length as defined by [ARG_MAX](https://www.in-ulm.de/~mascheck/various/argmax/) on the runner. If you need to attest to a large number of files that exceeds the maximum length, use the `base64-subjects-as-file` input option instead. This option requires that you save the ouput of the sha256sum command into a file:
+
+```shell
+sha256sum artifact1 artifact2 ... | base64 -w0 > large_digests_file.text
+```
+
+The you must then share this file with the generator using the [actions/generator/generic/create-base64-subjects-from-file Action](https://github.com/slsa-framework/slsa-github-generator/tree/main/actions/generator/generic/create-base64-subjects-from-file):
+
+```yaml
+build:
+  outputs:
+    subjects-as-file: ${{ steps.hashes.outputs.handle }}
+  ...
+    uses: slsa-framework/slsa-github-generator/actions/generator/generic/create-base64-subjects-from-file@v1.7.0
+    id: hashes
+    with:
+      path: large_digests_file.text
+provenance:
+  permissions:
+    actions: read # Needed for detection of GitHub Actions environment.
+    id-token: write # Needed for provenance signing and ID
+    contents: write # Needed for release uploads
+  uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v1.7.0
+  with:
+    base64-subjects-as-file: "${{ needs.build.outputs.subjects-as-file }}"
+```
+
 **Note**: Make sure that you reference the generator with a semantic version of the form `@vX.Y.Z`.
 More information [here](/README.md#referencing-slsa-builders-and-generators).
 

internal/builders/generic/attest.go

@@ -38,7 +38,7 @@ func attestCmd(provider slsa.ClientProvider, check func(error),
 	signer signing.Signer, tlog signing.TransparencyLog,
 ) *cobra.Command {
 	var attPath string
-	var subjects string
+	var subjectsFilename string
 
 	c := &cobra.Command{
 		Use:   "attest",
@@ -51,9 +51,10 @@ run in the context of a Github Actions workflow.`,
 			ghContext, err := github.GetWorkflowContext()
 			check(err)
 
-			parsedSubjects, err := parseSubjects(subjects)
+			subjectsBytes, err := utils.SafeReadFile(subjectsFilename)
+			check(err)
+			parsedSubjects, err := parseSubjects(string(subjectsBytes))
 			check(err)
-
 			if len(parsedSubjects) == 0 {
 				check(errors.New("expected at least one subject"))
 			}
@@ -133,9 +134,8 @@ run in the context of a Github Actions workflow.`,
 		"Path to write the signed provenance.",
 	)
 	c.Flags().StringVarP(
-		&subjects, "subjects", "s", "",
-		"Formatted list of subjects in the same format as sha256sum (base64 encoded).",
+		&subjectsFilename, "subjects-filename", "f", "",
+		"Filename containing a formatted list of subjects in the same format as sha256sum (base64 encoded).",
 	)
-
 	return c
 }

internal/builders/generic/attest_test.go

@@ -208,6 +208,18 @@ func TestParseSubjects(t *testing.T) {
 	}
 }
 
+func createTmpFile(content string) (string, error) {
+	file, err := os.CreateTemp(".", "test-")
+	if err != nil {
+		return "", err
+	}
+	defer file.Close()
+	if _, err := file.Write([]byte(content)); err != nil {
+		return "", err
+	}
+	return file.Name(), nil
+}
+
 // Test_attestCmd tests the attest command.
 func Test_attestCmd_default_single_artifact(t *testing.T) {
 	t.Setenv("GITHUB_CONTEXT", "{}")
@@ -231,10 +243,15 @@ func Test_attestCmd_default_single_artifact(t *testing.T) {
 		}
 	}()
 
+	fn, err := createTmpFile(base64.StdEncoding.EncodeToString([]byte(testHash)))
+	if err != nil {
+		t.Errorf("unexpected failure: %v", err)
+	}
+	defer os.Remove(fn)
 	c := attestCmd(&slsa.NilClientProvider{}, checkTest(t), &testutil.TestSigner{}, &testutil.TestTransparencyLog{})
 	c.SetOut(new(bytes.Buffer))
 	c.SetArgs([]string{
-		"--subjects", base64.StdEncoding.EncodeToString([]byte(testHash)),
+		"--subjects-filename", fn,
 	})
 	if err := c.Execute(); err != nil {
 		t.Errorf("unexpected failure: %v", err)
@@ -268,12 +285,17 @@ func Test_attestCmd_default_multi_artifact(t *testing.T) {
 		}
 	}()
 
+	fn, err := createTmpFile(base64.StdEncoding.EncodeToString([]byte(
+		`b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c  artifact1
+b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c  artifact2`)))
+	if err != nil {
+		t.Errorf("unexpected failure: %v", err)
+	}
+	defer os.Remove(fn)
 	c := attestCmd(&slsa.NilClientProvider{}, checkTest(t), &testutil.TestSigner{}, &testutil.TestTransparencyLog{})
 	c.SetOut(new(bytes.Buffer))
 	c.SetArgs([]string{
-		"--subjects", base64.StdEncoding.EncodeToString([]byte(
-			`b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c  artifact1
-b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c  artifact2`)),
+		"--subjects-filename", fn,
 	})
 	if err := c.Execute(); err != nil {
 		t.Errorf("unexpected failure: %v", err)
@@ -307,10 +329,15 @@ func Test_attestCmd_custom_provenance_name(t *testing.T) {
 		}
 	}()
 
+	fn, err := createTmpFile(base64.StdEncoding.EncodeToString([]byte(testHash)))
+	if err != nil {
+		t.Errorf("unexpected failure: %v", err)
+	}
+	defer os.Remove(fn)
 	c := attestCmd(&slsa.NilClientProvider{}, checkTest(t), &testutil.TestSigner{}, &testutil.TestTransparencyLog{})
 	c.SetOut(new(bytes.Buffer))
 	c.SetArgs([]string{
-		"--subjects", base64.StdEncoding.EncodeToString([]byte(testHash)),
+		"--subjects-filename", fn,
 		"--signature", "custom.intoto.jsonl",
 	})
 	if err := c.Execute(); err != nil {
@@ -357,10 +384,15 @@ func Test_attestCmd_invalid_extension(t *testing.T) {
 		}
 	}
 
+	fn, err := createTmpFile(base64.StdEncoding.EncodeToString([]byte(testHash)))
+	if err != nil {
+		t.Errorf("unexpected failure: %v", err)
+	}
+	defer os.Remove(fn)
 	c := attestCmd(&slsa.NilClientProvider{}, check, &testutil.TestSigner{}, &testutil.TestTransparencyLog{})
 	c.SetOut(new(bytes.Buffer))
 	c.SetArgs([]string{
-		"--subjects", base64.StdEncoding.EncodeToString([]byte(testHash)),
+		"--subjects-filename", fn,
 		"--signature", "invalid_name",
 	})
 	if err := c.Execute(); err != nil {
@@ -405,10 +437,15 @@ func Test_attestCmd_invalid_path(t *testing.T) {
 		}
 	}
 
+	fn, err := createTmpFile(base64.StdEncoding.EncodeToString([]byte(testHash)))
+	if err != nil {
+		t.Errorf("unexpected failure: %v", err)
+	}
+	defer os.Remove(fn)
 	c := attestCmd(&slsa.NilClientProvider{}, check, &testutil.TestSigner{}, &testutil.TestTransparencyLog{})
 	c.SetOut(new(bytes.Buffer))
 	c.SetArgs([]string{
-		"--subjects", base64.StdEncoding.EncodeToString([]byte(testHash)),
+		"--subjects-filename", fn,
 		"--signature", "/provenance.intoto.jsonl",
 	})
 	if err := c.Execute(); err != nil {
@@ -443,10 +480,15 @@ func Test_attestCmd_subdirectory_artifact(t *testing.T) {
 		}
 	}()
 
+	fn, err := createTmpFile(base64.StdEncoding.EncodeToString([]byte(testHash)))
+	if err != nil {
+		t.Errorf("unexpected failure: %v", err)
+	}
+	defer os.Remove(fn)
 	c := attestCmd(&slsa.NilClientProvider{}, checkTest(t), &testutil.TestSigner{}, &testutil.TestTransparencyLog{})
 	c.SetOut(new(bytes.Buffer))
 	c.SetArgs([]string{
-		"--subjects", base64.StdEncoding.EncodeToString([]byte(testHash)),
+		"--subjects-filename", fn,
 	})
 	if err := c.Execute(); err != nil {
 		t.Errorf("unexpected failure: %v", err)

internal/builders/generic/generic.go

@@ -74,10 +74,10 @@ var (
 )
 
 // parseSubjects parses the value given to the subjects option.
-func parseSubjects(b64str string) ([]intoto.Subject, error) {
+func parseSubjects(b64Str string) ([]intoto.Subject, error) {
 	var parsed []intoto.Subject
 
-	subjects, err := base64.StdEncoding.DecodeString(b64str)
+	subjects, err := base64.StdEncoding.DecodeString(b64Str)
 	if err != nil {
 		return nil, fmt.Errorf("%w: error decoding subjects (is it base64 encoded?): %w", errBase64, err)
 	}