diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 8573c36a6732..db6b504c810f 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -54,8 +54,8 @@ jobs: name: Upload pants.log uses: actions/upload-artifact@v3 with: - name: pants-log-wheels-Linux-ARM64 - path: .pants.d/pants.log + name: logs-wheels-Linux-ARM64 + path: .pants.d/*.log - env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -113,8 +113,8 @@ jobs: name: Upload pants.log uses: actions/upload-artifact@v3 with: - name: pants-log-wheels-Linux-x86_64 - path: .pants.d/pants.log + name: logs-wheels-Linux-x86_64 + path: .pants.d/*.log - env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -172,8 +172,8 @@ jobs: name: Upload pants.log uses: actions/upload-artifact@v3 with: - name: pants-log-wheels-macOS10-15-x86_64 - path: .pants.d/pants.log + name: logs-wheels-macOS10-15-x86_64 + path: .pants.d/*.log - env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -231,8 +231,8 @@ jobs: name: Upload pants.log uses: actions/upload-artifact@v3 with: - name: pants-log-wheels-macOS11-ARM64 - path: .pants.d/pants.log + name: logs-wheels-macOS11-ARM64 + path: .pants.d/*.log - env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 4434e050ecb7..65c181764a25 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -76,8 +76,8 @@ jobs: name: Upload pants.log uses: actions/upload-artifact@v3 with: - name: pants-log-bootstrap-Linux-ARM64 - path: .pants.d/pants.log + name: logs-bootstrap-Linux-ARM64 + path: .pants.d/*.log - name: Upload native binaries uses: actions/upload-artifact@v3 with: @@ -161,8 +161,8 @@ jobs: name: Upload pants.log uses: actions/upload-artifact@v3 with: - name: pants-log-bootstrap-Linux-x86_64 - path: .pants.d/pants.log + name: logs-bootstrap-Linux-x86_64 + path: .pants.d/*.log - name: Upload native binaries uses: actions/upload-artifact@v3 with: @@ -258,8 +258,8 @@ jobs: name: Upload pants.log uses: actions/upload-artifact@v3 with: - name: pants-log-bootstrap-macOS11-x86_64 - path: .pants.d/pants.log + name: logs-bootstrap-macOS11-x86_64 + path: .pants.d/*.log - name: Upload native binaries uses: actions/upload-artifact@v3 with: @@ -325,8 +325,8 @@ jobs: name: Upload pants.log uses: actions/upload-artifact@v3 with: - name: pants-log-wheels-Linux-ARM64 - path: .pants.d/pants.log + name: logs-wheels-Linux-ARM64 + path: .pants.d/*.log timeout-minutes: 90 build_wheels_linux_x86_64: container: @@ -379,8 +379,8 @@ jobs: name: Upload pants.log uses: actions/upload-artifact@v3 with: - name: pants-log-wheels-Linux-x86_64 - path: .pants.d/pants.log + name: logs-wheels-Linux-x86_64 + path: .pants.d/*.log timeout-minutes: 90 build_wheels_macos10_15_x86_64: env: @@ -434,8 +434,8 @@ jobs: name: Upload pants.log uses: actions/upload-artifact@v3 with: - name: pants-log-wheels-macOS10-15-x86_64 - path: .pants.d/pants.log + name: logs-wheels-macOS10-15-x86_64 + path: .pants.d/*.log timeout-minutes: 90 build_wheels_macos11_arm64: env: @@ -489,8 +489,8 @@ jobs: name: Upload pants.log uses: actions/upload-artifact@v3 with: - name: pants-log-wheels-macOS11-ARM64 - path: .pants.d/pants.log + name: logs-wheels-macOS11-ARM64 + path: .pants.d/*.log timeout-minutes: 90 check_labels: if: github.repository_owner == 'pantsbuild' @@ -585,8 +585,8 @@ jobs: name: Upload pants.log uses: actions/upload-artifact@v3 with: - name: pants-log-lint-Linux-x86_64 - path: .pants.d/pants.log + name: logs-lint-Linux-x86_64 + path: .pants.d/*.log timeout-minutes: 30 merge_ok: if: always() @@ -667,12 +667,21 @@ jobs: ' - continue-on-error: true + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + if: always() + name: Upload test reports + run: './build-support/bin/copy_to_s3.py --src-prefix=dist/test/reports --dst-prefix=s3://logs.pantsbuild.org/test/reports/Linux-ARM64/2023-06-07/${GITHUB_REF}/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT} --path= + + ' + - continue-on-error: true if: always() name: Upload pants.log uses: actions/upload-artifact@v3 with: - name: pants-log-python-test-Linux-ARM64 - path: .pants.d/pants.log + name: logs-python-test-Linux-ARM64 + path: .pants.d/*.log timeout-minutes: 90 test_python_linux_x86_64_0: env: {} @@ -744,12 +753,21 @@ jobs: ' - continue-on-error: true + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + if: always() + name: Upload test reports + run: './build-support/bin/copy_to_s3.py --src-prefix=dist/test/reports --dst-prefix=s3://logs.pantsbuild.org/test/reports/Linux-x86_64/2023-06-07/${GITHUB_REF}/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT} --path= + + ' + - continue-on-error: true if: always() name: Upload pants.log uses: actions/upload-artifact@v3 with: - name: pants-log-python-test-0_10-Linux-x86_64 - path: .pants.d/pants.log + name: logs-python-test-0_10-Linux-x86_64 + path: .pants.d/*.log timeout-minutes: 90 test_python_linux_x86_64_1: env: {} @@ -821,12 +839,21 @@ jobs: ' - continue-on-error: true + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + if: always() + name: Upload test reports + run: './build-support/bin/copy_to_s3.py --src-prefix=dist/test/reports --dst-prefix=s3://logs.pantsbuild.org/test/reports/Linux-x86_64/2023-06-07/${GITHUB_REF}/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT} --path= + + ' + - continue-on-error: true if: always() name: Upload pants.log uses: actions/upload-artifact@v3 with: - name: pants-log-python-test-1_10-Linux-x86_64 - path: .pants.d/pants.log + name: logs-python-test-1_10-Linux-x86_64 + path: .pants.d/*.log timeout-minutes: 90 test_python_linux_x86_64_2: env: {} @@ -898,12 +925,21 @@ jobs: ' - continue-on-error: true + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + if: always() + name: Upload test reports + run: './build-support/bin/copy_to_s3.py --src-prefix=dist/test/reports --dst-prefix=s3://logs.pantsbuild.org/test/reports/Linux-x86_64/2023-06-07/${GITHUB_REF}/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT} --path= + + ' + - continue-on-error: true if: always() name: Upload pants.log uses: actions/upload-artifact@v3 with: - name: pants-log-python-test-2_10-Linux-x86_64 - path: .pants.d/pants.log + name: logs-python-test-2_10-Linux-x86_64 + path: .pants.d/*.log timeout-minutes: 90 test_python_linux_x86_64_3: env: {} @@ -975,12 +1011,21 @@ jobs: ' - continue-on-error: true + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + if: always() + name: Upload test reports + run: './build-support/bin/copy_to_s3.py --src-prefix=dist/test/reports --dst-prefix=s3://logs.pantsbuild.org/test/reports/Linux-x86_64/2023-06-07/${GITHUB_REF}/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT} --path= + + ' + - continue-on-error: true if: always() name: Upload pants.log uses: actions/upload-artifact@v3 with: - name: pants-log-python-test-3_10-Linux-x86_64 - path: .pants.d/pants.log + name: logs-python-test-3_10-Linux-x86_64 + path: .pants.d/*.log timeout-minutes: 90 test_python_linux_x86_64_4: env: {} @@ -1052,12 +1097,21 @@ jobs: ' - continue-on-error: true + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + if: always() + name: Upload test reports + run: './build-support/bin/copy_to_s3.py --src-prefix=dist/test/reports --dst-prefix=s3://logs.pantsbuild.org/test/reports/Linux-x86_64/2023-06-07/${GITHUB_REF}/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT} --path= + + ' + - continue-on-error: true if: always() name: Upload pants.log uses: actions/upload-artifact@v3 with: - name: pants-log-python-test-4_10-Linux-x86_64 - path: .pants.d/pants.log + name: logs-python-test-4_10-Linux-x86_64 + path: .pants.d/*.log timeout-minutes: 90 test_python_linux_x86_64_5: env: {} @@ -1129,12 +1183,21 @@ jobs: ' - continue-on-error: true + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + if: always() + name: Upload test reports + run: './build-support/bin/copy_to_s3.py --src-prefix=dist/test/reports --dst-prefix=s3://logs.pantsbuild.org/test/reports/Linux-x86_64/2023-06-07/${GITHUB_REF}/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT} --path= + + ' + - continue-on-error: true if: always() name: Upload pants.log uses: actions/upload-artifact@v3 with: - name: pants-log-python-test-5_10-Linux-x86_64 - path: .pants.d/pants.log + name: logs-python-test-5_10-Linux-x86_64 + path: .pants.d/*.log timeout-minutes: 90 test_python_linux_x86_64_6: env: {} @@ -1206,12 +1269,21 @@ jobs: ' - continue-on-error: true + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + if: always() + name: Upload test reports + run: './build-support/bin/copy_to_s3.py --src-prefix=dist/test/reports --dst-prefix=s3://logs.pantsbuild.org/test/reports/Linux-x86_64/2023-06-07/${GITHUB_REF}/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT} --path= + + ' + - continue-on-error: true if: always() name: Upload pants.log uses: actions/upload-artifact@v3 with: - name: pants-log-python-test-6_10-Linux-x86_64 - path: .pants.d/pants.log + name: logs-python-test-6_10-Linux-x86_64 + path: .pants.d/*.log timeout-minutes: 90 test_python_linux_x86_64_7: env: {} @@ -1283,12 +1355,21 @@ jobs: ' - continue-on-error: true + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + if: always() + name: Upload test reports + run: './build-support/bin/copy_to_s3.py --src-prefix=dist/test/reports --dst-prefix=s3://logs.pantsbuild.org/test/reports/Linux-x86_64/2023-06-07/${GITHUB_REF}/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT} --path= + + ' + - continue-on-error: true if: always() name: Upload pants.log uses: actions/upload-artifact@v3 with: - name: pants-log-python-test-7_10-Linux-x86_64 - path: .pants.d/pants.log + name: logs-python-test-7_10-Linux-x86_64 + path: .pants.d/*.log timeout-minutes: 90 test_python_linux_x86_64_8: env: {} @@ -1360,12 +1441,21 @@ jobs: ' - continue-on-error: true + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + if: always() + name: Upload test reports + run: './build-support/bin/copy_to_s3.py --src-prefix=dist/test/reports --dst-prefix=s3://logs.pantsbuild.org/test/reports/Linux-x86_64/2023-06-07/${GITHUB_REF}/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT} --path= + + ' + - continue-on-error: true if: always() name: Upload pants.log uses: actions/upload-artifact@v3 with: - name: pants-log-python-test-8_10-Linux-x86_64 - path: .pants.d/pants.log + name: logs-python-test-8_10-Linux-x86_64 + path: .pants.d/*.log timeout-minutes: 90 test_python_linux_x86_64_9: env: {} @@ -1437,12 +1527,21 @@ jobs: ' - continue-on-error: true + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + if: always() + name: Upload test reports + run: './build-support/bin/copy_to_s3.py --src-prefix=dist/test/reports --dst-prefix=s3://logs.pantsbuild.org/test/reports/Linux-x86_64/2023-06-07/${GITHUB_REF}/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT} --path= + + ' + - continue-on-error: true if: always() name: Upload pants.log uses: actions/upload-artifact@v3 with: - name: pants-log-python-test-9_10-Linux-x86_64 - path: .pants.d/pants.log + name: logs-python-test-9_10-Linux-x86_64 + path: .pants.d/*.log timeout-minutes: 90 test_python_macos11_x86_64: env: @@ -1482,12 +1581,21 @@ jobs: ' - continue-on-error: true + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + if: always() + name: Upload test reports + run: './build-support/bin/copy_to_s3.py --src-prefix=dist/test/reports --dst-prefix=s3://logs.pantsbuild.org/test/reports/macOS11-x86_64/2023-06-07/${GITHUB_REF}/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT} --path= + + ' + - continue-on-error: true if: always() name: Upload pants.log uses: actions/upload-artifact@v3 with: - name: pants-log-python-test-macOS11-x86_64 - path: .pants.d/pants.log + name: logs-python-test-macOS11-x86_64 + path: .pants.d/*.log timeout-minutes: 90 name: Pull Request CI 'on': diff --git a/build-support/bin/copy_to_s3.py b/build-support/bin/copy_to_s3.py index d7e7a2b5227d..b245e4c657a4 100755 --- a/build-support/bin/copy_to_s3.py +++ b/build-support/bin/copy_to_s3.py @@ -36,12 +36,17 @@ def main() -> None: help="The AWS region to connect to.", default="us-east-1", ) + parser.add_argument( + "--acl", + help="An optional ACL to set on copied objects.", + ) options = parser.parse_args() perform_copy( src_prefix=options.src_prefix, dst_prefix=options.dst_prefix, path=options.path, - dst_region=options.dst_region, + region=options.region, + acl=options.acl, ) @@ -65,7 +70,8 @@ def perform_copy( src_prefix: str, dst_prefix: str, path: str, - dst_region: str, + region: str, + acl: str | None = None, aws_cli_symlink_path: str | None = None, ) -> None: """Recursively copy the files at src_prefix/src_path to S3. @@ -75,14 +81,15 @@ def perform_copy( :param path: The relpath under the src_prefix to copy. src_prefix/path will be (recursively) copied to dst_prefix/path. If empty, the entire src_prefix will be copied. - :param dst_region: The AWS region to access (should be the one the bucket is in). + :param region: The AWS region to access (should be the one the bucket is in). + :param acl: An optional ACL to set on the copied objects. :param aws_cli_symlink_path: If specified, symlink the aws cli into this dir. Otherwise, it will be synlinked into the system standard Path. """ if shutil.which("aws") is None: _install_aws_cli(symlink_path=aws_cli_symlink_path) _validate_authentication() - _copy(src_prefix=src_prefix, dst_prefix=dst_prefix, path=path, dst_region=dst_region) + _copy(src_prefix=src_prefix, dst_prefix=dst_prefix, path=path, region=region, acl=acl) def _install_aws_cli(symlink_path: str | None = None) -> None: @@ -105,7 +112,7 @@ def _validate_relpath(path_str: str, descr: str) -> None: raise ValueError(f"{descr} `{path_str}` must be a relative path with no parent refs") -def _copy(src_prefix: str, dst_prefix: str, path: str, dst_region: str) -> None: +def _copy(src_prefix: str, dst_prefix: str, path: str, region: str, acl: str | None = None) -> None: _validate_relpath(src_prefix, "src_prefix") _validate_relpath(path, "path") if not dst_prefix.startswith("s3://"): @@ -116,27 +123,29 @@ def _copy(src_prefix: str, dst_prefix: str, path: str, dst_region: str) -> None: copy_from = os.path.join(src_prefix, path) copy_to = f"{dst_prefix}/{path}" + if not os.path.exists(copy_from): + logger.warning(f"Local path {copy_from} does not exist. Skipping copy to s3.") + return + # NB: we use the sync command to avoid transferring files that have not changed. See # https://github.com/pantsbuild/pants/issues/7258. - _run( - [ - "aws", - "--region", - dst_region, - "s3", - "sync", - # This instructs the sync command to ignore timestamps, which we must do to allow - # distinct shards—which may finish building their wheels at different times—to not - # overwrite otherwise-identical wheels. - "--size-only", - # Turn off the dynamic progress display, which clutters the CI output. - "--no-progress", - "--acl", - "public-read", - copy_from, - copy_to, - ] - ) + cmd = [ + "aws", + "--region", + region, + "s3", + "sync", + # This instructs the sync command to ignore timestamps, which we must do to allow + # distinct shards—which may finish building their wheels at different times—to not + # overwrite otherwise-identical wheels. + "--size-only", + # Turn off the dynamic progress display, which clutters the CI output. + "--no-progress", + *(["--acl", acl] if acl else []), + copy_from, + copy_to, + ] + _run(cmd) if __name__ == "__main__": diff --git a/build-support/bin/deploy_to_s3.py b/build-support/bin/deploy_to_s3.py index 7c6690efd131..d630c4600dcb 100755 --- a/build-support/bin/deploy_to_s3.py +++ b/build-support/bin/deploy_to_s3.py @@ -40,7 +40,8 @@ def perform_deploy(*, aws_cli_symlink_path: str | None = None, scope: str | None src_prefix="dist/deploy", dst_prefix="s3://binaries.pantsbuild.org", path=scope or "", - dst_region="us-east-1", + region="us-east-1", + acl="public-read", aws_cli_symlink_path=aws_cli_symlink_path, ) diff --git a/build-support/bin/generate_github_workflows.py b/build-support/bin/generate_github_workflows.py index 4c2948d76e24..93d93c74a49f 100644 --- a/build-support/bin/generate_github_workflows.py +++ b/build-support/bin/generate_github_workflows.py @@ -6,6 +6,7 @@ import argparse import os from dataclasses import dataclass +from datetime import datetime, timezone from enum import Enum from pathlib import Path from textwrap import dedent @@ -545,8 +546,38 @@ def upload_log_artifacts(self, name: str) -> Step: "if": "always()", "continue-on-error": True, "with": { - "name": f"pants-log-{name.replace('/', '_')}-{self.platform_name()}", - "path": ".pants.d/pants.log", + "name": f"logs-{name.replace('/', '_')}-{self.platform_name()}", + "path": ".pants.d/*.log", + }, + } + + def upload_test_reports(self) -> Step: + date_str = datetime.now(timezone.utc).date().isoformat() + # The path purposely doesn't include job ID, as we want to aggregate test reports + # across all jobs/shards in a workflow. We do, however, qualify by run attempt, so we + # apture separate reports for tests that flake between attempts on the same workflow run. + s3_path = ( + "test/reports/" + + self.platform_name() + + "/" + + date_str + + "/${GITHUB_REF}/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}" + ) + return { + "name": "Upload test reports", + "if": "always()", + "continue-on-error": True, + "run": dedent( + f"""\ + ./build-support/bin/copy_to_s3.py \ + --src-prefix=dist/test/reports \ + --dst-prefix=s3://logs.pantsbuild.org/{s3_path} \ + --path= + """ + ), + "env": { + "AWS_SECRET_ACCESS_KEY": f"{gha_expr('secrets.AWS_SECRET_ACCESS_KEY')}", + "AWS_ACCESS_KEY_ID": f"{gha_expr('secrets.AWS_ACCESS_KEY_ID')}", }, } @@ -685,6 +716,7 @@ def test_jobs( "name": human_readable_step_name, "run": pants_args_str, }, + helper.upload_test_reports(), helper.upload_log_artifacts(name=log_name), ], } diff --git a/pants.ci.toml b/pants.ci.toml index c779c9326426..bff508e257e0 100644 --- a/pants.ci.toml +++ b/pants.ci.toml @@ -1,6 +1,9 @@ [GLOBAL] colors = true +[test] +report = true + [pytest] args = ["--no-header", "--noskip", "-vv"]