From e673f6e508cc5887191f89ad04df308f46a6c809 Mon Sep 17 00:00:00 2001 From: Samuel Chiang Date: Tue, 17 Oct 2023 11:13:24 -0700 Subject: [PATCH] Slight build fix and migrate to GHA for MacOS ARM CI (#1245) The FIPS Shared build on MacOSX wasn't working if OSX_VERSION_MIN_FLAG wasn't defined, so this fixes that. Github Actions also finally released MacOS ARM runners. We can deprecate the Codebuild->SSM->ec2 workaround for now. --- .github/workflows/osx.yml | 23 +++++ crypto/fipsmodule/CMakeLists.txt | 8 +- .../cdk/aws_lc_ec2_test_framework_ci_stack.py | 20 ---- .../codebuild/ec2_test_framework_omnibus.yaml | 11 --- .../ci/cdk/cdk/ssm/m1_tests_ssm_document.yaml | 34 ------- .../ci/codebuild/macos-aarch/run_m1_tests.yml | 13 --- tests/ci/run_m1_ec2_instance.sh | 96 ------------------- 7 files changed, 28 insertions(+), 177 deletions(-) delete mode 100644 tests/ci/cdk/cdk/ssm/m1_tests_ssm_document.yaml delete mode 100644 tests/ci/codebuild/macos-aarch/run_m1_tests.yml delete mode 100755 tests/ci/run_m1_ec2_instance.sh diff --git a/.github/workflows/osx.yml b/.github/workflows/osx.yml index d3dee44d2a..fb941f2eed 100644 --- a/.github/workflows/osx.yml +++ b/.github/workflows/osx.yml @@ -32,3 +32,26 @@ jobs: - name: Build ${{ env.PACKAGE_NAME }} with FIPS mode run: | ./tests/ci/run_fips_tests.sh + + macOS-ARM: + runs-on: macos-latest-xlarge + steps: + - uses: actions/checkout@v2 + - name: Install dependencies + run: | + brew install ninja golang + - name: Build ${{ env.PACKAGE_NAME }} + run: | + ./tests/ci/run_posix_tests.sh + + macOS-ARM-FIPS: + runs-on: macos-latest-xlarge + steps: + - uses: actions/checkout@v2 + - name: Install dependencies + run: | + brew install ninja golang + - name: Build ${{ env.PACKAGE_NAME }} with FIPS mode + run: | + ./tests/ci/run_fips_tests.sh + diff --git a/crypto/fipsmodule/CMakeLists.txt b/crypto/fipsmodule/CMakeLists.txt index db325bc60c..fb3a90f1e4 100644 --- a/crypto/fipsmodule/CMakeLists.txt +++ b/crypto/fipsmodule/CMakeLists.txt @@ -458,17 +458,19 @@ elseif(FIPS_SHARED) # generate the output object file where all the code in the __text section # and all the read-only data in the __const section are between the # respective start and end markers. + if (CMAKE_OSX_DEPLOYMENT_TARGET) + set(OSX_VERSION_MIN_FLAG "-mmacosx-version-min=${CMAKE_OSX_DEPLOYMENT_TARGET}") + endif() add_custom_command( OUTPUT fips_apple_start.o - COMMAND ${CMAKE_C_COMPILER} -arch ${CMAKE_SYSTEM_PROCESSOR} -isysroot ${CMAKE_OSX_SYSROOT} -mmacosx-version-min=${CMAKE_OSX_DEPLOYMENT_TARGET} -c ${CMAKE_CURRENT_SOURCE_DIR}/fips_shared_library_marker.c -DAWSLC_FIPS_SHARED_START -o fips_apple_start.o + COMMAND ${CMAKE_C_COMPILER} -arch ${CMAKE_SYSTEM_PROCESSOR} -isysroot ${CMAKE_OSX_SYSROOT} ${OSX_VERSION_MIN_FLAG} -c ${CMAKE_CURRENT_SOURCE_DIR}/fips_shared_library_marker.c -DAWSLC_FIPS_SHARED_START -o fips_apple_start.o DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/fips_shared_library_marker.c ) add_custom_command( OUTPUT fips_apple_end.o - COMMAND ${CMAKE_C_COMPILER} -arch ${CMAKE_SYSTEM_PROCESSOR} -isysroot ${CMAKE_OSX_SYSROOT} -mmacosx-version-min=${CMAKE_OSX_DEPLOYMENT_TARGET} -c ${CMAKE_CURRENT_SOURCE_DIR}/fips_shared_library_marker.c -DAWSLC_FIPS_SHARED_END -o fips_apple_end.o + COMMAND ${CMAKE_C_COMPILER} -arch ${CMAKE_SYSTEM_PROCESSOR} -isysroot ${CMAKE_OSX_SYSROOT} ${OSX_VERSION_MIN_FLAG} -c ${CMAKE_CURRENT_SOURCE_DIR}/fips_shared_library_marker.c -DAWSLC_FIPS_SHARED_END -o fips_apple_end.o DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/fips_shared_library_marker.c ) - add_custom_command( OUTPUT bcm.o COMMAND ${CMAKE_LINKER} -r fips_apple_start.o -force_load $ fips_apple_end.o -keep_private_externs -o bcm.o diff --git a/tests/ci/cdk/cdk/aws_lc_ec2_test_framework_ci_stack.py b/tests/ci/cdk/cdk/aws_lc_ec2_test_framework_ci_stack.py index bdf52129ed..81be51dfd2 100644 --- a/tests/ci/cdk/cdk/aws_lc_ec2_test_framework_ci_stack.py +++ b/tests/ci/cdk/cdk/aws_lc_ec2_test_framework_ci_stack.py @@ -98,26 +98,6 @@ def __init__(self, vpc=vpc, security_group_name='codebuild_ec2_sg') - # MacOS EC2 tag names must be specific for use in general tests/ci/run_m1_ec2_instance.sh script. - # Dedicated Hosts are required for Mac ec2 instances. - cfn_host = ec2.CfnHost(self, id="{}-dedicated-host".format(id), - availability_zone="us-west-2a", - auto_placement="off", - instance_type="mac2.metal") - Tags.of(cfn_host).add("Name", "{}-dedicated-host".format(id)) - # AMI is for M1 MacOS Monterey. - ami_id="ami-084c6ab9d03ad4d46" - macos_arm_instance = ec2.CfnInstance(self, "aws-lc-ci-macos-arm-ec2-instance", - availability_zone="us-west-2a", - tenancy="host", - host_id=cfn_host.attr_host_id, - iam_instance_profile="{}-ec2-profile".format(id), - image_id=ami_id, - instance_type="mac2.metal", - security_group_ids=[security_group.security_group_id], - subnet_id=selection.subnet_ids[0], - tags=[CfnTag(key="Name",value="aws-lc-ci-macos-arm-ec2-instance")]) - # Define logs for SSM. log_group_name = "{}-cw-logs".format(id) log_group = logs.CfnLogGroup(self, log_group_name, diff --git a/tests/ci/cdk/cdk/codebuild/ec2_test_framework_omnibus.yaml b/tests/ci/cdk/cdk/codebuild/ec2_test_framework_omnibus.yaml index 53e498b434..8b590f97ea 100644 --- a/tests/ci/cdk/cdk/codebuild/ec2_test_framework_omnibus.yaml +++ b/tests/ci/cdk/cdk/codebuild/ec2_test_framework_omnibus.yaml @@ -6,17 +6,6 @@ version: 0.2 # Doc for batch https://docs.aws.amazon.com/codebuild/latest/userguide/batch-build-buildspec.html#build-spec.batch.build-list batch: build-list: - # Actual tests are ran on an m1 ec2 instance via SSM Commands. - # MacOS is special since the ec2 instance can't be spun up and closed on the fly. - # TODO: Migrate this to Github Actions when https://github.com/actions/runner-images/issues/2187 is resolved. - - identifier: macos_arm_tests - buildspec: ./tests/ci/codebuild/macos-aarch/run_m1_tests.yml - env: - type: LINUX_CONTAINER - privileged-mode: true - compute-type: BUILD_GENERAL1_LARGE - image: 620771051181.dkr.ecr.us-west-2.amazonaws.com/aws-lc-docker-images-linux-x86:ubuntu-20.04_clang-7x-bm-framework_latest - # Actual tests are ran on an Graviton3 ec2 instance via SSM Commands. - identifier: graviton3_tests buildspec: ./tests/ci/codebuild/common/run_ec2_target.yml diff --git a/tests/ci/cdk/cdk/ssm/m1_tests_ssm_document.yaml b/tests/ci/cdk/cdk/ssm/m1_tests_ssm_document.yaml deleted file mode 100644 index 5003889a7f..0000000000 --- a/tests/ci/cdk/cdk/ssm/m1_tests_ssm_document.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 OR ISC - ---- -schemaVersion: '2.2' -description: aws-lc:bmFrameworkEc2Benchmark -mainSteps: - - action: aws:runShellScript - name: runShellScript - inputs: - timeoutSeconds: '7200' - runCommand: - - export HOME="/var/root" - - export SOURCE={SOURCE} - - eval $(/opt/homebrew/bin/brew shellenv) - - sudo su - - BUILD_ROOT=$(mktemp -d) - - cd ${BUILD_ROOT} - - trap "rm -rf ${BUILD_ROOT}" EXIT - # Check if the source code is on S3, otherwise treat the source as a PR. - - > - if [[ "${SOURCE:0:16}" == 'aws-lc-codebuild' ]]; then - aws s3api get-object --bucket {S3_BUCKET} --key "${SOURCE##{S3_BUCKET}/}" aws-lc-pr.zip - unzip aws-lc-pr.zip - else - git clone {SOURCE} aws-lc-pr - cd aws-lc-pr - git fetch origin pull/{PR_NUM}/head:temp - git checkout temp - fi - - chmod +x ./tests/ci/run_posix_tests.sh - - ./tests/ci/run_posix_tests.sh - - chmod +x ./tests/ci/run_fips_tests.sh - - ./tests/ci/run_fips_tests.sh diff --git a/tests/ci/codebuild/macos-aarch/run_m1_tests.yml b/tests/ci/codebuild/macos-aarch/run_m1_tests.yml deleted file mode 100644 index 0bca090d3f..0000000000 --- a/tests/ci/codebuild/macos-aarch/run_m1_tests.yml +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 OR ISC - -version: 0.2 - -env: - variables: - GOPROXY: https://proxy.golang.org,direct - -phases: - build: - commands: - - ./tests/ci/run_m1_ec2_instance.sh diff --git a/tests/ci/run_m1_ec2_instance.sh b/tests/ci/run_m1_ec2_instance.sh deleted file mode 100755 index ce77e3e0d0..0000000000 --- a/tests/ci/run_m1_ec2_instance.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env bash -set -exo pipefail -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 OR ISC - -# Please run from project root folder! -# You'll want to set the codebuild env variables set if running locally -source tests/ci/common_ssm_setup.sh - -# cleanup code -cleanup() { - set +e - # delete the various documents that we created - for name in ${ssm_document_names};do - aws ssm delete-document --name "${name}" - done -} - -# we wanna run the cleanup code on exit -trap cleanup EXIT - -# print some information for reference -echo GitHub PR Number: "${CODEBUILD_WEBHOOK_TRIGGER}" -echo GitHub Branch Name: "${CODEBUILD_WEBHOOK_HEAD_REF}" -AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) -echo AWS Account ID: "${AWS_ACCOUNT_ID}" -echo GitHub Repo Link: "${CODEBUILD_SOURCE_REPO_URL}" -export cloudwatch_group_name="aws-lc-ci-ec2-test-framework-cw-logs" -export s3_bucket_name="aws-lc-codebuild" - -# get information for ec2 instances -ec2_instance="$(aws ec2 describe-instances --filter "Name=tag:Name,Values=aws-lc-ci-macos-arm-ec2-instance" "Name=instance-state-name,Values=running" --query Reservations[*].Instances[*].InstanceId --output text)" - -generate_ssm_document_file() { - # use sed to replace placeholder values inside preexisting document - sed -e "s,{AWS_ACCOUNT_ID},${AWS_ACCOUNT_ID},g" \ - -e "s,{PR_NUM},${CODEBUILD_WEBHOOK_TRIGGER//pr\/},g" \ - -e "s,{SOURCE},${CODEBUILD_SOURCE_REPO_URL},g" \ - -e "s,{S3_BUCKET},${s3_bucket_name},g" \ - tests/ci/cdk/cdk/ssm/m1_tests_ssm_document.yaml \ - >tests/ci/cdk/cdk/ssm/macos_arm_ssm_document.yaml -} - -# create the ssm documents that will be used for the various ssm commands -generate_ssm_document_file - -# Create, and run ssm command. -ssm_doc_name=$(create_ssm_document "macos_arm") -ssm_document_names="${ssm_doc_name}" - -m1_ssm_command_id=$(run_ssm_command "${ssm_doc_name}" "${ec2_instance}" ${cloudwatch_group_name}) - -run_url="https://${AWS_REGION}.console.aws.amazon.com/cloudwatch/home?region=${AWS_REGION}\ -#logsV2:log-groups/log-group/${cloudwatch_group_name}/log-events/\ -${m1_ssm_command_id}\$252F${ec2_instance}\$252FrunShellScript\$252Fstdout" - -echo "Actual Run in EC2 can be observered at CloudWatch URL: ${run_url}" - -# Give some time for the commands to run -done=false -success=false -for i in {1..45}; do - echo "${i}: Continue to wait 2 min for SSM commands to finish." - sleep 120 - - ssm_command_status="$(aws ssm list-commands --command-id "${m1_ssm_command_id}" --query Commands[*].Status --output text)" - ssm_target_count="$(aws ssm list-commands --command-id "${m1_ssm_command_id}" --query Commands[*].TargetCount --output text)" - ssm_completed_count="$(aws ssm list-commands --command-id "${m1_ssm_command_id}" --query Commands[*].CompletedCount --output text)" - if [[ ${ssm_command_status} == 'Success' && ${ssm_completed_count} == "${ssm_target_count}" ]]; then - echo "SSM command ${m1_ssm_command_id} finished successfully." - success=true - done=true - elif [[ ${ssm_command_status} == 'Failed' && ${ssm_completed_count} == "${ssm_target_count}" ]]; then - echo "SSM command ${m1_ssm_command_id} failed." - done=true - else - # Still running. - done=false - fi - - # if after the loop finish and done is still true, then we're done - if [ "${done}" = true ]; then - echo "M1 SSM command has finished." - - # if success is still true here, then none of the commands failed - if [ "${success}" == true ]; then - echo "M1 SSM command succeeded!" - exit 0 - else - echo "M1 SSM command failed!" - exit 1 - fi - break - fi -done -exit 1 \ No newline at end of file