From 561cbc5d2aa7205ffa70a1463369a10aa59c876f Mon Sep 17 00:00:00 2001 From: Hogan Bobertz Date: Wed, 28 Jun 2023 04:35:47 -0400 Subject: [PATCH] chore: build noctilucent WASM library in a container (#26123) Due to the addition of noctilucent to cdk, contributors needed to download rust/rustup to be able to build the cdk. This uses the pre-existing dependency on Docker/Finch to containerize the process in order to not incur any further dependencies for contributors to manage. ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- CONTRIBUTING.md | 8 ++-- packages/aws-cdk/.gitignore | 3 -- packages/aws-cdk/.npmignore | 4 +- packages/aws-cdk/generate.sh | 44 +++++++------------ .../aws-cdk/lib/vendor/noctilucent/.gitignore | 4 ++ .../aws-cdk/lib/vendor/noctilucent/Dockerfile | 41 +++++++++++++++++ packages/aws-cdk/vendor/README.md | 13 ------ 7 files changed, 68 insertions(+), 49 deletions(-) create mode 100644 packages/aws-cdk/lib/vendor/noctilucent/.gitignore create mode 100644 packages/aws-cdk/lib/vendor/noctilucent/Dockerfile delete mode 100644 packages/aws-cdk/vendor/README.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ee4493a63fe4c..1356abe3eb3e6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -207,7 +207,7 @@ eval $(gp env -e) ### Amazon CodeCatalyst Dev Environments -Dev Environments are cloud-based development environments. +Dev Environments are cloud-based development environments. [Amazon CodeCatalyst](https://aws.amazon.com/codecatalyst/) allows you to checkout your linked Github repositories in your Dev Environments with your favorite local IDEs such as VSCode or JetBrains. @@ -220,7 +220,7 @@ $ NODE_OPTIONS=--max-old-space-size=8192 npx lerna run build --scope=aws-cdk-lib You may [configure your Dev Env](https://docs.aws.amazon.com/codecatalyst/latest/userguide/devenvironment-devfile.html) with the `devfile.yaml` to further customize your Dev Env for CDK development. -Read the links below for more details: +Read the links below for more details: - [Dev Environments in CodeCatalyst](https://docs.aws.amazon.com/codecatalyst/latest/userguide/devenvironment.html) - [Using GitHub repositories in CodeCatalyst](https://docs.aws.amazon.com/codecatalyst/latest/userguide/extensions-github.html) - [Setting up to use the AWS CLI with CodeCatalyst](https://docs.aws.amazon.com/codecatalyst/latest/userguide/set-up-cli.html) @@ -429,7 +429,7 @@ $ npx cdk -a test/aws-eks/test/sample.js deploy This allows you to iterate your development and ensure a minimal sample app would successfully deploy as you expect. You have the freedom to interact with it just as a common CDK app such as viewing differences with `npx cdk diff` -or pass context variables with `npx cdk deploy -c`. You can rapidly iterate your testing with repeated deployments +or pass context variables with `npx cdk deploy -c`. You can rapidly iterate your testing with repeated deployments by importing existing resource such as existing VPC. This can save a lot of time and help you focus on the core changes. ```ts @@ -439,7 +439,7 @@ const vpc = ec2.Vpc.fromLookup(stack, 'Vpc', { isDefault: true }); As this is for testing only, do not commit `sample.ts` and `sample.js` to your PR branch. Alternatively, you can write this test as a new integration test like `integ.my-test.ts` and deploy it -using `yarn integ --no-clean`. This may be useful when you need to publish a new +using `yarn integ --no-clean`. This may be useful when you need to publish a new integration test: ```console diff --git a/packages/aws-cdk/.gitignore b/packages/aws-cdk/.gitignore index 899d0fdac592b..86263e936e436 100644 --- a/packages/aws-cdk/.gitignore +++ b/packages/aws-cdk/.gitignore @@ -39,6 +39,3 @@ test/integ/cli/*.d.ts .DS_Store junit.xml - -# Exclude the noctilucent WASM package -lib/vendor/noctilucent/ diff --git a/packages/aws-cdk/.npmignore b/packages/aws-cdk/.npmignore index c1bf792f00614..a5f61214eee06 100644 --- a/packages/aws-cdk/.npmignore +++ b/packages/aws-cdk/.npmignore @@ -29,5 +29,5 @@ tsconfig.json **/cdk.out junit.xml -# exclude noctilucent source -/vendor/noctilucent/ +generate.sh +lib/vendor/noctilucent/Dockerfile diff --git a/packages/aws-cdk/generate.sh b/packages/aws-cdk/generate.sh index 036695896515e..aa3ef3136a431 100755 --- a/packages/aws-cdk/generate.sh +++ b/packages/aws-cdk/generate.sh @@ -15,30 +15,20 @@ cat > build-info.json </dev/null 2>/dev/null; then - echo "installing wasm-pack, this may take a while..." - cargo install wasm-pack - fi - - pkgroot=$(cd $(dirname -- "$0") && pwd) - - cd vendor/noctilucent - wasm-pack build --target nodejs \ - --out-dir="${pkgroot}/lib/vendor/noctilucent" \ - --out-name=index - - cd ../../lib/vendor/noctilucent - rm package.json -) +# Build noctilucent package in a Docker/Finch VM +NOCTILUCENT_GIT="https://github.com/iph/noctilucent.git" +NOCTILUCENT_COMMIT_ID="6da7c9fade55f8443bba7b8fdfcd4ebfe5208fb1" +if [ "$(cat lib/vendor/noctilucent/.version 2>/dev/null || echo '')" == "${NOCTILUCENT_GIT}:${NOCTILUCENT_COMMIT_ID}" ] +then + echo "⏭️ Noctilucent WASM binary is up-to date, skipping build..." + echo "ℹ️ Delete lib/vendor/noctilucent/.version to force a rebuild." +else + echo "⏳ Building Noctilucent WASM binary for embedding... This will take a while..." + ${CDK_DOCKER:-docker} build --rm \ + --build-arg NOCTILUCENT_GIT="${NOCTILUCENT_GIT}" \ + --build-arg NOCTILUCENT_COMMIT_ID="${NOCTILUCENT_COMMIT_ID}" \ + --file lib/vendor/noctilucent/Dockerfile \ + --target wasm \ + --output type=local,dest=lib/vendor/noctilucent \ + lib/vendor/noctilucent +fi diff --git a/packages/aws-cdk/lib/vendor/noctilucent/.gitignore b/packages/aws-cdk/lib/vendor/noctilucent/.gitignore new file mode 100644 index 0000000000000..2f2a446da42e4 --- /dev/null +++ b/packages/aws-cdk/lib/vendor/noctilucent/.gitignore @@ -0,0 +1,4 @@ +# Ignore all files in this directory except the Dockerfile +/* +!/.gitignore +!/Dockerfile diff --git a/packages/aws-cdk/lib/vendor/noctilucent/Dockerfile b/packages/aws-cdk/lib/vendor/noctilucent/Dockerfile new file mode 100644 index 0000000000000..b0071c6c2b0c9 --- /dev/null +++ b/packages/aws-cdk/lib/vendor/noctilucent/Dockerfile @@ -0,0 +1,41 @@ +FROM public.ecr.aws/debian/debian:buster-slim as build + +# Install basic pre-requisites +RUN apt-get update \ + && apt-get install -y build-essential curl git libssl-dev openssl pkg-config zsh + +# Make sure we use the correct shell going forward +SHELL ["/bin/zsh", "-c"] + +# Install Rustup +ENV RUSTUP_HOME=/usr/local/rustup +ENV CARGO_HOME=/usr/local/cargo +RUN set -eo pipefail \ + && curl -fSsL "https://sh.rustup.rs" | sh -s -- -y --no-modify-path --profile=minimal \ + && echo "source ${CARGO_HOME}/env" >> /etc/profile.d/cargo.sh \ + && chmod -R a+rw ${CARGO_HOME} +ENV PATH=$PATH:${CARGO_HOME}/bin + +# Install Node +RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \ + && apt-get install -y nodejs + +# Install wasm-pack +RUN cargo install wasm-opt wasm-pack + +ARG NOCTILUCENT_GIT +ARG NOCTILUCENT_COMMIT_ID + +# Check out noctilucent +RUN git clone "${NOCTILUCENT_GIT}" "${TMPDIR}/noctilucent" \ + && git -C "${TMPDIR}/noctilucent" checkout -b wasm "${NOCTILUCENT_COMMIT_ID}" + +# Build noctilucent to WASM +RUN cd "${TMPDIR}/noctilucent" \ + && wasm-pack build --target=nodejs --out-name=index --out-dir=/wasm-out \ + && rm --force /wasm-out/.gitignore /wasm-out/README.md /wasm-out/package.json \ + && echo "${NOCTILUCENT_GIT}:${NOCTILUCENT_COMMIT_ID}" > /wasm-out/.version + +#################################################################################################### +FROM scratch as wasm +COPY --from=build /wasm-out / diff --git a/packages/aws-cdk/vendor/README.md b/packages/aws-cdk/vendor/README.md deleted file mode 100644 index 1c9974a89f7e8..0000000000000 --- a/packages/aws-cdk/vendor/README.md +++ /dev/null @@ -1,13 +0,0 @@ -## Vendored-in dependencies - -The dependencies in this directory are checked out using the `gen` script. -This will fetch and clone the noctilucent crate and generate the wasm code if -that has not been done already, ensuring the dependencies are adequately -checked out. - -In order to update the notcilucent crate, run the ./generate.sh script. If you wish -to update to a different noctilucent commit hash instead of the one provided, modify -the hash in the generate.sh script and then rerun ./generate.sh - -The `THIRD_PARTY_LICENSES` file might need updating accordingly, which can be -automatically done by running `yarn pkglint`.