diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index 11076e73..3d8f4b2e 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -32,6 +32,9 @@ jobs: DOCKER_TAG=$SHA7 echo "DOCKER_TAG=${DOCKER_TAG}" >> $GITHUB_ENV + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -52,6 +55,7 @@ jobs: uses: docker/build-push-action@v5 with: context: ${{ env.IMAGE }} + platforms: linux/amd64,linux/arm64 push: true tags: | ${{env.DOCKER_ORG}}/${{ env.IMAGE }}:master diff --git a/.github/workflows/CondaLock.yml b/.github/workflows/CondaLock.yml index 62104cfb..1662cd6b 100644 --- a/.github/workflows/CondaLock.yml +++ b/.github/workflows/CondaLock.yml @@ -37,39 +37,37 @@ jobs: # Could run as single step in parallel, but would complicate logs... - name: Run conda-lock ${{ matrix.IMAGE }} run: | - cd ${{ matrix.IMAGE }} - if [ ${{ matrix.IMAGE }} = "base-notebook" ]; then - conda-lock lock -f environment.yml -p linux-64 - elif [ ${{ matrix.IMAGE }} = "pangeo-notebook" ]; then - conda-lock lock -f environment.yml -f ../base-notebook/environment.yml -p linux-64 - else - conda-lock lock -f environment.yml -f ../pangeo-notebook/environment.yml -f ../base-notebook/environment.yml -p linux-64 - fi - conda-lock render -k explicit + cd base-notebook + conda-lock lock -p linux-64 -p linux-aarch64 -p osx-64 -p osx-arm64 + conda-lock render -k explicit -p linux-64 ../generate-packages-list.py conda-linux-64.lock > packages.txt - - name: Upload lockfiles - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.image }} - path: ${{ matrix.image }} - + - name: Run conda-lock pangeo-notebook + timeout-minutes: 5 + continue-on-error: true + run: | + cd pangeo-notebook + conda-lock lock -f environment.yml -f ../base-notebook/environment.yml -p linux-64 -p linux-aarch64 -p osx-64 -p osx-arm64 + conda-lock render -k explicit -p linux-64 + ../generate-packages-list.py conda-linux-64.lock > packages.txt - # Each job will commit files, so we know it succeeds based on commits - commit-lockfiles: - needs: condalock - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - with: - token: ${{ secrets.PANGEOBOT_TOKEN }} - # These lines are critical, otherwise Pangeo-bot pushes changes directly to master from PRs! - repository: ${{ github.event.client_payload.pull_request.head.repo.full_name }} - ref: ${{ github.event.client_payload.pull_request.head.ref }} + - name: Run conda-lock ml-notebook + timeout-minutes: 5 + continue-on-error: true + run: | + cd ml-notebook + conda-lock lock -f environment.yml -f ../pangeo-notebook/environment.yml -f ../base-notebook/environment.yml -p linux-64 + conda-lock render -k explicit -p linux-64 + ../generate-packages-list.py conda-linux-64.lock > packages.txt - # Download all artifacts from previous matrix job - - uses: actions/download-artifact@v4 + - name: Run conda-lock pytorch-notebook + timeout-minutes: 5 + continue-on-error: true + run: | + cd pytorch-notebook + conda-lock lock -f environment.yml -f ../pangeo-notebook/environment.yml -f ../base-notebook/environment.yml -p linux-64 + conda-lock render -k explicit -p linux-64 + ../generate-packages-list.py conda-linux-64.lock > packages.txt - name: Commit condalock files to PR run: | diff --git a/.github/workflows/Test.yml b/.github/workflows/Test.yml index 5dbeb862..d5fac077 100644 --- a/.github/workflows/Test.yml +++ b/.github/workflows/Test.yml @@ -42,6 +42,9 @@ jobs: sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc df -h + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 with: @@ -51,6 +54,7 @@ jobs: uses: docker/build-push-action@v5 with: context: base-image + platforms: linux/amd64,linux/arm64 tags: localhost:5000/pangeo/base-image:PR push: true cache-from: type=gha @@ -60,6 +64,7 @@ jobs: uses: docker/build-push-action@v5 with: context: ${{ matrix.IMAGE }} + platforms: linux/amd64,linux/arm64 push: true tags: localhost:5000/${{env.DOCKER_ORG}}/${{ matrix.IMAGE }}:PR build-args: PANGEO_BASE_IMAGE_TAG=PR diff --git a/Makefile b/Makefile index 3e51a4a2..15561bb1 100644 --- a/Makefile +++ b/Makefile @@ -6,28 +6,31 @@ TESTDIR=/srv/test .PHONY: base-image base-image : cd base-image ; \ - docker build -t pangeo/base-image:master . + docker build -t pangeo/base-image:master --progress=plain --platform linux/amd64,linux/arm64 . .PHONY: base-notebook base-notebook : base-image cd base-notebook ; \ - conda-lock lock --mamba -k explicit -f environment.yml -p linux-64; \ + conda-lock lock -p linux-64 -p linux-aarch64 -p osx-64 -p osx-arm64; \ + conda-lock render -k explicit -p linux-64; \ ../generate-packages-list.py conda-linux-64.lock > packages.txt; \ - docker build -t pangeo/base-notebook:master . ; \ + docker build -t pangeo/base-notebook:master . --progress=plain --platform linux/amd64,linux/arm64 ; \ docker run -w $(TESTDIR) -v $(PWD):$(TESTDIR) pangeo/base-notebook:master ./run_tests.sh base-notebook .PHONY: pangeo-notebook pangeo-notebook : base-image cd pangeo-notebook ; \ - conda-lock lock --mamba -k explicit -f environment.yml -f ../base-notebook/environment.yml -p linux-64; \ + conda-lock lock -f environment.yml -f ../base-notebook/environment.yml -p linux-64 -p linux-aarch64 -p osx-64 -p osx-arm64; \ + conda-lock render -k explicit -p linux-64; \ ../generate-packages-list.py conda-linux-64.lock > packages.txt; \ - docker build -t pangeo/pangeo-notebook:master . ; \ + docker build -t pangeo/pangeo-notebook:master . --progress=plain --platform linux/amd64,linux/arm64 ; \ docker run -w $(TESTDIR) -v $(PWD):$(TESTDIR) pangeo/pangeo-notebook:master ./run_tests.sh pangeo-notebook .PHONY: ml-notebook ml-notebook : base-image cd ml-notebook ; \ - conda-lock lock --mamba -k explicit -f environment.yml -f ../pangeo-notebook/environment.yml -f ../base-notebook/environment.yml -p linux-64; \ + conda-lock lock -f environment.yml -f ../pangeo-notebook/environment.yml -f ../base-notebook/environment.yml -p linux-64; \ + conda-lock render -k explicit -p linux-64; \ ../generate-packages-list.py conda-linux-64.lock > packages.txt; \ docker build -t pangeo/ml-notebook:master . ; \ docker run -w $(TESTDIR) -v $(PWD):$(TESTDIR) pangeo/ml-notebook:master ./run_tests.sh ml-notebook @@ -35,7 +38,8 @@ ml-notebook : base-image .PHONY: pytorch-notebook pytorch-notebook : base-image cd pytorch-notebook ; \ - conda-lock lock --mamba -k explicit -f environment.yml -f ../pangeo-notebook/environment.yml -f ../base-notebook/environment.yml -p linux-64; \ + conda-lock lock -f environment.yml -f ../pangeo-notebook/environment.yml -f ../base-notebook/environment.yml -p linux-64; \ + conda-lock render -k explicit -p linux-64; \ ../generate-packages-list.py conda-linux-64.lock > packages.txt; \ docker build -t pangeo/pytorch-notebook:master . ; \ docker run -w $(TESTDIR) -v $(PWD):$(TESTDIR) pangeo/pytorch-notebook:master ./run_tests.sh pytorch-notebook diff --git a/base-image/Dockerfile b/base-image/Dockerfile index 28855ea4..7fe3439c 100644 --- a/base-image/Dockerfile +++ b/base-image/Dockerfile @@ -67,7 +67,7 @@ WORKDIR ${HOME} # Install latest mambaforge in ${CONDA_DIR} RUN echo "Installing Miniforge..." \ - && URL="https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-x86_64.sh" \ + && URL="https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-$(uname -m).sh" \ && wget --quiet ${URL} -O installer.sh \ && /bin/bash installer.sh -u -b -p ${CONDA_DIR} \ && rm installer.sh \ @@ -144,8 +144,10 @@ ONBUILD USER ${NB_USER} # We want to keep our images as reproducible as possible. If a lock # file with exact versions of all required packages is present, we use # it to install packages. conda-lock (https://github.com/conda-incubator/conda-lock) -# is used to generate this conda-linux-64.lock file from a given environment.yml -# file - so we get the exact same versions each time the image is built. This +# is used to generate this conda-lock.yml file from a given environment.yml +# file - so we get the exact same versions each time the image is built. Note that +# different packages may be used for different CPU architectures, but still, +# the same dockerfile can be used to build different architecture images. This # also lets us see what packages have changed between two images by diffing # the contents of the lock file between those image versions. # If a lock file is not present, we use the environment.yml file. And @@ -157,13 +159,11 @@ ONBUILD USER ${NB_USER} ONBUILD RUN echo "Checking for 'conda-lock.yml' 'conda-linux-64.lock' or 'environment.yml'..." \ ; [ -d binder ] && cd binder \ ; [ -d .binder ] && cd .binder \ - ; if test -f "conda-lock.yml" ; then \ - conda-lock install --name ${CONDA_ENV} conda-lock.yml \ - ; elif test -f "conda-linux-64.lock" ; then \ - mamba create --name ${CONDA_ENV} --file conda-linux-64.lock \ - ; elif test -f "environment.yml" ; then \ + ; if test -f "conda-lock.yml" ; then echo "Using conda-lock.yml" & \ + conda-lock install --name ${CONDA_ENV} \ + ; elif test -f "environment.yml" ; then echo "Using environment.yml" & \ mamba env create --name ${CONDA_ENV} -f environment.yml \ - ; else echo "No conda-lock.yml, conda-linux-64.lock, or environment.yml! *creating default env*" ; \ + ; else echo "No conda-lock.yml or environment.yml! *creating default env*" ; \ mamba create --name ${CONDA_ENV} pangeo-notebook \ ; fi \ && mamba clean -yaf \