From 925aab43f17f1256ca1986554baf23ba4dba28c5 Mon Sep 17 00:00:00 2001 From: epwalsh Date: Sat, 14 Nov 2020 11:02:41 -0800 Subject: [PATCH 01/10] Docker builds for other CUDA versions --- CHANGELOG.md | 1 + Dockerfile | 7 ++++++- Dockerfile.test | 5 +++++ Makefile | 15 +++++++++++---- 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ecfd9c7952..1c6ded556ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added Docker builds for other torch-supported versions of CUDA. - Adds [`allennlp-semparse`](https://github.com/allenai/allennlp-semparse) as an official, default plugin. ### Fixed diff --git a/Dockerfile b/Dockerfile index caee7e661bc..152a7dd9411 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,9 +18,14 @@ LABEL com.nvidia.volumes.needed="nvidia_driver" WORKDIR /stage/allennlp +# Install torch first. This build arg should be in the from of a version requirement, +# like '==1.7' or '==1.7+cu102'. +ARG TORCH +RUN pip install --no-cache-dir torch${TORCH} -f https://download.pytorch.org/whl/torch_stable.html + # Install the wheel of AllenNLP. COPY dist dist/ -RUN pip install $(ls dist/*.whl) +RUN pip install --no-cache-dir $(ls dist/*.whl) # TODO(epwalsh): In PyTorch 1.7, dataclasses is an unconditional dependency, when it should # only be a conditional dependency for Python < 3.7. # This has been fixed on PyTorch master branch, so we should be able to diff --git a/Dockerfile.test b/Dockerfile.test index b4837d3c564..bf23ce402aa 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -17,6 +17,11 @@ LABEL com.nvidia.volumes.needed="nvidia_driver" WORKDIR /stage/allennlp +# Install torch first. This build arg should be in the from of a version requirement, +# like '==1.7' or '==1.7+cu102'. +ARG TORCH +RUN pip install --no-cache-dir torch${TORCH} -f https://download.pytorch.org/whl/torch_stable.html + # Installing AllenNLP's dependencies is the most time-consuming part of building # this Docker image, so we make use of layer caching here by adding the minimal files # necessary to install the dependencies. Since most of the dependencies are defined diff --git a/Makefile b/Makefile index 487c6eb568e..3e335f16a30 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,8 @@ MD_DOCS_EXTRAS = $(addprefix $(MD_DOCS_ROOT),README.md CHANGELOG.md CONTRIBUTING DOCKER_TAG = latest DOCKER_IMAGE_NAME = allennlp/allennlp:$(DOCKER_TAG) DOCKER_TEST_IMAGE_NAME = allennlp/test:$(DOCKER_TAG) +# Our self-hosted runner currently has CUDA 11.0. +DOCKER_TEST_TORCH_VERSION = '==1.7.0+cu110' DOCKER_RUN_CMD = docker run --rm \ -v $$HOME/.allennlp:/root/.allennlp \ -v $$HOME/.cache/torch:/root/.cache/torch \ @@ -139,9 +141,10 @@ clean : .PHONY : docker-image docker-image : docker build \ - --pull \ - -f Dockerfile \ - -t $(DOCKER_IMAGE_NAME) . + --pull \ + -f Dockerfile \ + --build-arg TORCH=$(DOCKER_TORCH_VERSION) \ + -t $(DOCKER_IMAGE_NAME) . .PHONY : docker-run docker-run : @@ -149,7 +152,11 @@ docker-run : .PHONY : docker-test-image docker-test-image : - docker build --pull -f Dockerfile.test -t $(DOCKER_TEST_IMAGE_NAME) . + docker build \ + --pull \ + -f Dockerfile.test \ + --build-arg TORCH=$(DOCKER_TEST_TORCH_VERSION) \ + -t $(DOCKER_TEST_IMAGE_NAME) . .PHONY : docker-test-run docker-test-run : From cd871251b7832253d8124d5665ff58f1af69850a Mon Sep 17 00:00:00 2001 From: epwalsh Date: Mon, 16 Nov 2020 11:47:23 -0800 Subject: [PATCH 02/10] merge PR and master workflows --- .github/workflows/{master.yml => ci.yml} | 62 +++------ .github/workflows/pull_request.yml | 159 ----------------------- Dockerfile | 26 ++-- Dockerfile.test | 28 ++-- 4 files changed, 46 insertions(+), 229 deletions(-) rename .github/workflows/{master.yml => ci.yml} (84%) delete mode 100644 .github/workflows/pull_request.yml diff --git a/.github/workflows/master.yml b/.github/workflows/ci.yml similarity index 84% rename from .github/workflows/master.yml rename to .github/workflows/ci.yml index 51cfcdcd1e1..3ff86ebecad 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,9 @@ -name: Master +name: CI on: + pull_request: + branches: + - master push: branches: - master @@ -37,8 +40,6 @@ jobs: check_core: name: Check Core - # Don't run nightly builds for forks. - if: github.repository == 'allenai/allennlp' || github.event_name != 'schedule' runs-on: ubuntu-latest strategy: matrix: @@ -86,7 +87,7 @@ jobs: make test-with-cov - name: Upload coverage to Codecov - if: matrix.python == '3.7' && github.repository == 'allenai/allennlp' && github.event_name == 'push' + if: matrix.python == '3.7' && github.repository == 'allenai/allennlp' && (github.event_name == 'push' || github.event_name == 'pull_request') uses: codecov/codecov-action@v1 with: file: ./coverage.xml @@ -105,8 +106,6 @@ jobs: check_models: name: Check Models - # Don't run nightly builds for forks. - if: github.repository == 'allenai/allennlp' || github.event_name != 'schedule' runs-on: ubuntu-latest strategy: matrix: @@ -152,8 +151,6 @@ jobs: # Builds package distribution files for PyPI. build_package: name: Build Package - # Don't run nightly builds for forks. - if: github.repository == 'allenai/allennlp' || github.event_name != 'schedule' runs-on: ubuntu-latest steps: @@ -220,8 +217,6 @@ jobs: # Tests installing from the distribution files. test_package: name: Test Package - # Don't run nightly builds for forks. - if: github.repository == 'allenai/allennlp' || github.event_name != 'schedule' needs: [build_package] # needs the package artifact created from 'build_package' job. runs-on: ubuntu-latest strategy: @@ -264,10 +259,8 @@ jobs: # Builds Docker image from the core distribution files and uploads to Docker Hub. docker: name: Docker - # Don't run nightly builds for forks. - if: github.repository == 'allenai/allennlp' || github.event_name != 'schedule' - needs: [build_package] # needs the package artifact created from 'build_package' job. - runs-on: ubuntu-latest + if: github.repository == 'allenai/allennlp' + runs-on: [self-hosted, GPU] steps: - uses: actions/checkout@v2 @@ -280,12 +273,6 @@ jobs: echo "DOCKER_IMAGE_NAME=allennlp/commit:$GITHUB_SHA" >> $GITHUB_ENV; fi - - name: Download core package - uses: actions/download-artifact@v1 - with: - name: core-package - path: dist - - name: Build image run: | make docker-image DOCKER_IMAGE_NAME=$DOCKER_IMAGE_NAME @@ -294,16 +281,16 @@ jobs: run: | make docker-run DOCKER_IMAGE_NAME=$DOCKER_IMAGE_NAME ARGS='test-install' - - name: Upload master image - # Only run this for pushes to master on the main repo, not forks. - if: github.repository == 'allenai/allennlp' && github.event_name == 'push' + - name: Upload commit image + # Only run this for pushes to master branch. + if: github.event_name == 'push' run: | docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }} docker push $DOCKER_IMAGE_NAME - name: Upload release image - # Only run this for releases on the main repo, not forks. - if: github.repository == 'allenai/allennlp' && github.event_name == 'release' + # Only run this for releases. + if: github.event_name == 'release' run: | docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }} @@ -318,16 +305,15 @@ jobs: # allennlp-docs repo. docs: name: Docs - # Don't run nightly builds for forks. - if: github.repository == 'allenai/allennlp' || github.event_name != 'schedule' + # Don't run for forks. + if: github.repository == 'allenai/allennlp' runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 - name: Setup SSH Client 🔑 - # Only run this on main repo (not forks). - if: github.repository == 'allenai/allennlp' + if: github.event_name == 'release' || github.event_name == 'push' uses: webfactory/ssh-agent@v0.2.0 with: ssh-private-key: ${{ secrets.DOCS_DEPLOY_KEY }} @@ -355,14 +341,14 @@ jobs: ./scripts/build_docs.sh - name: Configure Git - # Only run this on main repo (not forks). - if: github.repository == 'allenai/allennlp' + if: github.event_name == 'release' || github.event_name == 'push' run: | git config --global user.email "ai2service@allenai.org" git config --global user.name "ai2service" git config --global push.default simple - name: Set target folders + if: github.event_name == 'release' || github.event_name == 'push' run: | if [[ $GITHUB_EVENT_NAME == 'release' ]]; then echo "DOCS_FOLDER=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV; @@ -371,9 +357,7 @@ jobs: fi - name: Stage docs - # Only run this on main repo (not forks) for commits and releases but not for - # nightly builds. - if: github.repository == 'allenai/allennlp' && github.event_name != 'schedule' + if: github.event_name == 'release' || github.event_name == 'push' run: | echo "Staging docs to $DOCS_FOLDER" @@ -386,9 +370,7 @@ jobs: cp -r site/* ~/allennlp-docs/$DOCS_FOLDER - name: Update shortcuts - # Only run this on main repo (not forks) for commits and releases but not for - # nightly builds. - if: github.repository == 'allenai/allennlp' && github.event_name == 'release' + if: github.event_name == 'release' run: | # Fail immediately if any step fails. set -e @@ -427,9 +409,7 @@ jobs: EOL - name: Deploy docs - # Only run this on main repo (not forks) for commits and releases but not for - # nightly builds. - if: github.repository == 'allenai/allennlp' && github.event_name != 'schedule' + if: github.event_name == 'release' || github.event_name == 'push' run: | # And push them up to GitHub cd ~/allennlp-docs/ @@ -447,7 +427,7 @@ jobs: name: PyPI needs: [check_core, check_models, gpu_checks, build_package, test_package, docker, docs] # Only publish to PyPI on releases and nightly builds to "allenai/allennlp" (not forks). - if: github.repository == 'allenai/allennlp' && github.event_name != 'push' + if: github.repository == 'allenai/allennlp' && (github.event_name == 'release' || github.event == 'schedule') runs-on: ubuntu-latest steps: diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml deleted file mode 100644 index d06fd1c8b46..00000000000 --- a/.github/workflows/pull_request.yml +++ /dev/null @@ -1,159 +0,0 @@ -name: PR - -on: - pull_request: - branches: - - master - -jobs: - changelog: - name: CHANGELOG - runs-on: ubuntu-latest - - steps: - # Note that checkout@v2 will not work with the git command below! - - uses: actions/checkout@v1 - - - name: Debugging info - run: | - git remote -v - - - name: Check that CHANGELOG has been updated - run: | - # If this step fails, this means you haven't updated the CHANGELOG.md - # file with notes on your contribution. - git diff --name-only $(git merge-base origin/master HEAD) | grep '^CHANGELOG.md$' && echo "Thanks for helping keep our CHANGELOG up-to-date!" - - gpu_checks: - name: GPU Checks - runs-on: [self-hosted, GPU] - timeout-minutes: 15 - - steps: - - uses: actions/checkout@v2 - - - name: Set Docker tag - run: | - echo "DOCKER_TAG=$GITHUB_SHA" >> $GITHUB_ENV - - - name: Build test image - run: | - make docker-test-image DOCKER_TAG=$DOCKER_TAG - - - name: Run GPU tests - run: | - make docker-test-run DOCKER_TAG=$DOCKER_TAG ARGS='gpu-test' - - check_core: - name: Check Core - runs-on: ubuntu-latest - strategy: - matrix: - python: ['3.7', '3.8'] - - steps: - - uses: actions/checkout@v2 - - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python }} - - - uses: actions/cache@v2 - with: - path: ${{ env.pythonLocation }} - key: ${{ runner.os }}-pydeps-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('dev-requirements.txt') }} - - - name: Install requirements - run: | - make install - - - name: Debug info - run: | - pip freeze - - - name: Format - if: always() - run: | - make format - - - name: Lint - if: always() - run: | - make lint - - - name: Type check - if: always() - run: | - make typecheck - - - name: Run tests - if: always() - run: | - make test-with-cov - - - name: Ensure docs can build - if: always() - run: | - make build-docs - - - name: Upload coverage to Codecov - if: matrix.python == '3.7' && github.repository == 'allenai/allennlp' - uses: codecov/codecov-action@v1 - with: - file: ./coverage.xml - # Ignore codecov failures as the codecov server is not - # very reliable but we don't want to report a failure - # in the github UI just because the coverage report failed to - # be published. - fail_ci_if_error: false - - - name: Clean up - if: always() - run: | - pip uninstall --yes allennlp - - check_models: - name: Check Models - runs-on: ubuntu-latest - strategy: - matrix: - python: ['3.7', '3.8'] - - steps: - - uses: actions/checkout@v2 - - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python }} - - - uses: actions/cache@v2 - with: - path: ${{ env.pythonLocation }} - key: ${{ runner.os }}-pydeps-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('dev-requirements.txt') }} - - - name: Install requirements - run: | - make install - - - name: Debug info - run: | - pip freeze - - - name: Pull and install models repo - env: - ALLENNLP_VERSION_OVERRIDE: "" # Don't replace the core library. - run: | - git clone https://github.com/allenai/allennlp-models.git - cd allennlp-models - pip install --upgrade --upgrade-strategy eager -e . -r dev-requirements.txt - - - name: Run models tests - run: | - cd allennlp-models && make test - - - name: Clean up - if: always() - run: | - pip uninstall --yes allennlp allennlp-models diff --git a/Dockerfile b/Dockerfile index 152a7dd9411..9fd8c1e25e1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,22 +18,28 @@ LABEL com.nvidia.volumes.needed="nvidia_driver" WORKDIR /stage/allennlp -# Install torch first. This build arg should be in the from of a version requirement, +# Install torch first. This build arg should be in the form of a version requirement, # like '==1.7' or '==1.7+cu102'. ARG TORCH RUN pip install --no-cache-dir torch${TORCH} -f https://download.pytorch.org/whl/torch_stable.html -# Install the wheel of AllenNLP. -COPY dist dist/ -RUN pip install --no-cache-dir $(ls dist/*.whl) -# TODO(epwalsh): In PyTorch 1.7, dataclasses is an unconditional dependency, when it should -# only be a conditional dependency for Python < 3.7. -# This has been fixed on PyTorch master branch, so we should be able to -# remove this check with the next PyTorch release. -RUN pip uninstall -y dataclasses +# Installing AllenNLP's dependencies is the most time-consuming part of building +# this Docker image, so we make use of layer caching here by adding the minimal files +# necessary to install the dependencies. +COPY allennlp/version.py allennlp/version.py +COPY setup.py . +RUN touch allennlp/__init__.py \ + && touch README.md \ + && pip install --no-cache-dir -e . + +# Now add the full package source and re-install just the package. +COPY allennlp allennlp +RUN pip install --no-cache-dir --no-deps -e . + +WORKDIR /app/ # Copy wrapper script to allow beaker to run resumable training workloads. -COPY scripts/ai2_internal/resumable_train.sh /stage/allennlp +COPY scripts/ai2_internal/resumable_train.sh . LABEL maintainer="allennlp-contact@allenai.org" diff --git a/Dockerfile.test b/Dockerfile.test index bf23ce402aa..2b22ec8cd39 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -17,32 +17,22 @@ LABEL com.nvidia.volumes.needed="nvidia_driver" WORKDIR /stage/allennlp -# Install torch first. This build arg should be in the from of a version requirement, +# Install torch first. This build arg should be in the form of a version requirement, # like '==1.7' or '==1.7+cu102'. ARG TORCH RUN pip install --no-cache-dir torch${TORCH} -f https://download.pytorch.org/whl/torch_stable.html # Installing AllenNLP's dependencies is the most time-consuming part of building # this Docker image, so we make use of layer caching here by adding the minimal files -# necessary to install the dependencies. Since most of the dependencies are defined -# in the setup.py file, we create a "shell" package of allennlp using the same setup file -# and then pip install it, after which we uninstall it so that we'll only have the dependencies -# installed. +# necessary to install the dependencies. COPY allennlp/version.py allennlp/version.py -RUN touch allennlp/__init__.py && touch README.md -COPY setup.py setup.py -COPY dev-requirements.txt dev-requirements.txt - -# Now install deps by installing the shell package, and then uninstall it so we can -# re-install the full package below. -RUN pip install --no-cache-dir -e . && \ - pip install --no-cache-dir -r dev-requirements.txt && \ - pip uninstall -y typing && \ - pip uninstall -y allennlp && \ - rm -rf allennlp/ - -# Now add the full package source and re-install just the package. +COPY setup.py . +COPY dev-requirements.txt . +RUN touch allennlp/__init__.py \ + && touch README.md \ + && pip install --no-cache-dir -r dev-requirements.txt -e . + +# Now add the full package source. COPY . . -RUN pip install --no-cache-dir --no-deps -e . ENTRYPOINT ["make"] From 765b4e8c1210dcfdde97c2dc7ef72fcfce39f06f Mon Sep 17 00:00:00 2001 From: epwalsh Date: Mon, 16 Nov 2020 12:35:17 -0800 Subject: [PATCH 03/10] build for different CUDA versions --- .github/workflows/ci.yml | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3ff86ebecad..22863a2b9d8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -258,32 +258,51 @@ jobs: # Builds Docker image from the core distribution files and uploads to Docker Hub. docker: - name: Docker + name: Docker (CUDA ${{ matrix.cuda }}) if: github.repository == 'allenai/allennlp' runs-on: [self-hosted, GPU] + strategy: + matrix: + cuda: ['10.2', '11.0'] steps: - uses: actions/checkout@v2 + - name: Set torch version + env: + CUDA: matrix.cuda + run: | + if [[ $CUDA == '10.2' ]]; then + echo "DOCKER_TORCH_VERSION='==1.7.0'" >> $GITHUB_ENV; + elif [[ $CUDA == '11.0' ]]; then + echo "DOCKER_TORCH_VERSION='==1.7.0+cu110'" >> $GITHUB_ENV; + else + echo "Unhandled CUDA version $CUDA"; + exit 1; + fi + - name: Set Docker image name + env: + CUDA: matrix.cuda run: | + echo "DOCKER_CUDA_TAG=-cuda$CUDA" >> $GITHUB_ENV; if [[ $GITHUB_EVENT_NAME == 'release' ]]; then - echo "DOCKER_IMAGE_NAME=allennlp/allennlp:${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV; + echo "DOCKER_IMAGE_NAME=allennlp/allennlp:${GITHUB_REF#refs/tags/}-${DOCKER_CUDA_TAG}" >> $GITHUB_ENV; else - echo "DOCKER_IMAGE_NAME=allennlp/commit:$GITHUB_SHA" >> $GITHUB_ENV; + echo "DOCKER_IMAGE_NAME=allennlp/commit:${GITHUB_SHA}-${DOCKER_CUDA_TAG}" >> $GITHUB_ENV; fi - name: Build image run: | - make docker-image DOCKER_IMAGE_NAME=$DOCKER_IMAGE_NAME + make docker-image DOCKER_IMAGE_NAME=$DOCKER_IMAGE_NAME DOCKER_TORCH_VERSION=$DOCKER_TORCH_VERSION - name: Test image run: | make docker-run DOCKER_IMAGE_NAME=$DOCKER_IMAGE_NAME ARGS='test-install' - name: Upload commit image - # Only run this for pushes to master branch. - if: github.event_name == 'push' + # Only upload the 10.2 image. + if: github.event_name == 'push' && matrix.cuda == '10.2' run: | docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }} docker push $DOCKER_IMAGE_NAME @@ -293,12 +312,13 @@ jobs: if: github.event_name == 'release' run: | docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }} + docker push $DOCKER_IMAGE_NAME - # Tag with latest. + - name: Upload latest image + # CUDA 10.2 is currently our default. + if: github.event_name == 'release' && matrix.cuda == '10.2' + run: | docker tag $DOCKER_IMAGE_NAME allennlp/allennlp:latest - - # Push both tags. - docker push $DOCKER_IMAGE_NAME docker push allennlp/allennlp:latest # Builds the API documentation and pushes it to the appropriate folder in the From edf5c3db3cb8deb019f873344d685387dda593f1 Mon Sep 17 00:00:00 2001 From: epwalsh Date: Mon, 16 Nov 2020 12:36:19 -0800 Subject: [PATCH 04/10] fix --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22863a2b9d8..267b403f494 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -270,7 +270,7 @@ jobs: - name: Set torch version env: - CUDA: matrix.cuda + CUDA: ${{ matrix.cuda }} run: | if [[ $CUDA == '10.2' ]]; then echo "DOCKER_TORCH_VERSION='==1.7.0'" >> $GITHUB_ENV; From def107a5bdc0b54a5f7922374b9930bc51565a22 Mon Sep 17 00:00:00 2001 From: epwalsh Date: Mon, 16 Nov 2020 12:48:47 -0800 Subject: [PATCH 05/10] fix --- .github/workflows/ci.yml | 5 +++-- Dockerfile | 4 ++-- Dockerfile.test | 7 ++++--- Makefile | 6 +++--- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 267b403f494..70a902a31e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -272,10 +272,11 @@ jobs: env: CUDA: ${{ matrix.cuda }} run: | + # Check the install instructions on https://pytorch.org/ to keep these up-to-date. if [[ $CUDA == '10.2' ]]; then - echo "DOCKER_TORCH_VERSION='==1.7.0'" >> $GITHUB_ENV; + echo "DOCKER_TORCH_VERSION='torch==1.7.0'" >> $GITHUB_ENV; elif [[ $CUDA == '11.0' ]]; then - echo "DOCKER_TORCH_VERSION='==1.7.0+cu110'" >> $GITHUB_ENV; + echo "DOCKER_TORCH_VERSION='torch==1.7.0+cu110 -f https://download.pytorch.org/whl/torch_stable.html'" >> $GITHUB_ENV; else echo "Unhandled CUDA version $CUDA"; exit 1; diff --git a/Dockerfile b/Dockerfile index 9fd8c1e25e1..b62eb4f0e13 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,9 +19,9 @@ LABEL com.nvidia.volumes.needed="nvidia_driver" WORKDIR /stage/allennlp # Install torch first. This build arg should be in the form of a version requirement, -# like '==1.7' or '==1.7+cu102'. +# like 'torch==1.7' or 'torch==1.7+cu102 -f https://download.pytorch.org/whl/torch_stable.html'. ARG TORCH -RUN pip install --no-cache-dir torch${TORCH} -f https://download.pytorch.org/whl/torch_stable.html +RUN pip install --no-cache-dir ${TORCH} # Installing AllenNLP's dependencies is the most time-consuming part of building # this Docker image, so we make use of layer caching here by adding the minimal files diff --git a/Dockerfile.test b/Dockerfile.test index 2b22ec8cd39..beb9128fcd6 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -18,9 +18,9 @@ LABEL com.nvidia.volumes.needed="nvidia_driver" WORKDIR /stage/allennlp # Install torch first. This build arg should be in the form of a version requirement, -# like '==1.7' or '==1.7+cu102'. +# like 'torch==1.7' or 'torch==1.7+cu102 -f https://download.pytorch.org/whl/torch_stable.html'. ARG TORCH -RUN pip install --no-cache-dir torch${TORCH} -f https://download.pytorch.org/whl/torch_stable.html +RUN pip install --no-cache-dir ${TORCH} # Installing AllenNLP's dependencies is the most time-consuming part of building # this Docker image, so we make use of layer caching here by adding the minimal files @@ -32,7 +32,8 @@ RUN touch allennlp/__init__.py \ && touch README.md \ && pip install --no-cache-dir -r dev-requirements.txt -e . -# Now add the full package source. +# Now add the full package source and re-install just the package. COPY . . +RUN pip install --no-cache-dir --no-deps -e . ENTRYPOINT ["make"] diff --git a/Makefile b/Makefile index 3e335f16a30..ed40113f0a4 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ DOCKER_TAG = latest DOCKER_IMAGE_NAME = allennlp/allennlp:$(DOCKER_TAG) DOCKER_TEST_IMAGE_NAME = allennlp/test:$(DOCKER_TAG) # Our self-hosted runner currently has CUDA 11.0. -DOCKER_TEST_TORCH_VERSION = '==1.7.0+cu110' +DOCKER_TEST_TORCH_VERSION = 'torch==1.7.0+cu110 -f https://download.pytorch.org/whl/torch_stable.html' DOCKER_RUN_CMD = docker run --rm \ -v $$HOME/.allennlp:/root/.allennlp \ -v $$HOME/.cache/torch:/root/.cache/torch \ @@ -148,7 +148,7 @@ docker-image : .PHONY : docker-run docker-run : - $(DOCKER_RUN_CMD) $(DOCKER_IMAGE_NAME) $(ARGS) + $(DOCKER_RUN_CMD) --gpus all $(DOCKER_IMAGE_NAME) $(ARGS) .PHONY : docker-test-image docker-test-image : @@ -160,4 +160,4 @@ docker-test-image : .PHONY : docker-test-run docker-test-run : - $(DOCKER_RUN_CMD) --gpus 2 $(DOCKER_TEST_IMAGE_NAME) $(ARGS) + $(DOCKER_RUN_CMD) --gpus all $(DOCKER_TEST_IMAGE_NAME) $(ARGS) From c5ee174e0fe04b797864a4a5ccac8ab7ebe08a06 Mon Sep 17 00:00:00 2001 From: epwalsh Date: Mon, 16 Nov 2020 12:52:27 -0800 Subject: [PATCH 06/10] fix --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 70a902a31e7..d5f48dd322d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -284,9 +284,9 @@ jobs: - name: Set Docker image name env: - CUDA: matrix.cuda + CUDA: ${{ matrix.cuda }} run: | - echo "DOCKER_CUDA_TAG=-cuda$CUDA" >> $GITHUB_ENV; + echo "DOCKER_CUDA_TAG=cuda$CUDA" >> $GITHUB_ENV; if [[ $GITHUB_EVENT_NAME == 'release' ]]; then echo "DOCKER_IMAGE_NAME=allennlp/allennlp:${GITHUB_REF#refs/tags/}-${DOCKER_CUDA_TAG}" >> $GITHUB_ENV; else @@ -295,7 +295,7 @@ jobs: - name: Build image run: | - make docker-image DOCKER_IMAGE_NAME=$DOCKER_IMAGE_NAME DOCKER_TORCH_VERSION=$DOCKER_TORCH_VERSION + make docker-image DOCKER_IMAGE_NAME="$DOCKER_IMAGE_NAME" DOCKER_TORCH_VERSION="$DOCKER_TORCH_VERSION" - name: Test image run: | From 3f2c3f7fd6d5634d41a821645240345bb3f1851c Mon Sep 17 00:00:00 2001 From: epwalsh Date: Mon, 16 Nov 2020 12:59:09 -0800 Subject: [PATCH 07/10] fix --- .github/workflows/ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d5f48dd322d..3bb7ea971b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -286,11 +286,10 @@ jobs: env: CUDA: ${{ matrix.cuda }} run: | - echo "DOCKER_CUDA_TAG=cuda$CUDA" >> $GITHUB_ENV; if [[ $GITHUB_EVENT_NAME == 'release' ]]; then - echo "DOCKER_IMAGE_NAME=allennlp/allennlp:${GITHUB_REF#refs/tags/}-${DOCKER_CUDA_TAG}" >> $GITHUB_ENV; + echo "DOCKER_IMAGE_NAME=allennlp/allennlp:${GITHUB_REF#refs/tags/}-cuda${CUDA}" >> $GITHUB_ENV; else - echo "DOCKER_IMAGE_NAME=allennlp/commit:${GITHUB_SHA}-${DOCKER_CUDA_TAG}" >> $GITHUB_ENV; + echo "DOCKER_IMAGE_NAME=allennlp/commit:${GITHUB_SHA}-cuda${CUDA}" >> $GITHUB_ENV; fi - name: Build image From e60cdddae64346f95a6724c304906be25368a172 Mon Sep 17 00:00:00 2001 From: epwalsh Date: Mon, 16 Nov 2020 14:52:48 -0800 Subject: [PATCH 08/10] clean up --- .github/workflows/ci.yml | 2 +- README.md | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3bb7ea971b0..c103b6a5079 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -282,7 +282,7 @@ jobs: exit 1; fi - - name: Set Docker image name + - name: Set image name env: CUDA: ${{ matrix.cuda }} run: | diff --git a/README.md b/README.md index f3eb9a05a8a..2f1b5034a43 100644 --- a/README.md +++ b/README.md @@ -204,20 +204,27 @@ whether you will leverage a GPU or just run on a CPU. Docker provides more isolation and consistency, and also makes it easy to distribute your environment to a compute cluster. -Once you have [installed Docker](https://docs.docker.com/engine/installation/) -just run the following command to get an environment that will run on either the cpu or gpu. +AllenNLP provides [official Docker images](https://hub.docker.com/r/allennlp/allennlp) with the library and all of its dependencies installed. + +Once you have [installed Docker](https://docs.docker.com/engine/installation/), +you should also install the [NVIDIA Container Toolkit](https://github.com/NVIDIA/nvidia-docker) +if you have GPUs available. + +Then run the following command to get an environment that will run on GPU: ```bash mkdir -p $HOME/.allennlp/ -docker run --rm -v $HOME/.allennlp:/root/.allennlp allennlp/allennlp:latest +docker run --rm --gpus all -v $HOME/.allennlp:/root/.allennlp allennlp/allennlp:latest ``` You can test the Docker environment with ```bash -docker run --rm -v $HOME/.allennlp:/root/.allennlp allennlp/allennlp:latest test-install +docker run --rm --gpus all -v $HOME/.allennlp:/root/.allennlp allennlp/allennlp:latest test-install ``` +If you don't have GPUs available, just omit the `--gpus all` flag. + ### Installing from source You can also install AllenNLP by cloning our git repository: From b343c6f3f0152ac3024dac61d60c17a01ca32ff0 Mon Sep 17 00:00:00 2001 From: epwalsh Date: Mon, 16 Nov 2020 15:10:28 -0800 Subject: [PATCH 09/10] clean up --- Makefile | 1 + README.md | 61 +++++++++++++++++++------------------------------------ 2 files changed, 22 insertions(+), 40 deletions(-) diff --git a/Makefile b/Makefile index ed40113f0a4..d686e1744c1 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,7 @@ MD_DOCS_EXTRAS = $(addprefix $(MD_DOCS_ROOT),README.md CHANGELOG.md CONTRIBUTING DOCKER_TAG = latest DOCKER_IMAGE_NAME = allennlp/allennlp:$(DOCKER_TAG) DOCKER_TEST_IMAGE_NAME = allennlp/test:$(DOCKER_TAG) +DOCKER_TORCH_VERSION = 'torch==1.7.0' # Our self-hosted runner currently has CUDA 11.0. DOCKER_TEST_TORCH_VERSION = 'torch==1.7.0+cu110 -f https://download.pytorch.org/whl/torch_stable.html' DOCKER_RUN_CMD = docker run --rm \ diff --git a/README.md b/README.md index 2f1b5034a43..cc18f103239 100644 --- a/README.md +++ b/README.md @@ -225,6 +225,26 @@ docker run --rm --gpus all -v $HOME/.allennlp:/root/.allennlp allennlp/allennlp: If you don't have GPUs available, just omit the `--gpus all` flag. +#### Building your own Docker image + +For various reasons you may need to create your own AllenNLP Docker image, such as if you need a different version +of PyTorch. To do so, just run `make docker-image` from the root of your local clone of AllenNLP. + +By default this builds an image with the tag `allennlp/allennlp`, but you can change this to anything you want +by setting the `DOCKER_TAG` flag when you call `make`. For example, +`make docker-image DOCKER_TAG=my-allennlp`. + +If you want to use a different version of PyTorch, set the flag `DOCKER_TORCH_VERSION` to something like +`torch==1.7.0` or `torch==1.7.0+cu110 -f https://download.pytorch.org/whl/torch_stable.html`. +The value of this flag will passed directly to `pip install`. + +After building the image you should be able to see it listed by running `docker images allennlp`. + +``` +REPOSITORY TAG IMAGE ID CREATED SIZE +allennlp/allennlp latest b66aee6cb593 5 minutes ago 2.38GB +``` + ### Installing from source You can also install AllenNLP by cloning our git repository: @@ -233,7 +253,7 @@ You can also install AllenNLP by cloning our git repository: git clone https://github.com/allenai/allennlp.git ``` -Create a Python 3.7 virtual environment, and install AllenNLP in `editable` mode by running: +Create a Python 3.7 or 3.8 virtual environment, and install AllenNLP in `editable` mode by running: ```bash pip install --editable . @@ -254,45 +274,6 @@ with the `allennlp` command (whether you installed from `pip` or from source). `allennlp` has various subcommands such as `train`, `evaluate`, and `predict`. To see the full usage information, run `allennlp --help`. -## Docker images - -AllenNLP releases Docker images to [Docker Hub](https://hub.docker.com/r/allennlp/) for each release. For information on how to run these releases, see [Installing using Docker](#installing-using-docker). - -### Building a Docker image - -For various reasons you may need to create your own AllenNLP Docker image. -The same image can be used either with a CPU or a GPU. - -First, you need to [install Docker](https://www.docker.com/get-started). -Then you will need a wheel of allennlp in the `dist/` directory. -You can either obtain a pre-built wheel from a PyPI release or build a new wheel from -source. - -PyPI release wheels can be downloaded by going to https://pypi.org/project/allennlp/#history, -clicking on the desired release, and then clicking "Download files" in the left sidebar. -After downloading, make you sure you put the wheel in the `dist/` directory -(which may not exist if you haven't built a wheel from source yet). - -To build a wheel from source, just run `python setup.py wheel`. - -*Before building the image, make sure you only have one wheel in the `dist/` directory.* - -Once you have your wheel, run `make docker-image`. By default this builds an image -with the tag `allennlp/allennlp`. You can change this to anything you want -by setting the `DOCKER_TAG` flag when you call `make`. For example, -`make docker-image DOCKER_TAG=my-allennlp`. - -You should now be able to see this image listed by running `docker images allennlp`. - -``` -REPOSITORY TAG IMAGE ID CREATED SIZE -allennlp/allennlp latest b66aee6cb593 5 minutes ago 2.38GB -``` - -### Running the Docker image - -You can run the image with `docker run --rm -it allennlp/allennlp:latest`. The `--rm` flag cleans up the image on exit and the `-it` flags make the session interactive so you can use the bash shell the Docker image starts. - You can test your installation by running `allennlp test-install`. ## Issues From 775f27eb001a7deccf598b8f0d8b1bff357b23b0 Mon Sep 17 00:00:00 2001 From: epwalsh Date: Mon, 16 Nov 2020 15:42:34 -0800 Subject: [PATCH 10/10] add CHANGELOG check back in --- .github/workflows/ci.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c103b6a5079..17664a13a4a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,20 @@ on: - cron: '37 11 * * 1,2,3,4,5' # early morning (11:37 UTC / 4:37 AM PDT) Monday - Friday jobs: + changelog: + name: CHANGELOG + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + + steps: + - uses: actions/checkout@v1 + + - name: Check that CHANGELOG has been updated + run: | + # If this step fails, this means you haven't updated the CHANGELOG.md + # file with notes on your contribution. + git diff --name-only $(git merge-base origin/master HEAD) | grep '^CHANGELOG.md$' && echo "Thanks for helping keep our CHANGELOG up-to-date!" + gpu_checks: name: GPU Checks if: github.repository == 'allenai/allennlp' # self-hosted runner only available on main repo