Update unittest and packaging to use UPLOAD_CHANNEL (#2846)

pytorch · Oct 20, 2020 · 3d103d1 · 3d103d1
1 parent 82acfef
commit 3d103d1
Show file tree

Hide file tree

Showing 6 changed files with 53 additions and 12 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -134,6 +134,7 @@ jobs:
     resource_class: 2xlarge+
     steps:
       - checkout_merge
+      - designate_upload_channel
       - run: packaging/build_wheel.sh
       - store_artifacts:
           path: dist
@@ -149,6 +150,7 @@ jobs:
     resource_class: 2xlarge+
     steps:
       - checkout_merge
+      - designate_upload_channel
       - run: packaging/build_conda.sh
       - store_artifacts:
           path: /opt/conda/conda-bld/linux-64
@@ -164,6 +166,7 @@ jobs:
     executor: windows-cpu
     steps:
       - checkout_merge
+      - designate_upload_channel
       - run:
           name: Build conda packages
           command: |
@@ -189,6 +192,7 @@ jobs:
     executor: windows-cpu
     steps:
       - checkout_merge
+      - designate_upload_channel
       - run:
           name: Build wheel packages
           command: |
@@ -211,6 +215,7 @@ jobs:
       xcode: "9.4.1"
     steps:
       - checkout_merge
+      - designate_upload_channel
       - run:
           # Cannot easily deduplicate this as source'ing activate
           # will set environment variables which we need to propagate
@@ -233,6 +238,7 @@ jobs:
       xcode: "9.4.1"
     steps:
       - checkout_merge
+      - designate_upload_channel
       - run:
           command: |
             curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
@@ -295,6 +301,7 @@ jobs:
     steps:
       - attach_workspace:
           at: ~/workspace
+      - designate_upload_channel
       - run:
           name: install binaries
           command: |
@@ -313,6 +320,7 @@ jobs:
     steps:
       - attach_workspace:
           at: ~/workspace
+      - designate_upload_channel
       - run:
           name: install binaries
           command: |
@@ -333,6 +341,7 @@ jobs:
       image_name: torchvision/smoke_test
     steps:
       - checkout
+      - designate_upload_channel
       - run:
           name: Build and push Docker image
           no_output_timeout: "1h"
@@ -352,6 +361,7 @@ jobs:
     steps:
       - attach_workspace:
           at: ~/workspace
+      - designate_upload_channel
       - run:
           name: install binaries
           command: |
@@ -377,6 +387,7 @@ jobs:
     steps:
       - attach_workspace:
           at: ~/workspace
+      - designate_upload_channel
       - run:
           name: install binaries
           command: |
@@ -400,6 +411,7 @@ jobs:
     resource_class: 2xlarge+
     steps:
       - checkout
+      - designate_upload_channel
       - run:
           name: Generate cache key
           # This will refresh cache on Sundays, nightly build should generate new cache.
@@ -440,6 +452,7 @@ jobs:
       image_name: "pytorch/manylinux-cuda101"
     steps:
       - checkout
+      - designate_upload_channel
       - run:
           name: Generate cache key
           # This will refresh cache on Sundays, nightly build should generate new cache.
@@ -461,7 +474,7 @@ jobs:
             - env
       - run:
           name: Install torchvision
-          command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
+          command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e UPLOAD_CHANNEL "${image_name}" .circleci/unittest/linux/scripts/install.sh
       - run:
           name: Run tests
           command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh
@@ -477,6 +490,7 @@ jobs:
       name: windows-cpu
     steps:
       - checkout
+      - designate_upload_channel
       - run:
           name: Generate cache key
           # This will refresh cache on Sundays, nightly build should generate new cache.
@@ -516,6 +530,7 @@ jobs:
       CUDA_VERSION: "10.1"
     steps:
       - checkout
+      - designate_upload_channel
       - run:
           name: Generate cache key
           # This will refresh cache on Sundays, nightly build should generate new cache.
@@ -554,6 +569,7 @@ jobs:
     resource_class: large
     steps:
       - checkout
+      - designate_upload_channel
       - run:
           name: Install wget
           command: HOMEBREW_NO_AUTO_UPDATE=1 brew install wget
@@ -596,6 +612,7 @@ jobs:
     resource_class: 2xlarge+
     steps:
       - checkout_merge
+      - designate_upload_channel
       - run:
           name: Setup conda
           command: .circleci/unittest/linux/scripts/setup_env.sh
@@ -613,19 +630,21 @@ jobs:
       CU_VERSION: << parameters.cu_version >>
     steps:
       - checkout_merge
+      - designate_upload_channel
       - run:
           name: Setup conda
           command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh
       - run:
           name: Build torchvision C++ distribution and test
-          command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh
+          command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -e UPLOAD_CHANNEL -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh
 
   cmake_macos_cpu:
     <<: *binary_common
     macos:
       xcode: "9.0"
     steps:
       - checkout_merge
+      - designate_upload_channel
       - run:
           command: |
             curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
@@ -640,6 +659,7 @@ jobs:
       name: windows-cpu
     steps:
       - checkout_merge
+      - designate_upload_channel
       - run:
           command: |
             set -ex
@@ -652,6 +672,7 @@ jobs:
       name: windows-gpu
     steps:
       - checkout_merge
+      - designate_upload_channel
       - run:
           command: |
             set -ex

diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in
@@ -134,6 +134,7 @@ jobs:
     resource_class: 2xlarge+
     steps:
       - checkout_merge
+      - designate_upload_channel
       - run: packaging/build_wheel.sh
       - store_artifacts:
           path: dist
@@ -149,6 +150,7 @@ jobs:
     resource_class: 2xlarge+
     steps:
       - checkout_merge
+      - designate_upload_channel
       - run: packaging/build_conda.sh
       - store_artifacts:
           path: /opt/conda/conda-bld/linux-64
@@ -164,6 +166,7 @@ jobs:
     executor: windows-cpu
     steps:
       - checkout_merge
+      - designate_upload_channel
       - run:
           name: Build conda packages
           command: |
@@ -189,6 +192,7 @@ jobs:
     executor: windows-cpu
     steps:
       - checkout_merge
+      - designate_upload_channel
       - run:
           name: Build wheel packages
           command: |
@@ -211,6 +215,7 @@ jobs:
       xcode: "9.4.1"
     steps:
       - checkout_merge
+      - designate_upload_channel
       - run:
           # Cannot easily deduplicate this as source'ing activate
           # will set environment variables which we need to propagate
@@ -233,6 +238,7 @@ jobs:
       xcode: "9.4.1"
     steps:
       - checkout_merge
+      - designate_upload_channel
       - run:
           command: |
             curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
@@ -295,6 +301,7 @@ jobs:
     steps:
       - attach_workspace:
           at: ~/workspace
+      - designate_upload_channel
       - run:
           name: install binaries
           command: |
@@ -313,6 +320,7 @@ jobs:
     steps:
       - attach_workspace:
           at: ~/workspace
+      - designate_upload_channel
       - run:
           name: install binaries
           command: |
@@ -333,6 +341,7 @@ jobs:
       image_name: torchvision/smoke_test
     steps:
       - checkout
+      - designate_upload_channel
       - run:
           name: Build and push Docker image
           no_output_timeout: "1h"
@@ -352,6 +361,7 @@ jobs:
     steps:
       - attach_workspace:
           at: ~/workspace
+      - designate_upload_channel
       - run:
           name: install binaries
           command: |
@@ -377,6 +387,7 @@ jobs:
     steps:
       - attach_workspace:
           at: ~/workspace
+      - designate_upload_channel
       - run:
           name: install binaries
           command: |
@@ -400,6 +411,7 @@ jobs:
     resource_class: 2xlarge+
     steps:
       - checkout
+      - designate_upload_channel
       - run:
           name: Generate cache key
           # This will refresh cache on Sundays, nightly build should generate new cache.
@@ -440,6 +452,7 @@ jobs:
       image_name: "pytorch/manylinux-cuda101"
     steps:
       - checkout
+      - designate_upload_channel
       - run:
           name: Generate cache key
           # This will refresh cache on Sundays, nightly build should generate new cache.
@@ -461,7 +474,7 @@ jobs:
             - env
       - run:
           name: Install torchvision
-          command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
+          command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e UPLOAD_CHANNEL "${image_name}" .circleci/unittest/linux/scripts/install.sh
       - run:
           name: Run tests
           command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh
@@ -477,6 +490,7 @@ jobs:
       name: windows-cpu
     steps:
       - checkout
+      - designate_upload_channel
       - run:
           name: Generate cache key
           # This will refresh cache on Sundays, nightly build should generate new cache.
@@ -516,6 +530,7 @@ jobs:
       CUDA_VERSION: "10.1"
     steps:
       - checkout
+      - designate_upload_channel
       - run:
           name: Generate cache key
           # This will refresh cache on Sundays, nightly build should generate new cache.
@@ -554,6 +569,7 @@ jobs:
     resource_class: large
     steps:
       - checkout
+      - designate_upload_channel
       - run:
           name: Install wget
           command: HOMEBREW_NO_AUTO_UPDATE=1 brew install wget
@@ -596,6 +612,7 @@ jobs:
     resource_class: 2xlarge+
     steps:
       - checkout_merge
+      - designate_upload_channel
       - run:
           name: Setup conda
           command: .circleci/unittest/linux/scripts/setup_env.sh
@@ -613,19 +630,21 @@ jobs:
       CU_VERSION: << parameters.cu_version >>
     steps:
       - checkout_merge
+      - designate_upload_channel
       - run:
           name: Setup conda
           command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh
       - run:
           name: Build torchvision C++ distribution and test
-          command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh
+          command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -e UPLOAD_CHANNEL -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh
 
   cmake_macos_cpu:
     <<: *binary_common
     macos:
       xcode: "9.0"
     steps:
       - checkout_merge
+      - designate_upload_channel
       - run:
           command: |
             curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
@@ -640,6 +659,7 @@ jobs:
       name: windows-cpu
     steps:
       - checkout_merge
+      - designate_upload_channel
       - run:
           command: |
             set -ex
@@ -652,6 +672,7 @@ jobs:
       name: windows-gpu
     steps:
       - checkout_merge
+      - designate_upload_channel
       - run:
           command: |
             set -ex

diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh
@@ -23,7 +23,7 @@ else
     cudatoolkit="cudatoolkit=${version}"
 fi
 printf "Installing PyTorch with %s\n" "${cudatoolkit}"
-conda install -y -c pytorch-nightly pytorch "${cudatoolkit}"
+conda install -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch "${cudatoolkit}"
 
 printf "* Installing torchvision\n"
-python setup.py develop
+python setup.py develop
diff --git a/.circleci/unittest/windows/scripts/install.sh b/.circleci/unittest/windows/scripts/install.sh
@@ -25,7 +25,7 @@ else
     cudatoolkit="cudatoolkit=${version}"
 fi
 printf "Installing PyTorch with %s\n" "${cudatoolkit}"
-conda install -y -c pytorch-nightly pytorch "${cudatoolkit}"
+conda install -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch "${cudatoolkit}"
 
 printf "* Installing torchvision\n"
-"$this_dir/vc_env_helper.bat" python setup.py develop
+"$this_dir/vc_env_helper.bat" python setup.py develop
diff --git a/packaging/build_cmake.sh b/packaging/build_cmake.sh
@@ -23,7 +23,7 @@ fi
 setup_visual_studio_constraint
 setup_junit_results_folder
 
-conda install -yq pytorch=$PYTORCH_VERSION $CONDA_CUDATOOLKIT_CONSTRAINT $CONDA_CPUONLY_FEATURE  -c pytorch-nightly
+conda install -yq pytorch=$PYTORCH_VERSION $CONDA_CUDATOOLKIT_CONSTRAINT $CONDA_CPUONLY_FEATURE  -c "pytorch-${UPLOAD_CHANNEL}"
 TORCH_PATH=$(dirname $(python -c "import torch; print(torch.__file__)"))
 
 if [[ "$(uname)" == Darwin || "$OSTYPE" == "msys" ]]; then