Reuse CI scripts for local installation process (#3698)

This commit streamlines the CI installation script, with the main purpose of making it easier to replicate on a Linux/MacOSX dev box the exact same environment that exists on CI. Apply safe defaults for environment variables if the script is not invoked by travis Do not install miniconda if the 'conda' command is already available Explicitly state conda channel every time to avoid relying on ~/.condarc Recommend using conda in developer docs
dask · Apr 21, 2020 · 83c27fa · 83c27fa
1 parent 35dc940
commit 83c27fa
Show file tree

Hide file tree

Showing 6 changed files with 60 additions and 49 deletions.
diff --git a/.github/workflows/ci-windows.yaml b/.github/workflows/ci-windows.yaml
@@ -19,8 +19,8 @@ jobs:
         with:
           miniconda-version: "latest"
           python-version: ${{ matrix.python-version }}
-          environment-file: continuous_integration/environment.yml
-          activate-environment: testenv
+          environment-file: continuous_integration/environment-windows.yml
+          activate-environment: dask-distributed
           auto-activate-base: false
 
       - name: Install tornado

diff --git a/.travis.yml b/.travis.yml
@@ -6,9 +6,9 @@ dist: trusty
 
 env:
   matrix:
-    - PYTHON=3.6 TESTS=true COVERAGE=true PACKAGES="scikit-learn lz4" TORNADO=5 CRICK=true
-    - PYTHON=3.7 TESTS=true PACKAGES="scikit-learn python-snappy python-blosc" TORNADO=6
-    - PYTHON=3.8 TESTS=true PACKAGES="scikit-learn python-snappy python-blosc" TORNADO=6
+    - PYTHON=3.6 TESTS=true COVERAGE=true PACKAGES="lz4" TORNADO=5 CRICK=true
+    - PYTHON=3.7 TESTS=true PACKAGES="python-snappy python-blosc" TORNADO=6
+    - PYTHON=3.8 TESTS=true PACKAGES="python-snappy python-blosc" TORNADO=6
 
 matrix:
   fast_finish: true
@@ -18,7 +18,7 @@ matrix:
     python: 3.6
     env: LINT=true
   - os: osx
-    env: PYTHON=3.7 TESTS=true PACKAGES="scikit-learn python-snappy python-blosc" TORNADO=6
+    env: PYTHON=3.7 TESTS=true PACKAGES="python-snappy python-blosc" TORNADO=6
     if: type != pull_request OR commit_message =~ test-osx  # Skip on PRs unless the commit message contains "test-osx"
 
   allow_failures:

diff --git a/continuous_integration/environment.yml → ...nuous_integration/environment-windows.yml b/continuous_integration/environment.yml → ...nuous_integration/environment-windows.yml
@@ -1,6 +1,7 @@
-name: testenv
+name: dask-distributed
 channels:
   - conda-forge
+  - defaults
 dependencies:
   - zstandard
   - bokeh!=2.0.0
@@ -16,18 +17,18 @@ dependencies:
   - prometheus_client
   - psutil
   - pytest
+  - pytest-asyncio
+  - pytest-repeat
+  - pytest-timeout
+  - pytest-faulthandler
   - requests
+  - sortedcollections
   - toolz
   - tblib
   - zict
   - fsspec
   - pip
   - pip:
-      - pytest-repeat
-      - pytest-timeout
-      - pytest-faulthandler
-      - sortedcollections
-      - pytest-asyncio
       - git+https://github.com/dask/dask
       - git+https://github.com/joblib/joblib.git
       - git+https://github.com/dask/zict
diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh
@@ -5,6 +5,12 @@
 
 # Note we disable progress bars to make Travis log loading much faster
 
+# Set default variable values if unset
+# (useful when this script is not invoked by Travis)
+: ${PYTHON:=3.8}
+: ${TORNADO:=6}
+: ${PACKAGES:=python-snappy python-blosc}
+
 # Install conda
 case "$(uname -s)" in
     'Darwin')
@@ -16,18 +22,17 @@ case "$(uname -s)" in
     *)  ;;
 esac
 
-wget https://repo.continuum.io/miniconda/$MINICONDA_FILENAME -O miniconda.sh
-bash miniconda.sh -b -p $HOME/miniconda
-export PATH="$HOME/miniconda/bin:$PATH"
-conda config --set always_yes yes --set changeps1 no
-conda update -q conda
+if ! which conda; then
+  wget https://repo.continuum.io/miniconda/$MINICONDA_FILENAME -O miniconda.sh
+  bash miniconda.sh -b -p $HOME/miniconda
+  export PATH="$HOME/miniconda/bin:$PATH"
+fi
 
-# Create conda environment
-conda create -q -n test-environment python=$PYTHON
-source activate test-environment
+conda config --set always_yes yes --set quiet yes --set changeps1 no
+conda update conda
 
-# Install dependencies
-conda install -c conda-forge -q \
+# Create conda environment
+conda create -n dask-distributed -c conda-forge -c defaults \
     asyncssh \
     bokeh \
     click \
@@ -39,45 +44,49 @@ conda install -c conda-forge -q \
     ipywidgets \
     joblib \
     jupyter_client \
-    msgpack-python>=0.6.0 \
+    'msgpack-python>=0.6.0' \
     netcdf4 \
     paramiko \
     prometheus_client \
     psutil \
-    pytest>=4 \
+    'pytest>=4' \
+    pytest-asyncio \
+    pytest-faulthandler \
+    pytest-repeat \
     pytest-timeout \
     python=$PYTHON \
     requests \
+    scikit-learn \
     scipy \
-    tblib>=1.5.0 \
+    sortedcollections \
+    'tblib>=1.5.0' \
     toolz \
     tornado=$TORNADO \
     zstandard \
     $PACKAGES
 
+source activate dask-distributed
+
 # stacktrace is not currently avaiable for Python 3.8.
 # Remove the version check block below when it is avaiable.
 if [[ $PYTHON != 3.8 ]]; then
     # For low-level profiler, install libunwind and stacktrace from conda-forge
     # For stacktrace we use --no-deps to avoid upgrade of python
-    conda install -c defaults -c conda-forge libunwind
-    conda install --no-deps -c defaults -c numba -c conda-forge stacktrace
-fi;
-
-python -m pip install -q "pytest>=4" pytest-repeat pytest-faulthandler pytest-asyncio
+    conda install -c conda-forge -c defaults libunwind
+    conda install --no-deps -c conda-forge -c defaults -c numba stacktrace
+fi
 
 python -m pip install -q git+https://github.com/dask/dask.git --upgrade --no-deps
 python -m pip install -q git+https://github.com/joblib/joblib.git --upgrade --no-deps
 python -m pip install -q git+https://github.com/intake/filesystem_spec.git --upgrade --no-deps
 python -m pip install -q git+https://github.com/dask/s3fs.git --upgrade --no-deps
 python -m pip install -q git+https://github.com/dask/zict.git --upgrade --no-deps
-python -m pip install -q sortedcollections --no-deps
 python -m pip install -q keras --upgrade --no-deps
 
 if [[ $CRICK == true ]]; then
-    conda install -q cython
+    conda install -c conda-forge -c defaults cython
     python -m pip install -q git+https://github.com/jcrist/crick.git
-fi;
+fi
 
 # Install distributed
 python -m pip install --no-deps -e .

diff --git a/continuous_integration/travis/run_tests.sh b/continuous_integration/travis/run_tests.sh
@@ -19,7 +19,7 @@ echo "--"
 ulimit -a -H
 
 if [[ $COVERAGE == true ]]; then
-    coverage run $(which py.test) distributed -m "not avoid_travis" $PYTEST_OPTIONS;
+    coverage run $(which py.test) distributed -m "not avoid_travis" $PYTEST_OPTIONS
 else
-    py.test -m "not avoid_travis" distributed $PYTEST_OPTIONS;
-fi;
+    py.test -m "not avoid_travis" distributed $PYTEST_OPTIONS
+fi
diff --git a/docs/source/develop.rst b/docs/source/develop.rst
@@ -12,25 +12,26 @@ guidelines`_ in the main documentation.
 Install
 -------
 
-After setting up an environment as described in the `Dask developer
-guidelines`_ you can clone this repository with git::
+Clone this repository with git::
 
    git clone git@github.com:dask/distributed.git
+   cd distributed
 
-and install it from source::
+Install all dependencies:
 
-   cd distributed
-   python setup.py install
+On Linux / MacOSX::
 
-Using conda, for example::
+    source continuous_integration/travis/install.sh
 
-   git clone git@github.com:{your-fork}/distributed.git
-   cd distributed
-   conda create -y -n distributed python=3.6
-   conda activate distributed
-   python -m pip install -U -r requirements.txt
-   python -m pip install -U -r dev-requirements.txt
-   python -m pip install -e .
+On Windows:
+
+1. Install anaconda or miniconda
+2. ::
+
+    conda create -n dask-distributed -c conda-forge -c defaults python=3.8 tornado=6
+    conda activate dask-distributed
+    conda env update --file continuous_integration/environment-windows.yml
+    python -m pip install .
 
 To keep a fork in sync with the upstream source::