Re-organize requirements files (mlflow#5073)

* clean up

Signed-off-by: harupy <hkawamura0130@gmail.com>

* revert unrelated changes

Signed-off-by: harupy <hkawamura0130@gmail.com>

* move doc-requirements.txt

Signed-off-by: harupy <hkawamura0130@gmail.com>

* fix

Signed-off-by: harupy <hkawamura0130@gmail.com>

* fix again

Signed-off-by: harupy <hkawamura0130@gmail.com>

* remove dev-requirements.txt

Signed-off-by: harupy <hkawamura0130@gmail.com>

.circleci/config.yml

@@ -105,7 +105,7 @@ jobs:
           name: Install Python dependencies
           command: |
             pip --version
-            pip install --progress-bar off -r doc-requirements.txt .
+            pip install --progress-bar off -r requirements/doc-requirements.txt .
       - run:
           name: Build documentation
           working_directory: docs

.dockerignore

@@ -8,10 +8,10 @@ examples
 
 dev
 # required for Docker build
-!dev/small-requirements.txt
-!dev/large-requirements.txt
-!dev/lint-requirements.txt
-!dev/extra-ml-requirements.txt
+!requirements/small-requirements.txt
+!requirements/large-requirements.txt
+!requirements/lint-requirements.txt
+!requirements/extra-ml-requirements.txt
 
 tests
 # required for Docker build

.github/actions/cache-pip/action.yml

@@ -11,6 +11,6 @@ runs:
     - uses: actions/cache@v2
       with:
         path: ~/.cache/pip
-        key: ${{ steps.get-cache-key-prefix.outputs.prefix }}-pip-${{ hashFiles('**/*-requirements.txt') }}
+        key: ${{ steps.get-cache-key-prefix.outputs.prefix }}-pip-${{ hashFiles('requirements/*-requirements.txt') }}
         restore-keys: |
           ${{ steps.get-cache-key-prefix.outputs.prefix }}-pip-

.github/workflows/cross-version-tests.yml

@@ -146,7 +146,7 @@ jobs:
           python --version
           pip install --upgrade pip wheel
           pip install -e .
-          pip install -r dev/small-requirements.txt
+          pip install -r requirements/small-requirements.txt
       - name: Install ${{ matrix.package }} ${{ matrix.version }}
         env:
           CACHE_DIR: /home/runner/.cache/wheels

.github/workflows/master.yml

@@ -40,7 +40,7 @@ jobs:
         INSTALL_SMALL_PYTHON_DEPS: true
       run: |
         source ./dev/install-common-deps.sh
-        pip install -r ./dev/lint-requirements.txt
+        pip install -r requirements/lint-requirements.txt
     - name: Run tests
       run: |
         ./lint.sh
@@ -410,7 +410,7 @@ jobs:
           python-version: 3.7
       - name: Install python dependencies
         run: |
-          pip install -r dev/small-requirements.txt
+          pip install -r requirements/small-requirements.txt
           pip install --no-dependencies tests/resources/mlflow-test-plugin
           pip install -e .[extras]
       - name: Run python tests

CONTRIBUTING.rst

@@ -147,8 +147,7 @@ If you plan on doing development and testing, you will also need to install the
 
 .. code-block:: bash
 
-    pip install -r doc-requirements.txt
-    pip install -r test-requirements.txt
+    pip install -r requirements/test-requirements.txt
     pip install -e .[extras]  # installs mlflow from current checkout
     pip install -e tests/resources/mlflow-test-plugin # installs `mlflow-test-plugin` that is required for running certain MLflow tests
 
@@ -355,7 +354,7 @@ Then, verify that the unit tests & linter pass before submitting a pull request
 Python tests are split into "small" & "large" categories, with new tests falling into the "small"
 category by default. Tests that take 10 or more seconds to run should be marked as large tests
 via the ``@pytest.mark.large`` annotation. Dependencies for small and large tests can be added to
-``dev/small-requirements.txt`` and ``dev/large-requirements.txt``, respectively.
+``requirements/small-requirements.txt`` and ``requirements/large-requirements.txt``, respectively.
 
 We use `pytest <https://docs.pytest.org/en/latest/contents.html>`_ to run Python tests.
 You can run tests for one or more test directories or files via
@@ -391,7 +390,7 @@ If you are adding new framework flavor support, you'll need to modify ``pytest``
   a. Add your tests to the ignore list, where the other frameworks are ignored
   b. Add a pytest command for your tests along with the other framework tests (as a separate command to avoid OOM issues)
 
-4. ``dev/large-requirements.txt``: add your framework and version to the list of requirements
+4. ``requirements/large-requirements.txt``: add your framework and version to the list of requirements
 
 You can see an example of a `flavor PR <https://github.com/mlflow/mlflow/pull/2136/files>`_.
 

Dockerfile

@@ -11,8 +11,7 @@ RUN apt-get update && \
     cmake protobuf-compiler &&  \
     conda install python=3.6 && \
     # install required python packages
-    pip install -r dev-requirements.txt --no-cache-dir && \
-    pip install -r test-requirements.txt --no-cache-dir && \
+    pip install -r requirements/test-requirements.txt --no-cache-dir && \
     # install mlflow in editable form
     pip install --no-cache-dir -e . && \
     # mkdir required to support install openjdk-11-jre-headless

dev/install-common-deps.sh

@@ -32,13 +32,13 @@ if [[ "$INSTALL_SMALL_PYTHON_DEPS" == "true" ]]; then
   # When downloading large packages from PyPI, the connection is sometimes aborted by the
   # remote host. See https://github.com/pypa/pip/issues/8510.
   # As a workaround, we retry installation of large packages.
-  retry-with-backoff pip install -r ./dev/small-requirements.txt
+  retry-with-backoff pip install -r requirements/small-requirements.txt
 fi
 if [[ "$INSTALL_SKINNY_PYTHON_DEPS" == "true" ]]; then
-  retry-with-backoff pip install -r ./dev/skinny-requirements.txt
+  retry-with-backoff pip install -r requirements/skinny-requirements.txt
 fi
 if [[ "$INSTALL_LARGE_PYTHON_DEPS" == "true" ]]; then
-  retry-with-backoff pip install -r ./dev/large-requirements.txt
+  retry-with-backoff pip install -r requirements/large-requirements.txt
 
   # Install prophet's dependencies beforehand, otherwise pip would fail to build a wheel for prophet
   if [[ -z "$(pip cache list prophet --format abspath)" ]]; then
@@ -49,7 +49,7 @@ if [[ "$INSTALL_LARGE_PYTHON_DEPS" == "true" ]]; then
     rm -rf $tmp_dir
   fi
 
-  retry-with-backoff pip install -r ./dev/extra-ml-requirements.txt
+  retry-with-backoff pip install -r requirements/extra-ml-requirements.txt
 fi
 
 # Install `mlflow-test-plugin` without dependencies

dev/setup-spark-datasource-autologging.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 # Test Spark autologging against the Spark 3.0 preview. This script is temporary and should be
 # removed once Spark 3.0 is released in favor of simply updating all tests to run against Spark 3.0
-# (i.e. updating the pyspark dependency version in dev/large-requirements.txt)
+# (i.e. updating the pyspark dependency version in requirements/large-requirements.txt)
 set -ex
 
 # Build Java package

lint.sh

@@ -11,7 +11,7 @@ black --check .
 
 if [ $? -ne 0 ]; then
   echo 'Run this command to apply Black formatting:'
-  echo '$ pip install $(cat dev/lint-requirements.txt | grep "black==") && black .'
+  echo '$ pip install $(cat requirements/lint-requirements.txt | grep "black==") && black .'
 fi
 
 echo -e "\n========== pylint ==========\n"

requirements/test-requirements.txt

@@ -0,0 +1,5 @@
+-r small-requirements.txt
+-r large-requirements.txt
+-r extra-ml-requirements.txt
+-r lint-requirements.txt
+-r doc-requirements.txt