revert: pip-compile on windows, Windows-compatible temp dir, unify us…

…er local dependencies (#141)

Signed-off-by: ThibaultFy <50656860+ThibaultFy@users.noreply.github.com>

CHANGELOG.md

@@ -14,7 +14,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Check and test on string used as metric name in test data nodes ([#122](https://github.com/Substra/substrafl/pull/122)).
 - Add default exclusion patterns when copying file to avoid creating large Docker images ([#118](https://github.com/Substra/substrafl/pull/118))
 - Add the possibility to force the Dependency editable_mode through the environment variable SUBSTRA_FORCE_EDITABLE_MODE ([#131](https://github.com/Substra/substrafl/pull/131))
-- Check on the Python version used before generating the Dockerfile ([#123])(https://github.com/Substra/substrafl/pull/123)).
 
 ## Changed
 
@@ -60,11 +59,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Way to copy function files ([#118](https://github.com/Substra/substrafl/pull/118))
 - `download_train_task_models_by_rank` uses new function `list_task_output_assets` instead of using `value` that has been removed ([#129](https://github.com/Substra/substrafl/pull/129))
-- Python dependencies are resolved using pip compile during function registration ([#123])(https://github.com/Substra/substrafl/pull/123)).
-- BREAKING: local_dependencies is renamed local_installable_dependencies([#123])(https://github.com/Substra/substrafl/pull/123)).
-- BREAKING: local_installable_dependencies are now limited to local modules or Python wheels (no support for bdist, sdist...)([#123])(https://github.com/Substra/substrafl/pull/123)).
 
 ### Fixed
+
 - New dependencies copy method in Docker mode.([#130](https://github.com/Substra/substrafl/pull/130))
 
 ## [0.36.0](https://github.com/Substra/substrafl/releases/tag/0.36.0) - 2023-05-11

benchmark/camelyon/workflows.py

@@ -89,7 +89,7 @@ def substrafl_fed_avg(
     # Dependencies
     base = Path(__file__).parent
     dependencies = Dependency(
-        pypi_dependencies=["torch", "numpy", "scikit-learn"],
+        pypi_dependencies=["torch", "numpy", "sklearn"],
         local_code=[base / "common", base / "weldon_fedavg.py"],
         editable_mode=False,
     )

docs/api/remote.rst

@@ -27,7 +27,7 @@ Register
 ^^^^^^^^
 
 .. automodule:: substrafl.remote.register.register
-.. automodule:: substrafl.remote.register.manage_dependencies
+.. automodule:: substrafl.remote.register.generate_wheel
 
 
 Serializers

docs/conf.py

@@ -193,8 +193,6 @@
     ("py:class", "substra.sdk.schemas.FunctionOutputSpec"),
     ("py:class", "substra.sdk.schemas.FunctionInputSpec"),
     ("py:class", "ComputePlan"),
-    ("py:class", "Path"),
-    ("py:class", "module"),
 ]
 
 html_css_files = [

setup.py

@@ -49,7 +49,6 @@
         "wheel",
         "six",
         "packaging",
-        "pip-tools",
     ],
     extras_require={
         "dev": [

substrafl/dependency/schemas.py

@@ -21,11 +21,10 @@ class Dependency(BaseModel):
             Dockerfiles submitted to Substra platform will be taken from pypi.
             If set to True, it will be the one installed in editable mode from your python environment.
             Defaults to False.
-        pypi_dependencies (List[str]): Python packages installable from PyPI.
-        local_installable_dependencies (List[pathlib.Path]): Local installable packages.
-            Each one can either be a wheel or a local folder. If it's a local folder, the command
-            `python -m pip wheel .` will be run, so each folder needs to be a valid Python module (containing a valid
-            `setup.py` or `pyproject.toml`). See the documentation of pip wheel for more details.
+        dependencies (List[str]): Python packages installable from pypi.
+        local_dependencies (List[pathlib.Path]): Local installable packages. The command
+            `pip install -e .` will be executed in each of those folders hence a `setup.py` must be present in each
+            folder.
         local_code (List[pathlib.Path]): Local relative imports used by your script. All files / folders will be
             pasted to the level of the running script.
         excluded_paths (List[pathlib.Path]): Local paths excluded from `local_dependencies` / `local_code`.
@@ -41,7 +40,7 @@ class Dependency(BaseModel):
 
     editable_mode: bool = False
     pypi_dependencies: List[str] = Field(default_factory=list)
-    local_installable_dependencies: List[Path] = Field(default_factory=list)
+    local_dependencies: List[Path] = Field(default_factory=list)
     local_code: List[Path] = Field(default_factory=list)
     excluded_paths: List[Path] = Field(default_factory=list)
     excluded_regex: List[str] = Field(default_factory=list)
@@ -50,7 +49,7 @@ class Dependency(BaseModel):
     class Config:
         arbitrary_types_allowed = True
 
-    @validator("local_installable_dependencies", "local_code")
+    @validator("local_dependencies", "local_code")
     def resolve_path(cls, v):  # noqa: N805
         """Resolve list of local code paths and check if they exist."""
         not_existing_paths = list()
@@ -68,9 +67,9 @@ def resolve_path(cls, v):  # noqa: N805
 
         return resolved_paths
 
-    @validator("local_installable_dependencies")
+    @validator("local_dependencies")
     def check_setup(cls, v):  # noqa: N805
-        """Check the presence of a setup.py file or a pyproject.toml in the provided paths."""
+        """Check the presence of a setup.py file in the provided paths."""
         not_installable = list()
         for dependency_path in v:
             installable_dir = (dependency_path / "setup.py").is_file() or (dependency_path / "pyproject.toml").is_file()
@@ -87,7 +86,7 @@ def check_setup(cls, v):  # noqa: N805
     def copy_dependencies_local_package(self, *, dest_dir: Path) -> List[Path]:
         return path_management.copy_paths(
             dest_dir=dest_dir,
-            src=self.local_installable_dependencies,
+            src=self.local_dependencies,
             force_included=self.force_included_paths,
             excluded=self.excluded_paths,
             excluded_regex=self.excluded_regex,

substrafl/exceptions.py

@@ -10,18 +10,6 @@ class SubstraToolsDeprecationWarning(DeprecationWarning):
     """The substratools version used is deprecated."""
 
 
-class UnsupportedPythonVersionError(Exception):
-    """The Python version used is not supported by Substra."""
-
-
-class InvalidUserModuleError(Exception):
-    """The local folder passed by the user as a dependency is not a valid Python module."""
-
-
-class InvalidDependenciesError(Exception):
-    """The set of constraints given on dependencies cannot be solved or is otherwise invalid (wrong package name)."""
-
-
 class CriterionReductionError(Exception):
     """The criterion reduction must be set to 'mean' to use the Newton-Raphson strategy."""
 

substrafl/experiment.py

@@ -256,7 +256,6 @@ def execute_experiment(
             Otherwise measuring of performance will follow the EvaluationStrategy. Defaults to None.
         aggregation_node (typing.Optional[AggregationNode]): For centralized strategy, the aggregation
             node, where all the shared tasks occurs else None.
-        evaluation_strategy: Optional[EvaluationStrategy]
         num_rounds (int): The number of time your strategy will be executed
         dependencies (Dependency, Optional): Dependencies of the experiment. It must be defined from
             the SubstraFL Dependency class. Defaults None.

substrafl/remote/register/generate_wheel.py

@@ -0,0 +1,142 @@
+"""
+Generate wheels for the Substra algo.
+"""
+import logging
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+from typing import List
+
+logger = logging.getLogger(__name__)
+
+LOCAL_WHEELS_FOLDER = Path.home() / ".substrafl"
+
+
+def local_lib_wheels(lib_modules: List, *, operation_dir: Path, python_major_minor: str, dest_dir: str) -> str:
+    """Prepares the private modules from lib_modules list to be installed in a Docker image and generates the
+    appropriated install command for a dockerfile. It first creates the wheel for each library. Each of the
+    libraries must be already installed in the correct version locally. Use command:
+    ``pip install -e library-name`` in the directory of each library.
+
+    This allows one user to use custom version of the passed modules.
+
+    Args:
+        lib_modules (list): list of modules to be installed.
+        operation_dir (pathlib.Path): Path to the operation directory
+        python_major_minor (str): version which is to be used in the dockerfile. Eg: '3.8'
+        dest_dir (str): relative directory where the wheels are saved
+
+    Returns:
+        str: dockerfile command for installing the given modules
+    """
+    install_cmds = []
+    wheels_dir = operation_dir / dest_dir
+    wheels_dir.mkdir(exist_ok=True, parents=True)
+    for lib_module in lib_modules:
+        if not (Path(lib_module.__file__).parents[1] / "setup.py").exists():
+            msg = ", ".join([lib.__name__ for lib in lib_modules])
+            raise NotImplementedError(
+                f"You must install {msg} in editable mode.\n" "eg `pip install -e substra` in the substra directory"
+            )
+        lib_name = lib_module.__name__
+        lib_path = Path(lib_module.__file__).parents[1]
+        wheel_name = f"{lib_name}-{lib_module.__version__}-py3-none-any.whl"
+
+        wheel_path = LOCAL_WHEELS_FOLDER / wheel_name
+        # Recreate the wheel only if it does not exist
+        if wheel_path.exists():
+            logger.warning(
+                f"Existing wheel {wheel_path} will be used to build {lib_name}. "
+                "It may lead to errors if you are using an unreleased version of this lib: "
+                "if it's the case, you can delete the wheel and it will be re-generated."
+            )
+        else:
+            # if the right version of substra or substratools is not found, it will search if they are already
+            # installed in 'dist' and take them from there.
+            # sys.executable takes the Python interpreter run by the code and not the default one on the computer
+            extra_args: list = list()
+            if lib_name == "substrafl":
+                extra_args = [
+                    "--find-links",
+                    operation_dir / "dist/substra",
+                    "--find-links",
+                    operation_dir / "dist/substratools",
+                ]
+            subprocess.check_output(
+                [
+                    sys.executable,
+                    "-m",
+                    "pip",
+                    "wheel",
+                    ".",
+                    "-w",
+                    LOCAL_WHEELS_FOLDER,
+                    "--no-deps",
+                ]
+                + extra_args,
+                cwd=str(lib_path),
+            )
+
+        shutil.copy(wheel_path, wheels_dir / wheel_name)
+
+        # Necessary command to install the wheel in the docker image
+        force_reinstall = "--force-reinstall " if lib_name in ["substratools", "substra"] else ""
+        install_cmd = (
+            f"COPY {dest_dir}/{wheel_name} . \n"
+            + f"RUN python{python_major_minor} -m pip install {force_reinstall}{wheel_name}\n"
+        )
+        install_cmds.append(install_cmd)
+
+    return "\n".join(install_cmds)
+
+
+def pypi_lib_wheels(lib_modules: List, *, operation_dir: Path, python_major_minor: str, dest_dir: str) -> str:
+    """Retrieves lib_modules' wheels to be installed in a Docker image and generates
+    the appropriated install command for a dockerfile.
+
+    Args:
+        lib_modules (list): list of modules to be installed.
+        operation_dir (pathlib.Path): Path to the operation directory
+        python_major_minor (str): version which is to be used in the dockerfile. Eg: '3.8'
+        dest_dir (str): relative directory where the wheels are saved
+
+    Returns:
+        str: dockerfile command for installing the given modules
+    """
+    install_cmds = []
+    wheels_dir = operation_dir / dest_dir
+    wheels_dir.mkdir(exist_ok=True, parents=True)
+
+    LOCAL_WHEELS_FOLDER.mkdir(exist_ok=True)
+
+    for lib_module in lib_modules:
+        wheel_name = f"{lib_module.__name__}-{lib_module.__version__}-py3-none-any.whl"
+
+        # Download only if exists
+        if not ((LOCAL_WHEELS_FOLDER / wheel_name).exists()):
+            subprocess.check_output(
+                [
+                    sys.executable,
+                    "-m",
+                    "pip",
+                    "download",
+                    "--only-binary",
+                    ":all:",
+                    "--python-version",
+                    python_major_minor,
+                    "--no-deps",
+                    "--implementation",
+                    "py",
+                    "-d",
+                    LOCAL_WHEELS_FOLDER,
+                    f"{lib_module.__name__}=={lib_module.__version__}",
+                ]
+            )
+
+        # Get wheel name based on current version
+        shutil.copy(LOCAL_WHEELS_FOLDER / wheel_name, wheels_dir / wheel_name)
+        install_cmd = f"COPY {dest_dir}/{wheel_name} . \n RUN python{python_major_minor} -m pip install {wheel_name}\n"
+        install_cmds.append(install_cmd)
+
+    return "\n".join(install_cmds)