fix: pip-compile on windows (#139)

Signed-off-by: SdgJlbl <sarah.diot-girard@owkin.com>
Co-authored-by: ThibaultFy <50656860+ThibaultFy@users.noreply.github.com>

benchmark/camelyon/workflows.py

@@ -89,7 +89,7 @@ def substrafl_fed_avg(
     # Dependencies
     base = Path(__file__).parent
     dependencies = Dependency(
-        pypi_dependencies=["torch", "numpy", "sklearn"],
+        pypi_dependencies=["torch", "numpy", "scikit-learn"],
         local_code=[base / "common", base / "weldon_fedavg.py"],
         editable_mode=False,
     )

substrafl/remote/register/manage_dependencies.py

@@ -2,14 +2,17 @@
 Utility functions to manage dependencies (building wheels, compiling requirement...)
 """
 import logging
+import os
 import re
 import shutil
 import subprocess
 import sys
 import tempfile
 from pathlib import Path
+from pathlib import PurePosixPath
 from types import ModuleType
 from typing import List
+from typing import Union
 
 from substrafl.dependency import Dependency
 from substrafl.exceptions import InvalidDependenciesError
@@ -38,7 +41,7 @@ def build_user_dependency_wheel(lib_path: Path, operation_dir: Path) -> str:
                 "-m",
                 "pip",
                 "wheel",
-                str(lib_path) + "/",
+                str(lib_path) + os.sep,
                 "--no-deps",
             ],
             cwd=str(operation_dir),
@@ -164,7 +167,7 @@ def get_pypi_dependencies_versions(lib_modules: List) -> List[str]:
     return [f"{lib_module.__name__}=={lib_module.__version__}" for lib_module in lib_modules]
 
 
-def compile_requirements(dependency_list: List[str], *, operation_dir: Path, sub_dir: Path) -> None:
+def compile_requirements(dependency_list: List[Union[str, Path]], *, operation_dir: Path, sub_dir: Path) -> None:
     """Compile a list of requirements using pip-compile to generate a set of fully pinned third parties requirements
 
     Writes down a `requirements.in` file with the list of explicit dependencies, then generates a `requirements.txt`
@@ -186,23 +189,29 @@ def compile_requirements(dependency_list: List[str], *, operation_dir: Path, sub
 
     requirements = ""
     for dependency in dependency_list:
-        if dependency.__str__().endswith(".whl"):
-            requirements += f"file:{dependency}\n"
+        if str(dependency).endswith(".whl"):
+            # pip compile require '/', even on windows. The double conversion resolves that.
+            requirements += f"file:{PurePosixPath(Path(dependency))}\n"
         else:
             requirements += f"{dependency}\n"
-
     requirements_in.write_text(requirements)
+    command = [
+        sys.executable,
+        "-m",
+        "piptools",
+        "compile",
+        "--resolver=backtracking",
+        str(requirements_in),
+    ]
     try:
-        subprocess.check_output(
-            [
-                sys.executable,
-                "-m",
-                "piptools",
-                "compile",
-                "--resolver=backtracking",
-                requirements_in,
-            ],
+        subprocess.run(
+            command,
             cwd=operation_dir,
+            check=True,
+            capture_output=True,
+            text=True,
         )
     except subprocess.CalledProcessError as e:
-        raise InvalidDependenciesError from e
+        raise InvalidDependenciesError(
+            f"Error in command {' '.join(command)}\nstdout: {e.stdout}\nstderr: {e.stderr}"
+        ) from e

substrafl/remote/register/register.py

@@ -10,6 +10,7 @@
 import warnings
 from distutils import util
 from pathlib import Path
+from pathlib import PurePosixPath
 from platform import python_version
 
 import substra
@@ -139,8 +140,9 @@ def _get_base_docker_image(python_major_minor: str, editable_mode: bool) -> str:
     return substratools_image
 
 
-def _generate_copy_local_files(local_files: typing.List[str]) -> str:
-    return "\n".join([f"COPY {file} {file}" for file in local_files])
+def _generate_copy_local_files(local_files: typing.List[Path]) -> str:
+    # In Dockerfiles, we need to always have '/'. PurePosixPath resolves that.
+    return "\n".join([f"COPY {PurePosixPath(file)} {PurePosixPath(file)}" for file in local_files])
 
 
 def _create_dockerfile(install_libraries: bool, dependencies: Dependency, operation_dir: Path, method_name: str) -> str:

tests/dependency/installable_library/setup.py

@@ -4,4 +4,5 @@
 setup(
     name="substrafltestlibrary",
     version="0.0.1",
+    packages=["substrafltestlibrary"],
 )

tests/dependency/installable_library2/setup.py

@@ -4,4 +4,5 @@
 setup(
     name="substrafltestlibrary2",
     version="0.0.1",
+    packages=["substrafltestlibrary2"],
 )

tests/dependency/test_dependency.py

@@ -2,8 +2,6 @@
 import tempfile
 import uuid
 from pathlib import Path
-from unittest.mock import MagicMock
-from unittest.mock import patch
 
 import numpy as np
 import pytest
@@ -299,14 +297,17 @@ def train(
         utils.wait(client, train_task)
 
     @pytest.mark.docker_only
-    @patch("substrafl.remote.register.register.local_lib_wheels", MagicMock(return_value="INSTALL IN EDITABLE MODE"))
     def test_force_editable_mode(
         self,
+        mocker,
         monkeypatch,
         network,
         session_dir,
         dummy_algo_class,
     ):
+        mocker.patch("substrafl.remote.register.register.local_lib_wheels", return_value=["INSTALL IN EDITABLE MODE"])
+        mocker.patch("substrafl.remote.register.register.compile_requirements")
+
         client = network.clients[0]
         algo_deps = Dependency(pypi_dependencies=["pytest"], editable_mode=False)