diff --git a/benchmark/camelyon/workflows.py b/benchmark/camelyon/workflows.py index 04e781e7..086570a5 100644 --- a/benchmark/camelyon/workflows.py +++ b/benchmark/camelyon/workflows.py @@ -89,7 +89,7 @@ def substrafl_fed_avg( # Dependencies base = Path(__file__).parent dependencies = Dependency( - pypi_dependencies=["torch", "numpy", "scikit-learn"], + pypi_dependencies=["torch", "numpy", "sklearn"], local_code=[base / "common", base / "weldon_fedavg.py"], editable_mode=False, ) diff --git a/substrafl/remote/register/manage_dependencies.py b/substrafl/remote/register/manage_dependencies.py index 58687633..8355823d 100644 --- a/substrafl/remote/register/manage_dependencies.py +++ b/substrafl/remote/register/manage_dependencies.py @@ -2,17 +2,14 @@ Utility functions to manage dependencies (building wheels, compiling requirement...) """ import logging -import os import re import shutil import subprocess import sys import tempfile from pathlib import Path -from pathlib import PurePosixPath from types import ModuleType from typing import List -from typing import Union from substrafl.dependency import Dependency from substrafl.exceptions import InvalidDependenciesError @@ -41,7 +38,7 @@ def build_user_dependency_wheel(lib_path: Path, operation_dir: Path) -> str: "-m", "pip", "wheel", - str(lib_path) + os.sep, + str(lib_path) + "/", "--no-deps", ], cwd=str(operation_dir), @@ -167,7 +164,7 @@ def get_pypi_dependencies_versions(lib_modules: List) -> List[str]: return [f"{lib_module.__name__}=={lib_module.__version__}" for lib_module in lib_modules] -def compile_requirements(dependency_list: List[Union[str, Path]], *, operation_dir: Path, sub_dir: Path) -> None: +def compile_requirements(dependency_list: List[str], *, operation_dir: Path, sub_dir: Path) -> None: """Compile a list of requirements using pip-compile to generate a set of fully pinned third parties requirements Writes down a `requirements.in` file with the list of explicit dependencies, then generates a `requirements.txt` @@ -189,29 +186,23 @@ def compile_requirements(dependency_list: List[Union[str, Path]], *, operation_d requirements = "" for dependency in dependency_list: - if str(dependency).endswith(".whl"): - # pip compile require '/', even on windows. The double conversion resolves that. - requirements += f"file:{PurePosixPath(Path(dependency))}\n" + if dependency.__str__().endswith(".whl"): + requirements += f"file:{dependency}\n" else: requirements += f"{dependency}\n" + requirements_in.write_text(requirements) - command = [ - sys.executable, - "-m", - "piptools", - "compile", - "--resolver=backtracking", - str(requirements_in), - ] try: - subprocess.run( - command, + subprocess.check_output( + [ + sys.executable, + "-m", + "piptools", + "compile", + "--resolver=backtracking", + requirements_in, + ], cwd=operation_dir, - check=True, - capture_output=True, - text=True, ) except subprocess.CalledProcessError as e: - raise InvalidDependenciesError( - f"Error in command {' '.join(command)}\nstdout: {e.stdout}\nstderr: {e.stderr}" - ) from e + raise InvalidDependenciesError from e diff --git a/substrafl/remote/register/register.py b/substrafl/remote/register/register.py index 9e933c4c..d871188d 100644 --- a/substrafl/remote/register/register.py +++ b/substrafl/remote/register/register.py @@ -10,7 +10,6 @@ import warnings from distutils import util from pathlib import Path -from pathlib import PurePosixPath from platform import python_version import substra @@ -140,9 +139,8 @@ def _get_base_docker_image(python_major_minor: str, editable_mode: bool) -> str: return substratools_image -def _generate_copy_local_files(local_files: typing.List[Path]) -> str: - # In Dockerfiles, we need to always have '/'. PurePosixPath resolves that. - return "\n".join([f"COPY {PurePosixPath(file)} {PurePosixPath(file)}" for file in local_files]) +def _generate_copy_local_files(local_files: typing.List[str]) -> str: + return "\n".join([f"COPY {file} {file}" for file in local_files]) def _create_dockerfile(install_libraries: bool, dependencies: Dependency, operation_dir: Path, method_name: str) -> str: diff --git a/tests/dependency/installable_library/setup.py b/tests/dependency/installable_library/setup.py index ec210da8..cf76a10f 100644 --- a/tests/dependency/installable_library/setup.py +++ b/tests/dependency/installable_library/setup.py @@ -4,5 +4,4 @@ setup( name="substrafltestlibrary", version="0.0.1", - packages=["substrafltestlibrary"], ) diff --git a/tests/dependency/installable_library2/setup.py b/tests/dependency/installable_library2/setup.py index 0c00ca26..3bfe6fb8 100644 --- a/tests/dependency/installable_library2/setup.py +++ b/tests/dependency/installable_library2/setup.py @@ -4,5 +4,4 @@ setup( name="substrafltestlibrary2", version="0.0.1", - packages=["substrafltestlibrary2"], ) diff --git a/tests/dependency/test_dependency.py b/tests/dependency/test_dependency.py index 8d642ebe..e27aead0 100644 --- a/tests/dependency/test_dependency.py +++ b/tests/dependency/test_dependency.py @@ -2,6 +2,8 @@ import tempfile import uuid from pathlib import Path +from unittest.mock import MagicMock +from unittest.mock import patch import numpy as np import pytest @@ -297,17 +299,14 @@ def train( utils.wait(client, train_task) @pytest.mark.docker_only + @patch("substrafl.remote.register.register.local_lib_wheels", MagicMock(return_value="INSTALL IN EDITABLE MODE")) def test_force_editable_mode( self, - mocker, monkeypatch, network, session_dir, dummy_algo_class, ): - mocker.patch("substrafl.remote.register.register.local_lib_wheels", return_value=["INSTALL IN EDITABLE MODE"]) - mocker.patch("substrafl.remote.register.register.compile_requirements") - client = network.clients[0] algo_deps = Dependency(pypi_dependencies=["pytest"], editable_mode=False)