From 1188b31287df42de615d25b1642ec894c2bb0c12 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Fri, 9 Oct 2020 06:34:43 +0200 Subject: [PATCH] locker: propagate cumulative markers to nested deps This change ensures that markers are propagated from top level dependencies to the deepest level by walking top to bottom instead of iterating over all available packages. In addition, we also compress any dependencies with the same name and constraint to provide a more concise representation. Resolves: #3112 #3160 (cherry picked from commit e78a67ba139c70ee1856834711ddaf14de0c926a) --- poetry/packages/locker.py | 86 ++++++++++++++------- tests/utils/test_exporter.py | 140 +++++++++++++++++++++++++++++++++++ 2 files changed, 199 insertions(+), 27 deletions(-) diff --git a/poetry/packages/locker.py b/poetry/packages/locker.py index 2a82fef816a..26c2b584504 100644 --- a/poetry/packages/locker.py +++ b/poetry/packages/locker.py @@ -215,10 +215,18 @@ def __get_locked_package( for dependency in project_requires: dependency = deepcopy(dependency) - if pinned_versions: - locked_package = __get_locked_package(dependency) - if locked_package: - dependency.set_constraint(locked_package.to_dependency().constraint) + locked_package = __get_locked_package(dependency) + if locked_package: + locked_dependency = locked_package.to_dependency() + locked_dependency.marker = dependency.marker.intersect( + locked_package.marker + ) + + if not pinned_versions: + locked_dependency.set_constraint(dependency.constraint) + + dependency = locked_dependency + project_level_dependencies.add(dependency.name) dependencies.append(dependency) @@ -226,19 +234,43 @@ def __get_locked_package( # return only with project level dependencies return dependencies - nested_dependencies = list() + nested_dependencies = dict() - for pkg in packages: # type: Package - for requirement in pkg.requires: # type: Dependency - if requirement.name in project_level_dependencies: + def __walk_level( + __dependencies, __level + ): # type: (List[Dependency], int) -> None + if not __dependencies: + return + + __next_level = [] + + for requirement in __dependencies: + __locked_package = __get_locked_package(requirement) + + if __locked_package: + for require in __locked_package.requires: + if require.marker.is_empty(): + require.marker = requirement.marker + else: + require.marker = require.marker.intersect( + requirement.marker + ) + + require.marker = require.marker.intersect( + __locked_package.marker + ) + __next_level.append(require) + + if requirement.name in project_level_dependencies and __level == 0: # project level dependencies take precedence continue - locked_package = __get_locked_package(requirement) - if locked_package: + if __locked_package: # create dependency from locked package to retain dependency metadata # if this is not done, we can end-up with incorrect nested dependencies - requirement = locked_package.to_dependency() + marker = requirement.marker + requirement = __locked_package.to_dependency() + requirement.marker = requirement.marker.intersect(marker) else: # we make a copy to avoid any side-effects requirement = deepcopy(requirement) @@ -251,26 +283,26 @@ def __get_locked_package( ) # dependencies use extra to indicate that it was activated via parent - # package's extras - marker = requirement.marker.without_extras() - for project_requirement in project_requires: - if ( - pkg.name == project_requirement.name - and project_requirement.constraint.allows(pkg.version) - ): - requirement.marker = marker.intersect( - project_requirement.marker - ) - break + # package's extras, this is not required for nested exports as we assume + # the resolver already selected this dependency + requirement.marker = requirement.marker.without_extras().intersect( + pkg.marker + ) + + key = (requirement.name, requirement.pretty_constraint) + if key not in nested_dependencies: + nested_dependencies[key] = requirement else: - # this dependency was not from a project requirement - requirement.marker = marker.intersect(pkg.marker) + nested_dependencies[key].marker = nested_dependencies[ + key + ].marker.intersect(requirement.marker) + + return __walk_level(__next_level, __level + 1) - if requirement not in nested_dependencies: - nested_dependencies.append(requirement) + __walk_level(dependencies, 0) return sorted( - itertools.chain(dependencies, nested_dependencies), + itertools.chain(dependencies, nested_dependencies.values()), key=lambda x: x.name.lower(), ) diff --git a/tests/utils/test_exporter.py b/tests/utils/test_exporter.py index a75fb3da502..66b9e81ff53 100644 --- a/tests/utils/test_exporter.py +++ b/tests/utils/test_exporter.py @@ -2,6 +2,7 @@ import pytest +from poetry.core.packages import dependency_from_pep_508 from poetry.core.toml.file import TOMLFile from poetry.factory import Factory from poetry.packages import Locker as BaseLocker @@ -175,6 +176,145 @@ def test_exporter_can_export_requirements_txt_with_standard_packages_and_markers assert expected == content +def test_exporter_can_export_requirements_txt_with_nested_packages_and_markers( + tmp_dir, poetry +): + poetry.locker.mock_lock_data( + { + "package": [ + { + "name": "a", + "version": "1.2.3", + "category": "main", + "optional": False, + "python-versions": "*", + "marker": "python_version < '3.7'", + "dependencies": {"b": ">=0.0.0", "c": ">=0.0.0"}, + }, + { + "name": "b", + "version": "4.5.6", + "category": "main", + "optional": False, + "python-versions": "*", + "marker": "platform_system == 'Windows'", + "dependencies": {"d": ">=0.0.0"}, + }, + { + "name": "c", + "version": "7.8.9", + "category": "main", + "optional": False, + "python-versions": "*", + "marker": "sys_platform == 'win32'", + "dependencies": {"d": ">=0.0.0"}, + }, + { + "name": "d", + "version": "0.0.1", + "category": "main", + "optional": False, + "python-versions": "*", + }, + ], + "metadata": { + "python-versions": "*", + "content-hash": "123456789", + "hashes": {"a": [], "b": [], "c": [], "d": []}, + }, + } + ) + set_package_requires(poetry, skip={"b", "c", "d"}) + + exporter = Exporter(poetry) + + exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt") + + with (Path(tmp_dir) / "requirements.txt").open(encoding="utf-8") as f: + content = f.read() + + expected = { + "a": dependency_from_pep_508("a==1.2.3; python_version < '3.7'"), + "b": dependency_from_pep_508( + "b==4.5.6; platform_system == 'Windows' and python_version < '3.7'" + ), + "c": dependency_from_pep_508( + "c==7.8.9; sys_platform == 'win32' and python_version < '3.7'" + ), + "d": dependency_from_pep_508( + "d==0.0.1; python_version < '3.7' and platform_system == 'Windows' and sys_platform == 'win32'" + ), + } + + for line in content.strip().split("\n"): + dependency = dependency_from_pep_508(line) + assert dependency.name in expected + expected_dependency = expected.pop(dependency.name) + assert dependency == expected_dependency + assert dependency.marker == expected_dependency.marker + + assert expected == {} + + +def test_exporter_can_export_requirements_txt_with_nested_packages_and_markers_any( + tmp_dir, poetry +): + poetry.locker.mock_lock_data( + { + "package": [ + { + "name": "a", + "version": "1.2.3", + "category": "main", + "optional": False, + "python-versions": "*", + }, + { + "name": "b", + "version": "4.5.6", + "category": "dev", + "optional": False, + "python-versions": "*", + "dependencies": {"a": ">=1.2.3"}, + }, + ], + "metadata": { + "python-versions": "*", + "content-hash": "123456789", + "hashes": {"a": [], "b": []}, + }, + } + ) + + poetry.package.requires = [ + Factory.create_dependency( + name="a", constraint=dict(version="^1.2.3", python="<3.8") + ), + ] + poetry.package.dev_requires = [ + Factory.create_dependency( + name="b", constraint=dict(version="^4.5.6"), category="dev" + ), + Factory.create_dependency(name="a", constraint=dict(version="^1.2.3")), + ] + + exporter = Exporter(poetry) + + exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt", dev=True) + + with (Path(tmp_dir) / "requirements.txt").open(encoding="utf-8") as f: + content = f.read() + + assert ( + content + == """\ +a==1.2.3 +a==1.2.3; python_version < "3.8" +b==4.5.6 +""" + ) + + def test_exporter_can_export_requirements_txt_with_standard_packages_and_hashes( tmp_dir, poetry ):