diff --git a/src/python/pants/backend/python/macros/poetry_requirements.py b/src/python/pants/backend/python/macros/poetry_requirements.py new file mode 100644 index 00000000000..b91c7303339 --- /dev/null +++ b/src/python/pants/backend/python/macros/poetry_requirements.py @@ -0,0 +1,280 @@ +# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +import itertools +import logging +import os +from pathlib import Path +from typing import Any, Iterable, Mapping, Optional + +import toml +from packaging.version import InvalidVersion, Version +from pkg_resources import Requirement + +from pants.base.build_environment import get_buildroot + +logger = logging.getLogger(__name__) + + +def get_max_caret(parsed_version: Version) -> str: + major = 0 + minor = 0 + micro = 0 + + if parsed_version.major != 0: + major = parsed_version.major + 1 + elif parsed_version.minor != 0: + minor = parsed_version.minor + 1 + elif parsed_version.micro != 0: + micro = parsed_version.micro + 1 + else: + base_len = len(parsed_version.base_version.split(".")) + if base_len >= 3: + micro = 1 + elif base_len == 2: + minor = 1 + elif base_len == 1: + major = 1 + + return f"{major}.{minor}.{micro}" + + +def get_max_tilde(parsed_version: Version) -> str: + major = 0 + minor = 0 + base_len = len(parsed_version.base_version.split(".")) + if base_len >= 2: + minor = int(str(parsed_version.minor)) + 1 + major = int(str(parsed_version.major)) + elif base_len == 1: + major = int(str(parsed_version.major)) + 1 + + return f"{major}.{minor}.0" + + +def parse_str_version(proj_name: str, attributes: str, fp: str) -> str: + valid_specifiers = "<>!~=" + pep440_reqs = [] + comma_split_reqs = (i.strip() for i in attributes.split(",")) + for req in comma_split_reqs: + is_caret = req[0] == "^" + # ~= is an acceptable default operator; however, ~ is not, and IS NOT the same as ~= + is_tilde = req[0] == "~" and req[1] != "=" + if is_caret or is_tilde: + try: + parsed_version = Version(req[1:]) + except InvalidVersion: + raise InvalidVersion( + ( + f'Failed to parse requirement {proj_name} = "{req}" in {fp}' + "loaded by the poetry_requirements macro.\n\nIf you believe this requirement is " + "valid, consider opening an issue at https://github.com/pantsbuild/pants/issues" + "so that we can update Pants's Poetry macro to support this." + ) + ) + + max_ver = get_max_caret(parsed_version) if is_caret else get_max_tilde(parsed_version) + min_ver = f"{parsed_version.public}" + pep440_reqs.append(f">={min_ver},<{max_ver}") + else: + pep440_reqs.append(req if req[0] in valid_specifiers else f"=={req}") + return f"{proj_name} {','.join(pep440_reqs)}" + + +def parse_python_constraint(constr: str | None, fp: str) -> str: + if constr is None: + return "" + valid_specifiers = "<>!~= " + or_and_split = [[j.strip() for j in i.split(",")] for i in constr.split("||")] + ver_parsed = [[parse_str_version("", j, fp) for j in i] for i in or_and_split] + + def conv_and(lst: list[str]) -> list: + return list(itertools.chain(*[i.split(",") for i in lst])) + + def prepend(version: str) -> str: + return ( + f"python_version{''.join(i for i in version if i in valid_specifiers)} '" + f"{''.join(i for i in version if i not in valid_specifiers)}'" + ) + + prepend_and_clean = [ + [prepend(".".join(j.split(".")[:2])) for j in conv_and(i)] for i in ver_parsed + ] + return ( + f"{'(' if len(or_and_split) > 1 else ''}" + f"{') or ('.join([' and '.join(i) for i in prepend_and_clean])}" + f"{')' if len(or_and_split) > 1 else ''}" + ) + + +def handle_dict_attr(proj_name: str, attributes: dict[str, str], fp: str) -> str: + def produce_match(sep: str, feat: Optional[str]) -> str: + return f"{sep}{feat}" if feat else "" + + git_lookup = attributes.get("git") + if git_lookup is not None: + rev_lookup = produce_match("#", attributes.get("rev")) + branch_lookup = produce_match("@", attributes.get("branch")) + tag_lookup = produce_match("@", attributes.get("tag")) + + return f"{proj_name} @ git+{git_lookup}{tag_lookup}{branch_lookup}{rev_lookup}" + + version_lookup = attributes.get("version") + path_lookup = attributes.get("path") + if path_lookup is not None: + return f"{proj_name} @ file://{path_lookup}" + url_lookup = attributes.get("url") + if url_lookup is not None: + return f"{proj_name} @ {url_lookup}" + if version_lookup is not None: + markers_lookup = produce_match(";", attributes.get("markers")) + python_lookup = parse_python_constraint(attributes.get("python"), fp) + version_parsed = parse_str_version(proj_name, version_lookup, fp) + return ( + f"{version_parsed}" + f"{markers_lookup}" + f"{' and ' if python_lookup and markers_lookup else (';' if python_lookup else '')}" + f"{python_lookup}" + ) + else: + raise AssertionError( + ( + f"{proj_name} is not formatted correctly; at" + " minimum provide either a version, url, path or git location for" + " your dependency. " + ) + ) + + +def parse_single_dependency( + proj_name: str, attributes: str | dict[str, Any] | list[dict[str, Any]], fp: str +) -> tuple[Requirement, ...]: + if isinstance(attributes, str): + # E.g. `foo = "~1.1~'. + return (Requirement.parse(parse_str_version(proj_name, attributes, fp)),) + elif isinstance(attributes, dict): + # E.g. `foo = {version = "~1.1"}`. + return (Requirement.parse(handle_dict_attr(proj_name, attributes, fp)),) + elif isinstance(attributes, list): + # E.g. ` foo = [{version = "1.1","python" = "2.7"}, {version = "1.1","python" = "2.7"}] + return tuple( + Requirement.parse(handle_dict_attr(proj_name, attr, fp)) for attr in attributes + ) + else: + raise AssertionError( + ( + "Error: invalid poetry requirement format. Expected " + " type of requirement attributes to be string," + f"dict, or list, but was of type {type(attributes).__name__}." + ) + ) + + +def parse_pyproject_toml(toml_contents: str, file_path: str) -> set[Requirement]: + parsed = toml.loads(toml_contents) + try: + poetry_vals = parsed["tool"]["poetry"] + except KeyError: + raise KeyError( + ( + f"No section `tool.poetry` found in {file_path}, which" + "is loaded by Pants from a `poetry_requirements` macro. " + "Did you mean to set up Poetry?" + ) + ) + dependencies = poetry_vals.get("dependencies", {}) + dev_dependencies = poetry_vals.get("dev-dependencies", {}) + if not dependencies and not dev_dependencies: + logger.warning( + ( + "No requirements defined in poetry.tools.dependencies and" + f" poetry.tools.dev-dependencies in {file_path}, which is loaded by Pants" + " from a poetry_requirements macro. Did you mean to populate these" + " with requirements?" + ) + ) + + return set( + itertools.chain.from_iterable( + parse_single_dependency(proj, attr, file_path) + for proj, attr in {**dependencies, **dev_dependencies}.items() + ) + ) + + +class PoetryRequirements: + """Translates dependencies specified in a pyproject.toml Poetry file to a set of + "python_requirements_library" targets. + + For example, if pyproject.toml contains the following entries under + poetry.tool.dependencies: `foo = ">1"` and `bar = ">2.4"`, + + python_requirement_library( + name="foo", + requirements=["foo>1"], + ) + + python_requirement_library( + name="bar", + requirements=["bar>2.4"], + ) + + See Poetry documentation for correct specification of pyproject.toml: + https://python-poetry.org/docs/pyproject/ + + You may also use the parameter `module_mapping` to teach Pants what modules each of your + requirements provide. For any requirement unspecified, Pants will default to the name of the + requirement. This setting is important for Pants to know how to convert your import + statements back into your dependencies. For example: + + python_requirements( + module_mapping={ + "ansicolors": ["colors"], + "setuptools": ["pkg_resources"], + } + ) + """ + + def __init__(self, parse_context): + self._parse_context = parse_context + + def __call__( + self, + pyproject_toml_relpath: str = "pyproject.toml", + *, + module_mapping: Optional[Mapping[str, Iterable[str]]] = None, + ) -> None: + """ + :param pyproject_toml_relpath: The relpath from this BUILD file to the requirements file. + Defaults to a `requirements.txt` file sibling to the BUILD file. + :param module_mapping: a mapping of requirement names to a list of the modules they provide. + For example, `{"ansicolors": ["colors"]}`. Any unspecified requirements will use the + requirement name as the default module, e.g. "Django" will default to + `modules=["django"]`. + """ + req_file_tgt = self._parse_context.create_object( + "_python_requirements_file", + name=pyproject_toml_relpath.replace(os.path.sep, "_"), + sources=[pyproject_toml_relpath], + ) + requirements_dep = f":{req_file_tgt.name}" + + req_file = Path(get_buildroot(), self._parse_context.rel_path, pyproject_toml_relpath) + requirements = parse_pyproject_toml( + req_file.read_text(), str(req_file.relative_to(get_buildroot())) + ) + for parsed_req in requirements: + req_module_mapping = ( + {parsed_req.project_name: module_mapping[parsed_req.project_name]} + if module_mapping and parsed_req.project_name in module_mapping + else None + ) + self._parse_context.create_object( + "python_requirement_library", + name=parsed_req.project_name, + requirements=[parsed_req], + module_mapping=req_module_mapping, + dependencies=[requirements_dep], + ) diff --git a/src/python/pants/backend/python/macros/poetry_requirements_test.py b/src/python/pants/backend/python/macros/poetry_requirements_test.py new file mode 100644 index 00000000000..c3e80250d32 --- /dev/null +++ b/src/python/pants/backend/python/macros/poetry_requirements_test.py @@ -0,0 +1,393 @@ +# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from textwrap import dedent +from typing import Any, Dict, Iterable + +import pytest +from packaging.version import Version +from pkg_resources import Requirement + +from pants.backend.python.macros.poetry_requirements import ( + PoetryRequirements, + get_max_caret, + get_max_tilde, + handle_dict_attr, + parse_pyproject_toml, + parse_single_dependency, + parse_str_version, +) +from pants.backend.python.target_types import PythonRequirementLibrary, PythonRequirementsFile +from pants.base.specs import AddressSpecs, DescendantAddresses, FilesystemSpecs, Specs +from pants.engine.addresses import Address +from pants.engine.internals.scheduler import ExecutionError +from pants.engine.target import Targets +from pants.testutil.rule_runner import QueryRule, RuleRunner + + +@pytest.mark.parametrize( + "test, exp", + [ + ("1.0.0-rc0", "2.0.0"), + ("1.2.3.dev0", "2.0.0"), + ("1.2.3-dev0", "2.0.0"), + ("1.2.3dev0", "2.0.0"), + ("1.2.3", "2.0.0"), + ("1.2", "2.0.0"), + ("1", "2.0.0"), + ("0.2.3", "0.3.0"), + ("0.0.3", "0.0.4"), + ("0.0", "0.1.0"), + ("0", "1.0.0"), + ], +) +def test_caret(test, exp) -> None: + version = Version(test) + assert get_max_caret(version) == exp + + +@pytest.mark.parametrize( + "test, exp", + [ + ("1.2.3", "1.3.0"), + ("1.2", "1.3.0"), + ("1", "2.0.0"), + ("0", "1.0.0"), + ("1.2.3.rc1", "1.3.0"), + ("1.2.3rc1", "1.3.0"), + ("1.2.3-rc1", "1.3.0"), + ], +) +def test_max_tilde(test, exp) -> None: + version = Version(test) + assert get_max_tilde(version) == exp + + +@pytest.mark.parametrize( + "test, exp", + [ + ("~1.0.0rc0", ">=1.0.0rc0,<1.1.0"), + ("^1.0.0rc0", ">=1.0.0rc0,<2.0.0"), + ("~1.2.3", ">=1.2.3,<1.3.0"), + ("^1.2.3", ">=1.2.3,<2.0.0"), + ("~=1.2.3", "~=1.2.3"), + ("1.2.3", "==1.2.3"), + (">1.2.3", ">1.2.3"), + ("~1.2, !=1.2.10", ">=1.2,<1.3.0,!=1.2.10"), + ], +) +def test_handle_str(test, exp) -> None: + assert parse_str_version("foo", test, "") == f"foo {exp}" + + +def test_handle_git() -> None: + def assert_git(extra_opts: Dict[str, str], suffix: str) -> None: + attr = {"git": "https://github.com/requests/requests.git", **extra_opts} + assert ( + handle_dict_attr("requests", attr, "") + == f"requests @ git+https://github.com/requests/requests.git{suffix}" + ) + + assert_git({}, "") + assert_git({"branch": "main"}, "@main") + assert_git({"tag": "v1.1.1"}, "@v1.1.1") + assert_git({"rev": "1a2b3c4d"}, "#1a2b3c4d") + + +def test_handle_path_arg() -> None: + attr = {"path": "../../my_py_proj.whl"} + assert handle_dict_attr("my_py_proj", attr, "") == "my_py_proj @ file://../../my_py_proj.whl" + + +def test_handle_url_arg() -> None: + attr = {"url": "https://my-site.com/mydep.whl"} + assert handle_dict_attr("my_py_proj", attr, "") == "my_py_proj @ https://my-site.com/mydep.whl" + + +def test_version_only() -> None: + attr = {"version": "1.2.3"} + assert handle_dict_attr("foo", attr, "") == "foo ==1.2.3" + + +def test_py_constraints() -> None: + def assert_py_constraints(py_req: str, suffix: str) -> None: + attr = {"version": "1.2.3", "python": py_req} + assert handle_dict_attr("foo", attr, "") == f"foo ==1.2.3;{suffix}" + + assert_py_constraints("3.6", "python_version == '3.6'") + assert_py_constraints("3.6 || 3.7", "(python_version == '3.6') or (python_version == '3.7')") + assert_py_constraints(">3.6,!=3.7", "python_version > '3.6' and python_version != '3.7'") + assert_py_constraints( + ">3.6 || 3.5,3.4", + "(python_version > '3.6') or (python_version == '3.5' and python_version == '3.4')", + ) + assert_py_constraints( + "~3.6 || ^3.7", + "(python_version >= '3.6' and python_version< '3.7') or (python_version >= '3.7' and python_version< '4.0')", + ) + + +def test_multi_version_const() -> None: + lst_attr = [{"version": "1.2.3", "python": "3.6"}, {"version": "1.2.4", "python": "3.7"}] + retval = parse_single_dependency("foo", lst_attr, "") + actual_reqs = ( + Requirement.parse("foo ==1.2.3; python_version == '3.6'"), + Requirement.parse("foo ==1.2.4; python_version == '3.7'"), + ) + assert retval == actual_reqs + + +def test_extended_form() -> None: + toml_black_str = """ + [tool.poetry.dependencies] + [tool.poetry.dependencies.black] + version = "19.10b0" + python = "3.6" + markers = "platform_python_implementation == 'CPython'" + [tool.poetry.dev-dependencies] + """ + retval = parse_pyproject_toml(toml_black_str, "/path/to/file") + actual_req = { + Requirement.parse( + 'black==19.10b0; platform_python_implementation == "CPython" and python_version == "3.6"' + ) + } + assert retval == actual_req + + +def test_parse_multi_reqs() -> None: + toml_str = """[tool.poetry] + name = "poetry_tinker" + version = "0.1.0" + description = "" + authors = ["Liam Wilson "] + + [tool.poetry.dependencies] + python = "^3.8" + junk = {url = "https://github.com/myrepo/junk.whl"} + poetry = {git = "https://github.com/python-poetry/poetry.git", tag = "v1.1.1"} + requests = {extras = ["security"], version = "^2.25.1", python = ">2.7"} + foo = [{version = ">=1.9", python = "^2.7"},{version = "^2.0", python = "3.4 || 3.5"}] + + [tool.poetry.dependencies.black] + version = "19.10b0" + python = "3.6" + markers = "platform_python_implementation == 'CPython'" + + [tool.poetry.dev-dependencies] + isort = ">=5.5.1,<5.6" + + [build-system] + requires = ["poetry-core>=1.0.0"] + build-backend = "poetry.core.masonry.api" + """ + retval = parse_pyproject_toml(toml_str, "/path/to/file") + actual_reqs = { + Requirement.parse("python<4.0.0,>=3.8"), + Requirement.parse("junk@ https://github.com/myrepo/junk.whl"), + Requirement.parse("poetry@ git+https://github.com/python-poetry/poetry.git@v1.1.1"), + Requirement.parse('requests<3.0.0,>=2.25.1; python_version > "2.7"'), + Requirement.parse('foo>=1.9; python_version >= "2.7" and python_version < "3.0"'), + Requirement.parse('foo<3.0.0,>=2.0; python_version == "3.4" or python_version == "3.5"'), + Requirement.parse( + 'black==19.10b0; platform_python_implementation == "CPython" and python_version == "3.6"' + ), + Requirement.parse("isort<5.6,>=5.5.1"), + } + assert retval == actual_reqs + + +@pytest.fixture +def rule_runner() -> RuleRunner: + return RuleRunner( + rules=[QueryRule(Targets, (Specs,))], + target_types=[PythonRequirementLibrary, PythonRequirementsFile], + context_aware_object_factories={"poetry_requirements": PoetryRequirements}, + ) + + +def assert_poetry_requirements( + rule_runner: RuleRunner, + build_file_entry: str, + pyproject_toml: str, + *, + expected_file_dep: PythonRequirementsFile, + expected_targets: Iterable[PythonRequirementLibrary], + pyproject_toml_relpath: str = "pyproject.toml", +) -> None: + rule_runner.add_to_build_file("", f"{build_file_entry}\n") + rule_runner.create_file(pyproject_toml_relpath, pyproject_toml) + targets = rule_runner.request( + Targets, + [Specs(AddressSpecs([DescendantAddresses("")]), FilesystemSpecs([]))], + ) + assert {expected_file_dep, *expected_targets} == set(targets) + + +def test_pyproject_toml(rule_runner: RuleRunner) -> None: + """This tests that we correctly create a new python_requirement_library for each entry in a + pyproject.toml file. + + Note that this just ensures proper targets are created; see prior tests for specific parsing + edge cases. + """ + assert_poetry_requirements( + rule_runner, + "poetry_requirements(module_mapping={'ansicolors': ['colors']})", + dedent( + """\ + [tool.poetry.dependencies] + Django = {version = "3.2", python = "3"} + Un-Normalized-PROJECT = "1.0.0" + [tool.poetry.dev-dependencies] + ansicolors = ">=1.18.0" + """ + ), + expected_file_dep=PythonRequirementsFile( + {"sources": ["pyproject.toml"]}, + address=Address("", target_name="pyproject.toml"), + ), + expected_targets=[ + PythonRequirementLibrary( + { + "dependencies": [":pyproject.toml"], + "requirements": [Requirement.parse("ansicolors>=1.18.0")], + "module_mapping": {"ansicolors": ["colors"]}, + }, + address=Address("", target_name="ansicolors"), + ), + PythonRequirementLibrary( + { + "dependencies": [":pyproject.toml"], + "requirements": [Requirement.parse("Django==3.2 ; python_version == '3'")], + }, + address=Address("", target_name="Django"), + ), + PythonRequirementLibrary( + { + "dependencies": [":pyproject.toml"], + "requirements": [Requirement.parse("Un_Normalized_PROJECT == 1.0.0")], + }, + address=Address("", target_name="Un-Normalized-PROJECT"), + ), + ], + ) + + +def test_relpath_override(rule_runner: RuleRunner) -> None: + assert_poetry_requirements( + rule_runner, + "poetry_requirements(pyproject_toml_relpath='subdir/pyproject.toml')", + dedent( + """\ + [tool.poetry.dependencies] + ansicolors = ">=1.18.0" + [tool.poetry.dev-dependencies] + """ + ), + pyproject_toml_relpath="subdir/pyproject.toml", + expected_file_dep=PythonRequirementsFile( + {"sources": ["subdir/pyproject.toml"]}, + address=Address("", target_name="subdir_pyproject.toml"), + ), + expected_targets=[ + PythonRequirementLibrary( + { + "dependencies": [":subdir_pyproject.toml"], + "requirements": [Requirement.parse("ansicolors>=1.18.0")], + }, + address=Address("", target_name="ansicolors"), + ), + ], + ) + + +def test_non_pep440_error(rule_runner: RuleRunner, caplog: Any) -> None: + with pytest.raises(ExecutionError) as exc: + assert_poetry_requirements( + rule_runner, + "poetry_requirements()", + """ + [tool.poetry.dependencies] + foo = "~r62b" + [tool.poetry.dev-dependencies] + """, + expected_file_dep=PythonRequirementsFile( + {"sources": ["pyproject.toml"]}, + address=Address("", target_name="pyproject.toml"), + ), + expected_targets=[], + ) + assert 'Failed to parse requirement foo = "~r62b" in pyproject.toml' in str(exc.value) + + +def test_no_req_defined_warning(rule_runner: RuleRunner, caplog: Any) -> None: + assert_poetry_requirements( + rule_runner, + "poetry_requirements()", + """ + [tool.poetry.dependencies] + [tool.poetry.dev-dependencies] + """, + expected_file_dep=PythonRequirementsFile( + {"sources": ["pyproject.toml"]}, + address=Address("", target_name="pyproject.toml"), + ), + expected_targets=[], + ) + assert "No requirements defined" in caplog.text + + +def test_bad_dict_format(rule_runner: RuleRunner) -> None: + with pytest.raises(ExecutionError) as exc: + assert_poetry_requirements( + rule_runner, + "poetry_requirements()", + """ + [tool.poetry.dependencies] + foo = {bad_req = "test"} + [tool.poetry.dev-dependencies] + """, + expected_file_dep=PythonRequirementsFile( + {"sources": ["pyproject.toml"]}, + address=Address("", target_name="pyproject.toml"), + ), + expected_targets=[], + ) + assert "not formatted correctly; at" in str(exc.value) + + +def test_bad_req_type(rule_runner: RuleRunner) -> None: + with pytest.raises(ExecutionError) as exc: + assert_poetry_requirements( + rule_runner, + "poetry_requirements()", + """ + [tool.poetry.dependencies] + foo = 4 + [tool.poetry.dev-dependencies] + """, + expected_file_dep=PythonRequirementsFile( + {"sources": ["pyproject.toml"]}, + address=Address("", target_name="pyproject.toml"), + ), + expected_targets=[], + ) + assert "was of type int" in str(exc.value) + + +def test_no_tool_poetry(rule_runner: RuleRunner) -> None: + with pytest.raises(ExecutionError) as exc: + assert_poetry_requirements( + rule_runner, + "poetry_requirements()", + """ + foo = 4 + """, + expected_file_dep=PythonRequirementsFile( + {"sources": ["pyproject.toml"]}, + address=Address("", target_name="pyproject.toml"), + ), + expected_targets=[], + ) + assert "`tool.poetry` found in pyproject.toml" in str(exc.value) diff --git a/src/python/pants/option/global_options.py b/src/python/pants/option/global_options.py index c36a9a9b446..cec1e47989f 100644 --- a/src/python/pants/option/global_options.py +++ b/src/python/pants/option/global_options.py @@ -1516,6 +1516,8 @@ def compute_pantsd_invalidation_globs( # macros should be adapted to allow this dependency to be automatically detected. "requirements.txt", "3rdparty/**/requirements.txt", + "pyproject.toml", + "3rdparty/**/pyproject.toml", *bootstrap_options.pantsd_invalidation_globs, ) )