From 7728308f1efa787f28a43162306908271f69a89c Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Wed, 8 Sep 2021 18:31:49 -0700 Subject: [PATCH 1/3] Warn if `[python-repos]` is set during lockfile generation # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- .../pants/backend/python/goals/lockfile.py | 22 +++++++++++++++++++ .../python/subsystems/python_tool_base.py | 6 ++++- src/python/pants/python/python_repos.py | 4 +++- 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/python/pants/backend/python/goals/lockfile.py b/src/python/pants/backend/python/goals/lockfile.py index ad81f28e53a..8d45338ba4a 100644 --- a/src/python/pants/backend/python/goals/lockfile.py +++ b/src/python/pants/backend/python/goals/lockfile.py @@ -47,6 +47,7 @@ from pants.engine.rules import Get, MultiGet, collect_rules, goal_rule, rule from pants.engine.target import TransitiveTargets, TransitiveTargetsRequest, UnexpandedTargets from pants.engine.unions import UnionMembership, union +from pants.python.python_repos import PythonRepos from pants.python.python_setup import PythonSetup from pants.util.logging import LogLevel from pants.util.ordered_set import FrozenOrderedSet @@ -327,7 +328,28 @@ async def generate_lockfiles_goal( union_membership: UnionMembership, generate_lockfiles_subsystem: GenerateLockfilesSubsystem, python_setup: PythonSetup, + python_repos: PythonRepos, ) -> GenerateLockfilesGoal: + failure_instructions = ( + "If lockfile generation fails, you can either disable lockfiles by setting " + "`[tool].lockfile = ''`, e.g. setting `[black].lockfile`, or you can manually " + "generate lockfiles, e.g. by using pip-compile or `pip freeze`. If manually generating " + "lockfiles, you should set `[python-setup].invalid_lockfile_behavior = 'ignore'." + ) + if python_repos.repos: + logger.warning( + "The option `[python-repos].repos` is configured, but it does not currently work with " + "lockfile generation because Pants is currently using Poetry for lockfile generation " + "and Poetry does not support `--find-links`. Lockfile generation may fail.\n\n" + f"{failure_instructions}" + ) + if python_repos.indexes != [python_repos.pypi_index]: + logger.warning( + "The option `[python-repos].indexes` is configured, but is not currently wired up to " + "lockfile generation. Lockfile generation may fail.\n\n" + f"{failure_instructions}" + ) + specified_user_resolves, specified_tool_sentinels = determine_resolves_to_generate( python_setup.resolves_to_lockfiles.keys(), union_membership[PythonToolLockfileSentinel], diff --git a/src/python/pants/backend/python/subsystems/python_tool_base.py b/src/python/pants/backend/python/subsystems/python_tool_base.py index ad68a5c4e22..e7ad7e14200 100644 --- a/src/python/pants/backend/python/subsystems/python_tool_base.py +++ b/src/python/pants/backend/python/subsystems/python_tool_base.py @@ -110,7 +110,11 @@ def register_options(cls, register): "To use a custom lockfile, set this option to a file path relative to the " f"build root, then run `./pants generate-lockfiles " f"--resolve={cls.options_scope}`.\n\n" - "" + "Lockfile generation currently does not wire up the `[python-repos]` options. " + "If lockfile generation fails, you can set this option to a manually " + "generated lockfile path, such as using pip-compile or `pip freeze` to " + "generate the lockfile. When manually maintaining lockfiles, set " + "`[python-setup].invalid_lockfile_behavior = 'ignore'`." ), ) diff --git a/src/python/pants/python/python_repos.py b/src/python/pants/python/python_repos.py index 476ec968efb..1c6988ae7c0 100644 --- a/src/python/pants/python/python_repos.py +++ b/src/python/pants/python/python_repos.py @@ -13,6 +13,8 @@ class PythonRepos(Subsystem): "custom cheeseshops when resolving requirements." ) + pypi_index = "https://pypi.org/simple/" + @classmethod def register_options(cls, register): super().register_options(register) @@ -30,7 +32,7 @@ def register_options(cls, register): "--indexes", advanced=True, type=list, - default=["https://pypi.org/simple/"], + default=[cls.pypi_index], help=( "URLs of code repository indexes to look for requirements. If set to an empty " "list, then Pex will use no indices (meaning it will not use PyPI). The values " From 6041ba7bf54dd5214b491e6a3a38216bbcb26dbd Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Thu, 9 Sep 2021 12:03:37 -0700 Subject: [PATCH 2/3] Review feedback # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- .../pants/backend/python/goals/lockfile.py | 31 +++++++++---------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/src/python/pants/backend/python/goals/lockfile.py b/src/python/pants/backend/python/goals/lockfile.py index 8d45338ba4a..0b5d9a0acfa 100644 --- a/src/python/pants/backend/python/goals/lockfile.py +++ b/src/python/pants/backend/python/goals/lockfile.py @@ -330,25 +330,10 @@ async def generate_lockfiles_goal( python_setup: PythonSetup, python_repos: PythonRepos, ) -> GenerateLockfilesGoal: - failure_instructions = ( - "If lockfile generation fails, you can either disable lockfiles by setting " - "`[tool].lockfile = ''`, e.g. setting `[black].lockfile`, or you can manually " - "generate lockfiles, e.g. by using pip-compile or `pip freeze`. If manually generating " - "lockfiles, you should set `[python-setup].invalid_lockfile_behavior = 'ignore'." - ) if python_repos.repos: - logger.warning( - "The option `[python-repos].repos` is configured, but it does not currently work with " - "lockfile generation because Pants is currently using Poetry for lockfile generation " - "and Poetry does not support `--find-links`. Lockfile generation may fail.\n\n" - f"{failure_instructions}" - ) + warn_python_repos("repos") if python_repos.indexes != [python_repos.pypi_index]: - logger.warning( - "The option `[python-repos].indexes` is configured, but is not currently wired up to " - "lockfile generation. Lockfile generation may fail.\n\n" - f"{failure_instructions}" - ) + warn_python_repos("indexes") specified_user_resolves, specified_tool_sentinels = determine_resolves_to_generate( python_setup.resolves_to_lockfiles.keys(), @@ -381,6 +366,18 @@ async def generate_lockfiles_goal( return GenerateLockfilesGoal(exit_code=0) +def warn_python_repos(option: str) -> None: + logger.warning( + f"The option `[python-repos].{option}` is configured, but it does not currently work " + "with lockfile generation. Lockfile generation will fail if the relevant requirements " + "cannot be located on PyPI.\n\n" + "If lockfile generation fails, you can either disable lockfiles by setting " + "`[tool].lockfile = ''`, e.g. setting `[black].lockfile`, or you can manually " + "generate lockfiles, e.g. by using pip-compile or `pip freeze`. If manually generating " + "lockfiles, you should set `[python-setup].invalid_lockfile_behavior = 'ignore'." + ) + + class AmbiguousResolveNamesError(Exception): def __init__(self, ambiguous_names: list[str]) -> None: if len(ambiguous_names) == 1: From 82db30c1229dd9cb6a4ed54b1b151baf61136beb Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Thu, 9 Sep 2021 12:13:45 -0700 Subject: [PATCH 3/3] Chris's rewording of manual lockfiles # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- src/python/pants/backend/python/goals/lockfile.py | 9 +++++---- .../pants/backend/python/subsystems/python_tool_base.py | 6 +++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/python/pants/backend/python/goals/lockfile.py b/src/python/pants/backend/python/goals/lockfile.py index 0b5d9a0acfa..8f2b2297c9b 100644 --- a/src/python/pants/backend/python/goals/lockfile.py +++ b/src/python/pants/backend/python/goals/lockfile.py @@ -371,10 +371,11 @@ def warn_python_repos(option: str) -> None: f"The option `[python-repos].{option}` is configured, but it does not currently work " "with lockfile generation. Lockfile generation will fail if the relevant requirements " "cannot be located on PyPI.\n\n" - "If lockfile generation fails, you can either disable lockfiles by setting " - "`[tool].lockfile = ''`, e.g. setting `[black].lockfile`, or you can manually " - "generate lockfiles, e.g. by using pip-compile or `pip freeze`. If manually generating " - "lockfiles, you should set `[python-setup].invalid_lockfile_behavior = 'ignore'." + "If lockfile generation fails, you can disable lockfiles by setting " + "`[tool].lockfile = ''`, e.g. setting `[black].lockfile`. You can also manually " + "generate a lockfile, such as by using pip-compile or `pip freeze`. Set the " + "`[tool].lockfile` option to the path you manually generated. When manually maintaining " + "lockfiles, set `[python-setup].invalid_lockfile_behavior = 'ignore'." ) diff --git a/src/python/pants/backend/python/subsystems/python_tool_base.py b/src/python/pants/backend/python/subsystems/python_tool_base.py index e7ad7e14200..fbf2dab1db7 100644 --- a/src/python/pants/backend/python/subsystems/python_tool_base.py +++ b/src/python/pants/backend/python/subsystems/python_tool_base.py @@ -111,9 +111,9 @@ def register_options(cls, register): f"build root, then run `./pants generate-lockfiles " f"--resolve={cls.options_scope}`.\n\n" "Lockfile generation currently does not wire up the `[python-repos]` options. " - "If lockfile generation fails, you can set this option to a manually " - "generated lockfile path, such as using pip-compile or `pip freeze` to " - "generate the lockfile. When manually maintaining lockfiles, set " + "If lockfile generation fails, you can manually generate a lockfile, such as " + "by using pip-compile or `pip freeze`. Set this option to the path to your " + "manually generated lockfile. When manually maintaining lockfiles, set " "`[python-setup].invalid_lockfile_behavior = 'ignore'`." ), )