From e1dd9812fcabf429549c22c2ef8c6b71aacab2c1 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Tue, 14 Sep 2021 11:12:29 -0700 Subject: [PATCH] Add new version of `LockfileMetadata` to support checking for identically specified requirements (#12782) This factors out versioning capabilities into `LockfileMetadata` so that it's possible to easily change the set of validation requirements for a lockfile. V1 represents the original lockfile version (where constraints and an invalidation digest are set). V2 allows for the old behaviour, but also allows specifying the input requirements for a lockfile, and verifying that the user requirements are ~a non-strict subset of~ identical to the input requirements. We decided to replace the requirements hex digest with requirements strings to allow us to test whether the lockfile produces a _compatible_ environment rather than an _identical_ environment, which will be useful for user lockfile support when we eventually enable that. In the meantime, tool lockfiles still test for an identical environment, but the extra data in the lockfile will allow for more fine-grained error messages in a future version. The implementations of `_from_json_dict` and `is_valid_for` are a bit repetitive; I can factor out the common behaviour with a bit of work, but given we expect to delete the V1 implementation before too long. Currently this _does not_ add the `platforms` capability to the header, but now it's going to be easy enough to bump the version number if we want to add more fields. Closes #12610 --- 3rdparty/python/lockfiles/flake8.txt | 8 +- 3rdparty/python/lockfiles/pytest.txt | 17 +- 3rdparty/python/lockfiles/user_reqs.txt | 26 +- .../python/mypy_protobuf_lockfile.txt | 6 +- .../pants/backend/python/goals/lockfile.py | 8 +- .../backend/python/lint/bandit/lockfile.txt | 8 +- .../backend/python/lint/black/lockfile.txt | 6 +- .../python/lint/docformatter/lockfile.txt | 6 +- .../backend/python/lint/flake8/lockfile.txt | 6 +- .../backend/python/lint/isort/lockfile.txt | 6 +- .../backend/python/lint/pylint/lockfile.txt | 6 +- .../backend/python/lint/yapf/lockfile.txt | 7 +- .../subsystems/coverage_py_lockfile.txt | 6 +- .../python/subsystems/ipython_lockfile.txt | 12 +- .../python/subsystems/lambdex_lockfile.txt | 6 +- .../python/subsystems/pytest_lockfile.txt | 7 +- .../python/subsystems/python_tool_base.py | 3 + .../python/subsystems/setuptools_lockfile.txt | 7 +- .../python/typecheck/mypy/lockfile.txt | 6 +- .../python/util_rules/lockfile_metadata.py | 283 +++++++++++++++--- .../util_rules/lockfile_metadata_test.py | 71 ++++- .../pants/backend/python/util_rules/pex.py | 24 +- .../python/util_rules/pex_from_targets.py | 2 + .../backend/python/util_rules/pex_test.py | 47 ++- 24 files changed, 488 insertions(+), 96 deletions(-) diff --git a/3rdparty/python/lockfiles/flake8.txt b/3rdparty/python/lockfiles/flake8.txt index 261735016be..0c065995c15 100644 --- a/3rdparty/python/lockfiles/flake8.txt +++ b/3rdparty/python/lockfiles/flake8.txt @@ -4,10 +4,14 @@ # # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- # { -# "version": 1, -# "requirements_invalidation_digest": "815457a1baf6226c993e5468ccdf64c69fe7214d3d9237911c762733e0130526", +# "version": 2, # "valid_for_interpreter_constraints": [ # "CPython<3.10,>=3.7" +# ], +# "generated_with_requirements": [ +# "flake8-2020<1.7.0,>=1.6.0", +# "flake8-pantsbuild<3,>=2.0", +# "flake8<4.0,>=3.9.2" # ] # } # --- END PANTS LOCKFILE METADATA --- diff --git a/3rdparty/python/lockfiles/pytest.txt b/3rdparty/python/lockfiles/pytest.txt index 8170eb6bd21..536731e62c1 100644 --- a/3rdparty/python/lockfiles/pytest.txt +++ b/3rdparty/python/lockfiles/pytest.txt @@ -4,10 +4,17 @@ # # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- # { -# "version": 1, -# "requirements_invalidation_digest": "a2057d396be0480d2586be2da25a021149a4f96276b853dd92cc63cfc3ae8503", +# "version": 2, # "valid_for_interpreter_constraints": [ # "CPython<3.10,>=3.7" +# ], +# "generated_with_requirements": [ +# "ipdb", +# "pygments", +# "pytest-cov<2.13,>=2.12.1", +# "pytest-html", +# "pytest-icdiff", +# "pytest<6.3,>=6.2.4" # ] # } # --- END PANTS LOCKFILE METADATA --- @@ -80,9 +87,9 @@ coverage==5.5; python_version >= "2.7" and python_full_version < "3.0.0" or pyth --hash=sha256:f030f8873312a16414c0d8e1a1ddff2d3235655a2174e3648b4fa66b3f2f1079 \ --hash=sha256:2a3859cb82dcbda1cfd3e6f71c27081d18aa251d20a17d87d26d4cd216fb0af4 \ --hash=sha256:ebe78fe9a0e874362175b02371bdfbee64d8edc42a044253ddf4ee7d3c15212c -decorator==5.0.9; python_version >= "3.7" \ - --hash=sha256:6e5c199c16f7a9f0e3a61a4a54b3d27e7dad0dbdde92b944426cb20914376323 \ - --hash=sha256:72ecfba4320a893c53f9706bebb2d55c270c1e51a28789361aa93e4a21319ed5 +decorator==5.1.0; python_version >= "3.7" \ + --hash=sha256:7b12e7c3c6ab203a29e157335e9122cb03de9ab7264b137594103fd4a683b374 \ + --hash=sha256:e59913af105b9860aa2c8d3272d9de5a56a4e608db9a2f167a8480b323d529a7 icdiff==2.0.4; python_version >= "3.6" \ --hash=sha256:c72572e5ce087bc7a7748af2664764d4a805897caeefb665bdc12677fefb2212 importlib-metadata==4.8.1; python_version < "3.8" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6") \ diff --git a/3rdparty/python/lockfiles/user_reqs.txt b/3rdparty/python/lockfiles/user_reqs.txt index 3321e75861b..902c323f2d8 100644 --- a/3rdparty/python/lockfiles/user_reqs.txt +++ b/3rdparty/python/lockfiles/user_reqs.txt @@ -4,10 +4,32 @@ # # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- # { -# "version": 1, -# "requirements_invalidation_digest": "2570870f165f546315ceb7ba2667cc764def9348baf7002ddb0a81c831cfd0fa", +# "version": 2, # "valid_for_interpreter_constraints": [ # "CPython<3.10,>=3.7" +# ], +# "generated_with_requirements": [ +# "PyYAML<5.5,>=5.4", +# "ansicolors==1.1.8", +# "fasteners==0.16", +# "freezegun==1.1.0", +# "humbug==0.2.6", +# "ijson==3.1.4", +# "packaging==21.0", +# "pex==2.1.48", +# "psutil==5.8.0", +# "pystache==0.5.4", +# "pytest<6.3,>=6.0.1", +# "requests[security]>=2.25.1", +# "setproctitle==1.2.2", +# "setuptools<58.0,>=56.0.0", +# "toml==0.10.2", +# "types-PyYAML==5.4.3", +# "types-freezegun==0.1.4", +# "types-requests==2.25.0", +# "types-setuptools==57.0.0", +# "types-toml==0.1.3", +# "typing-extensions==3.7.4.3" # ] # } # --- END PANTS LOCKFILE METADATA --- diff --git a/src/python/pants/backend/codegen/protobuf/python/mypy_protobuf_lockfile.txt b/src/python/pants/backend/codegen/protobuf/python/mypy_protobuf_lockfile.txt index 8850b880312..bf2a80581ef 100644 --- a/src/python/pants/backend/codegen/protobuf/python/mypy_protobuf_lockfile.txt +++ b/src/python/pants/backend/codegen/protobuf/python/mypy_protobuf_lockfile.txt @@ -4,10 +4,12 @@ # # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- # { -# "version": 1, -# "requirements_invalidation_digest": "babed61947e74aedbe0cdbaefdaec172db6d4a9d27e12acc80be5ab623e3acdf", +# "version": 2, # "valid_for_interpreter_constraints": [ # "CPython>=3.6" +# ], +# "generated_with_requirements": [ +# "mypy-protobuf==2.4" # ] # } # --- END PANTS LOCKFILE METADATA --- diff --git a/src/python/pants/backend/python/goals/lockfile.py b/src/python/pants/backend/python/goals/lockfile.py index 8f2b2297c9b..43df848d9a9 100644 --- a/src/python/pants/backend/python/goals/lockfile.py +++ b/src/python/pants/backend/python/goals/lockfile.py @@ -9,6 +9,8 @@ from pathlib import PurePath from typing import ClassVar, Iterable, Sequence, cast +from pkg_resources import Requirement + from pants.backend.python.subsystems.poetry import ( POETRY_LAUNCHER, PoetrySubsystem, @@ -233,7 +235,11 @@ async def generate_lockfile( initial_lockfile_digest_contents = await Get( DigestContents, Digest, poetry_export_result.output_digest ) - metadata = LockfileMetadata(req.requirements_hex_digest, req.interpreter_constraints) + # TODO(#12314) Improve error message on `Requirement.parse` + metadata = LockfileMetadata.new( + req.interpreter_constraints, + {Requirement.parse(i) for i in req.requirements}, + ) lockfile_with_header = metadata.add_header_to_lockfile( initial_lockfile_digest_contents[0].content, regenerate_command=( diff --git a/src/python/pants/backend/python/lint/bandit/lockfile.txt b/src/python/pants/backend/python/lint/bandit/lockfile.txt index 05cdba495b6..1c2c3029e15 100644 --- a/src/python/pants/backend/python/lint/bandit/lockfile.txt +++ b/src/python/pants/backend/python/lint/bandit/lockfile.txt @@ -4,10 +4,14 @@ # # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- # { -# "version": 1, -# "requirements_invalidation_digest": "b18a9f4d6a8cb4aafb414e9c8108d39dca053f8910ac801c147a932cd37e0040", +# "version": 2, # "valid_for_interpreter_constraints": [ # "CPython<4,>=3.6" +# ], +# "generated_with_requirements": [ +# "GitPython==3.1.18", +# "bandit<1.8,>=1.7.0", +# "setuptools" # ] # } # --- END PANTS LOCKFILE METADATA --- diff --git a/src/python/pants/backend/python/lint/black/lockfile.txt b/src/python/pants/backend/python/lint/black/lockfile.txt index 92cda9ec879..dad1e3542c3 100644 --- a/src/python/pants/backend/python/lint/black/lockfile.txt +++ b/src/python/pants/backend/python/lint/black/lockfile.txt @@ -4,10 +4,12 @@ # # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- # { -# "version": 1, -# "requirements_invalidation_digest": "1ba5f97d92f33b13e0cd56a960380fa6f616fae215d715ddd7a7ebf99795c890", +# "version": 2, # "valid_for_interpreter_constraints": [ # "CPython>=3.6.2" +# ], +# "generated_with_requirements": [ +# "black==21.8b0" # ] # } # --- END PANTS LOCKFILE METADATA --- diff --git a/src/python/pants/backend/python/lint/docformatter/lockfile.txt b/src/python/pants/backend/python/lint/docformatter/lockfile.txt index bd919a30549..83c9208218a 100644 --- a/src/python/pants/backend/python/lint/docformatter/lockfile.txt +++ b/src/python/pants/backend/python/lint/docformatter/lockfile.txt @@ -4,10 +4,12 @@ # # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- # { -# "version": 1, -# "requirements_invalidation_digest": "4ee50d37e5a334d7d496e9f7147dbbf19ee2eb0faaea33e46a553b6c692c7672", +# "version": 2, # "valid_for_interpreter_constraints": [ # "CPython>=3.6" +# ], +# "generated_with_requirements": [ +# "docformatter<1.5,>=1.4" # ] # } # --- END PANTS LOCKFILE METADATA --- diff --git a/src/python/pants/backend/python/lint/flake8/lockfile.txt b/src/python/pants/backend/python/lint/flake8/lockfile.txt index 9cc406bd707..bb8eb8df091 100644 --- a/src/python/pants/backend/python/lint/flake8/lockfile.txt +++ b/src/python/pants/backend/python/lint/flake8/lockfile.txt @@ -4,10 +4,12 @@ # # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- # { -# "version": 1, -# "requirements_invalidation_digest": "ebd4b2326fca44e0d0b362def86bddc1b57e6f6d6c4f077fef5e1218286d3883", +# "version": 2, # "valid_for_interpreter_constraints": [ # "CPython<4,>=3.6" +# ], +# "generated_with_requirements": [ +# "flake8<4.0,>=3.9.2" # ] # } # --- END PANTS LOCKFILE METADATA --- diff --git a/src/python/pants/backend/python/lint/isort/lockfile.txt b/src/python/pants/backend/python/lint/isort/lockfile.txt index 06e21bd2569..178c005996c 100644 --- a/src/python/pants/backend/python/lint/isort/lockfile.txt +++ b/src/python/pants/backend/python/lint/isort/lockfile.txt @@ -4,10 +4,12 @@ # # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- # { -# "version": 1, -# "requirements_invalidation_digest": "56fe21a2ac24d6f257d95eb97829e8c3cb65e2156cd987348ce837c055763302", +# "version": 2, # "valid_for_interpreter_constraints": [ # "CPython<4,>=3.7" +# ], +# "generated_with_requirements": [ +# "isort[colors,pyproject]<6.0,>=5.9.3" # ] # } # --- END PANTS LOCKFILE METADATA --- diff --git a/src/python/pants/backend/python/lint/pylint/lockfile.txt b/src/python/pants/backend/python/lint/pylint/lockfile.txt index 7adf520c2be..320e5e3c27c 100644 --- a/src/python/pants/backend/python/lint/pylint/lockfile.txt +++ b/src/python/pants/backend/python/lint/pylint/lockfile.txt @@ -4,10 +4,12 @@ # # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- # { -# "version": 1, -# "requirements_invalidation_digest": "b098df4cb8b4729e91282a0894f8804314269decfafe1a00d65b254074c63b8b", +# "version": 2, # "valid_for_interpreter_constraints": [ # "CPython<4,>=3.6" +# ], +# "generated_with_requirements": [ +# "pylint<2.7,>=2.6.2" # ] # } # --- END PANTS LOCKFILE METADATA --- diff --git a/src/python/pants/backend/python/lint/yapf/lockfile.txt b/src/python/pants/backend/python/lint/yapf/lockfile.txt index e7aaedb0858..d7a7785c75f 100644 --- a/src/python/pants/backend/python/lint/yapf/lockfile.txt +++ b/src/python/pants/backend/python/lint/yapf/lockfile.txt @@ -4,10 +4,13 @@ # # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- # { -# "version": 1, -# "requirements_invalidation_digest": "7b946efbcd8abc2a50c548b71c84b34e44eabf0430246b309d63d6200dc5715b", +# "version": 2, # "valid_for_interpreter_constraints": [ # "CPython>=3.6" +# ], +# "generated_with_requirements": [ +# "toml", +# "yapf==0.31.0" # ] # } # --- END PANTS LOCKFILE METADATA --- diff --git a/src/python/pants/backend/python/subsystems/coverage_py_lockfile.txt b/src/python/pants/backend/python/subsystems/coverage_py_lockfile.txt index 05f7e6309d1..174e0828d15 100644 --- a/src/python/pants/backend/python/subsystems/coverage_py_lockfile.txt +++ b/src/python/pants/backend/python/subsystems/coverage_py_lockfile.txt @@ -4,10 +4,12 @@ # # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- # { -# "version": 1, -# "requirements_invalidation_digest": "ed5d33bbada4d1d96c7d57e100c72bdb5f26d6fed4f1e77b0e74f2ea5e43e642", +# "version": 2, # "valid_for_interpreter_constraints": [ # "CPython<4,>=3.6" +# ], +# "generated_with_requirements": [ +# "coverage[toml]<5.6,>=5.5" # ] # } # --- END PANTS LOCKFILE METADATA --- diff --git a/src/python/pants/backend/python/subsystems/ipython_lockfile.txt b/src/python/pants/backend/python/subsystems/ipython_lockfile.txt index 081dbbcda74..f79a08edf18 100644 --- a/src/python/pants/backend/python/subsystems/ipython_lockfile.txt +++ b/src/python/pants/backend/python/subsystems/ipython_lockfile.txt @@ -4,10 +4,12 @@ # # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- # { -# "version": 1, -# "requirements_invalidation_digest": "fe82372002915f2550a0b25fea5b6f360c639252ad607a978e7e2f6cbd94c99a", +# "version": 2, # "valid_for_interpreter_constraints": [ # "CPython<4,>=3.6" +# ], +# "generated_with_requirements": [ +# "ipython==7.16.1" # ] # } # --- END PANTS LOCKFILE METADATA --- @@ -21,9 +23,9 @@ backcall==0.2.0; python_version >= "3.6" \ colorama==0.4.4; python_version >= "3.6" and python_full_version < "3.0.0" and sys_platform == "win32" or sys_platform == "win32" and python_version >= "3.6" and python_full_version >= "3.5.0" \ --hash=sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2 \ --hash=sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b -decorator==5.0.9; python_version >= "3.6" \ - --hash=sha256:6e5c199c16f7a9f0e3a61a4a54b3d27e7dad0dbdde92b944426cb20914376323 \ - --hash=sha256:72ecfba4320a893c53f9706bebb2d55c270c1e51a28789361aa93e4a21319ed5 +decorator==5.1.0; python_version >= "3.6" \ + --hash=sha256:7b12e7c3c6ab203a29e157335e9122cb03de9ab7264b137594103fd4a683b374 \ + --hash=sha256:e59913af105b9860aa2c8d3272d9de5a56a4e608db9a2f167a8480b323d529a7 ipython-genutils==0.2.0; python_version >= "3.6" \ --hash=sha256:72dd37233799e619666c9f639a9da83c34013a73e8bbc79a7a6348d93c61fab8 \ --hash=sha256:eb2e116e75ecef9d4d228fdc66af54269afa26ab4463042e33785b887c628ba8 diff --git a/src/python/pants/backend/python/subsystems/lambdex_lockfile.txt b/src/python/pants/backend/python/subsystems/lambdex_lockfile.txt index 1e6de50b28c..78a1884cef9 100644 --- a/src/python/pants/backend/python/subsystems/lambdex_lockfile.txt +++ b/src/python/pants/backend/python/subsystems/lambdex_lockfile.txt @@ -4,10 +4,12 @@ # # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- # { -# "version": 1, -# "requirements_invalidation_digest": "b3766b556f0c70fe89e235378292cb1369830e316df7ab0d4861a7dd0755f856", +# "version": 2, # "valid_for_interpreter_constraints": [ # "CPython<3.10,>=3.6" +# ], +# "generated_with_requirements": [ +# "lambdex==0.1.6" # ] # } # --- END PANTS LOCKFILE METADATA --- diff --git a/src/python/pants/backend/python/subsystems/pytest_lockfile.txt b/src/python/pants/backend/python/subsystems/pytest_lockfile.txt index 6047dec2e26..dbec20584b7 100644 --- a/src/python/pants/backend/python/subsystems/pytest_lockfile.txt +++ b/src/python/pants/backend/python/subsystems/pytest_lockfile.txt @@ -4,10 +4,13 @@ # # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- # { -# "version": 1, -# "requirements_invalidation_digest": "ac24b96fe05b827037bcb568a7578ecf36ae84929bb1409bc9dd0a89d5a2ec70", +# "version": 2, # "valid_for_interpreter_constraints": [ # "CPython<4,>=3.6" +# ], +# "generated_with_requirements": [ +# "pytest-cov<2.13,>=2.12.1", +# "pytest<6.3,>=6.2.4" # ] # } # --- END PANTS LOCKFILE METADATA --- diff --git a/src/python/pants/backend/python/subsystems/python_tool_base.py b/src/python/pants/backend/python/subsystems/python_tool_base.py index fbf2dab1db7..f568ad02266 100644 --- a/src/python/pants/backend/python/subsystems/python_tool_base.py +++ b/src/python/pants/backend/python/subsystems/python_tool_base.py @@ -19,6 +19,7 @@ from pants.engine.fs import FileContent from pants.option.errors import OptionsError from pants.option.subsystem import Subsystem +from pants.util.ordered_set import FrozenOrderedSet DEFAULT_TOOL_LOCKFILE = "" NO_TOOL_LOCKFILE = "" @@ -160,6 +161,7 @@ def pex_requirements( importlib.resources.read_binary(*self.default_lockfile_resource), ), lockfile_hex_digest=hex_digest, + req_strings=FrozenOrderedSet(requirements), options_scope_name=self.options_scope, uses_project_interpreter_constraints=(not self.register_interpreter_constraints), uses_source_plugins=self.uses_requirements_from_source_plugins, @@ -168,6 +170,7 @@ def pex_requirements( file_path=self.lockfile, file_path_description_of_origin=f"the option `[{self.options_scope}].lockfile`", lockfile_hex_digest=hex_digest, + req_strings=FrozenOrderedSet(requirements), options_scope_name=self.options_scope, uses_project_interpreter_constraints=(not self.register_interpreter_constraints), uses_source_plugins=self.uses_requirements_from_source_plugins, diff --git a/src/python/pants/backend/python/subsystems/setuptools_lockfile.txt b/src/python/pants/backend/python/subsystems/setuptools_lockfile.txt index fab730c3c10..91a7c98744a 100644 --- a/src/python/pants/backend/python/subsystems/setuptools_lockfile.txt +++ b/src/python/pants/backend/python/subsystems/setuptools_lockfile.txt @@ -4,10 +4,13 @@ # # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- # { -# "version": 1, -# "requirements_invalidation_digest": "a5ec2e69360b67f3262d8ecc191c0227f09216343e97e8c9e2a34ce567b07c29", +# "version": 2, # "valid_for_interpreter_constraints": [ # "CPython<4,>=3.6" +# ], +# "generated_with_requirements": [ +# "setuptools<58.0,>=50.3.0", +# "wheel<0.38,>=0.35.1" # ] # } # --- END PANTS LOCKFILE METADATA --- diff --git a/src/python/pants/backend/python/typecheck/mypy/lockfile.txt b/src/python/pants/backend/python/typecheck/mypy/lockfile.txt index 95a6ed68f53..963bfd8de0c 100644 --- a/src/python/pants/backend/python/typecheck/mypy/lockfile.txt +++ b/src/python/pants/backend/python/typecheck/mypy/lockfile.txt @@ -4,10 +4,12 @@ # # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- # { -# "version": 1, -# "requirements_invalidation_digest": "2bcd33a72af5d12ea4fcc7ada9dcaf7fa1017ba8286216e88c43ad9e1823ce75", +# "version": 2, # "valid_for_interpreter_constraints": [ # "CPython>=3.6" +# ], +# "generated_with_requirements": [ +# "mypy==0.910" # ] # } # --- END PANTS LOCKFILE METADATA --- diff --git a/src/python/pants/backend/python/util_rules/lockfile_metadata.py b/src/python/pants/backend/python/util_rules/lockfile_metadata.py index 72262509a68..670974534ff 100644 --- a/src/python/pants/backend/python/util_rules/lockfile_metadata.py +++ b/src/python/pants/backend/python/util_rules/lockfile_metadata.py @@ -7,7 +7,9 @@ import json from dataclasses import dataclass from enum import Enum -from typing import Any, Iterable +from typing import Any, Callable, Iterable, Set, Type, TypeVar + +from pkg_resources import Requirement from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints from pants.util.ordered_set import FrozenOrderedSet @@ -15,21 +17,70 @@ BEGIN_LOCKFILE_HEADER = b"# --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---" END_LOCKFILE_HEADER = b"# --- END PANTS LOCKFILE METADATA ---" -LOCKFILE_VERSION = 1 + +_concrete_metadata_classes: dict[int, Type[LockfileMetadata]] = {} + + +def _lockfile_metadata_version( + version: int, +) -> Callable[[Type[LockfileMetadata]], Type[LockfileMetadata]]: + """Decorator to register a Lockfile metadata version subclass with a given version number. + + The class must be a frozen dataclass + """ + + def _dec(cls: Type[LockfileMetadata]) -> Type[LockfileMetadata]: + + # Only frozen dataclasses may be registered as lockfile metadata: + cls_dataclass_params = getattr(cls, "__dataclass_params__", None) + if not cls_dataclass_params or not cls_dataclass_params.frozen: + raise ValueError( + "Classes registered with `_lockfile_metadata_version` may only be " + "frozen dataclasses" + ) + _concrete_metadata_classes[version] = cls + return cls + + return _dec class InvalidLockfileError(Exception): pass -@dataclass +@dataclass(frozen=True) class LockfileMetadata: - requirements_invalidation_digest: str + """Base class for metadata that is attached to a given lockfiles. + + This class, and provides the external API for serializing, deserializing, and validating the + contents of individual lockfiles. New versions of metadata implement a concrete subclass and + provide deserialization and validation logic, along with specialist serialization logic. + + To construct an instance of the most recent concrete subclass, call `LockfileMetadata.new()`. + """ + + _LockfileMetadataSubclass = TypeVar("_LockfileMetadataSubclass", bound="LockfileMetadata") + valid_for_interpreter_constraints: InterpreterConstraints - @classmethod + @staticmethod + def new( + valid_for_interpreter_constraints: InterpreterConstraints, + requirements: set[Requirement], + ) -> LockfileMetadata: + """Call the most recent version of the `LockfileMetadata` class to construct a concrete + instance. + + This static method should be used in place of the `LockfileMetadata` constructor. This gives + calling sites a predictable method to call to construct a new `LockfileMetadata` for + writing, while still allowing us to support _reading_ older, deprecated metadata versions. + """ + + return LockfileMetadataV2(valid_for_interpreter_constraints, requirements) + + @staticmethod def from_lockfile( - cls, lockfile: bytes, lockfile_path: str | None = None, resolve_name: str | None = None + lockfile: bytes, lockfile_path: str | None = None, resolve_name: str | None = None ) -> LockfileMetadata: """Parse all relevant metadata from the lockfile's header.""" in_metadata_block = False @@ -72,42 +123,34 @@ def from_lockfile( "be decoded. " + error_suffix ) - def get_or_raise(key: str) -> Any: - try: - return metadata[key] - except KeyError: - raise InvalidLockfileError( - f"Required key `{key}` is not present in metadata header for " - f"{lockfile_description}. {error_suffix}" - ) + concrete_class = _concrete_metadata_classes[metadata["version"]] - requirements_digest = get_or_raise("requirements_invalidation_digest") - if not isinstance(requirements_digest, str): - raise InvalidLockfileError( - f"Metadata value `requirements_invalidation_digest` in {lockfile_description} must " - "be a string. " + error_suffix - ) + return concrete_class._from_json_dict(metadata, lockfile_description, error_suffix) - try: - interpreter_constraints = InterpreterConstraints( - get_or_raise("valid_for_interpreter_constraints") - ) - except TypeError: - raise InvalidLockfileError( - f"Metadata value `valid_for_interpreter_constraints` in {lockfile_description} " - "must be a list of valid Python interpreter constraints strings. " + error_suffix - ) + @classmethod + def _from_json_dict( + cls: Type[_LockfileMetadataSubclass], + json_dict: dict[Any, Any], + lockfile_description: str, + error_suffix: str, + ) -> _LockfileMetadataSubclass: + """Construct a `LockfileMetadata` subclass from the supplied JSON dict. + + *** Not implemented. Subclasses should override. *** + + + `lockfile_description` is a detailed, human-readable description of the lockfile, which can + be read by the user to figure out which lockfile is broken in case of an error. - return LockfileMetadata(requirements_digest, interpreter_constraints) + `error_suffix` is a string describing how to fix the lockfile. + """ + + raise NotImplementedError( + "`LockfileMetadata._from_json_dict` should not be directly " "called." + ) def add_header_to_lockfile(self, lockfile: bytes, *, regenerate_command: str) -> bytes: - metadata_dict = { - "version": LOCKFILE_VERSION, - "requirements_invalidation_digest": self.requirements_invalidation_digest, - "valid_for_interpreter_constraints": [ - str(ic) for ic in self.valid_for_interpreter_constraints - ], - } + metadata_dict = self._header_dict() metadata_json = json.dumps(metadata_dict, ensure_ascii=True, indent=2).splitlines() metadata_as_a_comment = "\n".join(f"# {l}" for l in metadata_json).encode("ascii") header = b"%b\n%b\n%b" % (BEGIN_LOCKFILE_HEADER, metadata_as_a_comment, END_LOCKFILE_HEADER) @@ -119,14 +162,74 @@ def add_header_to_lockfile(self, lockfile: bytes, *, regenerate_command: str) -> return b"%b\n#\n%b\n\n%b" % (regenerate_command_bytes, header, lockfile) + def _header_dict(self) -> dict[Any, Any]: + """Produce a dictionary to be serialized into the lockfile header. + + Subclasses should call `super` and update the resulting dictionary. + """ + + version: int + for ver, cls in _concrete_metadata_classes.items(): + if isinstance(self, cls): + version = ver + break + else: + raise ValueError("Trying to serialize an unregistered `LockfileMetadata` subclass.") + + return { + "version": version, + "valid_for_interpreter_constraints": [ + str(ic) for ic in self.valid_for_interpreter_constraints + ], + } + def is_valid_for( self, expected_invalidation_digest: str | None, user_interpreter_constraints: InterpreterConstraints, interpreter_universe: Iterable[str], + user_requirements: Iterable[Requirement] | None, ) -> LockfileMetadataValidation: """Returns Truthy if this `LockfileMetadata` can be used in the current execution context.""" + + raise NotImplementedError("call `is_valid_for` on subclasses only") + + +@_lockfile_metadata_version(1) +@dataclass(frozen=True) +class LockfileMetadataV1(LockfileMetadata): + + requirements_invalidation_digest: str + + @classmethod + def _from_json_dict( + cls: Type[LockfileMetadataV1], + json_dict: dict[Any, Any], + lockfile_description: str, + error_suffix: str, + ) -> LockfileMetadataV1: + metadata = _get_metadata(json_dict, lockfile_description, error_suffix) + + interpreter_constraints = metadata( + "valid_for_interpreter_constraints", InterpreterConstraints, InterpreterConstraints + ) + requirements_digest = metadata("requirements_invalidation_digest", str, None) + + return LockfileMetadataV1(interpreter_constraints, requirements_digest) + + def _header_dict(self) -> dict[Any, Any]: + d = super()._header_dict() + d["requirements_invalidation_digest"] = self.requirements_invalidation_digest + return d + + def is_valid_for( + self, + expected_invalidation_digest: str | None, + user_interpreter_constraints: InterpreterConstraints, + interpreter_universe: Iterable[str], + _: Iterable[Requirement] | None, # User requirements are not used by V1 + ) -> LockfileMetadataValidation: failure_reasons: set[InvalidLockfileReason] = set() if expected_invalidation_digest is None: @@ -143,6 +246,70 @@ def is_valid_for( return LockfileMetadataValidation(failure_reasons) +@_lockfile_metadata_version(2) +@dataclass(frozen=True) +class LockfileMetadataV2(LockfileMetadata): + """Lockfile version that permits specifying a requirements as a set rather than a digest. + + Validity is tested by the set of requirements strings being the same in the user requirements as + those in the stored requirements. + """ + + requirements: set[Requirement] + + @classmethod + def _from_json_dict( + cls: Type[LockfileMetadataV2], + json_dict: dict[Any, Any], + lockfile_description: str, + error_suffix: str, + ) -> LockfileMetadataV2: + metadata = _get_metadata(json_dict, lockfile_description, error_suffix) + + requirements = metadata( + "generated_with_requirements", + Set[Requirement], + lambda l: {Requirement.parse(i) for i in l}, + ) + interpreter_constraints = metadata( + "valid_for_interpreter_constraints", InterpreterConstraints, InterpreterConstraints + ) + + return LockfileMetadataV2(interpreter_constraints, requirements) + + def _header_dict(self) -> dict[Any, Any]: + out = super()._header_dict() + + # Requirements need to be stringified then sorted so that tests are deterministic. Sorting + # followed by stringifying does not produce a meaningful result. + out["generated_with_requirements"] = ( + sorted(str(i) for i in self.requirements) if self.requirements is not None else None + ) + return out + + def is_valid_for( + self, + _: str | None, # Validation digests are not used by V2; this param will be deprecated + user_interpreter_constraints: InterpreterConstraints, + interpreter_universe: Iterable[str], + user_requirements: Iterable[Requirement] | None, + ) -> LockfileMetadataValidation: + failure_reasons: set[InvalidLockfileReason] = set() + + if user_requirements is None: + return LockfileMetadataValidation(failure_reasons) + + if self.requirements != set(user_requirements): + failure_reasons.add(InvalidLockfileReason.REQUIREMENTS_MISMATCH) + + if not self.valid_for_interpreter_constraints.contains( + user_interpreter_constraints, interpreter_universe + ): + failure_reasons.add(InvalidLockfileReason.INTERPRETER_CONSTRAINTS_MISMATCH) + + return LockfileMetadataValidation(failure_reasons) + + def calculate_invalidation_digest(requirements: Iterable[str]) -> str: """Returns an invalidation digest for the given requirements.""" m = hashlib.sha256() @@ -158,6 +325,7 @@ def calculate_invalidation_digest(requirements: Iterable[str]) -> str: class InvalidLockfileReason(Enum): INVALIDATION_DIGEST_MISMATCH = "invalidation_digest_mismatch" INTERPRETER_CONSTRAINTS_MISMATCH = "interpreter_constraints_mismatch" + REQUIREMENTS_MISMATCH = "requirements_mismatch" class LockfileMetadataValidation: @@ -170,3 +338,44 @@ def __init__(self, failure_reasons: Iterable[InvalidLockfileReason] = ()): def __bool__(self): return not self.failure_reasons + + +T = TypeVar("T") + + +def _get_metadata( + metadata: dict[Any, Any], + lockfile_description: str, + error_suffix: str, +) -> Callable[[str, Type[T], Callable[[Any], T] | None], T]: + """Returns a function that will get a given key from the `metadata` dict, and optionally do some + verification and post-processing to return a value of the correct type.""" + + def get_metadata(key: str, type_: Type[T], coerce: Callable[[Any], T] | None) -> T: + val: Any + try: + val = metadata[key] + except KeyError: + raise InvalidLockfileError( + f"Required key `{key}` is not present in metadata header for " + f"{lockfile_description}. {error_suffix}" + ) + + if not coerce: + if isinstance(val, type_): + return val + + raise InvalidLockfileError( + f"Metadata value `{key}` in {lockfile_description} must " + f"be a {type(type_).__name__}. {error_suffix}" + ) + else: + try: + return coerce(val) + except Exception: + raise InvalidLockfileError( + f"Metadata value `{key}` in {lockfile_description} must be able to " + f"be converted to a {type(type_).__name__}. {error_suffix}" + ) + + return get_metadata diff --git a/src/python/pants/backend/python/util_rules/lockfile_metadata_test.py b/src/python/pants/backend/python/util_rules/lockfile_metadata_test.py index 5d970149394..53c6e8656f3 100644 --- a/src/python/pants/backend/python/util_rules/lockfile_metadata_test.py +++ b/src/python/pants/backend/python/util_rules/lockfile_metadata_test.py @@ -7,18 +7,27 @@ from typing import Iterable import pytest +from pkg_resources import Requirement from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints from pants.backend.python.util_rules.lockfile_metadata import ( LockfileMetadata, + LockfileMetadataV1, + LockfileMetadataV2, calculate_invalidation_digest, ) +INTERPRETER_UNIVERSE = ["2.7", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10"] + + +def reqset(*a) -> set[Requirement]: + return {Requirement.parse(i) for i in a} + def test_metadata_header_round_trip() -> None: - input_metadata = LockfileMetadata( - "cab0c0c0c0c0dadacafec0c0c0c0cafedadabeefc0c0c0c0feedbeeffeedbeef", + input_metadata = LockfileMetadata.new( InterpreterConstraints(["CPython==2.7.*", "PyPy", "CPython>=3.6,<4,!=3.7.*"]), + reqset("ansicolors==0.1.0"), ) serialized_lockfile = input_metadata.add_header_to_lockfile( b"req1==1.0", regenerate_command="./pants lock" @@ -39,10 +48,12 @@ def test_add_header_to_lockfile() -> None: # # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- # { -# "version": 1, -# "requirements_invalidation_digest": "000faaafcacacaca", +# "version": 2, # "valid_for_interpreter_constraints": [ # "CPython>=3.7" +# ], +# "generated_with_requirements": [ +# "ansicolors==0.1.0" # ] # } # --- END PANTS LOCKFILE METADATA --- @@ -53,7 +64,7 @@ def test_add_header_to_lockfile() -> None: def line_by_line(b: bytes) -> list[bytes]: return [i for i in (j.strip() for j in b.splitlines()) if i] - metadata = LockfileMetadata("000faaafcacacaca", InterpreterConstraints([">=3.7"])) + metadata = LockfileMetadata.new(InterpreterConstraints([">=3.7"]), reqset("ansicolors==0.1.0")) result = metadata.add_header_to_lockfile(input_lockfile, regenerate_command="./pants lock") assert line_by_line(result) == line_by_line(expected) @@ -87,7 +98,7 @@ def assert_neq(left: Iterable[str], right: Iterable[str]) -> None: [">=3.5.5"], [">=3.5, <=3.6"], False, - ), # User ICs contain versions in the 3.6 range + ), # User ICs contain versions in the 3.7 range ("yes", "yes", [">=3.5.5, <=3.5.10"], [">=3.5, <=3.6"], True), ("yes", "no", [">=3.5.5, <=3.5.10"], [">=3.5, <=3.6"], False), # Digests do not match ( @@ -122,15 +133,57 @@ def assert_neq(left: Iterable[str], right: Iterable[str]) -> None: ), # Excluded version from expected ICs is not in a range specified ], ) -def test_is_valid_for(user_digest, expected_digest, user_ic, expected_ic, matches) -> None: - m = LockfileMetadata(expected_digest, InterpreterConstraints(expected_ic)) +def test_is_valid_for_v1(user_digest, expected_digest, user_ic, expected_ic, matches) -> None: + m: LockfileMetadata + m = LockfileMetadataV1(InterpreterConstraints(expected_ic), expected_digest) assert ( bool( m.is_valid_for( user_digest, InterpreterConstraints(user_ic), - ["2.7", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10"], + INTERPRETER_UNIVERSE, + set(), ) ) == matches ) + + +@pytest.mark.parametrize( + "user_ics_iter, expected_ics_iter, user_reqs, expected_reqs, matches", + [ + # Exact requirements match + [[">=3.5"], [">=3.5"], ["ansicolors==0.1.0"], ["ansicolors==0.1.0"], True], + # Version mismatch + [[">=3.5"], [">=3.5"], ["ansicolors==0.1.0"], ["ansicolors==0.1.1"], False], + # Range specifier mismatch + [[">=3.5"], [">=3.5"], ["ansicolors==0.1.0"], ["ansicolors>=0.1.0"], False], + # Requirements mismatch + [[">=3.5"], [">=3.5"], ["requests==1.0.0"], ["ansicolors==0.1.0"], False], + # Multiple requirements + [ + [">=3.5"], + [">=3.5"], + ["ansicolors==0.1.0", "requests==1.0.0"], + ["ansicolors==0.1.0", "requests==1.0.0"], + True, + ], + # Multiple requirements, order mismatch still passes + [ + [">=3.5"], + [">=3.5"], + ["ansicolors==0.1.0", "requests==1.0.0"], + ["requests==1.0.0", "ansicolors==0.1.0"], + True, + ], + # Exact requirements match, non-matching ICs (user includes 3.7 range and above) + [[">=3.5.5"], [">=3.5, <=3.6"], ["ansicolors==0.1.0"], ["ansicolors==0.1.0"], False], + ], +) +def test_is_valid_for_v2_only( + user_ics_iter, expected_ics_iter, user_reqs, expected_reqs, matches +) -> None: + user_ic = InterpreterConstraints(user_ics_iter) + expected_ic = InterpreterConstraints(expected_ics_iter) + m = LockfileMetadataV2(expected_ic, reqset(*expected_reqs)) + assert bool(m.is_valid_for("", user_ic, INTERPRETER_UNIVERSE, reqset(*user_reqs))) == matches diff --git a/src/python/pants/backend/python/util_rules/pex.py b/src/python/pants/backend/python/util_rules/pex.py index d4ded7fa3cf..0ee2fea5f7e 100644 --- a/src/python/pants/backend/python/util_rules/pex.py +++ b/src/python/pants/backend/python/util_rules/pex.py @@ -23,6 +23,7 @@ from pants.backend.python.util_rules import pex_cli from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints from pants.backend.python.util_rules.lockfile_metadata import ( + InvalidLockfileError, InvalidLockfileReason, LockfileMetadata, ) @@ -70,12 +71,14 @@ class Lockfile: file_path: str file_path_description_of_origin: str lockfile_hex_digest: str | None + req_strings: FrozenOrderedSet[str] | None @dataclass(frozen=True) class LockfileContent: file_content: FileContent lockfile_hex_digest: str | None + req_strings: FrozenOrderedSet[str] | None @dataclass(frozen=True) @@ -533,10 +536,19 @@ def _validate_metadata( python_setup: PythonSetup, ) -> None: + # TODO(#12314): Improve this message: `Requirement.parse` raises `InvalidRequirement`, which + # doesn't have mypy stubs at the moment; it may be hard to catch this exception and typecheck. + req_strings = ( + {Requirement.parse(i) for i in requirements.req_strings} + if requirements.req_strings is not None + else None + ) + validation = metadata.is_valid_for( requirements.lockfile_hex_digest, request.interpreter_constraints, python_setup.interpreter_universe, + req_strings, ) if validation: @@ -563,7 +575,13 @@ def tool_message_parts( "\n\n" ) - if InvalidLockfileReason.INVALIDATION_DIGEST_MISMATCH in validation.failure_reasons: + if any( + i == InvalidLockfileReason.INVALIDATION_DIGEST_MISMATCH + or i == InvalidLockfileReason.REQUIREMENTS_MISMATCH + for i in validation.failure_reasons + ): + # TODO(12314): Add message showing _which_ requirements diverged. + yield ( "- You have set different requirements than those used to generate the lockfile. " f"You can fix this by not setting `[{tool_name}].version`, " @@ -616,8 +634,8 @@ def tool_message_parts( if isinstance(requirements, (ToolCustomLockfile, ToolDefaultLockfile)): message = "".join(tool_message_parts(requirements)).strip() else: - # TODO: Replace with an actual value once user lockfiles are supported - assert False + # TODO(12314): Improve this message + raise InvalidLockfileError(f"{validation.failure_reasons}") if python_setup.invalid_lockfile_behavior == InvalidLockfileBehavior.error: raise ValueError(message) diff --git a/src/python/pants/backend/python/util_rules/pex_from_targets.py b/src/python/pants/backend/python/util_rules/pex_from_targets.py index cde0aaac455..1ddb99c9e5f 100644 --- a/src/python/pants/backend/python/util_rules/pex_from_targets.py +++ b/src/python/pants/backend/python/util_rules/pex_from_targets.py @@ -270,6 +270,7 @@ async def pex_from_targets(request: PexFromTargetsRequest, python_setup: PythonS ), # TODO(#12314): Hook up lockfile staleness check. lockfile_hex_digest=None, + req_strings=None, ), interpreter_constraints=interpreter_constraints, platforms=request.platforms, @@ -291,6 +292,7 @@ async def pex_from_targets(request: PexFromTargetsRequest, python_setup: PythonS # TODO(#12314): Hook up lockfile staleness check once multiple lockfiles # are supported. lockfile_hex_digest=None, + req_strings=None, ), interpreter_constraints=interpreter_constraints, platforms=request.platforms, diff --git a/src/python/pants/backend/python/util_rules/pex_test.py b/src/python/pants/backend/python/util_rules/pex_test.py index ee638c2a9f2..f4a0134d5fd 100644 --- a/src/python/pants/backend/python/util_rules/pex_test.py +++ b/src/python/pants/backend/python/util_rules/pex_test.py @@ -20,7 +20,11 @@ from pants.backend.python.target_types import EntryPoint, MainSpecification from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints -from pants.backend.python.util_rules.lockfile_metadata import LockfileMetadata +from pants.backend.python.util_rules.lockfile_metadata import ( + LockfileMetadata, + LockfileMetadataV1, + LockfileMetadataV2, +) from pants.backend.python.util_rules.pex import ( Lockfile, LockfileContent, @@ -46,6 +50,7 @@ from pants.python.python_setup import InvalidLockfileBehavior from pants.testutil.rule_runner import QueryRule, RuleRunner from pants.util.dirutil import safe_rmtree +from pants.util.ordered_set import FrozenOrderedSet @dataclass(frozen=True) @@ -601,13 +606,17 @@ def assert_description( LockfileContent( file_content=FileContent("lock.txt", b""), lockfile_hex_digest=None, + req_strings=None, ), expected="Building new.pex from lock.txt", ) assert_description( Lockfile( - file_path="lock.txt", file_path_description_of_origin="foo", lockfile_hex_digest=None + file_path="lock.txt", + file_path_description_of_origin="foo", + lockfile_hex_digest=None, + req_strings=None, ), expected="Building new.pex from lock.txt", ) @@ -693,6 +702,7 @@ def test_no_warning_on_valid_lockfile_with_content(rule_runner: RuleRunner, capl LOCKFILE_TYPES = (DEFAULT, FILE) BOOLEANS = (True, False) +VERSIONS = (1, 2) def _run_pex_for_lockfile_test( @@ -711,6 +721,8 @@ def _run_pex_for_lockfile_test( expected_digest, actual_constraints, expected_constraints, + actual_requirements, + expected_requirements, options_scope_name, ) = _metadata_validation_values( invalid_reqs, invalid_constraints, uses_source_plugins, uses_project_ic @@ -734,6 +746,7 @@ def _run_pex_for_lockfile_test( lockfile_type, lockfile, expected_digest, + expected_requirements, options_scope_name, uses_source_plugins, uses_project_ic, @@ -751,14 +764,15 @@ def _run_pex_for_lockfile_test( @pytest.mark.parametrize( - "lockfile_type,invalid_reqs,invalid_constraints,uses_source_plugins,uses_project_ic", + "lockfile_type,invalid_reqs,invalid_constraints,uses_source_plugins,uses_project_ic,version", [ - (lft, ir, ic, usp, upi) + (lft, ir, ic, usp, upi, v) for lft in LOCKFILE_TYPES for ir in BOOLEANS for ic in BOOLEANS for usp in BOOLEANS for upi in BOOLEANS + for v in VERSIONS if (ir or ic) ], ) @@ -769,6 +783,7 @@ def test_validate_metadata( invalid_constraints, uses_source_plugins, uses_project_ic, + version, caplog, ) -> None: class M: @@ -798,17 +813,29 @@ class M: expected_digest, actual_constraints, expected_constraints, + actual_requirements, + expected_requirements_, options_scope_name, ) = _metadata_validation_values( invalid_reqs, invalid_constraints, uses_source_plugins, uses_project_ic ) - metadata = LockfileMetadata(expected_digest, InterpreterConstraints([expected_constraints])) + metadata: LockfileMetadata + if version == 1: + metadata = LockfileMetadataV1( + InterpreterConstraints([expected_constraints]), expected_digest + ) + elif version == 2: + expected_requirements = {Requirement.parse(i) for i in expected_requirements_} + metadata = LockfileMetadataV2( + InterpreterConstraints([expected_constraints]), expected_requirements + ) requirements = _prepare_pex_requirements( rule_runner, lockfile_type, "lockfile_data_goes_here", actual_digest, + actual_requirements, options_scope_name, uses_source_plugins, uses_project_ic, @@ -862,12 +889,15 @@ class M: def _metadata_validation_values( invalid_reqs: bool, invalid_constraints: bool, uses_source_plugins: bool, uses_project_ic: bool -) -> tuple[str, str, str, str, str]: +) -> tuple[str, str, str, str, set[str], set[str], str]: actual_digest = "900d" expected_digest = actual_digest + actual_reqs = {"ansicolors==0.1.0"} + expected_reqs = actual_reqs if invalid_reqs: expected_digest = "baad" + expected_reqs = {"requests==3.0.0"} actual_constraints = "CPython>=3.6,<3.10" expected_constraints = actual_constraints @@ -889,6 +919,8 @@ def _metadata_validation_values( expected_digest, actual_constraints, expected_constraints, + actual_reqs, + expected_reqs, options_scope_name, ) @@ -898,6 +930,7 @@ def _prepare_pex_requirements( lockfile_type: str, lockfile: str, expected_digest: str, + expected_requirements: set[str], options_scope_name: str, uses_source_plugins: bool, uses_project_interpreter_constraints: bool, @@ -909,6 +942,7 @@ def _prepare_pex_requirements( file_path=file_path, file_path_description_of_origin="iceland", lockfile_hex_digest=expected_digest, + req_strings=FrozenOrderedSet(expected_requirements), options_scope_name=options_scope_name, uses_source_plugins=uses_source_plugins, uses_project_interpreter_constraints=uses_project_interpreter_constraints, @@ -918,6 +952,7 @@ def _prepare_pex_requirements( return ToolDefaultLockfile( file_content=content, lockfile_hex_digest=expected_digest, + req_strings=FrozenOrderedSet(expected_requirements), options_scope_name=options_scope_name, uses_source_plugins=uses_source_plugins, uses_project_interpreter_constraints=uses_project_interpreter_constraints,