Add new version of LockfileMetadata to support checking for identic…

…ally specified requirements (#12782)

This factors out versioning capabilities into `LockfileMetadata` so that it's possible to easily change the set of validation requirements for a lockfile. V1 represents the original lockfile version (where constraints and an invalidation digest are set). V2 allows for the old behaviour, but also allows specifying the input requirements for a lockfile, and verifying that the user requirements are ~a non-strict subset of~ identical to the input requirements.

We decided to replace the requirements hex digest with requirements strings to allow us to test whether the lockfile produces a _compatible_ environment rather than an _identical_ environment, which will be useful for user lockfile support when we eventually enable that. In the meantime, tool lockfiles still test for an identical environment, but the extra data in the lockfile will allow for more fine-grained error messages in a future version.

The implementations of  `_from_json_dict` and `is_valid_for` are a bit repetitive; I can factor out the common behaviour with a bit of work, but given we expect to delete the V1 implementation before too long.

Currently this _does not_ add the `platforms` capability to the header, but now it's going to be easy enough to bump the version number if we want to add more fields.

Closes #12610

3rdparty/python/lockfiles/flake8.txt

@@ -4,10 +4,14 @@
 #
 # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
 # {
-#   "version": 1,
-#   "requirements_invalidation_digest": "815457a1baf6226c993e5468ccdf64c69fe7214d3d9237911c762733e0130526",
+#   "version": 2,
 #   "valid_for_interpreter_constraints": [
 #     "CPython<3.10,>=3.7"
+#   ],
+#   "generated_with_requirements": [
+#     "flake8-2020<1.7.0,>=1.6.0",
+#     "flake8-pantsbuild<3,>=2.0",
+#     "flake8<4.0,>=3.9.2"
 #   ]
 # }
 # --- END PANTS LOCKFILE METADATA ---

3rdparty/python/lockfiles/pytest.txt

@@ -4,10 +4,17 @@
 #
 # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
 # {
-#   "version": 1,
-#   "requirements_invalidation_digest": "a2057d396be0480d2586be2da25a021149a4f96276b853dd92cc63cfc3ae8503",
+#   "version": 2,
 #   "valid_for_interpreter_constraints": [
 #     "CPython<3.10,>=3.7"
+#   ],
+#   "generated_with_requirements": [
+#     "ipdb",
+#     "pygments",
+#     "pytest-cov<2.13,>=2.12.1",
+#     "pytest-html",
+#     "pytest-icdiff",
+#     "pytest<6.3,>=6.2.4"
 #   ]
 # }
 # --- END PANTS LOCKFILE METADATA ---
@@ -80,9 +87,9 @@ coverage==5.5; python_version >= "2.7" and python_full_version < "3.0.0" or pyth
     --hash=sha256:f030f8873312a16414c0d8e1a1ddff2d3235655a2174e3648b4fa66b3f2f1079 \
     --hash=sha256:2a3859cb82dcbda1cfd3e6f71c27081d18aa251d20a17d87d26d4cd216fb0af4 \
     --hash=sha256:ebe78fe9a0e874362175b02371bdfbee64d8edc42a044253ddf4ee7d3c15212c
-decorator==5.0.9; python_version >= "3.7" \
-    --hash=sha256:6e5c199c16f7a9f0e3a61a4a54b3d27e7dad0dbdde92b944426cb20914376323 \
-    --hash=sha256:72ecfba4320a893c53f9706bebb2d55c270c1e51a28789361aa93e4a21319ed5
+decorator==5.1.0; python_version >= "3.7" \
+    --hash=sha256:7b12e7c3c6ab203a29e157335e9122cb03de9ab7264b137594103fd4a683b374 \
+    --hash=sha256:e59913af105b9860aa2c8d3272d9de5a56a4e608db9a2f167a8480b323d529a7
 icdiff==2.0.4; python_version >= "3.6" \
     --hash=sha256:c72572e5ce087bc7a7748af2664764d4a805897caeefb665bdc12677fefb2212
 importlib-metadata==4.8.1; python_version < "3.8" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6") \

3rdparty/python/lockfiles/user_reqs.txt

@@ -4,10 +4,32 @@
 #
 # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
 # {
-#   "version": 1,
-#   "requirements_invalidation_digest": "2570870f165f546315ceb7ba2667cc764def9348baf7002ddb0a81c831cfd0fa",
+#   "version": 2,
 #   "valid_for_interpreter_constraints": [
 #     "CPython<3.10,>=3.7"
+#   ],
+#   "generated_with_requirements": [
+#     "PyYAML<5.5,>=5.4",
+#     "ansicolors==1.1.8",
+#     "fasteners==0.16",
+#     "freezegun==1.1.0",
+#     "humbug==0.2.6",
+#     "ijson==3.1.4",
+#     "packaging==21.0",
+#     "pex==2.1.48",
+#     "psutil==5.8.0",
+#     "pystache==0.5.4",
+#     "pytest<6.3,>=6.0.1",
+#     "requests[security]>=2.25.1",
+#     "setproctitle==1.2.2",
+#     "setuptools<58.0,>=56.0.0",
+#     "toml==0.10.2",
+#     "types-PyYAML==5.4.3",
+#     "types-freezegun==0.1.4",
+#     "types-requests==2.25.0",
+#     "types-setuptools==57.0.0",
+#     "types-toml==0.1.3",
+#     "typing-extensions==3.7.4.3"
 #   ]
 # }
 # --- END PANTS LOCKFILE METADATA ---

src/python/pants/backend/codegen/protobuf/python/mypy_protobuf_lockfile.txt

@@ -4,10 +4,12 @@
 #
 # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
 # {
-#   "version": 1,
-#   "requirements_invalidation_digest": "babed61947e74aedbe0cdbaefdaec172db6d4a9d27e12acc80be5ab623e3acdf",
+#   "version": 2,
 #   "valid_for_interpreter_constraints": [
 #     "CPython>=3.6"
+#   ],
+#   "generated_with_requirements": [
+#     "mypy-protobuf==2.4"
 #   ]
 # }
 # --- END PANTS LOCKFILE METADATA ---

src/python/pants/backend/python/goals/lockfile.py

@@ -9,6 +9,8 @@
 from pathlib import PurePath
 from typing import ClassVar, Iterable, Sequence, cast
 
+from pkg_resources import Requirement
+
 from pants.backend.python.subsystems.poetry import (
     POETRY_LAUNCHER,
     PoetrySubsystem,
@@ -233,7 +235,11 @@ async def generate_lockfile(
     initial_lockfile_digest_contents = await Get(
         DigestContents, Digest, poetry_export_result.output_digest
     )
-    metadata = LockfileMetadata(req.requirements_hex_digest, req.interpreter_constraints)
+    # TODO(#12314) Improve error message on `Requirement.parse`
+    metadata = LockfileMetadata.new(
+        req.interpreter_constraints,
+        {Requirement.parse(i) for i in req.requirements},
+    )
     lockfile_with_header = metadata.add_header_to_lockfile(
         initial_lockfile_digest_contents[0].content,
         regenerate_command=(

src/python/pants/backend/python/lint/bandit/lockfile.txt

@@ -4,10 +4,14 @@
 #
 # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
 # {
-#   "version": 1,
-#   "requirements_invalidation_digest": "b18a9f4d6a8cb4aafb414e9c8108d39dca053f8910ac801c147a932cd37e0040",
+#   "version": 2,
 #   "valid_for_interpreter_constraints": [
 #     "CPython<4,>=3.6"
+#   ],
+#   "generated_with_requirements": [
+#     "GitPython==3.1.18",
+#     "bandit<1.8,>=1.7.0",
+#     "setuptools"
 #   ]
 # }
 # --- END PANTS LOCKFILE METADATA ---

src/python/pants/backend/python/lint/black/lockfile.txt

@@ -4,10 +4,12 @@
 #
 # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
 # {
-#   "version": 1,
-#   "requirements_invalidation_digest": "1ba5f97d92f33b13e0cd56a960380fa6f616fae215d715ddd7a7ebf99795c890",
+#   "version": 2,
 #   "valid_for_interpreter_constraints": [
 #     "CPython>=3.6.2"
+#   ],
+#   "generated_with_requirements": [
+#     "black==21.8b0"
 #   ]
 # }
 # --- END PANTS LOCKFILE METADATA ---

src/python/pants/backend/python/lint/docformatter/lockfile.txt

@@ -4,10 +4,12 @@
 #
 # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
 # {
-#   "version": 1,
-#   "requirements_invalidation_digest": "4ee50d37e5a334d7d496e9f7147dbbf19ee2eb0faaea33e46a553b6c692c7672",
+#   "version": 2,
 #   "valid_for_interpreter_constraints": [
 #     "CPython>=3.6"
+#   ],
+#   "generated_with_requirements": [
+#     "docformatter<1.5,>=1.4"
 #   ]
 # }
 # --- END PANTS LOCKFILE METADATA ---

src/python/pants/backend/python/lint/flake8/lockfile.txt

@@ -4,10 +4,12 @@
 #
 # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
 # {
-#   "version": 1,
-#   "requirements_invalidation_digest": "ebd4b2326fca44e0d0b362def86bddc1b57e6f6d6c4f077fef5e1218286d3883",
+#   "version": 2,
 #   "valid_for_interpreter_constraints": [
 #     "CPython<4,>=3.6"
+#   ],
+#   "generated_with_requirements": [
+#     "flake8<4.0,>=3.9.2"
 #   ]
 # }
 # --- END PANTS LOCKFILE METADATA ---

src/python/pants/backend/python/lint/isort/lockfile.txt

@@ -4,10 +4,12 @@
 #
 # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
 # {
-#   "version": 1,
-#   "requirements_invalidation_digest": "56fe21a2ac24d6f257d95eb97829e8c3cb65e2156cd987348ce837c055763302",
+#   "version": 2,
 #   "valid_for_interpreter_constraints": [
 #     "CPython<4,>=3.7"
+#   ],
+#   "generated_with_requirements": [
+#     "isort[colors,pyproject]<6.0,>=5.9.3"
 #   ]
 # }
 # --- END PANTS LOCKFILE METADATA ---

src/python/pants/backend/python/lint/pylint/lockfile.txt

@@ -4,10 +4,12 @@
 #
 # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
 # {
-#   "version": 1,
-#   "requirements_invalidation_digest": "b098df4cb8b4729e91282a0894f8804314269decfafe1a00d65b254074c63b8b",
+#   "version": 2,
 #   "valid_for_interpreter_constraints": [
 #     "CPython<4,>=3.6"
+#   ],
+#   "generated_with_requirements": [
+#     "pylint<2.7,>=2.6.2"
 #   ]
 # }
 # --- END PANTS LOCKFILE METADATA ---

src/python/pants/backend/python/lint/yapf/lockfile.txt

@@ -4,10 +4,13 @@
 #
 # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
 # {
-#   "version": 1,
-#   "requirements_invalidation_digest": "7b946efbcd8abc2a50c548b71c84b34e44eabf0430246b309d63d6200dc5715b",
+#   "version": 2,
 #   "valid_for_interpreter_constraints": [
 #     "CPython>=3.6"
+#   ],
+#   "generated_with_requirements": [
+#     "toml",
+#     "yapf==0.31.0"
 #   ]
 # }
 # --- END PANTS LOCKFILE METADATA ---

src/python/pants/backend/python/subsystems/coverage_py_lockfile.txt

@@ -4,10 +4,12 @@
 #
 # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
 # {
-#   "version": 1,
-#   "requirements_invalidation_digest": "ed5d33bbada4d1d96c7d57e100c72bdb5f26d6fed4f1e77b0e74f2ea5e43e642",
+#   "version": 2,
 #   "valid_for_interpreter_constraints": [
 #     "CPython<4,>=3.6"
+#   ],
+#   "generated_with_requirements": [
+#     "coverage[toml]<5.6,>=5.5"
 #   ]
 # }
 # --- END PANTS LOCKFILE METADATA ---

src/python/pants/backend/python/subsystems/ipython_lockfile.txt

@@ -4,10 +4,12 @@
 #
 # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
 # {
-#   "version": 1,
-#   "requirements_invalidation_digest": "fe82372002915f2550a0b25fea5b6f360c639252ad607a978e7e2f6cbd94c99a",
+#   "version": 2,
 #   "valid_for_interpreter_constraints": [
 #     "CPython<4,>=3.6"
+#   ],
+#   "generated_with_requirements": [
+#     "ipython==7.16.1"
 #   ]
 # }
 # --- END PANTS LOCKFILE METADATA ---
@@ -21,9 +23,9 @@ backcall==0.2.0; python_version >= "3.6" \
 colorama==0.4.4; python_version >= "3.6" and python_full_version < "3.0.0" and sys_platform == "win32" or sys_platform == "win32" and python_version >= "3.6" and python_full_version >= "3.5.0" \
     --hash=sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2 \
     --hash=sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b
-decorator==5.0.9; python_version >= "3.6" \
-    --hash=sha256:6e5c199c16f7a9f0e3a61a4a54b3d27e7dad0dbdde92b944426cb20914376323 \
-    --hash=sha256:72ecfba4320a893c53f9706bebb2d55c270c1e51a28789361aa93e4a21319ed5
+decorator==5.1.0; python_version >= "3.6" \
+    --hash=sha256:7b12e7c3c6ab203a29e157335e9122cb03de9ab7264b137594103fd4a683b374 \
+    --hash=sha256:e59913af105b9860aa2c8d3272d9de5a56a4e608db9a2f167a8480b323d529a7
 ipython-genutils==0.2.0; python_version >= "3.6" \
     --hash=sha256:72dd37233799e619666c9f639a9da83c34013a73e8bbc79a7a6348d93c61fab8 \
     --hash=sha256:eb2e116e75ecef9d4d228fdc66af54269afa26ab4463042e33785b887c628ba8

src/python/pants/backend/python/subsystems/lambdex_lockfile.txt

@@ -4,10 +4,12 @@
 #
 # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
 # {
-#   "version": 1,
-#   "requirements_invalidation_digest": "b3766b556f0c70fe89e235378292cb1369830e316df7ab0d4861a7dd0755f856",
+#   "version": 2,
 #   "valid_for_interpreter_constraints": [
 #     "CPython<3.10,>=3.6"
+#   ],
+#   "generated_with_requirements": [
+#     "lambdex==0.1.6"
 #   ]
 # }
 # --- END PANTS LOCKFILE METADATA ---

src/python/pants/backend/python/subsystems/pytest_lockfile.txt

@@ -4,10 +4,13 @@
 #
 # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
 # {
-#   "version": 1,
-#   "requirements_invalidation_digest": "ac24b96fe05b827037bcb568a7578ecf36ae84929bb1409bc9dd0a89d5a2ec70",
+#   "version": 2,
 #   "valid_for_interpreter_constraints": [
 #     "CPython<4,>=3.6"
+#   ],
+#   "generated_with_requirements": [
+#     "pytest-cov<2.13,>=2.12.1",
+#     "pytest<6.3,>=6.2.4"
 #   ]
 # }
 # --- END PANTS LOCKFILE METADATA ---

src/python/pants/backend/python/subsystems/python_tool_base.py

@@ -19,6 +19,7 @@
 from pants.engine.fs import FileContent
 from pants.option.errors import OptionsError
 from pants.option.subsystem import Subsystem
+from pants.util.ordered_set import FrozenOrderedSet
 
 DEFAULT_TOOL_LOCKFILE = "<default>"
 NO_TOOL_LOCKFILE = "<none>"
@@ -160,6 +161,7 @@ def pex_requirements(
                     importlib.resources.read_binary(*self.default_lockfile_resource),
                 ),
                 lockfile_hex_digest=hex_digest,
+                req_strings=FrozenOrderedSet(requirements),
                 options_scope_name=self.options_scope,
                 uses_project_interpreter_constraints=(not self.register_interpreter_constraints),
                 uses_source_plugins=self.uses_requirements_from_source_plugins,
@@ -168,6 +170,7 @@ def pex_requirements(
             file_path=self.lockfile,
             file_path_description_of_origin=f"the option `[{self.options_scope}].lockfile`",
             lockfile_hex_digest=hex_digest,
+            req_strings=FrozenOrderedSet(requirements),
             options_scope_name=self.options_scope,
             uses_project_interpreter_constraints=(not self.register_interpreter_constraints),
             uses_source_plugins=self.uses_requirements_from_source_plugins,

src/python/pants/backend/python/subsystems/setuptools_lockfile.txt

@@ -4,10 +4,13 @@
 #
 # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
 # {
-#   "version": 1,
-#   "requirements_invalidation_digest": "a5ec2e69360b67f3262d8ecc191c0227f09216343e97e8c9e2a34ce567b07c29",
+#   "version": 2,
 #   "valid_for_interpreter_constraints": [
 #     "CPython<4,>=3.6"
+#   ],
+#   "generated_with_requirements": [
+#     "setuptools<58.0,>=50.3.0",
+#     "wheel<0.38,>=0.35.1"
 #   ]
 # }
 # --- END PANTS LOCKFILE METADATA ---

src/python/pants/backend/python/typecheck/mypy/lockfile.txt

@@ -4,10 +4,12 @@
 #
 # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
 # {
-#   "version": 1,
-#   "requirements_invalidation_digest": "2bcd33a72af5d12ea4fcc7ada9dcaf7fa1017ba8286216e88c43ad9e1823ce75",
+#   "version": 2,
 #   "valid_for_interpreter_constraints": [
 #     "CPython>=3.6"
+#   ],
+#   "generated_with_requirements": [
+#     "mypy==0.910"
 #   ]
 # }
 # --- END PANTS LOCKFILE METADATA ---