Add complete platforms for Google Cloud Functions + auto-infer based …

…on runtime (#21248)

Closes #18195, closes #20515.

This adds `complete_platforms` for all Google Cloud Functions runtimes
and allows automatic selection of those platforms based on the provided
runtime. Additionally, the `generate_faas_complete_platforms.py` has
been updated to support generating `complete_platforms` JSON files for
GCF.

One complication while building this is that Google [publishes the
runtime image for each version of Python to a separate Docker
repository](https://cloud.google.com/functions/docs/concepts/execution-environment#python),
so the `known_runtime_docker_repo` abstraction that was used before
doesn't work for this case. The way I solved this is by pushing the
docker repo field down into `PythonFaaSKnownRuntime` instead, and
letting this field be customizable per-runtime.

Additionally, the tags used by GCF look like this:
`python37_20240728_3_7_17_RC00`. This follows the template of
`{runtime}_{date}_{python_version}_{signifier}`. This unfortunately
makes it very hard to find the "latest" version of the GCF image for a
given runtime, so I had to go into Google Container Registry to find the
latest tagged versions for each Python version.

build-support/bin/generate_faas_complete_platforms.py

@@ -24,16 +24,16 @@
 from pathlib import Path
 
 from pants.backend.awslambda.python.target_types import PythonAwsLambdaRuntime
+from pants.backend.google_cloud_function.python.target_types import PythonGoogleCloudFunctionRuntime
 from pants.backend.python.util_rules.faas import FaaSArchitecture, PythonFaaSRuntimeField
 from pants.base.build_environment import get_buildroot
 
-COMMAND = "pip install pex 1>&2 && pex3 interpreter inspect --markers --tags"
-
-
-RUNTIME_FIELDS = [
-    PythonAwsLambdaRuntime,
-    # TODO: what docker images to use for GCF?
-]
+# GCF images throw permissions errors when trying to install pex in the default site-packages.
+# Additionally, some of the images are missing a `pip` alias, so we need to use `python -m pip`.
+COMMAND = (
+    "python -m pip install --target=/tmp/subdir pex 1>&2 && "
+    + "PYTHONPATH=/tmp/subdir /tmp/subdir/bin/pex3 interpreter inspect --markers --tags"
+)
 
 
 def extract_complete_platform(repo: str, architecture: FaaSArchitecture, tag: str) -> object:
@@ -67,7 +67,7 @@ def run(runtime_field: type[PythonFaaSRuntimeField], python_base: Path) -> None:
     print(f"Generating for {runtime_field.__name__}, writing to {cp_dir}", file=sys.stderr)
     for rt in runtime_field.known_runtimes:
         cp = extract_complete_platform(
-            runtime_field.known_runtimes_docker_repo,
+            rt.docker_repo,
             FaaSArchitecture(rt.architecture) if rt.architecture else FaaSArchitecture.X86_64,
             rt.tag,
         )
@@ -79,18 +79,32 @@ def run(runtime_field: type[PythonFaaSRuntimeField], python_base: Path) -> None:
 
 def create_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(
-        description="Generates the complete platform JSON files for AWS Lambda and GCF"
+        description="Generates the complete platform JSON files for AWS Lambda and/or GCF"
+    )
+    parser.add_argument(
+        "--runtime",
+        choices=["lambda", "gcf", "all"],
+        default="all",
+        help="Choose which runtime(s) to generate complete platform files for",
     )
     return parser
 
 
 def main() -> None:
-    # The args are discarded, but the parser is useful
-    # to ensure that --help works.
-    create_parser().parse_args()
+    args = create_parser().parse_args()
 
     build_root = Path(get_buildroot()) / "src/python"
-    for runtime_field in RUNTIME_FIELDS:
+
+    # Type declaration needed for mypy
+    selected_runtimes: list[type[PythonFaaSRuntimeField]]
+    if args.runtime == "lambda":
+        selected_runtimes = [PythonAwsLambdaRuntime]
+    elif args.runtime == "gcf":
+        selected_runtimes = [PythonGoogleCloudFunctionRuntime]
+    else:
+        selected_runtimes = [PythonAwsLambdaRuntime, PythonGoogleCloudFunctionRuntime]
+
+    for runtime_field in selected_runtimes:
         run(runtime_field, build_root)
 
 

docs/notes/2.23.x.md

@@ -124,6 +124,8 @@ When building function-as-a-service targets like `python_google_cloud_function`,
 
 The `python_aws_lambda_function` and `python_aws_lambda_layer` targets now allow specifying the machine architecture to use for the Lambda function via [the new `architecture` field](https://www.pantsbuild.org/2.23/reference/targets/python_aws_lambda_function#architecture). Additionally, both targets now support automatically selecting the `complete_platforms` used for building PEX files based on the values of the `architecture` and `runtime` fields together.
 
+The `python_google_cloud_function` target can now automatically select the `complete_platforms` used for building PEX files based on the value of the `runtime` field.
+
 [The `pants.backend.python.providers.experimental.pyenv` backend](https://www.pantsbuild.org/2.23/reference/subsystems/pyenv-python-provider) now respects the patch version of the interpeter constraints.
 
 #### Terraform

src/python/pants/backend/awslambda/python/rules.py

@@ -73,9 +73,7 @@ async def package_python_aws_lambda_function(
             target_name=PythonAWSLambda.alias,
             complete_platforms=field_set.complete_platforms,
             runtime=field_set.runtime,
-            architecture=FaaSArchitecture(field_set.architecture.value)
-            if field_set.architecture.value
-            else None,
+            architecture=FaaSArchitecture(field_set.architecture.value),
             handler=field_set.handler,
             output_path=field_set.output_path,
             include_requirements=field_set.include_requirements.value,
@@ -99,9 +97,7 @@ async def package_python_aws_lambda_layer(
             target_name=PythonAWSLambdaLayer.alias,
             complete_platforms=field_set.complete_platforms,
             runtime=field_set.runtime,
-            architecture=FaaSArchitecture(field_set.architecture.value)
-            if field_set.architecture.value
-            else None,
+            architecture=FaaSArchitecture(field_set.architecture.value),
             output_path=field_set.output_path,
             include_requirements=field_set.include_requirements.value,
             include_sources=field_set.include_sources.value,

src/python/pants/backend/awslambda/python/rules_test.py

@@ -207,26 +207,26 @@ def handler(event, context):
     ("ics", "runtime", "architecture"),
     [
         pytest.param(
-            ["==3.7.*"],
+            ["==3.8.*"],
             None,
             FaaSArchitecture.X86_64,
             id="runtime inferred from ICs, x86_64 architecture",
         ),
         pytest.param(
             None,
-            "python3.7",
+            "python3.8",
             FaaSArchitecture.X86_64,
             id="runtime explicitly set, x86_64 architecture",
         ),
         pytest.param(
-            ["==3.7.*"],
+            ["==3.8.*"],
             None,
             FaaSArchitecture.ARM64,
             id="runtime inferred from ICs, ARM64 architecture",
         ),
         pytest.param(
             None,
-            "python3.7",
+            "python3.8",
             FaaSArchitecture.ARM64,
             id="runtime explicitly set, ARM64 architecture",
         ),
@@ -275,7 +275,7 @@ def handler(event, context):
         rule_runner,
         Address("src/python/foo/bar", target_name="lambda"),
         expected_extra_log_lines=(
-            "    Runtime: python3.7",
+            "    Runtime: python3.8",
             f"    Architecture: {architecture.value}",
             "    Handler: lambda_function.handler",
         ),
@@ -294,7 +294,7 @@ def handler(event, context):
         rule_runner,
         Address("src/python/foo/bar", target_name="slimlambda"),
         expected_extra_log_lines=(
-            "    Runtime: python3.7",
+            "    Runtime: python3.8",
             f"    Architecture: {architecture.value}",
             "    Handler: lambda_function.handler",
         ),
@@ -335,14 +335,14 @@ def handler(event, context):
                 python_aws_lambda_layer(
                     name='lambda',
                     dependencies=["./hello_world.py"],
-                    runtime="python3.7",
+                    runtime="python3.8",
                     architecture="{architecture.value}",
                 )
                 python_aws_lambda_layer(
                     name='slimlambda',
                     include_sources=False,
                     dependencies=["./hello_world.py"],
-                    runtime="python3.7",
+                    runtime="python3.8",
                     architecture="{architecture.value}",
                 )
                 """
@@ -354,7 +354,7 @@ def handler(event, context):
         rule_runner,
         Address("src/python/foo/bar", target_name="lambda"),
         expected_extra_log_lines=(
-            "    Runtime: python3.7",
+            "    Runtime: python3.8",
             f"    Architecture: {architecture.value}",
         ),
         layer=True,
@@ -372,7 +372,7 @@ def handler(event, context):
         rule_runner,
         Address("src/python/foo/bar", target_name="slimlambda"),
         expected_extra_log_lines=(
-            "    Runtime: python3.7",
+            "    Runtime: python3.8",
             f"    Architecture: {architecture.value}",
         ),
         layer=True,

src/python/pants/backend/awslambda/python/target_types.py

@@ -92,6 +92,8 @@ class PythonAwsLambdaIncludeSources(BoolField):
 
 class PythonAwsLambdaRuntime(PythonFaaSRuntimeField):
     PYTHON_RUNTIME_REGEX = r"python(?P<major>\d)\.(?P<minor>\d+)"
+    # https://gallery.ecr.aws/lambda/python
+    LAMBDA_DOCKER_REPO = "public.ecr.aws/lambda/python"
 
     help = help_text(
         """
@@ -105,21 +107,19 @@ class PythonAwsLambdaRuntime(PythonFaaSRuntimeField):
         """
     )
 
-    # https://gallery.ecr.aws/lambda/python
-    known_runtimes_docker_repo = "public.ecr.aws/lambda/python"
     known_runtimes = (
-        PythonFaaSKnownRuntime(3, 6, "3.6", FaaSArchitecture.X86_64),
-        PythonFaaSKnownRuntime(3, 7, "3.7", FaaSArchitecture.X86_64),
-        PythonFaaSKnownRuntime(3, 8, "3.8-x86_64", FaaSArchitecture.X86_64),
-        PythonFaaSKnownRuntime(3, 8, "3.8-arm64", FaaSArchitecture.ARM64),
-        PythonFaaSKnownRuntime(3, 9, "3.9-x86_64", FaaSArchitecture.X86_64),
-        PythonFaaSKnownRuntime(3, 9, "3.9-arm64", FaaSArchitecture.ARM64),
-        PythonFaaSKnownRuntime(3, 10, "3.10-x86_64", FaaSArchitecture.X86_64),
-        PythonFaaSKnownRuntime(3, 10, "3.10-arm64", FaaSArchitecture.ARM64),
-        PythonFaaSKnownRuntime(3, 11, "3.11-x86_64", FaaSArchitecture.X86_64),
-        PythonFaaSKnownRuntime(3, 11, "3.11-arm64", FaaSArchitecture.ARM64),
-        PythonFaaSKnownRuntime(3, 12, "3.12-x86_64", FaaSArchitecture.X86_64),
-        PythonFaaSKnownRuntime(3, 12, "3.12-arm64", FaaSArchitecture.ARM64),
+        PythonFaaSKnownRuntime(3, 6, LAMBDA_DOCKER_REPO, "3.6", FaaSArchitecture.X86_64),
+        PythonFaaSKnownRuntime(3, 7, LAMBDA_DOCKER_REPO, "3.7", FaaSArchitecture.X86_64),
+        PythonFaaSKnownRuntime(3, 8, LAMBDA_DOCKER_REPO, "3.8-x86_64", FaaSArchitecture.X86_64),
+        PythonFaaSKnownRuntime(3, 8, LAMBDA_DOCKER_REPO, "3.8-arm64", FaaSArchitecture.ARM64),
+        PythonFaaSKnownRuntime(3, 9, LAMBDA_DOCKER_REPO, "3.9-x86_64", FaaSArchitecture.X86_64),
+        PythonFaaSKnownRuntime(3, 9, LAMBDA_DOCKER_REPO, "3.9-arm64", FaaSArchitecture.ARM64),
+        PythonFaaSKnownRuntime(3, 10, LAMBDA_DOCKER_REPO, "3.10-x86_64", FaaSArchitecture.X86_64),
+        PythonFaaSKnownRuntime(3, 10, LAMBDA_DOCKER_REPO, "3.10-arm64", FaaSArchitecture.ARM64),
+        PythonFaaSKnownRuntime(3, 11, LAMBDA_DOCKER_REPO, "3.11-x86_64", FaaSArchitecture.X86_64),
+        PythonFaaSKnownRuntime(3, 11, LAMBDA_DOCKER_REPO, "3.11-arm64", FaaSArchitecture.ARM64),
+        PythonFaaSKnownRuntime(3, 12, LAMBDA_DOCKER_REPO, "3.12-x86_64", FaaSArchitecture.X86_64),
+        PythonFaaSKnownRuntime(3, 12, LAMBDA_DOCKER_REPO, "3.12-arm64", FaaSArchitecture.ARM64),
     )
 
     @classmethod

src/python/pants/backend/google_cloud_function/python/BUILD

@@ -1,7 +1,10 @@
 # Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-python_sources()
+python_sources(overrides={"target_types.py": {"dependencies": [":complete_platforms"]}})
+
+# generated by build-support/bin/generate_faas_complete_platforms.py
+resources(name="complete_platforms", sources=["complete_platform_*.json"])
 
 python_tests(name="target_types_test", sources=["target_types_test.py"])
 python_tests(