[internal] terraform: update tailor to use terraform_modules generator (

#12936)

Update the Terraform plugin's tailor support to use the `terraform_modules` generator instead of generating individual `terraform_module` targets.

[ci skip-rust]

[ci skip-build-wheels]

src/python/pants/backend/terraform/BUILD

@@ -5,7 +5,7 @@ python_library(dependencies=[":lockfile"])
 
 resources(name="lockfile", sources=["hcl2_lockfile.txt"])
 
-python_tests(name='tests', sources=["!dependency_inference_test.py"])
+python_tests(name='tests', sources=["*_test.py", "!dependency_inference_test.py"])
 python_tests(
   name="dependency_inference_test",
   sources=["dependency_inference_test.py"],

src/python/pants/backend/terraform/tailor.py

@@ -3,12 +3,12 @@
 
 from __future__ import annotations
 
-import os
 from dataclasses import dataclass
+from pathlib import PurePath
+from typing import Iterable
 
-from pants.backend.terraform.target_types import TerraformModule
+from pants.backend.terraform.target_types import TerraformModules
 from pants.core.goals.tailor import (
-    AllOwnedSources,
     PutativeTarget,
     PutativeTargets,
     PutativeTargetsRequest,
@@ -21,28 +21,82 @@
 from pants.util.logging import LogLevel
 
 
+def longest_common_prefix(x: tuple[str, ...], y: tuple[str, ...]) -> tuple[str, ...]:
+    """Find the longest common prefix between two sequences."""
+
+    i = j = 0
+    while i < len(x) and j < len(y):
+        if x[i] != y[j]:
+            break
+        i = i + 1
+        j = j + 1
+
+    return x[:i]
+
+
+def find_disjoint_longest_common_prefixes(
+    raw_values: Iterable[tuple[str, ...]]
+) -> set[tuple[str, ...]]:
+    values = sorted(raw_values)
+
+    if len(values) == 0:
+        return set()
+    elif len(values) == 1:
+        return set(values)
+
+    prefixes = set()
+    current_prefix = values[0]
+
+    i = 1
+    while i < len(values):
+        potential_prefix = longest_common_prefix(current_prefix, values[i])
+        if potential_prefix:
+            # If this item still has any common prefix with the current run of items, then
+            # update the prefix to this potential prefix.
+            current_prefix = potential_prefix
+        else:
+            # If there is no common prefix between this item and the current run of items, then
+            # this run of items with a common prefix has ended. Record the current prefix and make
+            # the current item be the next current prefix.
+            prefixes.add(current_prefix)
+            current_prefix = values[i]
+        i += 1
+
+    # Record any prefix from the last run of items.
+    if current_prefix:
+        prefixes.add(current_prefix)
+
+    return prefixes
+
+
 @dataclass(frozen=True)
 class PutativeTerraformTargetsRequest(PutativeTargetsRequest):
     pass
 
 
 @rule(level=LogLevel.DEBUG, desc="Determine candidate Terraform targets to create")
-async def find_putative_targets(
-    request: PutativeTerraformTargetsRequest, all_owned_sources: AllOwnedSources
+async def find_putative_terrform_modules_targets(
+    request: PutativeTerraformTargetsRequest,
 ) -> PutativeTargets:
     all_terraform_files = await Get(Paths, PathGlobs, request.search_paths.path_globs("*.tf"))
-    unowned_terraform_files = set(all_terraform_files.files) - set(all_owned_sources)
-
-    putative_targets = []
-    for dirname, filenames in group_by_dir(unowned_terraform_files).items():
-        putative_targets.append(
-            PutativeTarget.for_target_type(
-                TerraformModule,
-                dirname,
-                os.path.basename(dirname),
-                sorted(filenames),
-            )
+    directory_to_files = {
+        dir: files
+        for dir, files in group_by_dir(all_terraform_files.files).items()
+        if any(file.endswith(".tf") for file in files)
+    }
+    prefixes = find_disjoint_longest_common_prefixes(
+        [PurePath(dir).parts for dir in directory_to_files.keys()]
+    )
+
+    putative_targets = [
+        PutativeTarget.for_target_type(
+            TerraformModules,
+            str(PurePath(*dir_parts)),
+            "tf_mods",
+            [str(PurePath(*dir_parts).joinpath("**/*.tf"))],
         )
+        for dir_parts in prefixes
+    ]
 
     return PutativeTargets(putative_targets)
 

src/python/pants/backend/terraform/tailor_test.py

@@ -1,9 +1,13 @@
 # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
+from pathlib import PurePath
 
-from pants.backend.terraform.tailor import PutativeTerraformTargetsRequest
+from pants.backend.terraform.tailor import (
+    PutativeTerraformTargetsRequest,
+    find_disjoint_longest_common_prefixes,
+)
 from pants.backend.terraform.tailor import rules as terraform_tailor_rules
-from pants.backend.terraform.target_types import TerraformModule
+from pants.backend.terraform.target_types import TerraformModule, TerraformModules
 from pants.core.goals.tailor import (
     AllOwnedSources,
     PutativeTarget,
@@ -25,24 +29,29 @@ def test_find_putative_targets() -> None:
         ],
         target_types=[
             TerraformModule,
+            TerraformModules,
         ],
     )
     rule_runner.write_files(
         {
-            f"src/terraform/{fp}": ""
+            fp: ""
             for fp in (
-                "root.tf",
-                "owned-module/main.tf",
-                "owned-module/foo.tf",
-                "unowned-module/main.tf",
-                "unowned-module/bar.tf",
+                "prod/terraform/resources/foo/versions.tf",
+                "prod/terraform/resources/bar/versions.tf",
+                "prod/terraform/modules/bar/versions.tf",
+                "prod/terraform/modules/bar/hello/versions.tf",
+                "prod/terraform/modules/world/versions.tf",
+                "service1/src/terraform/versions.tf",
+                "service1/src/terraform/foo/versions.tf",
+                "service1/src/terraform/versions.tf",
+                "service2/src/terraform/versions.tf",
             )
         }
     )
     pts = rule_runner.request(
         PutativeTargets,
         [
-            PutativeTerraformTargetsRequest(PutativeTargetsSearchPaths(("src/",))),
+            PutativeTerraformTargetsRequest(PutativeTargetsSearchPaths(("",))),
             AllOwnedSources(
                 [
                     "src/terraform/root.tf",
@@ -56,32 +65,44 @@ def test_find_putative_targets() -> None:
         PutativeTargets(
             [
                 PutativeTarget.for_target_type(
-                    TerraformModule,
-                    "src/terraform",
-                    "terraform",
-                    [
-                        "root.tf",
-                    ],
+                    TerraformModules,
+                    "prod/terraform",
+                    "tf_mods",
+                    ("prod/terraform/**/*.tf",),
                 ),
                 PutativeTarget.for_target_type(
-                    TerraformModule,
-                    "src/terraform/owned-module",
-                    "owned-module",
-                    [
-                        "foo.tf",
-                        "main.tf",
-                    ],
+                    TerraformModules,
+                    "service1/src/terraform",
+                    "tf_mods",
+                    ("service1/src/terraform/**/*.tf",),
                 ),
                 PutativeTarget.for_target_type(
-                    TerraformModule,
-                    "src/terraform/unowned-module",
-                    "unowned-module",
-                    [
-                        "bar.tf",
-                        "main.tf",
-                    ],
+                    TerraformModules,
+                    "service2/src/terraform",
+                    "tf_mods",
+                    ("service2/src/terraform/**/*.tf",),
                 ),
             ]
         )
         == pts
     )
+
+
+def test_find_disjoint_longest_common_prefixes() -> None:
+    paths = [
+        "prod/terraform/resources/foo/versions.tf",
+        "prod/terraform/resources/bar/versions.tf",
+        "prod/terraform/modules/bar/versions.tf",
+        "prod/terraform/modules/bar/hello/versions.tf",
+        "prod/terraform/modules/world/versions.tf",
+        "service1/src/terraform/versions.tf",
+        "service1/src/terraform/foo/versions.tf",
+        "service1/src/terraform/versions.tf",
+        "service2/src/terraform/versions.tf",
+    ]
+    prefixes = find_disjoint_longest_common_prefixes([PurePath(p).parts[:-1] for p in paths])
+    assert prefixes == {
+        ("prod", "terraform"),
+        ("service1", "src", "terraform"),
+        ("service2", "src", "terraform"),
+    }

src/python/pants/backend/terraform/target_gen_test.py

@@ -9,8 +9,8 @@
     TerraformModules,
     TerraformModuleSources,
 )
-from pants.engine.addresses import Address
 from pants.core.util_rules import external_tool, source_files
+from pants.engine.addresses import Address
 from pants.engine.rules import QueryRule
 from pants.engine.target import GeneratedTargets
 from pants.testutil.rule_runner import RuleRunner
@@ -63,7 +63,7 @@ def test_target_generation(rule_runner: RuleRunner) -> None:
                         "versions.tf",
                     ),
                 },
-                generator_addr.create_generated("src/tf/")
+                generator_addr.create_generated("src/tf"),
             ),
         ],
     )