[interal] Use DownloadedExternalModules when analyzing external Go …

…packages (#13076)

Like #13070, this makes more progress on #12771.

We do not yet use `DownloadedExternalModules` for the sources digest used in `go_build_pkg.py`, only for the analysis of the package. That will be fixed in a followup.

To facilitate landing this, we remove the `go-pkg-debug` goal. It seems unlikely we'd want that for production, and you can simulate it by adding logging. It can be added back if we decide it's valuable once there is less churn.

[ci skip-rust]
[ci skip-build-wheels]

src/python/pants/backend/go/goals/custom_goals.py

@@ -5,17 +5,13 @@
 
 import logging
 
-from pants.backend.go.target_types import GoExternalPackageTarget, GoModSourcesField
+from pants.backend.go.target_types import GoModSourcesField
 from pants.backend.go.util_rules.build_go_pkg import BuildGoPackageRequest, BuiltGoPackage
-from pants.backend.go.util_rules.external_module import ResolveExternalGoPackageRequest
 from pants.backend.go.util_rules.go_mod import GoModInfo, GoModInfoRequest
 from pants.backend.go.util_rules.go_pkg import (
-    ResolvedGoPackage,
-    ResolveGoPackageRequest,
     is_first_party_package_target,
     is_third_party_package_target,
 )
-from pants.engine.console import Console
 from pants.engine.fs import MergeDigests, Snapshot, Workspace
 from pants.engine.goal import Goal, GoalSubsystem
 from pants.engine.internals.selectors import Get, MultiGet
@@ -71,35 +67,5 @@ async def run_go_build(targets: UnexpandedTargets) -> GoBuildGoal:
     return GoBuildGoal(exit_code=0)
 
 
-class GoPkgDebugSubsystem(GoalSubsystem):
-    name = "go-pkg-debug"
-    help = "Resolve a Go package and display its metadata"
-
-
-class GoPkgDebugGoal(Goal):
-    subsystem_cls = GoPkgDebugSubsystem
-
-
-@goal_rule
-async def run_go_pkg_debug(targets: UnexpandedTargets, console: Console) -> GoPkgDebugGoal:
-    first_party_requests = [
-        Get(ResolvedGoPackage, ResolveGoPackageRequest(address=tgt.address))
-        for tgt in targets
-        if is_first_party_package_target(tgt)
-    ]
-
-    third_party_requests = [
-        Get(ResolvedGoPackage, ResolveExternalGoPackageRequest(tgt))
-        for tgt in targets
-        if isinstance(tgt, GoExternalPackageTarget)
-    ]
-
-    resolved_packages = await MultiGet([*first_party_requests, *third_party_requests])  # type: ignore
-    for package in resolved_packages:
-        console.write_stdout(str(package) + "\n")
-
-    return GoPkgDebugGoal(exit_code=0)
-
-
 def rules():
     return collect_rules()

src/python/pants/backend/go/target_type_rules.py

@@ -195,7 +195,13 @@ async def inject_go_external_package_dependencies(
     wrapped_target = await Get(WrappedTarget, Address, request.dependencies_field.address)
     tgt = wrapped_target.target
     assert isinstance(tgt, GoExternalPackageTarget)
-    this_go_package = await Get(ResolvedGoPackage, ResolveExternalGoPackageRequest(tgt))
+
+    owning_go_mod = await Get(OwningGoMod, OwningGoModRequest(tgt.address))
+    go_mod_info = await Get(GoModInfo, GoModInfoRequest(owning_go_mod.address))
+
+    this_go_package = await Get(
+        ResolvedGoPackage, ResolveExternalGoPackageRequest(tgt, go_mod_info.stripped_digest)
+    )
 
     # Loop through all of the imports of this package and add dependencies on other packages and
     # external modules.

src/python/pants/backend/go/util_rules/build_go_pkg.py

@@ -23,6 +23,12 @@
     DownloadExternalModuleRequest,
     ResolveExternalGoPackageRequest,
 )
+from pants.backend.go.util_rules.go_mod import (
+    GoModInfo,
+    GoModInfoRequest,
+    OwningGoMod,
+    OwningGoModRequest,
+)
 from pants.backend.go.util_rules.go_pkg import (
     ResolvedGoPackage,
     ResolveGoPackageRequest,
@@ -76,6 +82,8 @@ async def build_target(
         source_files_subpath = target.address.spec_path
     elif is_third_party_package_target(target):
         assert isinstance(target, GoExternalPackageTarget)
+        owning_go_mod = await Get(OwningGoMod, OwningGoModRequest(target.address))
+        go_mod_info = await Get(GoModInfo, GoModInfoRequest(owning_go_mod.address))
         module_path = target[GoExternalModulePathField].value
         module, resolved_package = await MultiGet(
             Get(
@@ -85,7 +93,10 @@ async def build_target(
                     version=target[GoExternalModuleVersionField].value,
                 ),
             ),
-            Get(ResolvedGoPackage, ResolveExternalGoPackageRequest(target)),
+            Get(
+                ResolvedGoPackage,
+                ResolveExternalGoPackageRequest(target, go_mod_info.stripped_digest),
+            ),
         )
 
         source_files_digest = module.digest

src/python/pants/backend/go/util_rules/external_module.py

@@ -17,6 +17,7 @@
 from pants.backend.go.util_rules.sdk import GoSdkProcess
 from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior
 from pants.engine.collection import DeduplicatedCollection
+from pants.engine.engine_aware import EngineAwareParameter
 from pants.engine.fs import (
     EMPTY_DIGEST,
     CreateDigest,
@@ -216,32 +217,39 @@ async def download_external_module(
 
 
 @dataclass(frozen=True)
-class ResolveExternalGoPackageRequest:
+class ResolveExternalGoPackageRequest(EngineAwareParameter):
     tgt: GoExternalPackageTarget
+    go_mod_stripped_digest: Digest
+
+    def debug_hint(self) -> str:
+        return self.tgt[GoExternalPackageImportPathField].value
 
 
 @rule
-async def resolve_external_go_package(
+async def compute_external_go_package_info(
     request: ResolveExternalGoPackageRequest,
 ) -> ResolvedGoPackage:
+    # TODO: Extract the module we care about, rather than using everything. We also don't need the
+    #  root `go.sum` and `go.mod`.
+    downloaded_modules = await Get(
+        DownloadedExternalModules, DownloadExternalModulesRequest(request.go_mod_stripped_digest)
+    )
+
     module_path = request.tgt[GoExternalModulePathField].value
     module_version = request.tgt[GoExternalModuleVersionField].value
 
     import_path = request.tgt[GoExternalPackageImportPathField].value
     assert import_path.startswith(module_path)
     subpath = import_path[len(module_path) :]
 
-    downloaded_module = await Get(
-        DownloadedExternalModule,
-        DownloadExternalModuleRequest(module_path, module_version),
-    )
-
     json_result = await Get(
         ProcessResult,
         GoSdkProcess(
-            input_digest=downloaded_module.digest,
-            command=("list", "-json", f"./{subpath}"),
-            description="Resolve _go_external_package metadata.",
+            command=("list", "-mod=readonly", "-json", f"./{subpath}"),
+            env={"GOPROXY": "off"},
+            input_digest=downloaded_modules.digest,
+            working_dir=downloaded_modules.module_dir(module_path, module_version),
+            description=f"Determine metadata for Go external package {import_path}",
         ),
     )
 

src/python/pants/backend/go/util_rules/external_module_test.py

@@ -19,7 +19,7 @@
 )
 from pants.backend.go.util_rules.go_pkg import ResolvedGoPackage
 from pants.engine.addresses import Address
-from pants.engine.fs import Snapshot
+from pants.engine.fs import Digest, PathGlobs, Snapshot
 from pants.engine.internals.scheduler import ExecutionError
 from pants.engine.process import ProcessExecutionFailure
 from pants.engine.rules import QueryRule
@@ -229,10 +229,14 @@ def test_determine_external_package_info(rule_runner: RuleRunner) -> None:
             "BUILD": "go_mod(name='mod')",
         }
     )
+    input_digest = rule_runner.request(Digest, [PathGlobs(["go.mod", "go.sum"])])
     pkg_addr = Address("", target_name="mod", generated_name="github.com/google/go-cmp/cmp/cmpopts")
     tgt = rule_runner.get_target(pkg_addr)
     assert isinstance(tgt, GoExternalPackageTarget)
-    pkg_info = rule_runner.request(ResolvedGoPackage, [ResolveExternalGoPackageRequest(tgt)])
+
+    pkg_info = rule_runner.request(
+        ResolvedGoPackage, [ResolveExternalGoPackageRequest(tgt, input_digest)]
+    )
     assert pkg_info.address == pkg_addr
     assert pkg_info.module_address is None
     assert pkg_info.import_path == "github.com/google/go-cmp/cmp/cmpopts"