From 45a144c67edc8ff2fb5f0a7ae35f4da58e72b869 Mon Sep 17 00:00:00 2001 From: Patrick Lawson Date: Tue, 5 Oct 2021 19:23:08 -0400 Subject: [PATCH] Hoist generic JVM types out of java backend and into jvm subdir. (#13110) These are the most obvious types that were reused by the WIP Scala backend. Happy to change names or further refactor this, for now I'm trying to push these moves as I find them to avoid merge churn while I continue to work on Scala integration. [ci skip-rust] [ci skip-build-wheels] --- .../backend/experimental/java/register.py | 4 +- src/python/pants/backend/java/classpath.py | 3 +- .../pants/backend/java/compile/javac.py | 78 +--------------- .../pants/backend/java/compile/javac_test.py | 13 +-- .../import_parser_test.py | 2 +- .../java/dependency_inference/java_parser.py | 2 +- .../java_parser_launcher.py | 4 +- .../dependency_inference/java_parser_test.py | 4 +- .../java/dependency_inference/rules_test.py | 2 +- .../backend/java/package/deploy_jar_test.py | 32 +++---- src/python/pants/backend/java/test/junit.py | 2 +- .../pants/backend/java/test/junit_test.py | 2 +- src/python/pants/jvm/compile.py | 88 +++++++++++++++++++ .../java/util_rules.py => jvm/jdk_rules.py} | 0 .../jdk_rules_test.py} | 6 +- 15 files changed, 124 insertions(+), 118 deletions(-) create mode 100644 src/python/pants/jvm/compile.py rename src/python/pants/{backend/java/util_rules.py => jvm/jdk_rules.py} (100%) rename src/python/pants/{backend/java/util_rules_test.py => jvm/jdk_rules_test.py} (95%) diff --git a/src/python/pants/backend/experimental/java/register.py b/src/python/pants/backend/experimental/java/register.py index 68f12829eb9..fb5e2895122 100644 --- a/src/python/pants/backend/experimental/java/register.py +++ b/src/python/pants/backend/experimental/java/register.py @@ -2,7 +2,6 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). from pants.backend.java import classpath, tailor -from pants.backend.java import util_rules as java_util_rules from pants.backend.java.compile import javac from pants.backend.java.dependency_inference import ( import_parser, @@ -21,6 +20,7 @@ ) from pants.backend.java.target_types import rules as target_types_rules from pants.backend.java.test import junit +from pants.jvm import jdk_rules from pants.jvm import util_rules as jvm_util_rules from pants.jvm.goals import coursier from pants.jvm.resolve import coursier_fetch, coursier_setup @@ -55,6 +55,6 @@ def rules(): *dependency_inference_rules.rules(), *tailor.rules(), *jvm_util_rules.rules(), - *java_util_rules.rules(), + *jdk_rules.rules(), *target_types_rules(), ] diff --git a/src/python/pants/backend/java/classpath.py b/src/python/pants/backend/java/classpath.py index 56dfff391c0..dd7e53d12de 100644 --- a/src/python/pants/backend/java/classpath.py +++ b/src/python/pants/backend/java/classpath.py @@ -7,10 +7,11 @@ from dataclasses import dataclass from typing import Callable, Iterator -from pants.backend.java.compile.javac import CompiledClassfiles, CompileJavaSourceRequest +from pants.backend.java.compile.javac import CompileJavaSourceRequest from pants.engine.fs import AddPrefix, Digest, MergeDigests, Snapshot from pants.engine.rules import Get, MultiGet, collect_rules, rule from pants.engine.target import CoarsenedTargets, Targets +from pants.jvm.compile import CompiledClassfiles from pants.jvm.resolve.coursier_fetch import ( CoursierLockfileForTargetRequest, CoursierResolvedLockfile, diff --git a/src/python/pants/backend/java/compile/javac.py b/src/python/pants/backend/java/compile/javac.py index cc4a58607d8..cad47ecbb38 100644 --- a/src/python/pants/backend/java/compile/javac.py +++ b/src/python/pants/backend/java/compile/javac.py @@ -5,16 +5,13 @@ import logging from dataclasses import dataclass -from enum import Enum from itertools import chain from pants.backend.java.target_types import JavaSourceField -from pants.backend.java.util_rules import JdkSetup from pants.core.goals.check import CheckRequest, CheckResult, CheckResults from pants.core.util_rules.archive import ZipBinary from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest from pants.engine.addresses import Addresses -from pants.engine.engine_aware import EngineAwareReturnType from pants.engine.fs import ( EMPTY_DIGEST, AddPrefix, @@ -28,6 +25,8 @@ from pants.engine.rules import Get, MultiGet, collect_rules, rule from pants.engine.target import CoarsenedTarget, CoarsenedTargets, FieldSet, Sources, Targets from pants.engine.unions import UnionRule +from pants.jvm.compile import CompiledClassfiles, CompileResult, FallibleCompiledClassfiles +from pants.jvm.jdk_rules import JdkSetup from pants.jvm.resolve.coursier_fetch import ( CoursierLockfileForTargetRequest, CoursierResolvedLockfile, @@ -36,7 +35,6 @@ ) from pants.jvm.resolve.coursier_setup import Coursier from pants.util.logging import LogLevel -from pants.util.strutil import strip_v2_chroot_path logger = logging.getLogger(__name__) @@ -57,78 +55,6 @@ class CompileJavaSourceRequest: component: CoarsenedTarget -@dataclass(frozen=True) -class CompiledClassfiles: - """The outputs of a compilation contained in either zero or one JAR file. - - TODO: Rename this type to align with the guarantee about its content. - """ - - digest: Digest - - -class CompileResult(Enum): - SUCCEEDED = "succeeded" - FAILED = "failed" - DEPENDENCY_FAILED = "dependency failed" - - -@dataclass(frozen=True) -class FallibleCompiledClassfiles(EngineAwareReturnType): - description: str - result: CompileResult - output: CompiledClassfiles | None - exit_code: int - stdout: str | None = None - stderr: str | None = None - - @classmethod - def from_fallible_process_result( - cls, - description: str, - process_result: FallibleProcessResult, - output: CompiledClassfiles | None, - *, - strip_chroot_path: bool = False, - ) -> FallibleCompiledClassfiles: - def prep_output(s: bytes) -> str: - return strip_v2_chroot_path(s) if strip_chroot_path else s.decode() - - exit_code = process_result.exit_code - # TODO: Coursier renders this line on macOS. - stderr = "\n".join( - line - for line in prep_output(process_result.stderr).splitlines() - if line != "setrlimit to increase file descriptor limit failed, errno 22" - ) - return cls( - description=description, - result=(CompileResult.SUCCEEDED if exit_code == 0 else CompileResult.FAILED), - output=output, - exit_code=exit_code, - stdout=prep_output(process_result.stdout), - stderr=stderr, - ) - - def level(self) -> LogLevel: - return LogLevel.ERROR if self.exit_code != 0 else LogLevel.INFO - - def message(self) -> str: - message = self.description - message += ( - " succeeded." if self.exit_code == 0 else f" failed (exit code {self.exit_code})." - ) - if self.stdout: - message += f"\n{self.stdout}" - if self.stderr: - message += f"\n{self.stderr}" - return message - - def cacheable(self) -> bool: - # Failed compile outputs should be re-rendered in every run. - return self.exit_code == 0 - - @rule(desc="Compile with javac") async def compile_java_source( bash: BashBinary, diff --git a/src/python/pants/backend/java/compile/javac_test.py b/src/python/pants/backend/java/compile/javac_test.py index c58d9c67753..ec604d110b5 100644 --- a/src/python/pants/backend/java/compile/javac_test.py +++ b/src/python/pants/backend/java/compile/javac_test.py @@ -8,14 +8,7 @@ import pytest -from pants.backend.java import util_rules as java_util_rules -from pants.backend.java.compile.javac import ( - CompiledClassfiles, - CompileJavaSourceRequest, - CompileResult, - FallibleCompiledClassfiles, - JavacCheckRequest, -) +from pants.backend.java.compile.javac import CompileJavaSourceRequest, JavacCheckRequest from pants.backend.java.compile.javac import rules as javac_rules from pants.backend.java.target_types import JavaSourcesGeneratorTarget from pants.backend.java.target_types import rules as target_types_rules @@ -30,6 +23,8 @@ from pants.engine.process import Process, ProcessResult from pants.engine.rules import Get, MultiGet, rule from pants.engine.target import CoarsenedTarget, CoarsenedTargets, Targets +from pants.jvm import jdk_rules +from pants.jvm.compile import CompiledClassfiles, CompileResult, FallibleCompiledClassfiles from pants.jvm.goals.coursier import rules as coursier_rules from pants.jvm.resolve.coursier_fetch import ( Coordinate, @@ -61,7 +56,7 @@ def rule_runner() -> RuleRunner: *util_rules(), *target_types_rules(), *coursier_rules(), - *java_util_rules.rules(), + *jdk_rules.rules(), QueryRule(CheckResults, (JavacCheckRequest,)), QueryRule(FallibleCompiledClassfiles, (CompileJavaSourceRequest,)), QueryRule(CompiledClassfiles, (CompileJavaSourceRequest,)), diff --git a/src/python/pants/backend/java/dependency_inference/import_parser_test.py b/src/python/pants/backend/java/dependency_inference/import_parser_test.py index 561ff8b04ee..de132310756 100644 --- a/src/python/pants/backend/java/dependency_inference/import_parser_test.py +++ b/src/python/pants/backend/java/dependency_inference/import_parser_test.py @@ -19,12 +19,12 @@ ) from pants.backend.java.target_types import JavaSourceField, JavaSourcesGeneratorTarget from pants.backend.java.target_types import rules as java_target_rules -from pants.backend.java.util_rules import rules as java_util_rules from pants.core.util_rules import config_files, source_files from pants.core.util_rules.external_tool import rules as external_tool_rules from pants.engine.addresses import UnparsedAddressInputs from pants.engine.process import rules as process_rules from pants.engine.target import Targets +from pants.jvm.jdk_rules import rules as java_util_rules from pants.jvm.resolve.coursier_fetch import rules as coursier_fetch_rules from pants.jvm.resolve.coursier_setup import rules as coursier_setup_rules from pants.jvm.testutil import maybe_skip_jdk_test diff --git a/src/python/pants/backend/java/dependency_inference/java_parser.py b/src/python/pants/backend/java/dependency_inference/java_parser.py index 9bd9afa8dc5..5df7846e702 100644 --- a/src/python/pants/backend/java/dependency_inference/java_parser.py +++ b/src/python/pants/backend/java/dependency_inference/java_parser.py @@ -13,11 +13,11 @@ java_parser_artifact_requirements, ) from pants.backend.java.dependency_inference.types import JavaSourceDependencyAnalysis -from pants.backend.java.util_rules import JdkSetup from pants.core.util_rules.source_files import SourceFiles from pants.engine.fs import AddPrefix, Digest, DigestContents, MergeDigests from pants.engine.process import BashBinary, FallibleProcessResult, Process, ProcessExecutionFailure from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.jvm.jdk_rules import JdkSetup from pants.jvm.resolve.coursier_fetch import MaterializedClasspath, MaterializedClasspathRequest from pants.option.global_options import GlobalOptions from pants.util.logging import LogLevel diff --git a/src/python/pants/backend/java/dependency_inference/java_parser_launcher.py b/src/python/pants/backend/java/dependency_inference/java_parser_launcher.py index f3f06d7b527..779b10084a6 100644 --- a/src/python/pants/backend/java/dependency_inference/java_parser_launcher.py +++ b/src/python/pants/backend/java/dependency_inference/java_parser_launcher.py @@ -7,11 +7,11 @@ import pkg_resources -from pants.backend.java.compile.javac import CompiledClassfiles -from pants.backend.java.util_rules import JdkSetup from pants.engine.fs import CreateDigest, Digest, Directory, FileContent, MergeDigests, RemovePrefix from pants.engine.process import BashBinary, Process, ProcessResult from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.jvm.compile import CompiledClassfiles +from pants.jvm.jdk_rules import JdkSetup from pants.jvm.resolve.coursier_fetch import ( ArtifactRequirements, Coordinate, diff --git a/src/python/pants/backend/java/dependency_inference/java_parser_test.py b/src/python/pants/backend/java/dependency_inference/java_parser_test.py index 4b40ca23a3b..d230b3726ad 100644 --- a/src/python/pants/backend/java/dependency_inference/java_parser_test.py +++ b/src/python/pants/backend/java/dependency_inference/java_parser_test.py @@ -7,7 +7,6 @@ import pytest -from pants.backend.java import util_rules as java_util_rules from pants.backend.java.compile.javac import rules as javac_rules from pants.backend.java.dependency_inference.java_parser import ( FallibleJavaSourceDependencyAnalysisResult, @@ -25,6 +24,7 @@ from pants.engine.internals.scheduler import ExecutionError from pants.engine.process import ProcessExecutionFailure from pants.engine.target import Sources +from pants.jvm import jdk_rules from pants.jvm.resolve.coursier_fetch import CoursierResolvedLockfile from pants.jvm.resolve.coursier_fetch import rules as coursier_fetch_rules from pants.jvm.resolve.coursier_setup import rules as coursier_setup_rules @@ -47,7 +47,7 @@ def rule_runner() -> RuleRunner: *javac_rules(), *source_files.rules(), *util_rules(), - *java_util_rules.rules(), + *jdk_rules.rules(), QueryRule(FallibleJavaSourceDependencyAnalysisResult, (SourceFiles,)), QueryRule(JavaSourceDependencyAnalysis, (SourceFiles,)), QueryRule(SourceFiles, (SourceFilesRequest,)), diff --git a/src/python/pants/backend/java/dependency_inference/rules_test.py b/src/python/pants/backend/java/dependency_inference/rules_test.py index 15ba28878a9..d0749b10fe0 100644 --- a/src/python/pants/backend/java/dependency_inference/rules_test.py +++ b/src/python/pants/backend/java/dependency_inference/rules_test.py @@ -19,7 +19,6 @@ ) from pants.backend.java.target_types import rules as java_target_rules from pants.backend.java.test.junit import rules as junit_rules -from pants.backend.java.util_rules import rules as java_util_rules from pants.core.util_rules import config_files, source_files from pants.core.util_rules.external_tool import rules as external_tool_rules from pants.engine.addresses import Address, Addresses, UnparsedAddressInputs @@ -30,6 +29,7 @@ InferredDependencies, Targets, ) +from pants.jvm.jdk_rules import rules as java_util_rules from pants.jvm.resolve.coursier_fetch import rules as coursier_fetch_rules from pants.jvm.resolve.coursier_setup import rules as coursier_setup_rules from pants.jvm.testutil import maybe_skip_jdk_test diff --git a/src/python/pants/backend/java/package/deploy_jar_test.py b/src/python/pants/backend/java/package/deploy_jar_test.py index f762f2d1c06..0dbe51f30ac 100644 --- a/src/python/pants/backend/java/package/deploy_jar_test.py +++ b/src/python/pants/backend/java/package/deploy_jar_test.py @@ -7,21 +7,14 @@ import pytest -from pants.backend.java import util_rules as java_util_rules from pants.backend.java.classpath import Classpath from pants.backend.java.classpath import rules as classpath_rules -from pants.backend.java.compile.javac import ( - CompiledClassfiles, - CompileJavaSourceRequest, - FallibleCompiledClassfiles, - JavacCheckRequest, -) +from pants.backend.java.compile.javac import CompileJavaSourceRequest, JavacCheckRequest from pants.backend.java.compile.javac import rules as javac_rules from pants.backend.java.package.deploy_jar import DeployJarFieldSet from pants.backend.java.package.deploy_jar import rules as deploy_jar_rules from pants.backend.java.target_types import DeployJar, JavaSourcesGeneratorTarget from pants.backend.java.target_types import rules as target_types_rules -from pants.backend.java.util_rules import JdkSetup from pants.build_graph.address import Address from pants.core.goals.check import CheckResults from pants.core.goals.package import BuiltPackage @@ -30,7 +23,10 @@ from pants.engine.addresses import Addresses from pants.engine.fs import Digest, MergeDigests, Snapshot from pants.engine.process import BashBinary, Process, ProcessResult +from pants.jvm import jdk_rules +from pants.jvm.compile import CompiledClassfiles, FallibleCompiledClassfiles from pants.jvm.goals.coursier import rules as coursier_rules +from pants.jvm.jdk_rules import JdkSetup from pants.jvm.resolve.coursier_fetch import CoursierResolvedLockfile from pants.jvm.resolve.coursier_fetch import rules as coursier_fetch_rules from pants.jvm.resolve.coursier_setup import rules as coursier_setup_rules @@ -46,25 +42,25 @@ def rule_runner() -> RuleRunner: rules=[ # TODO: delete a few of these (they were copied from junit tests; not sure which # are needed) + *classpath_rules(), *config_files.rules(), *coursier_fetch_rules(), + *coursier_rules(), *coursier_setup_rules(), - *external_tool_rules(), - *source_files.rules(), - *classpath_rules(), *deploy_jar_rules(), + *external_tool_rules(), *javac_rules(), - *util_rules(), + *jdk_rules.rules(), + *source_files.rules(), *target_types_rules(), - *coursier_rules(), - *java_util_rules.rules(), + *util_rules(), + QueryRule(BashBinary, ()), + QueryRule(BuiltPackage, (DeployJarFieldSet,)), QueryRule(CheckResults, (JavacCheckRequest,)), - QueryRule(FallibleCompiledClassfiles, (CompileJavaSourceRequest,)), - QueryRule(CompiledClassfiles, (CompileJavaSourceRequest,)), QueryRule(Classpath, (Addresses,)), - QueryRule(BuiltPackage, (DeployJarFieldSet,)), + QueryRule(CompiledClassfiles, (CompileJavaSourceRequest,)), + QueryRule(FallibleCompiledClassfiles, (CompileJavaSourceRequest,)), QueryRule(JdkSetup, ()), - QueryRule(BashBinary, ()), QueryRule(ProcessResult, (Process,)), QueryRule(Snapshot, (Digest,)), ], diff --git a/src/python/pants/backend/java/test/junit.py b/src/python/pants/backend/java/test/junit.py index 7ee3edd07f5..5163b0cc821 100644 --- a/src/python/pants/backend/java/test/junit.py +++ b/src/python/pants/backend/java/test/junit.py @@ -7,13 +7,13 @@ from pants.backend.java.classpath import Classpath from pants.backend.java.subsystems.junit import JUnit from pants.backend.java.target_types import JavaTestSourceField -from pants.backend.java.util_rules import JdkSetup from pants.core.goals.test import TestDebugRequest, TestFieldSet, TestResult, TestSubsystem from pants.engine.addresses import Addresses from pants.engine.fs import Digest, DigestSubset, MergeDigests, PathGlobs, RemovePrefix, Snapshot from pants.engine.process import BashBinary, FallibleProcessResult, Process from pants.engine.rules import Get, collect_rules, rule from pants.engine.unions import UnionRule +from pants.jvm.jdk_rules import JdkSetup from pants.jvm.resolve.coursier_fetch import ( ArtifactRequirements, Coordinate, diff --git a/src/python/pants/backend/java/test/junit_test.py b/src/python/pants/backend/java/test/junit_test.py index dbdb1741c65..0c349ac2b8f 100644 --- a/src/python/pants/backend/java/test/junit_test.py +++ b/src/python/pants/backend/java/test/junit_test.py @@ -14,7 +14,6 @@ from pants.backend.java.target_types import rules as target_types_rules from pants.backend.java.test.junit import JavaTestFieldSet from pants.backend.java.test.junit import rules as junit_rules -from pants.backend.java.util_rules import rules as java_util_rules from pants.build_graph.address import Address from pants.core.goals.test import TestResult from pants.core.util_rules import config_files, source_files @@ -22,6 +21,7 @@ from pants.engine.addresses import Addresses from pants.engine.fs import FileDigest from pants.engine.target import CoarsenedTargets +from pants.jvm.jdk_rules import rules as java_util_rules from pants.jvm.resolve.coursier_fetch import ( Coordinate, Coordinates, diff --git a/src/python/pants/jvm/compile.py b/src/python/pants/jvm/compile.py new file mode 100644 index 00000000000..535b0cd2a00 --- /dev/null +++ b/src/python/pants/jvm/compile.py @@ -0,0 +1,88 @@ +# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from enum import Enum + +from pants.engine.engine_aware import EngineAwareReturnType +from pants.engine.fs import Digest +from pants.engine.process import FallibleProcessResult +from pants.util.logging import LogLevel +from pants.util.strutil import strip_v2_chroot_path + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class CompiledClassfiles: + """The outputs of a compilation contained in either zero or one JAR file. + + TODO: Rename this type to align with the guarantee about its content. + """ + + digest: Digest + + +class CompileResult(Enum): + SUCCEEDED = "succeeded" + FAILED = "failed" + DEPENDENCY_FAILED = "dependency failed" + + +@dataclass(frozen=True) +class FallibleCompiledClassfiles(EngineAwareReturnType): + description: str + result: CompileResult + output: CompiledClassfiles | None + exit_code: int + stdout: str | None = None + stderr: str | None = None + + @classmethod + def from_fallible_process_result( + cls, + description: str, + process_result: FallibleProcessResult, + output: CompiledClassfiles | None, + *, + strip_chroot_path: bool = False, + ) -> FallibleCompiledClassfiles: + def prep_output(s: bytes) -> str: + return strip_v2_chroot_path(s) if strip_chroot_path else s.decode() + + exit_code = process_result.exit_code + # TODO: Coursier renders this line on macOS. + stderr = "\n".join( + line + for line in prep_output(process_result.stderr).splitlines() + if line != "setrlimit to increase file descriptor limit failed, errno 22" + ) + return cls( + description=description, + result=(CompileResult.SUCCEEDED if exit_code == 0 else CompileResult.FAILED), + output=output, + exit_code=exit_code, + stdout=prep_output(process_result.stdout), + stderr=stderr, + ) + + def level(self) -> LogLevel: + return LogLevel.ERROR if self.exit_code != 0 else LogLevel.INFO + + def message(self) -> str: + message = self.description + message += ( + " succeeded." if self.exit_code == 0 else f" failed (exit code {self.exit_code})." + ) + if self.stdout: + message += f"\n{self.stdout}" + if self.stderr: + message += f"\n{self.stderr}" + return message + + def cacheable(self) -> bool: + # Failed compile outputs should be re-rendered in every run. + return self.exit_code == 0 diff --git a/src/python/pants/backend/java/util_rules.py b/src/python/pants/jvm/jdk_rules.py similarity index 100% rename from src/python/pants/backend/java/util_rules.py rename to src/python/pants/jvm/jdk_rules.py diff --git a/src/python/pants/backend/java/util_rules_test.py b/src/python/pants/jvm/jdk_rules_test.py similarity index 95% rename from src/python/pants/backend/java/util_rules_test.py rename to src/python/pants/jvm/jdk_rules_test.py index 82ddc0bcb0f..bf70d270846 100644 --- a/src/python/pants/backend/java/util_rules_test.py +++ b/src/python/pants/jvm/jdk_rules_test.py @@ -5,13 +5,13 @@ import pytest -from pants.backend.java.util_rules import JdkSetup -from pants.backend.java.util_rules import rules as java_util_rules from pants.core.util_rules import config_files, source_files from pants.core.util_rules.external_tool import rules as external_tool_rules from pants.engine.internals.scheduler import ExecutionError from pants.engine.process import BashBinary, Process, ProcessResult from pants.engine.process import rules as process_rules +from pants.jvm.jdk_rules import JdkSetup +from pants.jvm.jdk_rules import rules as jdk_rules from pants.jvm.resolve.coursier_fetch import rules as coursier_fetch_rules from pants.jvm.resolve.coursier_setup import rules as coursier_setup_rules from pants.jvm.testutil import maybe_skip_jdk_test @@ -29,7 +29,7 @@ def rule_runner() -> RuleRunner: *coursier_fetch_rules(), *external_tool_rules(), *util_rules(), - *java_util_rules(), + *jdk_rules(), *process_rules(), QueryRule(BashBinary, ()), QueryRule(JdkSetup, ()),