Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Replace include_special_cased_deps flag with should_traverse_deps_predicate #19272

Merged
merged 20 commits into from
Jun 13, 2023
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
c95aa60
Add `ShouldTraverseDepsPredicate`
cognifloyd Jun 8, 2023
c2c6f26
Handle SpecialCasedDependencies in `ShouldTraverseDepsPredicate`
cognifloyd Jun 8, 2023
ef7a09f
Replace `include_special_cased_deps=True` flag with `should_traverse_…
cognifloyd Jun 8, 2023
e30672c
Drop `include_special_cased_deps` flag
cognifloyd Jun 8, 2023
941c3c2
fmt
cognifloyd Jun 8, 2023
c1c9252
Add `ExplicitlyProvidedDependenciesRequest`
cognifloyd Jun 8, 2023
c1ed5a2
Add DepsTraversalPredicates class as a mini-namespace for predicates
cognifloyd Jun 8, 2023
39e5bde
Merge branch 'main' into cognifloyd/deps-predicate
cognifloyd Jun 8, 2023
434a8fb
Correct type hints for DepsTraversalPredicates methods
cognifloyd Jun 8, 2023
945b50b
comment typo
cognifloyd Jun 8, 2023
869b91f
Merge branch 'main' into cognifloyd/deps-predicate
cognifloyd Jun 8, 2023
f0a808c
Merge branch 'main' into cognifloyd/deps-predicate
cognifloyd Jun 8, 2023
a77b046
Use TypeAlias to move ShouldTraverseDepsPredicate closer to implement…
cognifloyd Jun 8, 2023
412e994
Merge branch 'main' into cognifloyd/deps-predicate
cognifloyd Jun 12, 2023
0b91c76
Turn ShouldTraverseDepsPredicate into a dataclass
cognifloyd Jun 13, 2023
57660a6
cleanup test
cognifloyd Jun 13, 2023
d0364f5
Add test for frozen ShouldTraverseDepsPredicate subclasses
cognifloyd Jun 13, 2023
b856127
Merge branch 'main' into cognifloyd/deps-predicate
cognifloyd Jun 13, 2023
7ce2bce
Improve comment on _callable
cognifloyd Jun 13, 2023
6a28cd6
reword comment about subclassing ShouldTraverseDepsPredicate
cognifloyd Jun 13, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions src/python/pants/backend/project_info/dependencies.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@
from pants.engine.console import Console
from pants.engine.goal import Goal, GoalSubsystem, LineOriented
from pants.engine.rules import Get, MultiGet, collect_rules, goal_rule
from pants.engine.target import AlwaysTraverseDeps
from pants.engine.target import Dependencies as DependenciesField
from pants.engine.target import (
DependenciesRequest,
DepsTraversalPredicates,
Targets,
TransitiveTargets,
TransitiveTargetsRequest,
Expand Down Expand Up @@ -46,7 +46,7 @@ async def dependencies(
transitive_targets = await Get(
TransitiveTargets,
TransitiveTargetsRequest(
addresses, should_traverse_deps_predicate=DepsTraversalPredicates.always
addresses, should_traverse_deps_predicate=AlwaysTraverseDeps()
),
)
targets = Targets(transitive_targets.dependencies)
Expand All @@ -61,7 +61,7 @@ async def dependencies(
Targets,
DependenciesRequest(
tgt.get(DependenciesField),
should_traverse_deps_predicate=DepsTraversalPredicates.always,
should_traverse_deps_predicate=AlwaysTraverseDeps(),
),
)
for tgt in target_roots
Expand Down
4 changes: 2 additions & 2 deletions src/python/pants/backend/project_info/dependents.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,9 @@
from pants.engine.rules import Get, MultiGet, collect_rules, goal_rule, rule
from pants.engine.target import (
AllUnexpandedTargets,
AlwaysTraverseDeps,
Dependencies,
DependenciesRequest,
DepsTraversalPredicates,
)
from pants.option.option_types import BoolOption
from pants.util.frozendict import FrozenDict
Expand All @@ -33,7 +33,7 @@ async def map_addresses_to_dependents(all_targets: AllUnexpandedTargets) -> Addr
Get(
Addresses,
DependenciesRequest(
tgt.get(Dependencies), should_traverse_deps_predicate=DepsTraversalPredicates.always
tgt.get(Dependencies), should_traverse_deps_predicate=AlwaysTraverseDeps()
),
)
for tgt in all_targets
Expand Down
4 changes: 2 additions & 2 deletions src/python/pants/backend/project_info/filedeps.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from pants.engine.goal import Goal, GoalSubsystem, LineOriented
from pants.engine.rules import Get, MultiGet, collect_rules, goal_rule
from pants.engine.target import (
DepsTraversalPredicates,
AlwaysTraverseDeps,
HydratedSources,
HydrateSourcesRequest,
SourcesField,
Expand Down Expand Up @@ -75,7 +75,7 @@ async def file_deps(
transitive_targets = await Get(
TransitiveTargets,
TransitiveTargetsRequest(
addresses, should_traverse_deps_predicate=DepsTraversalPredicates.always
addresses, should_traverse_deps_predicate=AlwaysTraverseDeps()
),
)
targets = transitive_targets.closure
Expand Down
6 changes: 3 additions & 3 deletions src/python/pants/backend/project_info/paths.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@
from pants.engine.goal import Goal, GoalSubsystem, Outputting
from pants.engine.rules import Get, MultiGet, collect_rules, goal_rule
from pants.engine.target import (
AlwaysTraverseDeps,
Dependencies,
DependenciesRequest,
DepsTraversalPredicates,
Targets,
TransitiveTargets,
TransitiveTargetsRequest,
Expand Down Expand Up @@ -117,15 +117,15 @@ async def paths(console: Console, paths_subsystem: PathsSubsystem) -> PathsGoal:
transitive_targets = await Get(
TransitiveTargets,
TransitiveTargetsRequest(
[root.address], should_traverse_deps_predicate=DepsTraversalPredicates.always
[root.address], should_traverse_deps_predicate=AlwaysTraverseDeps()
),
)

adjacent_targets_per_target = await MultiGet(
Get(
Targets,
DependenciesRequest(
tgt.get(Dependencies), should_traverse_deps_predicate=DepsTraversalPredicates.always
tgt.get(Dependencies), should_traverse_deps_predicate=AlwaysTraverseDeps()
),
)
for tgt in transitive_targets.closure
Expand Down
4 changes: 2 additions & 2 deletions src/python/pants/backend/project_info/peek.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,11 @@
from pants.engine.internals.dep_rules import DependencyRuleApplication, DependencyRuleSet
from pants.engine.rules import Get, MultiGet, collect_rules, goal_rule, rule
from pants.engine.target import (
AlwaysTraverseDeps,
Dependencies,
DependenciesRequest,
DependenciesRuleApplication,
DependenciesRuleApplicationRequest,
DepsTraversalPredicates,
Field,
HydratedSources,
HydrateSourcesRequest,
Expand Down Expand Up @@ -181,7 +181,7 @@ async def get_target_data(
Get(
Targets,
DependenciesRequest(
tgt.get(Dependencies), should_traverse_deps_predicate=DepsTraversalPredicates.always
tgt.get(Dependencies), should_traverse_deps_predicate=AlwaysTraverseDeps()
),
)
for tgt in sorted_targets
Expand Down
4 changes: 2 additions & 2 deletions src/python/pants/backend/visibility/lint.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,11 @@
from pants.engine.platform import Platform
from pants.engine.rules import Get, MultiGet, collect_rules, rule
from pants.engine.target import (
AlwaysTraverseDeps,
Dependencies,
DependenciesRequest,
DependenciesRuleApplication,
DependenciesRuleApplicationRequest,
DepsTraversalPredicates,
FieldSet,
)
from pants.util.logging import LogLevel
Expand All @@ -44,7 +44,7 @@ async def check_visibility_rule_violations(
Addresses,
DependenciesRequest(
field_set.dependencies,
should_traverse_deps_predicate=DepsTraversalPredicates.always,
should_traverse_deps_predicate=AlwaysTraverseDeps(),
),
)
for field_set in request.elements
Expand Down
15 changes: 10 additions & 5 deletions src/python/pants/engine/internals/graph_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,11 @@
from pants.engine.target import (
AllTargets,
AllUnexpandedTargets,
AlwaysTraverseDeps,
AsyncFieldMixin,
CoarsenedTargets,
Dependencies,
DependenciesRequest,
DepsTraversalPredicates,
ExplicitlyProvidedDependencies,
Field,
FieldDefaultFactoryRequest,
Expand All @@ -55,6 +55,7 @@
MultipleSourcesField,
OverridesField,
SecondaryOwnerMixin,
ShouldTraverseDepsPredicate,
SingleSourceField,
SourcesPaths,
SourcesPathsRequest,
Expand Down Expand Up @@ -318,7 +319,7 @@ def get_target(name: str) -> Target:
Targets,
[
DependenciesRequest(
root[Dependencies], should_traverse_deps_predicate=DepsTraversalPredicates.always
root[Dependencies], should_traverse_deps_predicate=AlwaysTraverseDeps()
)
],
)
Expand All @@ -336,7 +337,7 @@ def get_target(name: str) -> Target:
[
TransitiveTargetsRequest(
[root.address, d2.address],
should_traverse_deps_predicate=DepsTraversalPredicates.always,
should_traverse_deps_predicate=AlwaysTraverseDeps(),
)
],
)
Expand Down Expand Up @@ -417,13 +418,17 @@ def get_target(name: str) -> Target:
)
assert direct_deps == Targets([d1, d2, d3, d4])

@dataclass(frozen=True)
class SkipMeTagOrTraverse(ShouldTraverseDepsPredicate):
def __call__(self, target: Target, field: Dependencies | SpecialCasedDependencies) -> bool:
return "skip_deps" not in (target[Tags].value or [])

transitive_targets = transitive_targets_rule_runner.request(
TransitiveTargets,
[
TransitiveTargetsRequest(
[root.address, d2.address],
should_traverse_deps_predicate=lambda tgt, fld: "skip_deps"
not in (tgt[Tags].value or []),
should_traverse_deps_predicate=SkipMeTagOrTraverse(),
)
],
)
Expand Down
68 changes: 42 additions & 26 deletions src/python/pants/engine/target.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
get_type_hints,
)

from typing_extensions import Protocol, Self, TypeAlias, final
from typing_extensions import Protocol, Self, final

from pants.base.deprecated import warn_or_error
from pants.engine.addresses import Address, Addresses, UnparsedAddressInputs, assert_single_address
Expand Down Expand Up @@ -714,33 +714,51 @@ def expect_single(self) -> Target:
return self[0]


# If this predicate returns false, then the target's field's deps will be ignored.
ShouldTraverseDepsPredicate: TypeAlias = (
"Callable[[Target, Dependencies | SpecialCasedDependencies], bool]"
)
@dataclass(frozen=True)
class ShouldTraverseDepsPredicate(metaclass=ABCMeta):
"""This callable determines whether to traverse through deps of a given Target + Field.

This is a callable dataclass instead of a function to avoid issues with hashing closures.
Only the id is used when hashing a function; any closure vars are NOT accounted for.

class DepsTraversalPredicates:
"""All methods on this class are ShouldTraverseDepsPredicate implementations."""
NB: When subclassing a dataclass (like this one), generated methods on a dataclass are NOT
inherited because the @dataclass decorator only inspects the current class before generating
a function--the decorator does NOT inspect __mro__.
"""

@staticmethod
def if_dependencies_field(
target: Target, field: Dependencies | SpecialCasedDependencies
) -> bool:
"""This is the default ShouldTraverseDepsPredicate implementation.
# This _callable field ensures that __call__ is included in the __hash__ method generated by @dataclass.
cognifloyd marked this conversation as resolved.
Show resolved Hide resolved
_callable: Callable[
[Target, Dependencies | SpecialCasedDependencies], bool
] = dataclasses.field(init=False, repr=False)

This skips resolving dependencies for fields (like SpecialCasedDependencies) that are not
subclasses of Dependencies.
"""
def __post_init__(self):
object.__setattr__(self, "_callable", type(self).__call__)

@abstractmethod
def __call__(self, target: Target, field: Dependencies | SpecialCasedDependencies) -> bool:
"""If this predicate returns false, then the target's field's deps will be ignored."""


@dataclass(frozen=True)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

do the subclasses actually need the dataclass decorator at all?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Without it, aren't they unfrozen? I'll try dropping the decorator on these subclasses since they don't add any fields.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch. It works just fine without decorating the subclasses. I will have to add the decorator on the predicate that stops at package targets because that will add a field. These, however, don't need it.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I actually think we should still decorate:

  • It'll work correctly when someone adds a field later
  • When people copy+paste this code elsewhere they'll have it
    My 2c

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I adjusted one of the tests to make sure it works without the decorator. Doing that helped me resolve a bug with the type hint on _callable since the test made mypy inspect it more carefully.

I can re-add the decorator, I think I like it with the decorator a bit better as it is more consistent. But I've documented it as is for now.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it'll come down to preference then. I'm for not having them until needed, but won't object/block if you prefer to keep them.

class TraverseIfDependenciesField(ShouldTraverseDepsPredicate):
"""This is the default ShouldTraverseDepsPredicate implementation.

This skips resolving dependencies for fields (like SpecialCasedDependencies) that are not
subclasses of Dependencies.
"""

def __call__(self, target: Target, field: Dependencies | SpecialCasedDependencies) -> bool:
return isinstance(field, Dependencies)

@staticmethod
def always(target: Target, field: Dependencies | SpecialCasedDependencies) -> bool:
"""A predicate to use when a request needs all deps.

This includes deps from fields like SpecialCasedDependencies which are ignored in most
cases.
"""
@dataclass(frozen=True)
class AlwaysTraverseDeps(ShouldTraverseDepsPredicate):
"""A predicate to use when a request needs all deps.

This includes deps from fields like SpecialCasedDependencies which are ignored in most cases.
"""

def __call__(self, target: Target, field: Dependencies | SpecialCasedDependencies) -> bool:
return True


Expand Down Expand Up @@ -875,7 +893,7 @@ def __init__(
roots: Iterable[Address],
*,
expanded_targets: bool = False,
should_traverse_deps_predicate: ShouldTraverseDepsPredicate = DepsTraversalPredicates.if_dependencies_field,
should_traverse_deps_predicate: ShouldTraverseDepsPredicate = TraverseIfDependenciesField(),
) -> None:
object.__setattr__(self, "roots", tuple(roots))
object.__setattr__(self, "expanded_targets", expanded_targets)
Expand Down Expand Up @@ -914,7 +932,7 @@ def __init__(
self,
roots: Iterable[Address],
*,
should_traverse_deps_predicate: ShouldTraverseDepsPredicate = DepsTraversalPredicates.if_dependencies_field,
should_traverse_deps_predicate: ShouldTraverseDepsPredicate = TraverseIfDependenciesField(),
) -> None:
object.__setattr__(self, "roots", tuple(roots))
object.__setattr__(self, "should_traverse_deps_predicate", should_traverse_deps_predicate)
Expand Down Expand Up @@ -2502,9 +2520,7 @@ def unevaluated_transitive_excludes(self) -> UnparsedAddressInputs:
@dataclass(frozen=True)
class DependenciesRequest(EngineAwareParameter):
field: Dependencies
should_traverse_deps_predicate: ShouldTraverseDepsPredicate = (
DepsTraversalPredicates.if_dependencies_field
)
should_traverse_deps_predicate: ShouldTraverseDepsPredicate = TraverseIfDependenciesField()

def debug_hint(self) -> str:
return self.field.address.spec
Expand Down