From 7329d6d54e7d5f54887ae3acfb528a87a29f8580 Mon Sep 17 00:00:00 2001 From: Asher Foa Date: Mon, 4 Oct 2021 22:09:45 -0700 Subject: [PATCH] [internal] Run pyupgrade src/python/pants/[util,options,testutil] (#13107) doing this in small batches to minimize interruption (i.e. other people need to rebase/deal with conflicts) --- src/python/pants/option/arg_splitter.py | 2 +- src/python/pants/option/config.py | 22 ++++++------ src/python/pants/option/custom_types.py | 14 ++++---- src/python/pants/option/global_options.py | 8 ++--- .../pants/option/option_value_container.py | 19 +++++----- src/python/pants/option/options.py | 36 +++++++++---------- .../pants/option/options_bootstrapper.py | 14 ++++---- .../pants/option/options_fingerprinter.py | 2 +- src/python/pants/option/options_test.py | 12 +++---- src/python/pants/option/parser.py | 28 +++++++-------- src/python/pants/option/ranked_value.py | 4 +-- src/python/pants/option/subsystem.py | 8 ++--- src/python/pants/testutil/option_util.py | 6 ++-- .../pants/testutil/pants_integration_test.py | 13 ++++--- src/python/pants/testutil/rule_runner.py | 18 +++++----- src/python/pants/util/collections.py | 4 +-- src/python/pants/util/contextutil.py | 12 +++---- src/python/pants/util/dirutil.py | 8 ++--- src/python/pants/util/dirutil_test.py | 8 ++--- src/python/pants/util/eval.py | 7 ++-- src/python/pants/util/frozendict.py | 6 ++-- src/python/pants/util/logging.py | 2 +- src/python/pants/util/memo_test.py | 2 +- src/python/pants/util/ordered_set.py | 17 ++------- src/python/pants/util/osutil.py | 7 ++-- src/python/pants/util/socket.py | 2 +- src/python/pants/util/strutil.py | 6 ++-- 27 files changed, 136 insertions(+), 151 deletions(-) diff --git a/src/python/pants/option/arg_splitter.py b/src/python/pants/option/arg_splitter.py index 37125cd723b..8f8cd6292f7 100644 --- a/src/python/pants/option/arg_splitter.py +++ b/src/python/pants/option/arg_splitter.py @@ -104,7 +104,7 @@ def __init__(self, known_scope_infos: Iterable[ScopeInfo], buildroot: str) -> No # We check for prefixes in reverse order, so we match the longest prefix first. sorted_scope_infos = sorted( - [si for si in self._known_scope_infos if si.scope], + (si for si in self._known_scope_infos if si.scope), key=lambda si: si.scope, reverse=True, ) diff --git a/src/python/pants/option/config.py b/src/python/pants/option/config.py index 90caed03cf0..f704a5f03f3 100644 --- a/src/python/pants/option/config.py +++ b/src/python/pants/option/config.py @@ -12,7 +12,7 @@ from dataclasses import dataclass from functools import partial from hashlib import sha1 -from typing import Any, ClassVar, Dict, Iterable, List, Mapping, Sequence, Tuple, Union, cast +from typing import Any, ClassVar, Dict, Iterable, List, Mapping, Sequence, Union, cast import toml from typing_extensions import Protocol @@ -96,7 +96,7 @@ def load( @classmethod def _parse_toml( - cls, config_content: str, normalized_seed_values: Dict[str, str] + cls, config_content: str, normalized_seed_values: dict[str, str] ) -> _ConfigValues: """Attempt to parse as TOML, raising an exception on failure.""" toml_values = cast(Dict[str, Any], toml.loads(config_content)) @@ -107,7 +107,7 @@ def _parse_toml( return _ConfigValues(toml_values) @staticmethod - def _determine_seed_values(*, seed_values: SeedValues | None = None) -> Dict[str, str]: + def _determine_seed_values(*, seed_values: SeedValues | None = None) -> dict[str, str]: """We pre-populate several default values to allow %([key-name])s interpolation. This sets up those defaults and checks if the user overrode any of the values. @@ -115,7 +115,7 @@ def _determine_seed_values(*, seed_values: SeedValues | None = None) -> Dict[str safe_seed_values = seed_values or {} buildroot = cast(str, safe_seed_values.get("buildroot", get_buildroot())) - all_seed_values: Dict[str, str] = { + all_seed_values: dict[str, str] = { "buildroot": buildroot, "homedir": os.path.expanduser("~"), "user": getpass.getuser(), @@ -152,15 +152,15 @@ def get(self, section, option, type_=str, default=None): ) @abstractmethod - def configs(self) -> Sequence["_SingleFileConfig"]: + def configs(self) -> Sequence[_SingleFileConfig]: """Returns the underlying single-file configs represented by this object.""" @abstractmethod - def sources(self) -> List[str]: + def sources(self) -> list[str]: """Returns the sources of this config as a list of filenames.""" @abstractmethod - def sections(self) -> List[str]: + def sections(self) -> list[str]: """Returns the sections in this config (not including DEFAULT).""" @abstractmethod @@ -194,7 +194,7 @@ def get_source_for_option(self, section: str, option: str) -> str | None: class _ConfigValues: """The parsed contents of a TOML config file.""" - values: Dict[str, Any] + values: dict[str, Any] @staticmethod def _is_an_option(option_value: _TomlValue | dict) -> bool: @@ -212,7 +212,7 @@ def _possibly_interpolate_value( *, option: str, section: str, - section_values: Dict, + section_values: dict, ) -> str: """For any values with %(foo)s, substitute it with the corresponding value from DEFAULT or the same section.""" @@ -254,7 +254,7 @@ def _stringify_val( *, option: str, section: str, - section_values: Dict, + section_values: dict, interpolate: bool = True, list_prefix: str | None = None, ) -> str: @@ -489,7 +489,7 @@ class TomlSerializer: parsed: Mapping[str, dict[str, int | float | str | bool | list | dict]] def normalize(self) -> dict: - def normalize_section_value(option, option_value) -> Tuple[str, Any]: + def normalize_section_value(option, option_value) -> tuple[str, Any]: # With TOML, we store dict values as strings (for now). if isinstance(option_value, dict): option_value = str(option_value) diff --git a/src/python/pants/option/custom_types.py b/src/python/pants/option/custom_types.py index e9e1fb8aaf2..e6e630763b3 100644 --- a/src/python/pants/option/custom_types.py +++ b/src/python/pants/option/custom_types.py @@ -8,7 +8,7 @@ import re import shlex from enum import Enum -from typing import Dict, Iterable, List, Pattern, Sequence +from typing import Iterable, Pattern, Sequence from pants.option.errors import ParseError from pants.util.eval import parse_expression @@ -165,7 +165,7 @@ def _convert_list(val, member_type, is_enum): return [item if isinstance(item, member_type) else member_type(item) for item in converted] -def _flatten_shlexed_list(shlexed_args: Sequence[str]) -> List[str]: +def _flatten_shlexed_list(shlexed_args: Sequence[str]) -> list[str]: """Convert a list of shlexed args into a flattened list of individual args. For example, ['arg1 arg2=foo', '--arg3'] would be converted to ['arg1', 'arg2=foo', '--arg3']. @@ -203,7 +203,7 @@ def _get_modifier_expr_re(cls) -> Pattern[str]: return re.compile(r"(?<=\]|\))\s*,\s*(?=[+-](?:\[|\())") @classmethod - def _split_modifier_expr(cls, s: str) -> List[str]: + def _split_modifier_expr(cls, s: str) -> list[str]: # This check ensures that the first expression (before the first split point) is a modification. if s.startswith("+") or s.startswith("-"): return cls._get_modifier_expr_re().split(s) @@ -232,13 +232,13 @@ def merge(cls, components: Iterable[ListValueComponent]) -> ListValueComponent: raise ParseError(f"Unknown action for list value: {component._action}") return cls(action, appends, filters) - def __init__(self, action: str, appends: List, filters: List) -> None: + def __init__(self, action: str, appends: list, filters: list) -> None: self._action = action self._appends = appends self._filters = filters @property - def val(self) -> List: + def val(self) -> list: ret = list(self._appends) for x in self._filters: # Note: can't do ret.remove(x) because that only removes the first instance of x. @@ -315,7 +315,7 @@ class DictValueComponent: EXTEND = "EXTEND" @classmethod - def merge(cls, components: Iterable["DictValueComponent"]) -> DictValueComponent: + def merge(cls, components: Iterable[DictValueComponent]) -> DictValueComponent: """Merges components into a single component, applying their actions appropriately. This operation is associative: M(M(a, b), c) == M(a, M(b, c)) == M(a, b, c). @@ -334,7 +334,7 @@ def merge(cls, components: Iterable["DictValueComponent"]) -> DictValueComponent raise ParseError(f"Unknown action for dict value: {component.action}") return cls(action, val) - def __init__(self, action: str, val: Dict) -> None: + def __init__(self, action: str, val: dict) -> None: self.action = action self.val = val diff --git a/src/python/pants/option/global_options.py b/src/python/pants/option/global_options.py index 752bfb34b9e..91806883d46 100644 --- a/src/python/pants/option/global_options.py +++ b/src/python/pants/option/global_options.py @@ -14,7 +14,7 @@ from datetime import datetime from enum import Enum from pathlib import Path -from typing import Any, Dict, List, Tuple, cast +from typing import Any, cast from pants.base.build_environment import ( get_buildroot, @@ -334,8 +334,8 @@ class ExecutionOptions: remote_cache_rpc_concurrency: int remote_execution_address: str | None - remote_execution_extra_platform_properties: List[str] - remote_execution_headers: Dict[str, str] + remote_execution_extra_platform_properties: list[str] + remote_execution_headers: dict[str, str] remote_execution_overall_deadline_secs: int remote_execution_rpc_concurrency: int @@ -1556,7 +1556,7 @@ def add(absolute_path, include=False): @staticmethod def compute_pantsd_invalidation_globs( buildroot: str, bootstrap_options: OptionValueContainer - ) -> Tuple[str, ...]: + ) -> tuple[str, ...]: """Computes the merged value of the `--pantsd-invalidation-globs` option. Combines --pythonpath and --pants-config-files files that are in {buildroot} dir with those diff --git a/src/python/pants/option/option_value_container.py b/src/python/pants/option/option_value_container.py index 8d1895fc377..b081897e614 100644 --- a/src/python/pants/option/option_value_container.py +++ b/src/python/pants/option/option_value_container.py @@ -5,7 +5,7 @@ import copy from dataclasses import dataclass -from typing import Dict, Iterator, List, Optional +from typing import Iterator from pants.option.ranked_value import Rank, RankedValue, Value @@ -13,10 +13,10 @@ class OptionValueContainerBuilder: - def __init__(self, value_map: Optional[Dict[Key, RankedValue]] = None) -> None: - self._value_map: Dict[Key, RankedValue] = value_map if value_map else {} + def __init__(self, value_map: dict[Key, RankedValue] | None = None) -> None: + self._value_map: dict[Key, RankedValue] = value_map if value_map else {} - def update(self, other: "OptionValueContainerBuilder") -> None: + def update(self, other: OptionValueContainerBuilder) -> None: """Set other's values onto this object. For each key, highest ranked value wins. In a tie, other's value wins. @@ -60,9 +60,9 @@ class OptionValueContainer: See ranked_value.py for more details. """ - _value_map: Dict[Key, RankedValue] + _value_map: dict[Key, RankedValue] - def get_explicit_keys(self) -> List[Key]: + def get_explicit_keys(self) -> list[Key]: """Returns the keys for any values that were set explicitly (via flag, config, or env var).""" ret = [] @@ -108,13 +108,13 @@ def is_default(self, key: Key) -> bool: """ return self.get_rank(key) in (Rank.NONE, Rank.HARDCODED) - def get(self, key: Key, default: Optional[Value] = None): + def get(self, key: Key, default: Value | None = None): # Support dict-like dynamic access. See also __getitem__ below. if key not in self._value_map: return default return self._get_underlying_value(key) - def as_dict(self) -> Dict[Key, Value]: + def as_dict(self) -> dict[Key, Value]: return {key: self.get(key) for key in self._value_map} def to_builder(self) -> OptionValueContainerBuilder: @@ -143,5 +143,4 @@ def __getattr__(self, key: Key): def __iter__(self) -> Iterator[Key]: """Returns an iterator over all option names, in lexicographical order.""" - for name in sorted(self._value_map.keys()): - yield name + yield from sorted(self._value_map.keys()) diff --git a/src/python/pants/option/options.py b/src/python/pants/option/options.py index a710bb83520..46d40a24381 100644 --- a/src/python/pants/option/options.py +++ b/src/python/pants/option/options.py @@ -5,7 +5,7 @@ import dataclasses import logging -from typing import Dict, Iterable, List, Mapping, Optional, Sequence +from typing import Iterable, Mapping, Sequence from pants.base.build_environment import get_buildroot from pants.base.deprecated import warn_or_error @@ -79,7 +79,7 @@ def complete_scopes(cls, scope_infos: Iterable[ScopeInfo]) -> FrozenOrderedSet[S Also validates that scopes do not collide. """ ret: OrderedSet[ScopeInfo] = OrderedSet() - original_scopes: Dict[str, ScopeInfo] = {} + original_scopes: dict[str, ScopeInfo] = {} for si in sorted(scope_infos, key=lambda _si: _si.scope): if si.scope in original_scopes: raise cls.DuplicateScopeError( @@ -100,7 +100,7 @@ def create( config: Config, known_scope_infos: Iterable[ScopeInfo], args: Sequence[str], - bootstrap_option_values: Optional[OptionValueContainer] = None, + bootstrap_option_values: OptionValueContainer | None = None, allow_unknown_options: bool = False, ) -> Options: """Create an Options instance. @@ -131,7 +131,7 @@ def create( spec_files = bootstrap_option_values.spec_files if spec_files: for spec_file in spec_files: - with open(spec_file, "r") as f: + with open(spec_file) as f: split_args.specs.extend( [line for line in [line.strip() for line in f] if line] ) @@ -154,14 +154,14 @@ def create( def __init__( self, - goals: List[str], - scope_to_flags: Dict[str, List[str]], - specs: List[str], - passthru: List[str], - help_request: Optional[HelpRequest], - parser_by_scope: Dict[str, Parser], - bootstrap_option_values: Optional[OptionValueContainer], - known_scope_to_info: Dict[str, ScopeInfo], + goals: list[str], + scope_to_flags: dict[str, list[str]], + specs: list[str], + passthru: list[str], + help_request: HelpRequest | None, + parser_by_scope: dict[str, Parser], + bootstrap_option_values: OptionValueContainer | None, + known_scope_to_info: dict[str, ScopeInfo], allow_unknown_options: bool = False, ) -> None: """The low-level constructor for an Options instance. @@ -179,14 +179,14 @@ def __init__( self._allow_unknown_options = allow_unknown_options @property - def help_request(self) -> Optional[HelpRequest]: + def help_request(self) -> HelpRequest | None: """ :API: public """ return self._help_request @property - def specs(self) -> List[str]: + def specs(self) -> list[str]: """The specifications to operate on, e.g. the target addresses and the file names. :API: public @@ -194,7 +194,7 @@ def specs(self) -> List[str]: return self._specs @property - def goals(self) -> List[str]: + def goals(self) -> list[str]: """The requested goals, in the order specified on the cmd line. :API: public @@ -202,11 +202,11 @@ def goals(self) -> List[str]: return self._goals @property - def known_scope_to_info(self) -> Dict[str, ScopeInfo]: + def known_scope_to_info(self) -> dict[str, ScopeInfo]: return self._known_scope_to_info @property - def scope_to_flags(self) -> Dict[str, List[str]]: + def scope_to_flags(self) -> dict[str, list[str]]: return self._scope_to_flags def verify_configs(self, global_config: Config) -> None: @@ -396,7 +396,7 @@ def __getitem__(self, scope: str) -> OptionValueContainer: # Consider killing if tests consolidate on using TestOptions instead of the raw dicts. return self.for_scope(scope) - def bootstrap_option_values(self) -> Optional[OptionValueContainer]: + def bootstrap_option_values(self) -> OptionValueContainer | None: """Return the option values for bootstrap options. General code can also access these values in the global scope. But option registration code diff --git a/src/python/pants/option/options_bootstrapper.py b/src/python/pants/option/options_bootstrapper.py index 967cbc6732e..839650c82c6 100644 --- a/src/python/pants/option/options_bootstrapper.py +++ b/src/python/pants/option/options_bootstrapper.py @@ -8,7 +8,7 @@ import warnings from dataclasses import dataclass from pathlib import Path -from typing import Dict, Iterable, List, Mapping, Sequence, Set, Tuple, Type +from typing import Iterable, Mapping, Sequence from pants.base.build_environment import get_default_pants_config_file, pants_version from pants.base.exceptions import BuildConfigurationError @@ -30,9 +30,9 @@ class OptionsBootstrapper: """Holds the result of the first stage of options parsing, and assists with parsing full options.""" - env_tuples: Tuple[Tuple[str, str], ...] - bootstrap_args: Tuple[str, ...] - args: Tuple[str, ...] + env_tuples: tuple[tuple[str, str], ...] + bootstrap_args: tuple[str, ...] + args: tuple[str, ...] config: Config def __repr__(self) -> str: @@ -43,7 +43,7 @@ def __repr__(self) -> str: return f"OptionsBootstrapper(args={args}, env={env}, config={self.config})" @staticmethod - def get_config_file_paths(env, args) -> List[str]: + def get_config_file_paths(env, args) -> list[str]: """Get the location of the config files. The locations are specified by the --pants-config-files option. However we need to load the @@ -190,7 +190,7 @@ def is_bootstrap_option(arg: str) -> bool: ) @memoized_property - def env(self) -> Dict[str, str]: + def env(self) -> dict[str, str]: return dict(self.env_tuples) @memoized_property @@ -223,7 +223,7 @@ def _full_options( allow_unknown_options=allow_unknown_options, ) - distinct_subsystem_classes: Set[Type[Subsystem]] = set() + distinct_subsystem_classes: set[type[Subsystem]] = set() for ksi in known_scope_infos: if not ksi.subsystem_cls or ksi.subsystem_cls in distinct_subsystem_classes: continue diff --git a/src/python/pants/option/options_fingerprinter.py b/src/python/pants/option/options_fingerprinter.py index b1028e24929..73cb72a2704 100644 --- a/src/python/pants/option/options_fingerprinter.py +++ b/src/python/pants/option/options_fingerprinter.py @@ -150,7 +150,7 @@ def _fingerprint_dict_with_files(self, option_val): for k, v in option_val.items(): for sub_value in sorted(v.split(",")): if os.path.isfile(sub_value): - with open(sub_value, "r") as f: + with open(sub_value) as f: final[k].append(f.read()) else: final[k].append(sub_value) diff --git a/src/python/pants/option/options_test.py b/src/python/pants/option/options_test.py index f00ffea17ba..089610c4c1c 100644 --- a/src/python/pants/option/options_test.py +++ b/src/python/pants/option/options_test.py @@ -10,7 +10,7 @@ from enum import Enum from functools import partial from textwrap import dedent -from typing import Any, Callable, Dict, List, cast +from typing import Any, Callable, Dict, cast import pytest import toml @@ -764,7 +764,7 @@ def test_arg_scoping(self) -> None: def test_list_option(self) -> None: def check( *, - expected: List[int], + expected: list[int], flags: str = "", env_val: str | None = None, config_val: str | None = None, @@ -862,7 +862,7 @@ def check( def test_dict_list_option(self) -> None: def check( *, - expected: List[Dict[str, int]], + expected: list[dict[str, int]], flags: str = "", env_val: str | None = None, config_val: str | None = None, @@ -901,7 +901,7 @@ def check( def test_target_list_option(self) -> None: def check( *, - expected: List[str], + expected: list[str], flags: str = "", env_val: str | None = None, config_val: str | None = None, @@ -932,7 +932,7 @@ def check( def test_shell_str_list(self) -> None: def check( *, - expected: List[str], + expected: list[str], flags: str = "", env_val: str | None = None, config_val: str | None = None, @@ -965,7 +965,7 @@ def check( def test_dict_option(self) -> None: def check( *, - expected: Dict[str, str], + expected: dict[str, str], flags: str = "", config_val: str | None = None, ) -> None: diff --git a/src/python/pants/option/parser.py b/src/python/pants/option/parser.py index ea25456c756..c90eadbcb3d 100644 --- a/src/python/pants/option/parser.py +++ b/src/python/pants/option/parser.py @@ -13,7 +13,7 @@ from dataclasses import dataclass from enum import Enum from pathlib import Path -from typing import Any, DefaultDict, Dict, Iterable, List, Mapping, Set, Tuple, Type +from typing import Any, DefaultDict, Iterable, Mapping import yaml @@ -59,7 +59,7 @@ @dataclass(frozen=True) class OptionValueHistory: - ranked_values: Tuple[RankedValue] + ranked_values: tuple[RankedValue] @property def final_value(self) -> RankedValue: @@ -123,13 +123,13 @@ def __init__( self._scope = self._scope_info.scope # All option args registered with this parser. Used to prevent conflicts. - self._known_args: Set[str] = set() + self._known_args: set[str] = set() # List of (args, kwargs) registration pairs, exactly as captured at registration time. - self._option_registrations: List[Tuple[Tuple[str, ...], Dict[str, Any]]] = [] + self._option_registrations: list[tuple[tuple[str, ...], dict[str, Any]]] = [] # Map of dest -> history. - self._history: Dict[str, OptionValueHistory] = {} + self._history: dict[str, OptionValueHistory] = {} @property def scope_info(self) -> ScopeInfo: @@ -145,16 +145,16 @@ def history(self, dest: str) -> OptionValueHistory | None: @frozen_after_init @dataclass(unsafe_hash=True) class ParseArgsRequest: - flag_value_map: Dict[str, List[Any]] + flag_value_map: dict[str, list[Any]] namespace: OptionValueContainerBuilder - passthrough_args: List[str] + passthrough_args: list[str] allow_unknown_flags: bool def __init__( self, flags_in_scope: Iterable[str], namespace: OptionValueContainerBuilder, - passthrough_args: List[str], + passthrough_args: list[str], allow_unknown_flags: bool, ) -> None: """ @@ -199,7 +199,7 @@ def parse_args(self, parse_args_request: ParseArgsRequest) -> OptionValueContain flag_value_map = parse_args_request.flag_value_map namespace = parse_args_request.namespace - mutex_map: DefaultDict[str, List[str]] = defaultdict(list) + mutex_map: DefaultDict[str, list[str]] = defaultdict(list) for args, kwargs in self._option_registrations: self._validate(args, kwargs) dest = self.parse_dest(*args, **kwargs) @@ -386,7 +386,7 @@ def _validate(self, args, kwargs) -> None: """Validate option registration arguments.""" def error( - exception_type: Type[RegistrationError], + exception_type: type[RegistrationError], arg_name: str | None = None, **msg_kwargs, ) -> None: @@ -562,7 +562,7 @@ def expand(val_or_str): else: fromfile = val_or_str[1:] try: - with open(fromfile, "r") as fp: + with open(fromfile) as fp: s = fp.read().strip() if fromfile.endswith(".json"): return json.loads(s) @@ -570,7 +570,7 @@ def expand(val_or_str): return yaml.safe_load(s) else: return s - except (IOError, ValueError, yaml.YAMLError) as e: + except (OSError, ValueError, yaml.YAMLError) as e: raise FromfileError( f"Failed to read {dest} in {self._scope_str()} from file {fromfile}: {e!r}" ) @@ -650,14 +650,14 @@ def expand(val_or_str): # instances of RankedValue (so none will be None, although they may wrap a None value). ranked_vals = list(reversed(list(RankedValue.prioritized_iter(*values_to_rank)))) - def group(value_component_type, process_val_func) -> List[RankedValue]: + def group(value_component_type, process_val_func) -> list[RankedValue]: # We group any values that are merged together, so that the history can reflect # merges vs. replacements in a useful way. E.g., if we merge [a, b] and [c], # and then replace it with [d, e], the history will contain: # - [d, e] (from command-line flag) # - [a, b, c] (from env var, from config) # And similarly for dicts. - grouped: List[List[RankedValue]] = [[]] + grouped: list[list[RankedValue]] = [[]] for ranked_val in ranked_vals: if ranked_val.value and ranked_val.value.action == value_component_type.REPLACE: grouped.append([]) diff --git a/src/python/pants/option/ranked_value.py b/src/python/pants/option/ranked_value.py index 905476c32e2..9eef42e974f 100644 --- a/src/python/pants/option/ranked_value.py +++ b/src/python/pants/option/ranked_value.py @@ -22,7 +22,7 @@ class Rank(Enum): _rank: int def __new__(cls, rank: int, display: str) -> Rank: - member: "Rank" = object.__new__(cls) + member: Rank = object.__new__(cls) member._value_ = display member._rank = rank return member @@ -80,7 +80,7 @@ def prioritized_iter( config_default_val: ValueAndDetails, hardcoded_val: ValueAndDetails, default: ValueAndDetails, - ) -> Iterator["RankedValue"]: + ) -> Iterator[RankedValue]: """Yield the non-None values from highest-ranked to lowest, as RankedValue instances.""" if flag_val[0] is not None: yield RankedValue(Rank.FLAG, *flag_val) diff --git a/src/python/pants/option/subsystem.py b/src/python/pants/option/subsystem.py index 49cb456bc98..8d5eea14d0c 100644 --- a/src/python/pants/option/subsystem.py +++ b/src/python/pants/option/subsystem.py @@ -7,7 +7,7 @@ import inspect import re from abc import ABCMeta -from typing import Any, ClassVar, Optional, Type, TypeVar +from typing import Any, ClassVar, TypeVar from pants.engine.internals.selectors import Get, GetConstraints from pants.option.errors import OptionsError @@ -33,8 +33,8 @@ class Subsystem(metaclass=ABCMeta): # Option values can be read from the deprecated scope, but a deprecation warning will be issued. # The deprecation warning becomes an error at the given Pants version (which must therefore be # a valid semver). - deprecated_options_scope: Optional[str] = None - deprecated_options_scope_removal_version: Optional[str] = None + deprecated_options_scope: str | None = None + deprecated_options_scope_removal_version: str | None = None _scope_name_re = re.compile(r"^(?:[a-z0-9_])+(?:-(?:[a-z0-9_])+)*$") @@ -119,6 +119,6 @@ def __eq__(self, other: Any) -> bool: _T = TypeVar("_T", bound=Subsystem) -async def _construct_subsytem(subsystem_typ: Type[_T]) -> _T: +async def _construct_subsytem(subsystem_typ: type[_T]) -> _T: scoped_options = await Get(ScopedOptions, Scope(str(subsystem_typ.options_scope))) return subsystem_typ(scoped_options.options) diff --git a/src/python/pants/testutil/option_util.py b/src/python/pants/testutil/option_util.py index ecf7cf3ac77..4beb7581250 100644 --- a/src/python/pants/testutil/option_util.py +++ b/src/python/pants/testutil/option_util.py @@ -3,7 +3,7 @@ from __future__ import annotations -from typing import Iterable, Mapping, Type, TypeVar +from typing import Iterable, Mapping, TypeVar from pants.engine.goal import GoalSubsystem from pants.option.option_value_container import OptionValueContainer, OptionValueContainerBuilder @@ -37,7 +37,7 @@ def create_option_value_container( def create_goal_subsystem( - goal_subsystem_type: Type[_GS], + goal_subsystem_type: type[_GS], default_rank: Rank = Rank.NONE, **options: RankedValue | Value, ) -> _GS: @@ -56,7 +56,7 @@ def create_goal_subsystem( def create_subsystem( - subsystem_type: Type[_SS], default_rank: Rank = Rank.NONE, **options: RankedValue | Value + subsystem_type: type[_SS], default_rank: Rank = Rank.NONE, **options: RankedValue | Value ) -> _SS: """Creates a new subsystem instance populated with the given option values. diff --git a/src/python/pants/testutil/pants_integration_test.py b/src/python/pants/testutil/pants_integration_test.py index dda8d96227b..ad3f100d0da 100644 --- a/src/python/pants/testutil/pants_integration_test.py +++ b/src/python/pants/testutil/pants_integration_test.py @@ -9,7 +9,7 @@ import sys from contextlib import contextmanager from dataclasses import dataclass -from typing import Any, Iterator, List, Mapping, Optional, Union +from typing import Any, Iterator, List, Mapping, Union import pytest @@ -37,7 +37,7 @@ class PantsResult: workdir: str pid: Pid - def _format_unexpected_error_code_msg(self, msg: Optional[str]) -> str: + def _format_unexpected_error_code_msg(self, msg: str | None) -> str: details = [msg] if msg else [] details.append(" ".join(self.command)) details.append(f"exit_code: {self.exit_code}") @@ -49,10 +49,10 @@ def indent(content): details.append(f"stderr:\n\t{indent(self.stderr)}") return "\n".join(details) - def assert_success(self, msg: Optional[str] = None) -> None: + def assert_success(self, msg: str | None = None) -> None: assert self.exit_code == 0, self._format_unexpected_error_code_msg(msg) - def assert_failure(self, msg: Optional[str] = None) -> None: + def assert_failure(self, msg: str | None = None) -> None: assert self.exit_code != 0, self._format_unexpected_error_code_msg(msg) @@ -307,11 +307,10 @@ def render_logs(workdir: str) -> None: def read_pants_log(workdir: str) -> Iterator[str]: """Yields all lines from the pants log under the given workdir.""" # Surface the pants log for easy viewing via pytest's `-s` (don't capture stdio) option. - for line in _read_log(f"{workdir}/pants.log"): - yield line + yield from _read_log(f"{workdir}/pants.log") def _read_log(filename: str) -> Iterator[str]: - with open(filename, "r") as f: + with open(filename) as f: for line in f: yield line.rstrip() diff --git a/src/python/pants/testutil/rule_runner.py b/src/python/pants/testutil/rule_runner.py index 817c1364645..e166c3a8eca 100644 --- a/src/python/pants/testutil/rule_runner.py +++ b/src/python/pants/testutil/rule_runner.py @@ -14,7 +14,7 @@ from pprint import pformat from tempfile import mkdtemp from types import CoroutineType, GeneratorType -from typing import Any, Callable, Iterable, Iterator, Mapping, Sequence, Tuple, Type, TypeVar, cast +from typing import Any, Callable, Iterable, Iterator, Mapping, Sequence, TypeVar, cast from pants.base.build_root import BuildRoot from pants.base.specs_parser import SpecsParser @@ -225,7 +225,7 @@ def rules(self) -> FrozenOrderedSet[Rule | UnionRule]: return FrozenOrderedSet([*self.build_config.rules, *self.build_config.union_rules]) @property - def target_types(self) -> Tuple[Type[Target], ...]: + def target_types(self) -> tuple[type[Target], ...]: return self.build_config.target_types @property @@ -237,7 +237,7 @@ def new_session(self, build_id: str) -> None: """Mutates this RuleRunner to begin a new Session with the same Scheduler.""" self.scheduler = self.scheduler.scheduler.new_session(build_id) - def request(self, output_type: Type[_O], inputs: Iterable[Any]) -> _O: + def request(self, output_type: type[_O], inputs: Iterable[Any]) -> _O: result = assert_single_element( self.scheduler.product_request(output_type, [Params(*inputs)]) ) @@ -245,7 +245,7 @@ def request(self, output_type: Type[_O], inputs: Iterable[Any]) -> _O: def run_goal_rule( self, - goal: Type[Goal], + goal: type[Goal], *, global_args: Iterable[str] | None = None, args: Iterable[str] | None = None, @@ -439,8 +439,8 @@ def write_digest(self, digest: Digest, *, path_prefix: str | None = None) -> Non # `Callable[[InputType], OutputType]`. @dataclass(frozen=True) class MockGet: - output_type: Type - input_type: Type + output_type: type + input_type: type mock: Callable[[Any], Any] @@ -552,12 +552,12 @@ def get(product, subject): @contextmanager def stdin_context(content: bytes | str | None = None): if content is None: - yield open("/dev/null", "r") + yield open("/dev/null") else: with temporary_file(binary_mode=isinstance(content, bytes)) as stdin_file: stdin_file.write(content) stdin_file.close() - yield open(stdin_file.name, "r") + yield open(stdin_file.name) @contextmanager @@ -565,7 +565,7 @@ def mock_console( options_bootstrapper: OptionsBootstrapper, *, stdin_content: bytes | str | None = None, -) -> Iterator[Tuple[Console, StdioReader]]: +) -> Iterator[tuple[Console, StdioReader]]: global_bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope() colors = ( options_bootstrapper.full_options_for_scopes( diff --git a/src/python/pants/util/collections.py b/src/python/pants/util/collections.py index 7a9434ce98e..a0cca4ea851 100644 --- a/src/python/pants/util/collections.py +++ b/src/python/pants/util/collections.py @@ -5,7 +5,7 @@ import collections import collections.abc -from typing import Any, Iterable, List, MutableMapping, TypeVar +from typing import Any, Iterable, MutableMapping, TypeVar def recursively_update(d: MutableMapping, d2: MutableMapping) -> None: @@ -54,7 +54,7 @@ def ensure_list( raise ValueError( f"The value {val} (type {type(val)}) was not an iterable of {expected_type}." ) - result: List[_T] = [] + result: list[_T] = [] for i, x in enumerate(val): if not isinstance(x, expected_type): raise ValueError( diff --git a/src/python/pants/util/contextutil.py b/src/python/pants/util/contextutil.py index 2ce9b18c0f0..1d5b444c7fc 100644 --- a/src/python/pants/util/contextutil.py +++ b/src/python/pants/util/contextutil.py @@ -15,7 +15,7 @@ from pathlib import Path from queue import Queue from socketserver import TCPServer -from typing import IO, Any, Callable, Iterator, Mapping, Tuple, Type +from typing import IO, Any, Callable, Iterator, Mapping from colors import green @@ -84,7 +84,7 @@ def hermetic_environment_as(**kwargs: str | None) -> Iterator[None]: @contextmanager -def argv_as(args: Tuple[str, ...]) -> Iterator[None]: +def argv_as(args: tuple[str, ...]) -> Iterator[None]: """Temporarily set `sys.argv` to the supplied value.""" old_args = sys.argv try: @@ -274,8 +274,8 @@ def maybe_profiled(profile_path: str | None) -> Iterator[None]: @contextmanager -def http_server(handler_class: Type, ssl_context: ssl.SSLContext | None = None) -> Iterator[int]: - def serve(port_queue: "Queue[int]", shutdown_queue: "Queue[bool]") -> None: +def http_server(handler_class: type, ssl_context: ssl.SSLContext | None = None) -> Iterator[int]: + def serve(port_queue: Queue[int], shutdown_queue: Queue[bool]) -> None: httpd = TCPServer(("", 0), handler_class) httpd.timeout = 0.1 if ssl_context: @@ -285,8 +285,8 @@ def serve(port_queue: "Queue[int]", shutdown_queue: "Queue[bool]") -> None: while shutdown_queue.empty(): httpd.handle_request() - port_queue: "Queue[int]" = Queue() - shutdown_queue: "Queue[bool]" = Queue() + port_queue: Queue[int] = Queue() + shutdown_queue: Queue[bool] = Queue() t = threading.Thread(target=lambda: serve(port_queue, shutdown_queue)) t.daemon = True t.start() diff --git a/src/python/pants/util/dirutil.py b/src/python/pants/util/dirutil.py index 3ff70e04f8e..9d81eb754c8 100644 --- a/src/python/pants/util/dirutil.py +++ b/src/python/pants/util/dirutil.py @@ -13,7 +13,7 @@ import uuid from collections import defaultdict from contextlib import contextmanager -from typing import Any, Callable, DefaultDict, Iterator, Sequence, Set, overload +from typing import Any, Callable, DefaultDict, Iterator, Sequence, overload from typing_extensions import Literal @@ -138,7 +138,7 @@ def maybe_read_file(filename: str, binary_mode: bool = False) -> bytes | str | N """ try: return read_file(filename, binary_mode=binary_mode) - except IOError: + except OSError: return None @@ -194,7 +194,7 @@ def safe_walk(path: bytes | str, **kwargs: Any) -> Iterator[tuple[str, list[str] _MkdtempCleanerType = Callable[[], None] _MKDTEMP_CLEANER: _MkdtempCleanerType | None = None -_MKDTEMP_DIRS: DefaultDict[int, Set[str]] = defaultdict(set) +_MKDTEMP_DIRS: DefaultDict[int, set[str]] = defaultdict(set) _MKDTEMP_LOCK = threading.RLock() @@ -284,7 +284,7 @@ def safe_concurrent_rename(src: str, dst: str) -> None: safe_delete(dst) try: shutil.move(src, dst) - except IOError as e: + except OSError as e: if e.errno != errno.EEXIST: raise diff --git a/src/python/pants/util/dirutil_test.py b/src/python/pants/util/dirutil_test.py index b9565cd7d47..31cdfa6f3f1 100644 --- a/src/python/pants/util/dirutil_test.py +++ b/src/python/pants/util/dirutil_test.py @@ -9,7 +9,7 @@ import unittest.mock from contextlib import contextmanager from dataclasses import dataclass -from typing import Iterator, Tuple +from typing import Iterator from pants.util import dirutil from pants.util.contextutil import pushd, temporary_dir @@ -128,7 +128,7 @@ def test_safe_walk(self) -> None: self.assertTrue(all(isinstance(dirname, str) for dirname in dirs)) @contextmanager - def tree(self) -> Iterator[Tuple[str, str]]: + def tree(self) -> Iterator[tuple[str, str]]: # root/ # a/ # b/ @@ -160,7 +160,7 @@ def empty(cls, path: str) -> DirutilTest.File: @classmethod def read(cls, root: str, relpath: str) -> DirutilTest.File: - with open(os.path.join(root, relpath), "r") as fp: + with open(os.path.join(root, relpath)) as fp: return cls(relpath, fp.read()) @dataclass(frozen=True) @@ -388,7 +388,7 @@ def test_overwrite_link_file(self) -> None: self._create_and_check_link(self.source, self.link) # The link should have been deleted (over-written), not the file it pointed to. - with open(self.source, "r") as fp: + with open(self.source) as fp: self.assertEqual("evidence", fp.read()) def test_overwrite_link_dir(self) -> None: diff --git a/src/python/pants/util/eval.py b/src/python/pants/util/eval.py index f000185b93d..24e18097a80 100644 --- a/src/python/pants/util/eval.py +++ b/src/python/pants/util/eval.py @@ -4,14 +4,14 @@ from __future__ import annotations from textwrap import dedent -from typing import Any, Type +from typing import Any def parse_expression( val: str, acceptable_types: type | tuple[type, ...], name: str | None = None, - raise_type: Type[BaseException] = ValueError, + raise_type: type[BaseException] = ValueError, ) -> Any: """Attempts to parse the given `val` as a python expression of the specified `acceptable_types`. @@ -63,8 +63,7 @@ def iter_types(types): yield types elif isinstance(types, tuple): for item in types: - for typ in iter_types(item): - yield typ + yield from iter_types(item) else: raise ValueError( f"The given acceptable_types is not a valid type (tuple): {acceptable_types}" diff --git a/src/python/pants/util/frozendict.py b/src/python/pants/util/frozendict.py index 7ab9d16504c..d16a1725130 100644 --- a/src/python/pants/util/frozendict.py +++ b/src/python/pants/util/frozendict.py @@ -3,7 +3,7 @@ from __future__ import annotations -from typing import Any, Iterable, Iterator, Mapping, Tuple, TypeVar, overload +from typing import Any, Iterable, Iterator, Mapping, TypeVar, overload K = TypeVar("K") V = TypeVar("V") @@ -18,7 +18,7 @@ class FrozenDict(Mapping[K, V]): """ @overload - def __init__(self, __items: Iterable[Tuple[K, V]], **kwargs: V) -> None: + def __init__(self, __items: Iterable[tuple[K, V]], **kwargs: V) -> None: ... @overload @@ -29,7 +29,7 @@ def __init__(self, __other: Mapping[K, V], **kwargs: V) -> None: def __init__(self, **kwargs: V) -> None: ... - def __init__(self, *item: Mapping[K, V] | Iterable[Tuple[K, V]], **kwargs: V) -> None: + def __init__(self, *item: Mapping[K, V] | Iterable[tuple[K, V]], **kwargs: V) -> None: """Creates a `FrozenDict` with arguments accepted by `dict` that also must be hashable.""" if len(item) > 1: raise ValueError( diff --git a/src/python/pants/util/logging.py b/src/python/pants/util/logging.py index 48e863c4570..564ce7ec2ed 100644 --- a/src/python/pants/util/logging.py +++ b/src/python/pants/util/logging.py @@ -20,7 +20,7 @@ class LogLevel(Enum): _level: int def __new__(cls, value: str, level: int) -> LogLevel: - member: "LogLevel" = object.__new__(cls) + member: LogLevel = object.__new__(cls) member._value_ = value member._level = level return member diff --git a/src/python/pants/util/memo_test.py b/src/python/pants/util/memo_test.py index e933b35cf02..3c81f0f98f2 100644 --- a/src/python/pants/util/memo_test.py +++ b/src/python/pants/util/memo_test.py @@ -158,7 +158,7 @@ def square(num): self.assertEqual([2, 3, 2, 3], calculations) - class _Called(object): + class _Called: def __init__(self, increment): self._calls = 0 self._increment = increment diff --git a/src/python/pants/util/ordered_set.py b/src/python/pants/util/ordered_set.py index 87c0d323bf2..6ef76bda2a7 100644 --- a/src/python/pants/util/ordered_set.py +++ b/src/python/pants/util/ordered_set.py @@ -15,18 +15,7 @@ from __future__ import annotations import itertools -from typing import ( - AbstractSet, - Any, - Dict, - Hashable, - Iterable, - Iterator, - MutableSet, - Set, - TypeVar, - cast, -) +from typing import AbstractSet, Any, Hashable, Iterable, Iterator, MutableSet, Set, TypeVar, cast T = TypeVar("T") T_co = TypeVar("T_co", covariant=True) @@ -43,7 +32,7 @@ def __init__(self, iterable: Iterable[T] | None = None) -> None: # NB: Dictionaries are ordered in Python 3.6+. While this was not formalized until Python # 3.7, Python 3.6 uses this behavior; Pants requires CPython 3.6+ to run, so this # assumption is safe for us to rely on. - self._items: Dict[T, None] = {v: None for v in iterable or ()} + self._items: dict[T, None] = {v: None for v in iterable or ()} def __len__(self) -> int: """Returns the number of unique elements in the set.""" @@ -185,7 +174,7 @@ def clear(self) -> None: def difference_update(self, *others: Iterable[T]) -> None: """Update this OrderedSet to remove items from one or more other sets.""" - items_to_remove: Set[T] = set() + items_to_remove: set[T] = set() for other in others: items_as_set = set(other) items_to_remove |= items_as_set diff --git a/src/python/pants/util/osutil.py b/src/python/pants/util/osutil.py index 350953d62f0..064ae8ff7a2 100644 --- a/src/python/pants/util/osutil.py +++ b/src/python/pants/util/osutil.py @@ -9,7 +9,6 @@ import platform import posix from functools import reduce -from typing import Optional logger = logging.getLogger(__name__) @@ -43,7 +42,7 @@ def _compute_cpu_count() -> int: Pid = int -def get_arch_name(uname_result: Optional[posix.uname_result] = None) -> str: +def get_arch_name(uname_result: posix.uname_result | None = None) -> str: """ :API: public """ @@ -52,7 +51,7 @@ def get_arch_name(uname_result: Optional[posix.uname_result] = None) -> str: return uname_result.machine.lower() -def get_os_name(uname_result: Optional[posix.uname_result] = None) -> str: +def get_os_name(uname_result: posix.uname_result | None = None) -> str: """ :API: public """ @@ -119,7 +118,7 @@ def safe_kill(pid: Pid, signum: int) -> None: assert isinstance(signum, int) try: os.kill(pid, signum) - except (IOError, OSError) as e: + except OSError as e: if e.errno in [errno.ESRCH, errno.EPERM]: pass elif e.errno == errno.EINVAL: diff --git a/src/python/pants/util/socket.py b/src/python/pants/util/socket.py index 01d584ae62e..0bb00e36129 100644 --- a/src/python/pants/util/socket.py +++ b/src/python/pants/util/socket.py @@ -39,7 +39,7 @@ def _maybe_tune_socket(self, sock): try: # Disable Nagle's algorithm to improve latency. sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - except (OSError, IOError): + except OSError: # This can fail in tests where `socket.socketpair()` is used, or potentially # in odd environments - but we shouldn't ever crash over it. return diff --git a/src/python/pants/util/strutil.py b/src/python/pants/util/strutil.py index c1cceeaab06..72721b43301 100644 --- a/src/python/pants/util/strutil.py +++ b/src/python/pants/util/strutil.py @@ -6,7 +6,7 @@ import re import shlex import textwrap -from typing import Iterable, List, Sequence +from typing import Iterable, Sequence def ensure_binary(text_or_binary: bytes | str) -> bytes: @@ -27,7 +27,7 @@ def ensure_text(text_or_binary: bytes | str) -> str: raise TypeError(f"Argument is neither text nor binary type ({type(text_or_binary)})") -def safe_shlex_split(text_or_binary: bytes | str) -> List[str]: +def safe_shlex_split(text_or_binary: bytes | str) -> list[str]: """Split a string using shell-like syntax. Safe even on python versions whose shlex.split() method doesn't accept unicode. @@ -74,7 +74,7 @@ def create_path_env_var( prev_path = env.get(env_var, None) if prev_path is None: - path_dirs: List[str] = [] + path_dirs: list[str] = [] else: path_dirs = list(prev_path.split(delimiter))