Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add |= and | operators support for TypedDict #16249

Merged
merged 15 commits into from
Oct 23, 2023
19 changes: 15 additions & 4 deletions mypy/checker.py
Original file line number Diff line number Diff line change
Expand Up @@ -7452,14 +7452,25 @@ def infer_operator_assignment_method(typ: Type, operator: str) -> tuple[bool, st
"""
typ = get_proper_type(typ)
method = operators.op_methods[operator]
existing_method = None
if isinstance(typ, Instance):
if operator in operators.ops_with_inplace_method:
inplace_method = "__i" + method[2:]
if typ.type.has_readable_member(inplace_method):
return True, inplace_method
existing_method = _find_inplace_method(typ, method, operator)
elif isinstance(typ, TypedDictType):
existing_method = _find_inplace_method(typ.fallback, method, operator)

if existing_method is not None:
return True, existing_method
return False, method


def _find_inplace_method(inst: Instance, method: str, operator: str) -> str | None:
if operator in operators.ops_with_inplace_method:
inplace_method = "__i" + method[2:]
if inst.type.has_readable_member(inplace_method):
return inplace_method
return None


def is_valid_inferred_type(typ: Type, is_lvalue_final: bool = False) -> bool:
"""Is an inferred type valid and needs no further refinement?

Expand Down
21 changes: 19 additions & 2 deletions mypy/checkexpr.py
Original file line number Diff line number Diff line change
Expand Up @@ -3318,7 +3318,7 @@ def visit_ellipsis(self, e: EllipsisExpr) -> Type:
"""Type check '...'."""
return self.named_type("builtins.ellipsis")

def visit_op_expr(self, e: OpExpr) -> Type:
def visit_op_expr(self, e: OpExpr, *, allow_reverse: bool = True) -> Type:
"""Type check a binary operator expression."""
if e.analyzed:
# It's actually a type expression X | Y.
Expand Down Expand Up @@ -3366,6 +3366,21 @@ def visit_op_expr(self, e: OpExpr) -> Type:
return proper_left_type.copy_modified(
items=proper_left_type.items + [UnpackType(mapped)]
)
if is_named_instance(proper_left_type, "builtins.dict") and e.op == "|":
# This is a special case for `dict | TypedDict`.
# Before this change this operation was not allowed ude to typing limitations,
# however, it does perfect sense from runtime's point of view.
# So, what we do now?
sobolevn marked this conversation as resolved.
Show resolved Hide resolved
# 1. Find `dict | TypedDict` case
# 2. Switch `dict.__or__` to `TypedDict.__or__` (the same from typing's perspective)
# 3. Do not allow `dict.__ror__` to be executed, since this is a special case
# This can later be removed if `typeshed` can do this without special casing.
# https://github.com/python/mypy/pull/16249
proper_right_type = get_proper_type(self.accept(e.right))
if isinstance(proper_right_type, TypedDictType):
reverse_op = OpExpr(e.op, e.right, e.left)
reverse_op.set_line(e)
return self.visit_op_expr(reverse_op, allow_reverse=False)
if TYPE_VAR_TUPLE in self.chk.options.enable_incomplete_feature:
# Handle tuple[X, ...] + tuple[Y, Z] = tuple[*tuple[X, ...], Y, Z].
if (
Expand All @@ -3385,7 +3400,9 @@ def visit_op_expr(self, e: OpExpr) -> Type:

if e.op in operators.op_methods:
method = operators.op_methods[e.op]
result, method_type = self.check_op(method, left_type, e.right, e, allow_reverse=True)
result, method_type = self.check_op(
method, left_type, e.right, e, allow_reverse=allow_reverse
)
e.method_type = method_type
return result
else:
Expand Down
67 changes: 67 additions & 0 deletions test-data/unit/check-typeddict.test
Original file line number Diff line number Diff line change
Expand Up @@ -3236,3 +3236,70 @@ def foo(x: int) -> Foo: ...
f: Foo = {**foo("no")} # E: Argument 1 to "foo" has incompatible type "str"; expected "int"
[builtins fixtures/dict.pyi]
[typing fixtures/typing-typeddict.pyi]


[case testTypedDictWith__or__method]
from mypy_extensions import TypedDict

class Foo(TypedDict):
key: int

foo1: Foo = {'key': 1}
foo2: Foo = {'key': 2}

reveal_type(foo1 | foo2) # N: Revealed type is "TypedDict('__main__.Foo', {'key': builtins.int})"
reveal_type(foo1 | {'key': 1}) # N: Revealed type is "TypedDict('__main__.Foo', {'key': builtins.int})"
reveal_type(foo1 | {'key': 'a'}) # N: Revealed type is "builtins.dict[builtins.str, builtins.object]"
reveal_type(foo1 | {}) # N: Revealed type is "builtins.dict[builtins.str, builtins.object]"
[builtins fixtures/dict.pyi]
[typing fixtures/typing-typeddict-iror.pyi]

[case testTypedDictWith__or__method_error]
from mypy_extensions import TypedDict

class Foo(TypedDict):
key: int

foo: Foo = {'key': 1}

foo | 1
[out]
main:8: error: No overload variant of "__or__" of "TypedDict" matches argument type "int"
main:8: note: Possible overload variants:
main:8: note: def __or__(self, Foo, /) -> Foo
main:8: note: def __or__(self, Dict[str, object], /) -> Dict[str, object]
[builtins fixtures/dict.pyi]
[typing fixtures/typing-typeddict-iror.pyi]

[case testTypedDictWith__ror__method]
from mypy_extensions import TypedDict

class Foo(TypedDict):
key: int

foo: Foo = {'key': 1}

reveal_type({'key': 1} | foo) # N: Revealed type is "TypedDict('__main__.Foo', {'key': builtins.int})"
reveal_type({'key': 'a'} | foo) # N: Revealed type is "builtins.dict[builtins.str, builtins.object]"
reveal_type({} | foo) # N: Revealed type is "builtins.dict[builtins.str, builtins.object]"

1 | foo # E: Unsupported left operand type for | ("int")
[builtins fixtures/dict.pyi]
[typing fixtures/typing-typeddict-iror.pyi]

[case testTypedDictWith__ior__method]
from mypy_extensions import TypedDict

class Foo(TypedDict):
key: int
sobolevn marked this conversation as resolved.
Show resolved Hide resolved

foo: Foo = {'key': 1}
foo |= {'key': 2}

foo |= {} # E: Missing key "key" for TypedDict "Foo"
foo |= {'key': 'a', 'b': 'a'} # E: Extra key "b" for TypedDict "Foo" \
# E: Incompatible types (expression has type "str", TypedDict item "key" has type "int")
foo |= {'b': 2} # E: Missing key "key" for TypedDict "Foo" \
# E: Extra key "b" for TypedDict "Foo"
[builtins fixtures/dict.pyi]
[typing fixtures/typing-typeddict-iror.pyi]
66 changes: 66 additions & 0 deletions test-data/unit/fixtures/typing-typeddict-iror.pyi
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
# Test stub for typing module that includes TypedDict `|` operator.
# It only covers `__or__`, `__ror__`, and `__ior__`.
#
# We cannot define these methods in `typing-typeddict.pyi`,
# because they need `dict` with two type args,
# and not all tests using `[typing typing-typeddict.pyi]` have the proper
# `dict` stub.
#
# Keep in sync with `typeshed`'s definition.
from abc import ABCMeta

cast = 0
assert_type = 0
overload = 0
Any = 0
Union = 0
Optional = 0
TypeVar = 0
Generic = 0
Protocol = 0
Tuple = 0
Callable = 0
NamedTuple = 0
Final = 0
Literal = 0
TypedDict = 0
NoReturn = 0
Required = 0
NotRequired = 0
Self = 0

T = TypeVar('T')
T_co = TypeVar('T_co', covariant=True)
V = TypeVar('V')

# Note: definitions below are different from typeshed, variances are declared
# to silence the protocol variance checks. Maybe it is better to use type: ignore?

class Sized(Protocol):
def __len__(self) -> int: pass

class Iterable(Protocol[T_co]):
def __iter__(self) -> 'Iterator[T_co]': pass

class Iterator(Iterable[T_co], Protocol):
def __next__(self) -> T_co: pass

class Sequence(Iterable[T_co]):
# misc is for explicit Any.
def __getitem__(self, n: Any) -> T_co: pass # type: ignore[misc]

class Mapping(Iterable[T], Generic[T, T_co], metaclass=ABCMeta):
pass

# Fallback type for all typed dicts (does not exist at runtime).
class _TypedDict(Mapping[str, object]):
@overload
def __or__(self, __value: Self) -> Self: ...
@overload
def __or__(self, __value: dict[str, object]) -> dict[str, object]: ...
@overload
def __ror__(self, __value: Self) -> Self: ...
@overload
def __ror__(self, __value: dict[str, Any]) -> dict[str, object]: ...
# supposedly incompatible definitions of __or__ and __ior__
def __ior__(self, __value: Self) -> Self: ... # type: ignore[misc]
15 changes: 15 additions & 0 deletions test-data/unit/fixtures/typing-typeddict.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -71,3 +71,18 @@ class _TypedDict(Mapping[str, object]):
def pop(self, k: NoReturn, default: T = ...) -> object: ...
def update(self: T, __m: T) -> None: ...
def __delitem__(self, k: NoReturn) -> None: ...
# It is a bit of a lie:
# 1. it is only supported since 3.9 in runtime
# 2. `__or__` uses `dict[str, Any]`, not `Mapping[str, object]`
@overload
def __or__(self, __value: Self) -> Self: ...
@overload
def __or__(self, __value: Mapping[str, object]) -> Mapping[str, object]: ...
# TODO: re-enable after `__ror__` definition is fixed
sobolevn marked this conversation as resolved.
Show resolved Hide resolved
# https://github.com/python/typeshed/issues/10678
# @overload
# def __ror__(self, __value: Self) -> Self: ...
# @overload
# def __ror__(self, __value: dict[str, Any]) -> dict[str, object]: ...
# supposedly incompatible definitions of __or__ and __ior__
Copy link
Member

@AlexWaygood AlexWaygood Oct 13, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't understand this comment: see what I wrote in python/typeshed#10678 (comment). What do you think is broken in _TypedDict.__(r)or__?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Here's my understanding of this problem:

  1. dict.__or__ does not allow TypedDict, because it only allows dict (and TypedDict does not match it):
ex.py:7: error: No overload variant of "__or__" of "dict" matches argument type "Foo"  [operator]
ex.py:7: note: Possible overload variants:
ex.py:7: note:     def __or__(self, dict[str, int], /) -> dict[str, int]
ex.py:7: note:     def [_T1, _T2] __or__(self, dict[_T1, _T2], /) -> dict[str | _T1, int | _T2]
ex.py:7: note: Revealed type is "Any"
ex.py:8: error: No overload variant of "__or__" of "dict" matches argument type "Foo"  [operator]
ex.py:8: note: Possible overload variants:
ex.py:8: note:     def __or__(self, dict[str, str], /) -> dict[str, str]
ex.py:8: note:     def [_T1, _T2] __or__(self, dict[_T1, _T2], /) -> dict[str | _T1, str | _T2]
ex.py:8: note: Revealed type is "Any"
  1. mypy does the reverse checking for every __or__ operator, so even for foo | {'key': 1} it raises (without this PR):
ex.py:7: error: No overload variant of "__ror__" of "dict" matches argument type "Foo"  [operator]
ex.py:7: note: Possible overload variants:
ex.py:7: note:     def __ror__(self, dict[str, int], /) -> dict[str, int]
ex.py:7: note:     def [_T1, _T2] __ror__(self, dict[_T1, _T2], /) -> dict[str | _T1, int | _T2]
ex.py:7: note: Revealed type is "Any"

So, my problem definition is: dict.__or__ is not compatible with TypedDict and dict.__ror__ is also not compatible with TypedDict

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Here is the snippet the OP was complaining about in #16244:

from typing import TypedDict

class Foo(TypedDict):
    key: int
    
foo: Foo = {"key": 1}
foo |= {"key": 3}

mypy reports:

error: Incompatible types in assignment (expression has type "dict[str, object]", variable has type "Foo")  [assignment]

https://mypy-play.net/?mypy=latest&python=3.11&gist=9927e1732f6fce4595d29a19493ae603

Note: mypy does not report that |= cannot be used between foo and {"key: 3}. It understands that the |= operator can be used between the two objects. However, it is inferring that the type of the foo variable after the |-ing is dict[str, object] rather than Foo, which would be an incompatible reassignment of the foo variable, since the foo variable has been declared to always have type Foo.

That indicates that mypy is using the second overload of _TypedDict.__or__ here to infer the type of foo after the |= operation, rather than _TypedDict.__ior__ or the first overload of _TypedDict.__or__. If it used the _TypedDict.__ior__ annotations, or the first overload of _TypedDict.__or__, it would be able to correctly infer that the type of foo after the |= operation is still Foo, which is a reassignment that's compatible with the declared type of the foo variable:

@overload
def __or__(self, __value: typing_extensions.Self) -> typing_extensions.Self: ...
@overload
def __or__(self, __value: dict[str, Any]) -> dict[str, object]: ...
@overload
def __ror__(self, __value: typing_extensions.Self) -> typing_extensions.Self: ...
@overload
def __ror__(self, __value: dict[str, Any]) -> dict[str, object]: ...
# supposedly incompatible definitions of __or__ and __ior__
def __ior__(self, __value: typing_extensions.Self) -> typing_extensions.Self: ... # type: ignore[misc]

I think the issue is that in this snippet, mypy is not able currently to infer using type context (bidirectional type inference) that the object on the right-hand-side of the |= operator can be considered to be an instance of Foo. If it did so, then it would select _TypedDict.__ior__ or the first overload of _TypedDict.__or__ to infer the type of foo after the |= operation, and would correctly understand that the type of the foo variable does not change as a result of the operation.

def __ior__(self, __value: Self) -> Self: ... # type: ignore[misc]