From 2c3850afb19aced0f71dcc1fb3275467d98e6e83 Mon Sep 17 00:00:00 2001 From: Eric Wolf Date: Sun, 7 Aug 2022 15:44:23 +0200 Subject: [PATCH 01/11] Mostly working --- mypy/checkexpr.py | 67 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 49 insertions(+), 18 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index c445b1a9b714..8e72f57c1e78 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -642,11 +642,16 @@ def check_typeddict_call( self.chk.fail(message_registry.INVALID_TYPEDDICT_ARGS, context) return AnyType(TypeOfAny.from_error) - def validate_typeddict_kwargs(self, kwargs: DictExpr) -> "Optional[Dict[str, Expression]]": - item_args = [item[1] for item in kwargs.items] + def validate_typeddict_kwargs(self, kwargs: DictExpr) -> "Optional[List[Tuple[Optional[str], Expression]]]": + item_names: List[Optional[str]] = [] + item_args: List[Expression] = [] - item_names = [] # List[str] for item_name_expr, item_arg in kwargs.items: + # Allow unpacking with ** + if item_name_expr is None: + item_names.append(None) + item_args.append(item_arg) + continue literal_value = None if item_name_expr: key_type = self.accept(item_name_expr) @@ -659,14 +664,15 @@ def validate_typeddict_kwargs(self, kwargs: DictExpr) -> "Optional[Dict[str, Exp return None else: item_names.append(literal_value) - return dict(zip(item_names, item_args)) + item_args.append(item_arg) + return list(zip(item_names, item_args)) def match_typeddict_call_with_dict( self, callee: TypedDictType, kwargs: DictExpr, context: Context ) -> bool: validated_kwargs = self.validate_typeddict_kwargs(kwargs=kwargs) if validated_kwargs is not None: - return callee.required_keys <= set(validated_kwargs.keys()) <= set(callee.items.keys()) + return callee.required_keys <= set(dict(validated_kwargs).keys()) <= set(callee.items.keys()) else: return False @@ -682,30 +688,55 @@ def check_typeddict_call_with_dict( return AnyType(TypeOfAny.from_error) def check_typeddict_call_with_kwargs( - self, callee: TypedDictType, kwargs: Dict[str, Expression], context: Context + self, callee: TypedDictType, kwargs: List[Tuple[Optional[str], Expression]], context: Context ) -> Type: - if not (callee.required_keys <= set(kwargs.keys()) <= set(callee.items.keys())): + items: Dict[str, Tuple[Expression, Type]] = {} + for key, value in kwargs: + value_type = self.accept(value, callee.items.get(key) if key is not None else callee) + if key is not None: + items[key] = (value, value_type) + else: + # Spread operator was used; determine type of spread expression + proper_type = get_proper_type(value_type) + if isinstance(proper_type, TypedDictType): + # Spread TypedDict items directly into our items + unpacked = {k: (value, proper_type.items[k]) for k in proper_type.required_keys} + items.update(**unpacked) + else: + # If it's not a TypedDict, we can't check its items, so we just pretend there are none + # But we can at least make sure it's a Mapping + self.chk.check_subtype( + value_type, + self.chk.named_type("typing.Mapping"), + value, + message_registry.INCOMPATIBLE_TYPES, + f"unpacked expression has type", + f'expected', + code=codes.TYPEDDICT_ITEM, + ) + + if not (callee.required_keys <= set(items.keys()) <= set(callee.items.keys())): expected_keys = [ key for key in callee.items.keys() - if key in callee.required_keys or key in kwargs.keys() + if key in callee.required_keys or key in items.keys() ] - actual_keys = kwargs.keys() + actual_keys = items.keys() self.msg.unexpected_typeddict_keys( callee, expected_keys=expected_keys, actual_keys=list(actual_keys), context=context ) return AnyType(TypeOfAny.from_error) for (item_name, item_expected_type) in callee.items.items(): - if item_name in kwargs: - item_value = kwargs[item_name] - self.chk.check_simple_assignment( - lvalue_type=item_expected_type, - rvalue=item_value, - context=item_value, - msg=message_registry.INCOMPATIBLE_TYPES, - lvalue_name=f'TypedDict item "{item_name}"', - rvalue_name="expression", + if item_name in items: + item_value_expr, item_value_actual_type = items[item_name] + self.chk.check_subtype( + item_value_actual_type, + item_expected_type, + item_value_expr, + message_registry.INCOMPATIBLE_TYPES, + f'expression has type', + f'TypedDict item "{item_name}" has type', code=codes.TYPEDDICT_ITEM, ) From 0d32e6040264d0188c7c56781b55fe0cac6a2bc1 Mon Sep 17 00:00:00 2001 From: Eric Wolf Date: Sun, 7 Aug 2022 17:56:40 +0200 Subject: [PATCH 02/11] Clean up and fix edge cases --- mypy/checkexpr.py | 84 ++++++++++++++++++++++------------------------- 1 file changed, 40 insertions(+), 44 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 8e72f57c1e78..3dbd8f5e1424 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -617,13 +617,11 @@ def check_typeddict_call( args: List[Expression], context: Context, ) -> Type: - if len(args) >= 1 and all([ak == ARG_NAMED for ak in arg_kinds]): + if all([ak in {ARG_NAMED, ARG_STAR2} for ak in arg_kinds]): # ex: Point(x=42, y=1337) - assert all(arg_name is not None for arg_name in arg_names) - item_names = cast(List[str], arg_names) item_args = args return self.check_typeddict_call_with_kwargs( - callee, dict(zip(item_names, item_args)), context + callee, list(zip(arg_names, item_args)), context ) if len(args) == 1 and arg_kinds[0] == ARG_POS: @@ -635,22 +633,15 @@ def check_typeddict_call( # ex: Point(dict(x=42, y=1337)) return self.check_typeddict_call_with_dict(callee, unique_arg.analyzed, context) - if len(args) == 0: - # ex: EmptyDict() - return self.check_typeddict_call_with_kwargs(callee, {}, context) - self.chk.fail(message_registry.INVALID_TYPEDDICT_ARGS, context) return AnyType(TypeOfAny.from_error) - def validate_typeddict_kwargs(self, kwargs: DictExpr) -> "Optional[List[Tuple[Optional[str], Expression]]]": - item_names: List[Optional[str]] = [] - item_args: List[Expression] = [] - + def validate_typeddict_kwargs(self, kwargs: DictExpr) -> Optional[List[Tuple[Optional[str], Expression]]]: + items: List[Tuple[Optional[str], Expression]] = [] for item_name_expr, item_arg in kwargs.items: - # Allow unpacking with ** + # If the unpack operator (**) was used, name will be None if item_name_expr is None: - item_names.append(None) - item_args.append(item_arg) + items.append((None, item_arg)) continue literal_value = None if item_name_expr: @@ -663,9 +654,8 @@ def validate_typeddict_kwargs(self, kwargs: DictExpr) -> "Optional[List[Tuple[Op self.chk.fail(message_registry.TYPEDDICT_KEY_MUST_BE_STRING_LITERAL, key_context) return None else: - item_names.append(literal_value) - item_args.append(item_arg) - return list(zip(item_names, item_args)) + items.append((literal_value, item_arg)) + return items def match_typeddict_call_with_dict( self, callee: TypedDictType, kwargs: DictExpr, context: Context @@ -690,53 +680,59 @@ def check_typeddict_call_with_dict( def check_typeddict_call_with_kwargs( self, callee: TypedDictType, kwargs: List[Tuple[Optional[str], Expression]], context: Context ) -> Type: + # Infer types of item values and expand unpack operators items: Dict[str, Tuple[Expression, Type]] = {} - for key, value in kwargs: - value_type = self.accept(value, callee.items.get(key) if key is not None else callee) + actual_keys: List[str] = [] + for key, value_expr in kwargs: if key is not None: - items[key] = (value, value_type) + # Regular key and value + value_type = self.accept(value_expr, callee.items.get(key)) + items[key] = (value_expr, value_type) + actual_keys.append(key) else: - # Spread operator was used; determine type of spread expression + # Unpack operator (**) was used; unpack all items of the type of this expression into items list + value_type = self.accept(value_expr, callee) proper_type = get_proper_type(value_type) if isinstance(proper_type, TypedDictType): - # Spread TypedDict items directly into our items - unpacked = {k: (value, proper_type.items[k]) for k in proper_type.required_keys} - items.update(**unpacked) + # Only allow unpacking of TypedDicts + for nested_key, nested_value_type in proper_type.items.items(): + items[nested_key] = (value_expr, nested_value_type) + # Only add required keys to set of actual keys + # Non-required keys will still be items so that their values can be type-checked + actual_keys.extend(proper_type.required_keys) else: - # If it's not a TypedDict, we can't check its items, so we just pretend there are none - # But we can at least make sure it's a Mapping - self.chk.check_subtype( - value_type, - self.chk.named_type("typing.Mapping"), - value, - message_registry.INCOMPATIBLE_TYPES, - f"unpacked expression has type", - f'expected', + # Show error when trying to unpack anything else + assert not self.chk.check_subtype( + subtype=value_type, + supertype=self.chk.named_type("typing._TypedDict"), + context=value_expr, + msg=message_registry.INCOMPATIBLE_TYPES, + subtype_label="unpacked expression has type", + supertype_label="expected", code=codes.TYPEDDICT_ITEM, ) - if not (callee.required_keys <= set(items.keys()) <= set(callee.items.keys())): + if not (callee.required_keys <= set(actual_keys) <= set(callee.items.keys())): expected_keys = [ key for key in callee.items.keys() if key in callee.required_keys or key in items.keys() ] - actual_keys = items.keys() self.msg.unexpected_typeddict_keys( - callee, expected_keys=expected_keys, actual_keys=list(actual_keys), context=context + callee, expected_keys=expected_keys, actual_keys=actual_keys, context=context ) return AnyType(TypeOfAny.from_error) for (item_name, item_expected_type) in callee.items.items(): if item_name in items: - item_value_expr, item_value_actual_type = items[item_name] + item_value_expr, item_actual_type = items[item_name] self.chk.check_subtype( - item_value_actual_type, - item_expected_type, - item_value_expr, - message_registry.INCOMPATIBLE_TYPES, - f'expression has type', - f'TypedDict item "{item_name}" has type', + subtype=item_actual_type, + supertype=item_expected_type, + context=item_value_expr, + msg=message_registry.INCOMPATIBLE_TYPES, + subtype_label=f'expression has type', + supertype_label=f'TypedDict item "{item_name}" has type', code=codes.TYPEDDICT_ITEM, ) From aaa4b203424d3868d5d2c6ec16dc4b2f340c8922 Mon Sep 17 00:00:00 2001 From: Eric Wolf Date: Sun, 7 Aug 2022 18:11:15 +0200 Subject: [PATCH 03/11] Cleanup, comments, tests --- mypy/checkexpr.py | 42 ++-- test-data/unit/check-typeddictspread.test | 232 ++++++++++++++++++++++ 2 files changed, 258 insertions(+), 16 deletions(-) create mode 100644 test-data/unit/check-typeddictspread.test diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 3dbd8f5e1424..cd2495be8509 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -618,7 +618,7 @@ def check_typeddict_call( context: Context, ) -> Type: if all([ak in {ARG_NAMED, ARG_STAR2} for ak in arg_kinds]): - # ex: Point(x=42, y=1337) + # ex: Point(x=42, y=1337, **other_point) item_args = args return self.check_typeddict_call_with_kwargs( callee, list(zip(arg_names, item_args)), context @@ -637,6 +637,9 @@ def check_typeddict_call( return AnyType(TypeOfAny.from_error) def validate_typeddict_kwargs(self, kwargs: DictExpr) -> Optional[List[Tuple[Optional[str], Expression]]]: + """Validate kwargs for TypedDict constructor, e.g. Point({'x': 1, 'y': 2}). + Check that all items have string literal keys or are using the unpack operator (**) + """ items: List[Tuple[Optional[str], Expression]] = [] for item_name_expr, item_arg in kwargs.items: # If the unpack operator (**) was used, name will be None @@ -644,11 +647,10 @@ def validate_typeddict_kwargs(self, kwargs: DictExpr) -> Optional[List[Tuple[Opt items.append((None, item_arg)) continue literal_value = None - if item_name_expr: - key_type = self.accept(item_name_expr) - values = try_getting_str_literals(item_name_expr, key_type) - if values and len(values) == 1: - literal_value = values[0] + key_type = self.accept(item_name_expr) + values = try_getting_str_literals(item_name_expr, key_type) + if values and len(values) == 1: + literal_value = values[0] if literal_value is None: key_context = item_name_expr or item_arg self.chk.fail(message_registry.TYPEDDICT_KEY_MUST_BE_STRING_LITERAL, key_context) @@ -658,8 +660,9 @@ def validate_typeddict_kwargs(self, kwargs: DictExpr) -> Optional[List[Tuple[Opt return items def match_typeddict_call_with_dict( - self, callee: TypedDictType, kwargs: DictExpr, context: Context + self, callee: TypedDictType, kwargs: DictExpr ) -> bool: + """Check that kwargs is a valid set of TypedDict items, contains all required keys of callee, and has no extraneous keys""" validated_kwargs = self.validate_typeddict_kwargs(kwargs=kwargs) if validated_kwargs is not None: return callee.required_keys <= set(dict(validated_kwargs).keys()) <= set(callee.items.keys()) @@ -669,6 +672,7 @@ def match_typeddict_call_with_dict( def check_typeddict_call_with_dict( self, callee: TypedDictType, kwargs: DictExpr, context: Context ) -> Type: + """Check TypedDict constructor of format Point({'x': 1, 'y': 2})""" validated_kwargs = self.validate_typeddict_kwargs(kwargs=kwargs) if validated_kwargs is not None: return self.check_typeddict_call_with_kwargs( @@ -680,15 +684,17 @@ def check_typeddict_call_with_dict( def check_typeddict_call_with_kwargs( self, callee: TypedDictType, kwargs: List[Tuple[Optional[str], Expression]], context: Context ) -> Type: + """Check TypedDict constructor of format Point(x=1, y=2)""" # Infer types of item values and expand unpack operators items: Dict[str, Tuple[Expression, Type]] = {} - actual_keys: List[str] = [] + sure_keys: List[str] = [] + maybe_keys: List[str] = [] for key, value_expr in kwargs: if key is not None: # Regular key and value value_type = self.accept(value_expr, callee.items.get(key)) items[key] = (value_expr, value_type) - actual_keys.append(key) + sure_keys.append(key) else: # Unpack operator (**) was used; unpack all items of the type of this expression into items list value_type = self.accept(value_expr, callee) @@ -697,11 +703,12 @@ def check_typeddict_call_with_kwargs( # Only allow unpacking of TypedDicts for nested_key, nested_value_type in proper_type.items.items(): items[nested_key] = (value_expr, nested_value_type) - # Only add required keys to set of actual keys - # Non-required keys will still be items so that their values can be type-checked - actual_keys.extend(proper_type.required_keys) + if nested_key in proper_type.required_keys: + sure_keys.append(nested_key) + else: + maybe_keys.append(nested_key) else: - # Show error when trying to unpack anything else + # Fail when trying to unpack anything but TypedDict assert not self.chk.check_subtype( subtype=value_type, supertype=self.chk.named_type("typing._TypedDict"), @@ -711,18 +718,21 @@ def check_typeddict_call_with_kwargs( supertype_label="expected", code=codes.TYPEDDICT_ITEM, ) + actual_keys = [*sure_keys, *maybe_keys] - if not (callee.required_keys <= set(actual_keys) <= set(callee.items.keys())): + if not (callee.required_keys <= set(sure_keys) <= set(actual_keys) <= set(callee.items.keys())): expected_keys = [ key for key in callee.items.keys() if key in callee.required_keys or key in items.keys() ] + self.msg.unexpected_typeddict_keys( - callee, expected_keys=expected_keys, actual_keys=actual_keys, context=context + callee, expected_keys=expected_keys, actual_keys=sure_keys, context=context ) return AnyType(TypeOfAny.from_error) + # Check item value types for (item_name, item_expected_type) in callee.items.items(): if item_name in items: item_value_expr, item_actual_type = items[item_name] @@ -4024,7 +4034,7 @@ def find_typeddict_context( for item in context.items: item_context = self.find_typeddict_context(item, dict_expr) if item_context is not None and self.match_typeddict_call_with_dict( - item_context, dict_expr, dict_expr + item_context, dict_expr ): items.append(item_context) if len(items) == 1: diff --git a/test-data/unit/check-typeddictspread.test b/test-data/unit/check-typeddictspread.test new file mode 100644 index 000000000000..f3c76b51036b --- /dev/null +++ b/test-data/unit/check-typeddictspread.test @@ -0,0 +1,232 @@ +[case testTypedDictUnpackingSame] +from typing import TypedDict + +class Foo(TypedDict): + a: int + b: int + +foo1: Foo = {'a': 1, 'b': 1} +foo2: Foo = {**foo1, 'b': 2} +foo3 = Foo(**foo1, b=2) +foo4 = Foo({**foo1, 'b': 2}) + +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictUnpackingCompatible] +from typing import TypedDict + +class Foo(TypedDict): + a: int + +class Bar(TypedDict): + a: int + b: int + +foo: Foo = {'a': 1} +bar: Bar = {**foo, 'b': 2} + +[typing fixtures/typing-typeddict.pyi] + + + +[case testTypedDictUnpackingIncompatible] +from typing import TypedDict + +class Foo(TypedDict): + a: str + +class Bar(TypedDict): + a: int + b: int + +foo: Foo = {'a': 'x'} +bar: Bar = {**foo, 'b': 2} # E: Incompatible types (expression has type "str", TypedDict item "a" has type "int") + +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictUnpackingIncompatibleCorrected] +from typing import TypedDict + +class Foo(TypedDict): + a: int + b: str + +class Bar(TypedDict): + a: int + b: int + +foo: Foo = {'a': 1, 'b': 'x'} +bar: Bar = {**foo, 'b': 2} + +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictUnpackingMissingKey] +from typing import TypedDict + +class Foo(TypedDict): + a: int + +class Bar(TypedDict): + a: int + b: int + +foo: Foo = {'a': 1} +bar: Bar = {**foo} # E: Missing key "b" for TypedDict "Bar" + +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictUnpackingExtraKey] +from typing import TypedDict + +class Foo(TypedDict): + a: int + c: int + +class Bar(TypedDict): + a: int + b: int + +foo: Foo = {'a': 1, 'c': 1} +bar: Bar = {**foo, 'b': 1} # E: Extra key "c" for TypedDict "Bar" + +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictUnpackingNotRequiredKeyMissing] +from typing import TypedDict, NotRequired + +class Foo(TypedDict): + a: int + +class Bar(TypedDict): + a: int + b: NotRequired[int] + +foo: Foo = {'a': 1} +bar: Bar = {**foo} + +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictUnpackingRequiredKeyMissing] +from typing import TypedDict, NotRequired + +class Foo(TypedDict): + a: NotRequired[int] + +class Bar(TypedDict): + a: int + +foo: Foo = {'a': 1} +bar: Bar = {**foo} # E: Missing key "a" for TypedDict "Bar" + +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictUnpackingNotRequiredKeyIncompatible] +from typing import TypedDict, NotRequired + +class Foo(TypedDict): + a: NotRequired[str] + +class Bar(TypedDict): + a: NotRequired[int] + +foo: Foo = {} +bar: Bar = {**foo} # E: Incompatible types (expression has type "str", TypedDict item "a" has type "int") + +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictUnpackingNotRequiredKeyExtra] +from typing import TypedDict, NotRequired + +class Foo(TypedDict): + a: NotRequired[int] + +class Bar(TypedDict): + b: int + +foo: Foo = {'a': 1} +bar: Bar = {**foo, 'b': 2} # E: Incompatible types (expression has type "str", TypedDict item "a" has type "int") + +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictUnpackingMultiple] +from typing import TypedDict + +class Foo(TypedDict): + a: int + +class Bar(TypedDict): + b: int + +class Baz(TypedDict): + a: int + b: int + c: int + +foo: Foo = {'a': 1} +bar: Bar = {'b': 1} +baz: Baz = {**foo, **bar, 'c': 1} + +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictUnpackingNested] +from typing import TypedDict + +class Foo(TypedDict): + a: int + b: int + +class Bar(TypedDict): + c: Foo + d: int + +foo: Foo = {'a': 1, 'b': 1} +bar: Bar = {'c': foo, 'd': 1} +bar2: Bar = {**bar, 'c': {**bar['c'], 'b': 2}, 'd': 2} + +[builtins fixtures/tuple.pyi] +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictUnpackingNestedError] +from typing import TypedDict + +class Foo(TypedDict): + a: int + b: int + +class Bar(TypedDict): + c: Foo + d: int + +foo: Foo = {'a': 1, 'b': 1} +bar: Bar = {'c': foo, 'd': 1} +bar2: Bar = {**bar, 'c': {**bar['c'], 'b': 'wrong'}, 'd': 2} # E: Incompatible types (expression has type "str", TypedDict item "b" has type "int") + +[builtins fixtures/tuple.pyi] +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictUnpackingUntypedDict] +from typing import TypedDict + +class Bar(TypedDict): + pass + +foo: dict = {} +bar: Bar = {**foo} # E: Incompatible types (unpacked expression has type "Dict[Any, Any]", expected "_TypedDict") + +[builtins fixtures/tuple.pyi] +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] From 9dda0f64a8fbe04ad77663a71c8c273a59ee8a7d Mon Sep 17 00:00:00 2001 From: Eric Wolf Date: Sun, 7 Aug 2022 18:34:52 +0200 Subject: [PATCH 04/11] Fix key error messages --- mypy/checkexpr.py | 16 +++++++--------- mypy/messages.py | 21 ++++++++++++++------- test-data/unit/check-typeddictspread.test | 2 +- 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index cd2495be8509..42d3ade0eb14 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -718,17 +718,15 @@ def check_typeddict_call_with_kwargs( supertype_label="expected", code=codes.TYPEDDICT_ITEM, ) - actual_keys = [*sure_keys, *maybe_keys] - - if not (callee.required_keys <= set(sure_keys) <= set(actual_keys) <= set(callee.items.keys())): - expected_keys = [ - key - for key in callee.items.keys() - if key in callee.required_keys or key in items.keys() - ] + if not (callee.required_keys <= set(sure_keys) <= set(sure_keys + maybe_keys) <= set(callee.items.keys())): self.msg.unexpected_typeddict_keys( - callee, expected_keys=expected_keys, actual_keys=sure_keys, context=context + callee, + required_keys=list(callee.required_keys), + expected_keys=list(callee.items.keys()), + actual_sure_keys=sure_keys, + actual_maybe_keys=maybe_keys, + context=context ) return AnyType(TypeOfAny.from_error) diff --git a/mypy/messages.py b/mypy/messages.py index b0dff20fc8b2..db741d9aa52a 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -1579,17 +1579,24 @@ def explicit_any(self, ctx: Context) -> None: def unexpected_typeddict_keys( self, typ: TypedDictType, + required_keys: List[str], expected_keys: List[str], - actual_keys: List[str], + actual_sure_keys: List[str], + actual_maybe_keys: List[str], context: Context, ) -> None: - actual_set = set(actual_keys) + actual_keys = actual_sure_keys + actual_maybe_keys + actual_sure_set = set(actual_sure_keys) + actual_maybe_set = set(actual_maybe_keys) + actual_set = actual_sure_set | actual_maybe_set + required_set = set(required_keys) expected_set = set(expected_keys) + if not typ.is_anonymous(): # Generate simpler messages for some common special cases. - if actual_set < expected_set: + if actual_sure_set < required_set: # Use list comprehension instead of set operations to preserve order. - missing = [key for key in expected_keys if key not in actual_set] + missing = [key for key in required_keys if key not in actual_sure_set] self.fail( "Missing {} for TypedDict {}".format( format_key_list(missing, short=True), format_type(typ) @@ -1612,11 +1619,11 @@ def unexpected_typeddict_keys( ) return found = format_key_list(actual_keys, short=True) - if not expected_keys: + if not expected_set: self.fail(f"Unexpected TypedDict {found}", context) return - expected = format_key_list(expected_keys) - if actual_keys and actual_set < expected_set: + expected = format_key_list(required_keys) + if actual_keys and actual_set < required_set: found = f"only {found}" self.fail(f"Expected {expected} but found {found}", context, code=codes.TYPEDDICT_ITEM) diff --git a/test-data/unit/check-typeddictspread.test b/test-data/unit/check-typeddictspread.test index f3c76b51036b..bdcac311da3a 100644 --- a/test-data/unit/check-typeddictspread.test +++ b/test-data/unit/check-typeddictspread.test @@ -152,7 +152,7 @@ class Bar(TypedDict): b: int foo: Foo = {'a': 1} -bar: Bar = {**foo, 'b': 2} # E: Incompatible types (expression has type "str", TypedDict item "a" has type "int") +bar: Bar = {**foo, 'b': 2} # E: Extra key "a" for TypedDict "Bar" [typing fixtures/typing-typeddict.pyi] From e755d0e6611626fa68ecea2ddeb65c567064ab77 Mon Sep 17 00:00:00 2001 From: Eric Wolf Date: Sun, 7 Aug 2022 18:38:26 +0200 Subject: [PATCH 05/11] Format --- mypy/checkexpr.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 42d3ade0eb14..d5ac7ff8d2de 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -636,7 +636,9 @@ def check_typeddict_call( self.chk.fail(message_registry.INVALID_TYPEDDICT_ARGS, context) return AnyType(TypeOfAny.from_error) - def validate_typeddict_kwargs(self, kwargs: DictExpr) -> Optional[List[Tuple[Optional[str], Expression]]]: + def validate_typeddict_kwargs( + self, kwargs: DictExpr + ) -> Optional[List[Tuple[Optional[str], Expression]]]: """Validate kwargs for TypedDict constructor, e.g. Point({'x': 1, 'y': 2}). Check that all items have string literal keys or are using the unpack operator (**) """ @@ -659,13 +661,15 @@ def validate_typeddict_kwargs(self, kwargs: DictExpr) -> Optional[List[Tuple[Opt items.append((literal_value, item_arg)) return items - def match_typeddict_call_with_dict( - self, callee: TypedDictType, kwargs: DictExpr - ) -> bool: + def match_typeddict_call_with_dict(self, callee: TypedDictType, kwargs: DictExpr) -> bool: """Check that kwargs is a valid set of TypedDict items, contains all required keys of callee, and has no extraneous keys""" validated_kwargs = self.validate_typeddict_kwargs(kwargs=kwargs) if validated_kwargs is not None: - return callee.required_keys <= set(dict(validated_kwargs).keys()) <= set(callee.items.keys()) + return ( + callee.required_keys + <= set(dict(validated_kwargs).keys()) + <= set(callee.items.keys()) + ) else: return False @@ -682,7 +686,10 @@ def check_typeddict_call_with_dict( return AnyType(TypeOfAny.from_error) def check_typeddict_call_with_kwargs( - self, callee: TypedDictType, kwargs: List[Tuple[Optional[str], Expression]], context: Context + self, + callee: TypedDictType, + kwargs: List[Tuple[Optional[str], Expression]], + context: Context, ) -> Type: """Check TypedDict constructor of format Point(x=1, y=2)""" # Infer types of item values and expand unpack operators @@ -719,14 +726,19 @@ def check_typeddict_call_with_kwargs( code=codes.TYPEDDICT_ITEM, ) - if not (callee.required_keys <= set(sure_keys) <= set(sure_keys + maybe_keys) <= set(callee.items.keys())): + if not ( + callee.required_keys + <= set(sure_keys) + <= set(sure_keys + maybe_keys) + <= set(callee.items.keys()) + ): self.msg.unexpected_typeddict_keys( callee, required_keys=list(callee.required_keys), expected_keys=list(callee.items.keys()), actual_sure_keys=sure_keys, actual_maybe_keys=maybe_keys, - context=context + context=context, ) return AnyType(TypeOfAny.from_error) @@ -739,7 +751,7 @@ def check_typeddict_call_with_kwargs( supertype=item_expected_type, context=item_value_expr, msg=message_registry.INCOMPATIBLE_TYPES, - subtype_label=f'expression has type', + subtype_label=f"expression has type", supertype_label=f'TypedDict item "{item_name}" has type', code=codes.TYPEDDICT_ITEM, ) From 74c31eb3162311831e2855e5d6454148b3269b34 Mon Sep 17 00:00:00 2001 From: Eric Wolf Date: Sun, 7 Aug 2022 20:48:36 +0200 Subject: [PATCH 06/11] Improve tests and error messages --- mypy/checkexpr.py | 16 ++-- mypy/messages.py | 60 +++++-------- test-data/unit/check-typeddict.test | 10 +-- test-data/unit/check-typeddictspread.test | 102 +++++++--------------- 4 files changed, 63 insertions(+), 125 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index d5ac7ff8d2de..cd3281b29b49 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -640,11 +640,11 @@ def validate_typeddict_kwargs( self, kwargs: DictExpr ) -> Optional[List[Tuple[Optional[str], Expression]]]: """Validate kwargs for TypedDict constructor, e.g. Point({'x': 1, 'y': 2}). - Check that all items have string literal keys or are using the unpack operator (**) + Check that all items have string literal keys or are using unpack operator (**) """ items: List[Tuple[Optional[str], Expression]] = [] for item_name_expr, item_arg in kwargs.items: - # If the unpack operator (**) was used, name will be None + # If unpack operator (**) was used, name will be None if item_name_expr is None: items.append((None, item_arg)) continue @@ -662,7 +662,7 @@ def validate_typeddict_kwargs( return items def match_typeddict_call_with_dict(self, callee: TypedDictType, kwargs: DictExpr) -> bool: - """Check that kwargs is a valid set of TypedDict items, contains all required keys of callee, and has no extraneous keys""" + """Check that kwargs is valid set of TypedDict items, contains all required keys of callee, and has no extraneous keys""" validated_kwargs = self.validate_typeddict_kwargs(kwargs=kwargs) if validated_kwargs is not None: return ( @@ -695,7 +695,7 @@ def check_typeddict_call_with_kwargs( # Infer types of item values and expand unpack operators items: Dict[str, Tuple[Expression, Type]] = {} sure_keys: List[str] = [] - maybe_keys: List[str] = [] + maybe_keys: List[str] = [] # Will contain non-required items of unpacked TypedDicts for key, value_expr in kwargs: if key is not None: # Regular key and value @@ -707,7 +707,6 @@ def check_typeddict_call_with_kwargs( value_type = self.accept(value_expr, callee) proper_type = get_proper_type(value_type) if isinstance(proper_type, TypedDictType): - # Only allow unpacking of TypedDicts for nested_key, nested_value_type in proper_type.items.items(): items[nested_key] = (value_expr, nested_value_type) if nested_key in proper_type.required_keys: @@ -733,12 +732,7 @@ def check_typeddict_call_with_kwargs( <= set(callee.items.keys()) ): self.msg.unexpected_typeddict_keys( - callee, - required_keys=list(callee.required_keys), - expected_keys=list(callee.items.keys()), - actual_sure_keys=sure_keys, - actual_maybe_keys=maybe_keys, - context=context, + callee, actual_sure_keys=sure_keys, actual_maybe_keys=maybe_keys, context=context ) return AnyType(TypeOfAny.from_error) diff --git a/mypy/messages.py b/mypy/messages.py index db741d9aa52a..842500521c32 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -1579,8 +1579,6 @@ def explicit_any(self, ctx: Context) -> None: def unexpected_typeddict_keys( self, typ: TypedDictType, - required_keys: List[str], - expected_keys: List[str], actual_sure_keys: List[str], actual_maybe_keys: List[str], context: Context, @@ -1588,44 +1586,32 @@ def unexpected_typeddict_keys( actual_keys = actual_sure_keys + actual_maybe_keys actual_sure_set = set(actual_sure_keys) actual_maybe_set = set(actual_maybe_keys) - actual_set = actual_sure_set | actual_maybe_set + required_keys = [k for k in typ.items.keys() if k in typ.required_keys] required_set = set(required_keys) - expected_set = set(expected_keys) + expected_set = set(typ.items.keys()) if not typ.is_anonymous(): - # Generate simpler messages for some common special cases. - if actual_sure_set < required_set: - # Use list comprehension instead of set operations to preserve order. - missing = [key for key in required_keys if key not in actual_sure_set] - self.fail( - "Missing {} for TypedDict {}".format( - format_key_list(missing, short=True), format_type(typ) - ), - context, - code=codes.TYPEDDICT_ITEM, - ) - return - else: - extra = [key for key in actual_keys if key not in expected_set] - if extra: - # If there are both extra and missing keys, only report extra ones for - # simplicity. - self.fail( - "Extra {} for TypedDict {}".format( - format_key_list(extra, short=True), format_type(typ) - ), - context, - code=codes.TYPEDDICT_ITEM, - ) - return - found = format_key_list(actual_keys, short=True) - if not expected_set: - self.fail(f"Unexpected TypedDict {found}", context) - return - expected = format_key_list(required_keys) - if actual_keys and actual_set < required_set: - found = f"only {found}" - self.fail(f"Expected {expected} but found {found}", context, code=codes.TYPEDDICT_ITEM) + type_description = f" for TypedDict {format_type(typ)}" + else: + type_description = "" + + if actual_sure_set < required_set: + # Use list comprehension instead of set operations to preserve order. + missing = [key for key in required_keys if key not in actual_sure_set] + self.fail( + f"Missing {format_key_list(missing, short=True)}{type_description}", + context, + code=codes.TYPEDDICT_ITEM, + ) + else: + # If there are both extra and missing keys, only report extra ones for + # simplicity. + extra = [key for key in actual_keys if key not in expected_set] + self.fail( + f"Extra {format_key_list(extra, short=True)}{type_description}", + context, + code=codes.TYPEDDICT_ITEM, + ) def typeddict_key_must_be_string_literal(self, typ: TypedDictType, context: Context) -> None: self.fail( diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index 62ac5e31da45..efea43ec021e 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -943,7 +943,7 @@ T = TypeVar('T') def join(x: T, y: T) -> T: return x ab = join(A(x=1, y=1), B(x=1, y='')) if int(): - ab = {'x': 1, 'z': 1} # E: Expected TypedDict key "x" but found keys ("x", "z") + ab = {'x': 1, 'z': 1} # E: Extra key "z" [builtins fixtures/dict.pyi] [case testCannotCreateAnonymousTypedDictInstanceUsingDictLiteralWithMissingItems] @@ -955,7 +955,7 @@ T = TypeVar('T') def join(x: T, y: T) -> T: return x ab = join(A(x=1, y=1, z=1), B(x=1, y=1, z='')) if int(): - ab = {} # E: Expected TypedDict keys ("x", "y") but found no keys + ab = {} # E: Missing keys ("x", "y") [builtins fixtures/dict.pyi] @@ -1653,9 +1653,9 @@ a.update({'x': 1}) a.update({'x': ''}) # E: Incompatible types (expression has type "str", TypedDict item "x" has type "int") a.update({'x': 1, 'y': []}) a.update({'x': 1, 'y': [1]}) -a.update({'z': 1}) # E: Unexpected TypedDict key "z" -a.update({'z': 1, 'zz': 1}) # E: Unexpected TypedDict keys ("z", "zz") -a.update({'z': 1, 'x': 1}) # E: Expected TypedDict key "x" but found keys ("z", "x") +a.update({'z': 1}) # E: Extra key "z" +a.update({'z': 1, 'zz': 1}) # E: Extra keys ("z", "zz") +a.update({'z': 1, 'x': 1}) # E: Extra key "z" d = {'x': 1} a.update(d) # E: Argument 1 to "update" of "TypedDict" has incompatible type "Dict[str, int]"; expected "TypedDict({'x'?: int, 'y'?: List[int]})" [builtins fixtures/dict.pyi] diff --git a/test-data/unit/check-typeddictspread.test b/test-data/unit/check-typeddictspread.test index bdcac311da3a..e39f576b2b6c 100644 --- a/test-data/unit/check-typeddictspread.test +++ b/test-data/unit/check-typeddictspread.test @@ -1,4 +1,4 @@ -[case testTypedDictUnpackingSame] +[case testTypedDictUnpackSame] from typing import TypedDict class Foo(TypedDict): @@ -13,7 +13,7 @@ foo4 = Foo({**foo1, 'b': 2}) [typing fixtures/typing-typeddict.pyi] -[case testTypedDictUnpackingCompatible] +[case testTypedDictUnpackCompatible] from typing import TypedDict class Foo(TypedDict): @@ -30,23 +30,7 @@ bar: Bar = {**foo, 'b': 2} -[case testTypedDictUnpackingIncompatible] -from typing import TypedDict - -class Foo(TypedDict): - a: str - -class Bar(TypedDict): - a: int - b: int - -foo: Foo = {'a': 'x'} -bar: Bar = {**foo, 'b': 2} # E: Incompatible types (expression has type "str", TypedDict item "a" has type "int") - -[typing fixtures/typing-typeddict.pyi] - - -[case testTypedDictUnpackingIncompatibleCorrected] +[case testTypedDictUnpackIncompatible] from typing import TypedDict class Foo(TypedDict): @@ -57,46 +41,49 @@ class Bar(TypedDict): a: int b: int -foo: Foo = {'a': 1, 'b': 'x'} -bar: Bar = {**foo, 'b': 2} +foo: Foo = {'a': 1, 'b': 'a'} +bar1: Bar = {**foo, 'b': 2} # Incompatible item is overriden +bar2: Bar = {**foo, 'a': 2} # E: Incompatible types (expression has type "str", TypedDict item "b" has type "int") +[builtins fixtures/tuple.pyi] +[builtins fixtures/dict.pyi] [typing fixtures/typing-typeddict.pyi] -[case testTypedDictUnpackingMissingKey] -from typing import TypedDict +[case testTypedDictUnpackNotRequiredKeyIncompatible] +from typing import TypedDict, NotRequired class Foo(TypedDict): - a: int + a: NotRequired[str] class Bar(TypedDict): - a: int - b: int + a: NotRequired[int] -foo: Foo = {'a': 1} -bar: Bar = {**foo} # E: Missing key "b" for TypedDict "Bar" +foo: Foo = {} +bar: Bar = {**foo} # E: Incompatible types (expression has type "str", TypedDict item "a" has type "int") [typing fixtures/typing-typeddict.pyi] -[case testTypedDictUnpackingExtraKey] +[case testTypedDictUnpackMissingOrExtraKey] from typing import TypedDict class Foo(TypedDict): a: int - c: int class Bar(TypedDict): a: int b: int -foo: Foo = {'a': 1, 'c': 1} -bar: Bar = {**foo, 'b': 1} # E: Extra key "c" for TypedDict "Bar" +foo1: Foo = {'a': 1} +bar1: Bar = {'a': 1, 'b': 1} +foo2: Foo = {**bar1} # E: Extra key "b" for TypedDict "Foo" +bar2: Bar = {**foo1} # E: Missing key "b" for TypedDict "Bar" [typing fixtures/typing-typeddict.pyi] -[case testTypedDictUnpackingNotRequiredKeyMissing] +[case testTypedDictUnpackNotRequiredKeyExtra] from typing import TypedDict, NotRequired class Foo(TypedDict): @@ -106,13 +93,15 @@ class Bar(TypedDict): a: int b: NotRequired[int] -foo: Foo = {'a': 1} -bar: Bar = {**foo} +foo1: Foo = {'a': 1} +bar1: Bar = {'a': 1} +foo2: Foo = {**bar1} # E: Extra key "b" for TypedDict "Foo" +bar2: Bar = {**foo1} [typing fixtures/typing-typeddict.pyi] -[case testTypedDictUnpackingRequiredKeyMissing] +[case testTypedDictUnpackRequiredKeyMissing] from typing import TypedDict, NotRequired class Foo(TypedDict): @@ -127,37 +116,7 @@ bar: Bar = {**foo} # E: Missing key "a" for TypedDict "Bar" [typing fixtures/typing-typeddict.pyi] -[case testTypedDictUnpackingNotRequiredKeyIncompatible] -from typing import TypedDict, NotRequired - -class Foo(TypedDict): - a: NotRequired[str] - -class Bar(TypedDict): - a: NotRequired[int] - -foo: Foo = {} -bar: Bar = {**foo} # E: Incompatible types (expression has type "str", TypedDict item "a" has type "int") - -[typing fixtures/typing-typeddict.pyi] - - -[case testTypedDictUnpackingNotRequiredKeyExtra] -from typing import TypedDict, NotRequired - -class Foo(TypedDict): - a: NotRequired[int] - -class Bar(TypedDict): - b: int - -foo: Foo = {'a': 1} -bar: Bar = {**foo, 'b': 2} # E: Extra key "a" for TypedDict "Bar" - -[typing fixtures/typing-typeddict.pyi] - - -[case testTypedDictUnpackingMultiple] +[case testTypedDictUnpackMultiple] from typing import TypedDict class Foo(TypedDict): @@ -178,7 +137,7 @@ baz: Baz = {**foo, **bar, 'c': 1} [typing fixtures/typing-typeddict.pyi] -[case testTypedDictUnpackingNested] +[case testTypedDictUnpackNested] from typing import TypedDict class Foo(TypedDict): @@ -198,7 +157,7 @@ bar2: Bar = {**bar, 'c': {**bar['c'], 'b': 2}, 'd': 2} [typing fixtures/typing-typeddict.pyi] -[case testTypedDictUnpackingNestedError] +[case testTypedDictUnpackNestedError] from typing import TypedDict class Foo(TypedDict): @@ -213,12 +172,11 @@ foo: Foo = {'a': 1, 'b': 1} bar: Bar = {'c': foo, 'd': 1} bar2: Bar = {**bar, 'c': {**bar['c'], 'b': 'wrong'}, 'd': 2} # E: Incompatible types (expression has type "str", TypedDict item "b" has type "int") -[builtins fixtures/tuple.pyi] -[builtins fixtures/dict.pyi] + [typing fixtures/typing-typeddict.pyi] -[case testTypedDictUnpackingUntypedDict] +[case testTypedDictUnpackUntypedDict] from typing import TypedDict class Bar(TypedDict): From 4f89b41933694fe3bc44ee9a776a31840ea5eaf4 Mon Sep 17 00:00:00 2001 From: Eric Wolf Date: Sun, 7 Aug 2022 20:49:11 +0200 Subject: [PATCH 07/11] Merge tests --- test-data/unit/check-typeddict.test | 192 ++++++++++++++++++++++ test-data/unit/check-typeddictspread.test | 190 --------------------- 2 files changed, 192 insertions(+), 190 deletions(-) delete mode 100644 test-data/unit/check-typeddictspread.test diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index efea43ec021e..06f2fe16a716 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -2395,3 +2395,195 @@ def func(foo: Union[F1, F2]): # E: Argument 1 to "__setitem__" has incompatible type "int"; expected "str" [builtins fixtures/dict.pyi] [typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictUnpackSame] +from typing import TypedDict + +class Foo(TypedDict): + a: int + b: int + +foo1: Foo = {'a': 1, 'b': 1} +foo2: Foo = {**foo1, 'b': 2} +foo3 = Foo(**foo1, b=2) +foo4 = Foo({**foo1, 'b': 2}) + +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictUnpackCompatible] +from typing import TypedDict + +class Foo(TypedDict): + a: int + +class Bar(TypedDict): + a: int + b: int + +foo: Foo = {'a': 1} +bar: Bar = {**foo, 'b': 2} + +[typing fixtures/typing-typeddict.pyi] + + + +[case testTypedDictUnpackIncompatible] +from typing import TypedDict + +class Foo(TypedDict): + a: int + b: str + +class Bar(TypedDict): + a: int + b: int + +foo: Foo = {'a': 1, 'b': 'a'} +bar1: Bar = {**foo, 'b': 2} # Incompatible item is overriden +bar2: Bar = {**foo, 'a': 2} # E: Incompatible types (expression has type "str", TypedDict item "b" has type "int") + +[builtins fixtures/tuple.pyi] +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictUnpackNotRequiredKeyIncompatible] +from typing import TypedDict, NotRequired + +class Foo(TypedDict): + a: NotRequired[str] + +class Bar(TypedDict): + a: NotRequired[int] + +foo: Foo = {} +bar: Bar = {**foo} # E: Incompatible types (expression has type "str", TypedDict item "a" has type "int") + +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictUnpackMissingOrExtraKey] +from typing import TypedDict + +class Foo(TypedDict): + a: int + +class Bar(TypedDict): + a: int + b: int + +foo1: Foo = {'a': 1} +bar1: Bar = {'a': 1, 'b': 1} +foo2: Foo = {**bar1} # E: Extra key "b" for TypedDict "Foo" +bar2: Bar = {**foo1} # E: Missing key "b" for TypedDict "Bar" + +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictUnpackNotRequiredKeyExtra] +from typing import TypedDict, NotRequired + +class Foo(TypedDict): + a: int + +class Bar(TypedDict): + a: int + b: NotRequired[int] + +foo1: Foo = {'a': 1} +bar1: Bar = {'a': 1} +foo2: Foo = {**bar1} # E: Extra key "b" for TypedDict "Foo" +bar2: Bar = {**foo1} + +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictUnpackRequiredKeyMissing] +from typing import TypedDict, NotRequired + +class Foo(TypedDict): + a: NotRequired[int] + +class Bar(TypedDict): + a: int + +foo: Foo = {'a': 1} +bar: Bar = {**foo} # E: Missing key "a" for TypedDict "Bar" + +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictUnpackMultiple] +from typing import TypedDict + +class Foo(TypedDict): + a: int + +class Bar(TypedDict): + b: int + +class Baz(TypedDict): + a: int + b: int + c: int + +foo: Foo = {'a': 1} +bar: Bar = {'b': 1} +baz: Baz = {**foo, **bar, 'c': 1} + +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictUnpackNested] +from typing import TypedDict + +class Foo(TypedDict): + a: int + b: int + +class Bar(TypedDict): + c: Foo + d: int + +foo: Foo = {'a': 1, 'b': 1} +bar: Bar = {'c': foo, 'd': 1} +bar2: Bar = {**bar, 'c': {**bar['c'], 'b': 2}, 'd': 2} + +[builtins fixtures/tuple.pyi] +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictUnpackNestedError] +from typing import TypedDict + +class Foo(TypedDict): + a: int + b: int + +class Bar(TypedDict): + c: Foo + d: int + +foo: Foo = {'a': 1, 'b': 1} +bar: Bar = {'c': foo, 'd': 1} +bar2: Bar = {**bar, 'c': {**bar['c'], 'b': 'wrong'}, 'd': 2} # E: Incompatible types (expression has type "str", TypedDict item "b" has type "int") + + +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictUnpackUntypedDict] +from typing import TypedDict + +class Bar(TypedDict): + pass + +foo: dict = {} +bar: Bar = {**foo} # E: Incompatible types (unpacked expression has type "Dict[Any, Any]", expected "_TypedDict") + +[builtins fixtures/tuple.pyi] +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] diff --git a/test-data/unit/check-typeddictspread.test b/test-data/unit/check-typeddictspread.test deleted file mode 100644 index e39f576b2b6c..000000000000 --- a/test-data/unit/check-typeddictspread.test +++ /dev/null @@ -1,190 +0,0 @@ -[case testTypedDictUnpackSame] -from typing import TypedDict - -class Foo(TypedDict): - a: int - b: int - -foo1: Foo = {'a': 1, 'b': 1} -foo2: Foo = {**foo1, 'b': 2} -foo3 = Foo(**foo1, b=2) -foo4 = Foo({**foo1, 'b': 2}) - -[typing fixtures/typing-typeddict.pyi] - - -[case testTypedDictUnpackCompatible] -from typing import TypedDict - -class Foo(TypedDict): - a: int - -class Bar(TypedDict): - a: int - b: int - -foo: Foo = {'a': 1} -bar: Bar = {**foo, 'b': 2} - -[typing fixtures/typing-typeddict.pyi] - - - -[case testTypedDictUnpackIncompatible] -from typing import TypedDict - -class Foo(TypedDict): - a: int - b: str - -class Bar(TypedDict): - a: int - b: int - -foo: Foo = {'a': 1, 'b': 'a'} -bar1: Bar = {**foo, 'b': 2} # Incompatible item is overriden -bar2: Bar = {**foo, 'a': 2} # E: Incompatible types (expression has type "str", TypedDict item "b" has type "int") - -[builtins fixtures/tuple.pyi] -[builtins fixtures/dict.pyi] -[typing fixtures/typing-typeddict.pyi] - - -[case testTypedDictUnpackNotRequiredKeyIncompatible] -from typing import TypedDict, NotRequired - -class Foo(TypedDict): - a: NotRequired[str] - -class Bar(TypedDict): - a: NotRequired[int] - -foo: Foo = {} -bar: Bar = {**foo} # E: Incompatible types (expression has type "str", TypedDict item "a" has type "int") - -[typing fixtures/typing-typeddict.pyi] - - -[case testTypedDictUnpackMissingOrExtraKey] -from typing import TypedDict - -class Foo(TypedDict): - a: int - -class Bar(TypedDict): - a: int - b: int - -foo1: Foo = {'a': 1} -bar1: Bar = {'a': 1, 'b': 1} -foo2: Foo = {**bar1} # E: Extra key "b" for TypedDict "Foo" -bar2: Bar = {**foo1} # E: Missing key "b" for TypedDict "Bar" - -[typing fixtures/typing-typeddict.pyi] - - -[case testTypedDictUnpackNotRequiredKeyExtra] -from typing import TypedDict, NotRequired - -class Foo(TypedDict): - a: int - -class Bar(TypedDict): - a: int - b: NotRequired[int] - -foo1: Foo = {'a': 1} -bar1: Bar = {'a': 1} -foo2: Foo = {**bar1} # E: Extra key "b" for TypedDict "Foo" -bar2: Bar = {**foo1} - -[typing fixtures/typing-typeddict.pyi] - - -[case testTypedDictUnpackRequiredKeyMissing] -from typing import TypedDict, NotRequired - -class Foo(TypedDict): - a: NotRequired[int] - -class Bar(TypedDict): - a: int - -foo: Foo = {'a': 1} -bar: Bar = {**foo} # E: Missing key "a" for TypedDict "Bar" - -[typing fixtures/typing-typeddict.pyi] - - -[case testTypedDictUnpackMultiple] -from typing import TypedDict - -class Foo(TypedDict): - a: int - -class Bar(TypedDict): - b: int - -class Baz(TypedDict): - a: int - b: int - c: int - -foo: Foo = {'a': 1} -bar: Bar = {'b': 1} -baz: Baz = {**foo, **bar, 'c': 1} - -[typing fixtures/typing-typeddict.pyi] - - -[case testTypedDictUnpackNested] -from typing import TypedDict - -class Foo(TypedDict): - a: int - b: int - -class Bar(TypedDict): - c: Foo - d: int - -foo: Foo = {'a': 1, 'b': 1} -bar: Bar = {'c': foo, 'd': 1} -bar2: Bar = {**bar, 'c': {**bar['c'], 'b': 2}, 'd': 2} - -[builtins fixtures/tuple.pyi] -[builtins fixtures/dict.pyi] -[typing fixtures/typing-typeddict.pyi] - - -[case testTypedDictUnpackNestedError] -from typing import TypedDict - -class Foo(TypedDict): - a: int - b: int - -class Bar(TypedDict): - c: Foo - d: int - -foo: Foo = {'a': 1, 'b': 1} -bar: Bar = {'c': foo, 'd': 1} -bar2: Bar = {**bar, 'c': {**bar['c'], 'b': 'wrong'}, 'd': 2} # E: Incompatible types (expression has type "str", TypedDict item "b" has type "int") - - -[typing fixtures/typing-typeddict.pyi] - - -[case testTypedDictUnpackUntypedDict] -from typing import TypedDict - -class Bar(TypedDict): - pass - -foo: dict = {} -bar: Bar = {**foo} # E: Incompatible types (unpacked expression has type "Dict[Any, Any]", expected "_TypedDict") - -[builtins fixtures/tuple.pyi] -[builtins fixtures/dict.pyi] -[typing fixtures/typing-typeddict.pyi] From 63e4ca8a4885f4384cbbe2a2a5caf0dc7091bb19 Mon Sep 17 00:00:00 2001 From: Eric Wolf Date: Sun, 7 Aug 2022 20:54:01 +0200 Subject: [PATCH 08/11] Fix flake8 issues --- mypy/checkexpr.py | 2 +- mypy/messages.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index cd3281b29b49..e7bbe9ba916e 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -745,7 +745,7 @@ def check_typeddict_call_with_kwargs( supertype=item_expected_type, context=item_value_expr, msg=message_registry.INCOMPATIBLE_TYPES, - subtype_label=f"expression has type", + subtype_label="expression has type", supertype_label=f'TypedDict item "{item_name}" has type', code=codes.TYPEDDICT_ITEM, ) diff --git a/mypy/messages.py b/mypy/messages.py index 842500521c32..f69b977e1b99 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -1585,7 +1585,6 @@ def unexpected_typeddict_keys( ) -> None: actual_keys = actual_sure_keys + actual_maybe_keys actual_sure_set = set(actual_sure_keys) - actual_maybe_set = set(actual_maybe_keys) required_keys = [k for k in typ.items.keys() if k in typ.required_keys] required_set = set(required_keys) expected_set = set(typ.items.keys()) From df33b86c99fb2579fac9836480095bea4d0d5670 Mon Sep 17 00:00:00 2001 From: Eric Wolf Date: Sun, 7 Aug 2022 21:42:37 +0200 Subject: [PATCH 09/11] Tests and better error message for unions --- mypy/checkexpr.py | 17 ++++++------- mypy/message_registry.py | 3 +++ test-data/unit/check-typeddict.test | 39 ++++++++++++++++++++++++++++- 3 files changed, 49 insertions(+), 10 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index e7bbe9ba916e..c10436f5f1d5 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -20,7 +20,7 @@ from mypy.maptype import map_instance_to_supertype from mypy.meet import is_overlapping_types, narrow_declared_type from mypy.message_registry import ErrorMessage -from mypy.messages import MessageBuilder +from mypy.messages import MessageBuilder, format_type_bare from mypy.nodes import ( ARG_NAMED, ARG_POS, @@ -715,15 +715,14 @@ def check_typeddict_call_with_kwargs( maybe_keys.append(nested_key) else: # Fail when trying to unpack anything but TypedDict - assert not self.chk.check_subtype( - subtype=value_type, - supertype=self.chk.named_type("typing._TypedDict"), - context=value_expr, - msg=message_registry.INCOMPATIBLE_TYPES, - subtype_label="unpacked expression has type", - supertype_label="expected", - code=codes.TYPEDDICT_ITEM, + self.chk.fail( + ErrorMessage.format( + message_registry.TYPEDDICT_UNPACKING_MUST_BE_TYPEDDICT, + format_type_bare(value_type), + ), + value_expr, ) + return AnyType(TypeOfAny.from_error) if not ( callee.required_keys diff --git a/mypy/message_registry.py b/mypy/message_registry.py index 11c8696f73f4..ce9b6447e3f6 100644 --- a/mypy/message_registry.py +++ b/mypy/message_registry.py @@ -116,6 +116,9 @@ def format(self, *args: object, **kwargs: object) -> "ErrorMessage": TYPEDDICT_KEY_MUST_BE_STRING_LITERAL: Final = ErrorMessage( "Expected TypedDict key to be string literal" ) +TYPEDDICT_UNPACKING_MUST_BE_TYPEDDICT: Final = ErrorMessage( + "{} cannot be unpacked into TypedDict (must be TypedDict)" +) MALFORMED_ASSERT: Final = ErrorMessage("Assertion is always true, perhaps remove parentheses?") DUPLICATE_TYPE_SIGNATURES: Final = "Function has duplicate type signatures" DESCRIPTOR_SET_NOT_CALLABLE: Final = ErrorMessage("{}.__set__ is not callable") diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index 06f2fe16a716..aad547ef1b90 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -2582,7 +2582,44 @@ class Bar(TypedDict): pass foo: dict = {} -bar: Bar = {**foo} # E: Incompatible types (unpacked expression has type "Dict[Any, Any]", expected "_TypedDict") +bar: Bar = {**foo} # E: Dict[Any, Any] cannot be unpacked into TypedDict (must be TypedDict) + +[builtins fixtures/tuple.pyi] +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictUnpackIntoUnion] +from typing import TypedDict, Union + +class Foo(TypedDict): + a: int + +class Bar(TypedDict): + b: int + +# Would be great if this worked in the future +foo: Foo = {'a': 1} +foo_or_bar: Union[Foo, Bar] = {**foo} # E: Incompatible types in assignment (expression has type "Dict[str, object]", variable has type "Union[Foo, Bar]") + +[builtins fixtures/tuple.pyi] +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + + +[case testTypedDictUnpackFromUnion] +from typing import TypedDict, Union + +class Foo(TypedDict): + a: int + b: int + +class Bar(TypedDict): + b: int + +# Would be great if this worked in the future +foo_or_bar: Union[Foo, Bar] = {'b': 1} +foo: Bar = {**foo_or_bar} # E: Union[Foo, Bar] cannot be unpacked into TypedDict (must be TypedDict) [builtins fixtures/tuple.pyi] [builtins fixtures/dict.pyi] From 4b9a7394ab77a8deffd87dcfeb2af2192a8ae9bf Mon Sep 17 00:00:00 2001 From: Eric Wolf Date: Mon, 8 Aug 2022 22:14:11 +0200 Subject: [PATCH 10/11] Update error messages in tests --- test-data/unit/check-errorcodes.test | 2 +- test-data/unit/pythoneval.test | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test-data/unit/check-errorcodes.test b/test-data/unit/check-errorcodes.test index f1a6f3c77ada..ff29a33e89f6 100644 --- a/test-data/unit/check-errorcodes.test +++ b/test-data/unit/check-errorcodes.test @@ -452,7 +452,7 @@ class E(TypedDict): a: D = {'x': ''} # E: Incompatible types (expression has type "str", TypedDict item "x" has type "int") [typeddict-item] b: D = {'y': ''} # E: Extra key "y" for TypedDict "D" [typeddict-item] c = D(x=0) if int() else E(x=0, y=0) -c = {} # E: Expected TypedDict key "x" but found no keys [typeddict-item] +c = {} # E: Missing key "x" [typeddict-item] a['y'] = 1 # E: TypedDict "D" has no key "y" [typeddict-item] a['x'] = 'x' # E: Value of "x" has incompatible type "str"; expected "int" [typeddict-item] diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index a79e8743fb97..214eb1921d4b 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -1129,7 +1129,7 @@ _testTypedDictMappingMethods.py:10: note: Revealed type is "typing.ItemsView[bui _testTypedDictMappingMethods.py:11: note: Revealed type is "typing.ValuesView[builtins.object]" _testTypedDictMappingMethods.py:12: note: Revealed type is "TypedDict('_testTypedDictMappingMethods.Cell', {'value': builtins.int})" _testTypedDictMappingMethods.py:13: note: Revealed type is "builtins.int" -_testTypedDictMappingMethods.py:15: error: Unexpected TypedDict key "invalid" +_testTypedDictMappingMethods.py:15: error: Extra key "invalid" _testTypedDictMappingMethods.py:16: error: Key "value" of TypedDict "Cell" cannot be deleted _testTypedDictMappingMethods.py:21: note: Revealed type is "builtins.int" From 114e12adc959d87d9e0a1db46fe6ed5ff75785c0 Mon Sep 17 00:00:00 2001 From: Eric Wolf Date: Mon, 8 Aug 2022 22:22:29 +0200 Subject: [PATCH 11/11] Update according to feedback --- mypy/checkexpr.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index dd65cd9cc843..01ee003995b4 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -629,7 +629,7 @@ def check_typeddict_call( args: List[Expression], context: Context, ) -> Type: - if all([ak in {ARG_NAMED, ARG_STAR2} for ak in arg_kinds]): + if all(ak in {ARG_NAMED, ARG_STAR2} for ak in arg_kinds): # ex: Point(x=42, y=1337, **other_point) item_args = args return self.check_typeddict_call_with_kwargs( @@ -677,11 +677,7 @@ def match_typeddict_call_with_dict(self, callee: TypedDictType, kwargs: DictExpr """Check that kwargs is valid set of TypedDict items, contains all required keys of callee, and has no extraneous keys""" validated_kwargs = self.validate_typeddict_kwargs(kwargs=kwargs) if validated_kwargs is not None: - return ( - callee.required_keys - <= set(dict(validated_kwargs).keys()) - <= set(callee.items.keys()) - ) + return callee.required_keys <= dict(validated_kwargs).keys() <= callee.items.keys() else: return False