From 5cc88e88f3cde20bce0b3897c400ad3de365ef3f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 19 Sep 2024 13:04:40 +0100 Subject: [PATCH 1/2] [PEP 695] Allow Self return types with contravariance Fix variance inference in this fragment from a typing conformance test: ``` class ClassA[T1, T2, T3](list[T1]): def method1(self, a: T2) -> None: ... def method2(self) -> T3: ... ``` Previously T2 was incorrectly inferred as invariant due to `list` having methods that return `Self`. Be more flexible with return types to allow inferring contravariance for type variables even if there are `Self` return types, in particular. We could probably make this even more lenient, but after thinking about this for a while, I wasn't sure what the most general rule would be, so I decided to just make a tweak to support the likely most common use case (which is probably actually not that common either). Link to conformance test: https://github.com/python/typing/blob/main/conformance/tests/generics_variance_inference.py#L15C1-L20C12 --- mypy/subtypes.py | 27 +++++++++++++ test-data/unit/check-python312.test | 60 ++++++++++++++++++++++++++++- test-data/unit/pythoneval.test | 20 ++++++++++ 3 files changed, 106 insertions(+), 1 deletion(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 5c4471cc5b62..c8460aa266bf 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -2024,6 +2024,16 @@ def infer_variance(info: TypeInfo, i: int) -> bool: typ = find_member(member, self_type, self_type) if typ: + # It's okay for a method in a generic class with a contravariant type + # variable to return a generic instance of the class, if it doesn't involve + # variance (i.e. values of type variables are propagated). Our normal rules + # would disallow this. Replace such return types with 'Any' to allow this. + # + # This could probably be more lenient (e.g. allow self type be nested, don't + # require all type arguments to be identical to self_type), but this will + # hopefully cover the vast majority of such cases, including Self. + typ = erase_return_self_types(typ, self_type) + typ2 = expand_type(typ, {tvar.id: object_type}) if not is_subtype(typ, typ2): co = False @@ -2057,3 +2067,20 @@ def infer_class_variances(info: TypeInfo) -> bool: if not infer_variance(info, i): success = False return success + + +def erase_return_self_types(typ: Type, self_type: Instance) -> Type: + """If a typ is function-like and returns self_type, replace return type with Any.""" + proper_type = get_proper_type(typ) + if isinstance(proper_type, CallableType): + ret = get_proper_type(proper_type.ret_type) + if isinstance(ret, Instance) and ret == self_type: + return proper_type.copy_modified(ret_type=AnyType(TypeOfAny.implementation_artifact)) + elif isinstance(proper_type, Overloaded): + return Overloaded( + [ + cast(CallableType, erase_return_self_types(it, self_type)) + for it in proper_type.items + ] + ) + return typ diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index d0a39f7e56a6..78538518763c 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -233,7 +233,7 @@ b: Invariant[int] if int(): a = b # E: Incompatible types in assignment (expression has type "Invariant[int]", variable has type "Invariant[object]") if int(): - b = a # E: Incompatible types in assignment (expression has type "Invariant[object]", variable has type "Invariant[int]") + b = a c: Covariant[object] d: Covariant[int] @@ -424,6 +424,64 @@ inv3_1: Invariant3[float] = Invariant3[int](1) # E: Incompatible types in assig inv3_2: Invariant3[int] = Invariant3[float](1) # E: Incompatible types in assignment (expression has type "Invariant3[float]", variable has type "Invariant3[int]") [builtins fixtures/property.pyi] +[case testPEP695InferVarianceWithInheritedSelf] +# flags: --enable-incomplete-feature=NewGenericSyntax +from typing import overload, Self, TypeVar, Generic + +T = TypeVar("T") +S = TypeVar("S") + +class C(Generic[T]): + def f(self, x: T) -> Self: ... + def g(self) -> T: ... + +class D[T1, T2](C[T1]): + def m(self, x: T2) -> None: ... + +a1: D[int, int] = D[int, object]() +a2: D[int, object] = D[int, int]() # E: Incompatible types in assignment (expression has type "D[int, int]", variable has type "D[int, object]") +a3: D[int, int] = D[object, object]() # E: Incompatible types in assignment (expression has type "D[object, object]", variable has type "D[int, int]") +a4: D[object, int] = D[int, object]() # E: Incompatible types in assignment (expression has type "D[int, object]", variable has type "D[object, int]") + +[case testPEP695InferVarianceWithReturnSelf] +# flags: --enable-incomplete-feature=NewGenericSyntax +from typing import Self, overload + +class Cov[T]: + def f(self) -> Self: ... + +a1: Cov[int] = Cov[float]() # E: Incompatible types in assignment (expression has type "Cov[float]", variable has type "Cov[int]") +a2: Cov[float] = Cov[int]() + +class Contra[T]: + def f(self) -> Self: ... + def g(self, x: T) -> None: ... + +b1: Contra[int] = Contra[float]() +b2: Contra[float] = Contra[int]() # E: Incompatible types in assignment (expression has type "Contra[int]", variable has type "Contra[float]") + +class Cov2[T]: + @overload + def f(self, x): ... + @overload + def f(self) -> Self: ... + def f(self, x=None): ... + +c1: Cov2[int] = Cov2[float]() # E: Incompatible types in assignment (expression has type "Cov2[float]", variable has type "Cov2[int]") +c2: Cov2[float] = Cov2[int]() + +class Contra2[T]: + @overload + def f(self, x): ... + @overload + def f(self) -> Self: ... + def f(self, x=None): ... + + def g(self, x: T) -> None: ... + +d1: Contra2[int] = Contra2[float]() +d2: Contra2[float] = Contra2[int]() # E: Incompatible types in assignment (expression has type "Contra2[int]", variable has type "Contra2[float]") + [case testPEP695InheritInvariant] # flags: --enable-incomplete-feature=NewGenericSyntax diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index dbf228623d7c..d41e0509e721 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -2196,3 +2196,23 @@ type K4 = None | B[int] type L1 = Never type L2 = list[Never] + +[case testPEP695VarianceInferenceSpecialCaseWithTypeshed] +# flags: --python-version=3.12 --enable-incomplete-feature=NewGenericSyntax +class C1[T1, T2](list[T1]): + def m(self, a: T2) -> None: ... + +def func1(p: C1[int, object]): + x: C1[int, int] = p + +class C2[T1, T2, T3](dict[T2, T3]): + def m(self, a: T1) -> None: ... + +def func2(p: C2[object, int, int]): + x: C2[int, int, int] = p + +class C3[T1, T2](tuple[T1, ...]): + def m(self, a: T2) -> None: ... + +def func3(p: C3[int, object]): + x: C3[int, int] = p From ae0876380cf16463c283e9a269f23776cc6c1fb3 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 20 Sep 2024 14:33:20 +0100 Subject: [PATCH 2/2] New syntax does not need to be enabled with a flag --- test-data/unit/check-python312.test | 2 -- test-data/unit/pythoneval.test | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 7ef34bf43509..085cc052705d 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -394,7 +394,6 @@ inv3_2: Invariant3[int] = Invariant3[float](1) # E: Incompatible types in assig [builtins fixtures/property.pyi] [case testPEP695InferVarianceWithInheritedSelf] -# flags: --enable-incomplete-feature=NewGenericSyntax from typing import overload, Self, TypeVar, Generic T = TypeVar("T") @@ -413,7 +412,6 @@ a3: D[int, int] = D[object, object]() # E: Incompatible types in assignment (ex a4: D[object, int] = D[int, object]() # E: Incompatible types in assignment (expression has type "D[int, object]", variable has type "D[object, int]") [case testPEP695InferVarianceWithReturnSelf] -# flags: --enable-incomplete-feature=NewGenericSyntax from typing import Self, overload class Cov[T]: diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index 0d21f687da04..89f01bff963e 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -2198,7 +2198,7 @@ type L1 = Never type L2 = list[Never] [case testPEP695VarianceInferenceSpecialCaseWithTypeshed] -# flags: --python-version=3.12 --enable-incomplete-feature=NewGenericSyntax +# flags: --python-version=3.12 class C1[T1, T2](list[T1]): def m(self, a: T2) -> None: ...