Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

stubgen: Support TypedDict alternative syntax #14682

Merged
merged 13 commits into from
May 6, 2023
64 changes: 64 additions & 0 deletions mypy/stubgen.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@

import argparse
import glob
import keyword
import os
import os.path
import sys
Expand Down Expand Up @@ -80,6 +81,7 @@
ClassDef,
ComparisonExpr,
Decorator,
DictExpr,
EllipsisExpr,
Expression,
FloatExpr,
Expand Down Expand Up @@ -126,6 +128,7 @@
from mypy.traverser import all_yield_expressions, has_return_statement, has_yield_expression
from mypy.types import (
OVERLOAD_NAMES,
TPDICT_NAMES,
AnyType,
CallableType,
Instance,
Expand Down Expand Up @@ -405,6 +408,14 @@ def visit_tuple_expr(self, node: TupleExpr) -> str:
def visit_list_expr(self, node: ListExpr) -> str:
return f"[{', '.join(n.accept(self) for n in node.items)}]"

def visit_dict_expr(self, o: DictExpr) -> str:
dict_items = []
for key, value in o.items:
# This is currently only used for TypedDict where all keys are strings.
assert isinstance(key, StrExpr)
dict_items.append(f"{key.accept(self)}: {value.accept(self)}")
return f"{{{', '.join(dict_items)}}}"

def visit_ellipsis(self, node: EllipsisExpr) -> str:
return "..."

Expand Down Expand Up @@ -641,6 +652,7 @@ def visit_mypy_file(self, o: MypyFile) -> None:
"_typeshed": ["Incomplete"],
"typing": ["Any", "TypeVar"],
"collections.abc": ["Generator"],
"typing_extensions": ["TypedDict"],
}
for pkg, imports in known_imports.items():
for t in imports:
Expand Down Expand Up @@ -1003,6 +1015,13 @@ def visit_assignment_stmt(self, o: AssignmentStmt) -> None:
assert isinstance(o.rvalue, CallExpr)
self.process_namedtuple(lvalue, o.rvalue)
continue
if (
isinstance(lvalue, NameExpr)
and isinstance(o.rvalue, CallExpr)
and self.is_typeddict(o.rvalue)
):
self.process_typeddict(lvalue, o.rvalue)
continue
if (
isinstance(lvalue, NameExpr)
and not self.is_private_name(lvalue.name)
Expand Down Expand Up @@ -1071,6 +1090,51 @@ def process_namedtuple(self, lvalue: NameExpr, rvalue: CallExpr) -> None:
self.add(f"{self._indent} {item}: Incomplete\n")
self._state = CLASS

def is_typeddict(self, expr: CallExpr) -> bool:
callee = expr.callee
return (
isinstance(callee, NameExpr) and self.refers_to_fullname(callee.name, TPDICT_NAMES)
) or (
isinstance(callee, MemberExpr)
and isinstance(callee.expr, NameExpr)
and f"{callee.expr.name}.{callee.name}" in TPDICT_NAMES
)

def process_typeddict(self, lvalue: NameExpr, rvalue: CallExpr) -> None:
if self._state != EMPTY:
self.add("\n")
if isinstance(rvalue.args[1], DictExpr):
items: list[tuple[str, Expression]] = []
for attr_name, attr_type in rvalue.args[1].items:
if not isinstance(attr_name, StrExpr):
self.add(f"{self._indent}{lvalue.name}: Incomplete")
self.import_tracker.require_name("Incomplete")
return
items.append((attr_name.value, attr_type))
else:
self.add(f"{self._indent}{lvalue.name}: Incomplete")
self.import_tracker.require_name("Incomplete")
return
self.import_tracker.require_name("TypedDict")
p = AliasPrinter(self)
if any(not key.isidentifier() or keyword.iskeyword(key) for key, _ in items):
# Keep the call syntax if there are non-identifier or keyword keys.
self.add(f"{self._indent}{lvalue.name} = {rvalue.accept(p)}\n")
hamdanal marked this conversation as resolved.
Show resolved Hide resolved
self._state = VAR
else:
bases = "TypedDict"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Mypy actually now supports generic TypedDicts defined with the call syntax, in which case TypedDict wouldn't be the only base (the rewritten version using the class-based syntax would use multiple inheritance with Generic[T] in this playground example): https://mypy-play.net/?mypy=latest&python=3.11&gist=fbeb5bbd0c3036b7327fc65fff0c9a9d

I think it's reasonable not to handle generic TypedDicts defined using the call syntax in this PR, since they're unlikely to come up much. But probably worth a TODO comment?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

TIL, thank you.
I'll add a TODO for now. Generic TypedDict requires keeping track of TypeVars defined in the file which deserves its own PR IMO.

if len(rvalue.args) > 2:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What if a user does something like

Foo = TypedDict("Foo", {"a": int}, b=str, d=bytes)

Defining TypedDicts like this using keyword arguments is deprecated at runtime, and has never been supported by mypy — but according to the typing docs it is (or was until recently, when we deprecated it) a supported way of creating a new TypedDict type: https://docs.python.org/3/library/typing.html#typing.TypedDict

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Right again Alex. Indeed I forgot about the keyword syntax.
The example you gave however is invalid at runtime:

Python 3.10.6 (main, Mar 10 2023, 10:55:28) [GCC 11.3.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> from typing import TypedDict
>>> Foo = TypedDict("Foo", {"a": int}, b=str, d=bytes)
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/usr/lib/python3.10/typing.py", line 2435, in TypedDict
    raise TypeError("TypedDict takes either a dict or keyword arguments,"
TypeError: TypedDict takes either a dict or keyword arguments, but not both

this works:

>>> from typing import TypedDict
>>> Foo = TypedDict("Foo", a=int, b=str, d=bytes)
>>> Foo.__required_keys__
frozenset({'b', 'd', 'a'})
>>> Foo.__optional_keys__
frozenset()

I'll update the PR to add support for the keyword syntax.

bases += f", total={rvalue.args[2].accept(p)}"
self.add(f"{self._indent}class {lvalue.name}({bases}):")
if len(items) == 0:
self.add(" ...\n")
self._state = EMPTY_CLASS
else:
self.add("\n")
for key, key_type in items:
self.add(f"{self._indent} {key}: {key_type.accept(p)}\n")
self._state = CLASS

def is_alias_expression(self, expr: Expression, top_level: bool = True) -> bool:
"""Return True for things that look like target for an alias.

Expand Down
72 changes: 72 additions & 0 deletions test-data/unit/stubgen.test
Original file line number Diff line number Diff line change
Expand Up @@ -2793,3 +2793,75 @@ def f(x: str | None) -> None: ...
a: str | int

def f(x: str | None) -> None: ...

[case testTypeddict]
import typing, x
X = typing.TypedDict('X', {'a': int, 'b': str})
Y = typing.TypedDict('X', {'a': int, 'b': str}, total=False)
[out]
from typing_extensions import TypedDict

class X(TypedDict):
a: int
b: str

class Y(TypedDict, total=False):
a: int
b: str

[case testTypeddictWithNonIdentifierOrKeywordKeys]
from typing import TypedDict
X = TypedDict('X', {'a-b': int, 'c': str})
Y = TypedDict('X', {'a-b': int, 'c': str}, total=False)
Z = TypedDict('X', {'a': int, 'in': str})
[out]
from typing import TypedDict

X = TypedDict('X', {'a-b': int, 'c': str})

Y = TypedDict('X', {'a-b': int, 'c': str}, total=False)

Z = TypedDict('X', {'a': int, 'in': str})

[case testEmptyTypeddict]
import typing
X = typing.TypedDict('X', {})
[out]
from typing_extensions import TypedDict

class X(TypedDict): ...

[case testTypeddictWithUnderscore]
from typing import TypedDict as _TypedDict
def f(): ...
X = _TypedDict('X', {'a': int, 'b': str})
def g(): ...
[out]
from typing_extensions import TypedDict

def f() -> None: ...

class X(TypedDict):
a: int
b: str

def g() -> None: ...

[case testNotTypeddict]
from x import TypedDict
import y
X = TypedDict('X', {'a': int, 'b': str})
Y = y.TypedDict('Y', {'a': int, 'b': str})
[out]
from _typeshed import Incomplete

X: Incomplete
Y: Incomplete

[case testTypeddictWithWrongAttributesType]
from typing import TypedDict
T = TypeDict("T", {"a": int, **{"b": str, "c": bytes}})
[out]
from _typeshed import Incomplete

T: Incomplete