From 45e593122a3a3cacd28a8755b902ef1279da0392 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 1 Sep 2017 00:00:37 +0200 Subject: [PATCH 01/39] Add basic tests, more details will be added when they will not crash --- test-data/unit/check-classes.test | 186 ++++++++++++++++++++++++++++++ test-data/unit/pythoneval.test | 8 ++ 2 files changed, 194 insertions(+) diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 66c34b42b66f..6edcc6db26f7 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -3388,6 +3388,192 @@ reveal_type(mkdict(ExampleDict)) # E: Revealed type is '__main__.ExampleDict*[A -- Synthetic types crashes -- ----------------------- +[case testCrashOnSelfRecursiveNamedTupleVar] +from typing import NamedTuple + +N = NamedTuple('N', [('x', N)]) +n: N +[out] + +[case testCrashOnSelfRecursiveTypedDictVar] +from mypy_extensions import TypedDict + +A = TypedDict('A', {'a': 'A'}) +a: A +[out] + +[case testCrashInJoinOfSelfRecursiveNamedTuples] +from typing import NamedTuple + +class N(NamedTuple): + x: N +class M(NamedTuple): + x: M + +n: N +m: M +lst = [n, m] +[builtins fixtures/isinstancelist.pyi] +[out] + +[case testCorrectJoinOfSelfRecursiveTypedDicts] +from mypy_extensions import TypedDict + +class N(TypedDict): + x: N +class M(TypedDict): + x: M + +n: N +m: M +lst = [n, m] +reveal_type(lst) +[builtins fixtures/isinstancelist.pyi] +[out] + +[case testCrashInForwardRefToNamedTupleWithIsinstance] +from typing import Dict, NamedTuple + +NameDict = Dict[str, 'NameInfo'] +class NameInfo(NamedTuple): + ast: bool + +def parse_ast(name_dict: NameDict) -> None: + if isinstance(name_dict[''], int): + pass +[builtins fixtures/isinstancelist.pyi] +[out] + +[case testCrashInForwardRefToTypedDictWithIsinstance] +from mypy_extensions import TypedDict +from typing import Dict + +NameDict = Dict[str, 'NameInfo'] +class NameInfo(TypedDict): + ast: bool + +def parse_ast(name_dict: NameDict) -> None: + if isinstance(name_dict[''], int): + pass +[builtins fixtures/isinstancelist.pyi] +[out] + +[case testCorrectIsinstanceInForwardRefToNewType] +from typing import Dict, NewType + +NameDict = Dict[str, 'NameInfo'] +class NameInfoBase: + ast: bool +NameInfo = NewType('NameInfo', NameInfoBase) + +def parse_ast(name_dict: NameDict) -> None: + if isinstance(name_dict[''], int): + pass +[builtins fixtures/isinstancelist.pyi] +[out] + +[case testCorrectAttributeInForwardRefToNamedTuple] +from typing import NamedTuple + +def get_state(proc: 'Process') -> int: + return proc.state +class Process(NamedTuple): + state: int +[out] + +[case testCorrectItemTypeInForwardRefToTypedDict] +from mypy_extensions import TypedDict + +def get_state(proc: 'Process') -> int: + return proc['state'] +class Process(TypedDict): + state: int +[builtins fixtures/isinstancelist.pyi] +[out] + +[case testCorrectDoubleForwardNamedTuple] +from typing import NamedTuple + +class A(NamedTuple): + name: 'B' + year: int +class B(NamedTuple): + director: str + +x: A +reveal_type(x.name.director) # E: Revealed type is 'builtins.str' +[out] + +[case testCrashOnDoubleForwardTypedDict] +from mypy_extensions import TypedDict + +class A(TypedDict): + name: 'B' + year: int +class B(TypedDict): + director: str +[builtins fixtures/isinstancelist.pyi] +[out] + +[case testCrashOnForwardUnionOfNamedTuples] +from typing import Union, NamedTuple + +NodeType = Union['Foo', 'Bar'] +class Foo(NamedTuple): + pass +class Bar(NamedTuple): + pass + +def foo(node: NodeType): + x = node +[out] + +[case testCrashOnForwardUnionOfTypedDicts] +from mypy_extensions import TypedDict +from typing import Union + +NodeType = Union['Foo', 'Bar'] +class Foo(TypedDict): + pass +class Bar(TypedDict): + pass + +def foo(node: NodeType): + x = node +[builtins fixtures/isinstancelist.pyi] +[out] + +[case testSupportForwardUnionOfNewTypes] +from typing import Union, NewType + +NodeType = Union['Foo', 'Bar'] +Foo = NewType('Foo', int) +Bar = NewType('Bar', str) + +def foo(node: NodeType): + x = node +[out] + +[case testCrashOnComplexCheckWithNamedTupleUnion] +from typing import NamedTuple, Union + +AOrB = Union['A', 'B'] +class A(NamedTuple): + lat: float + lng: float + +class B(object): + def __init__(self, a: AOrB) -> None: + self.a = a + @property + def lat(self) -> float: + return self.a.lat + @property + def lng(self) -> float: + return self.a.lng +[builtins fixtures/property.pyi] +[out] + [case testCrashInvalidArgsSyntheticClassSyntax] from typing import List, NamedTuple from mypy_extensions import TypedDict diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index 4ea16032e0d7..7713bd344a0c 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -1380,6 +1380,14 @@ _testTypedDictMappingMethods.py:9: error: Revealed type is 'typing.AbstractSet[b _testTypedDictMappingMethods.py:10: error: Revealed type is 'typing.AbstractSet[Tuple[builtins.str*, builtins.int*]]' _testTypedDictMappingMethods.py:11: error: Revealed type is 'typing.ValuesView[builtins.int*]' +[case testCrashOnComplexCheckWithNamedTupleNext] +from typing import NamedTuple + +MyNamedTuple = NamedTuple('MyNamedTuple', [('parent', 'MyNamedTuple')]) +def foo(mymap) -> MyNamedTuple: + return next((mymap[key] for key in mymap), None) +[out] + [case testCanConvertTypedDictToAnySuperclassOfMapping] from mypy_extensions import TypedDict from typing import Sized, Iterable, Container From cb4caa596eda6e7d9e093604f158157cc7ef9afc Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 1 Sep 2017 10:06:19 +0200 Subject: [PATCH 02/39] Correct tests --- test-data/unit/check-classes.test | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 6edcc6db26f7..bea01229021b 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -3400,6 +3400,7 @@ from mypy_extensions import TypedDict A = TypedDict('A', {'a': 'A'}) a: A +[builtins fixtures/isinstancelist.pyi] [out] [case testCrashInJoinOfSelfRecursiveNamedTuples] @@ -3427,7 +3428,7 @@ class M(TypedDict): n: N m: M lst = [n, m] -reveal_type(lst) +reveal_type(lst[0]) # E: Revealed type is 'TypedDict({'x': object}, fallback=typing.Mapping[builtins.str, object])' [builtins fixtures/isinstancelist.pyi] [out] @@ -3559,17 +3560,17 @@ from typing import NamedTuple, Union AOrB = Union['A', 'B'] class A(NamedTuple): - lat: float - lng: float + lat: int + lng: int class B(object): def __init__(self, a: AOrB) -> None: self.a = a @property - def lat(self) -> float: + def lat(self) -> int: return self.a.lat @property - def lng(self) -> float: + def lng(self) -> int: return self.a.lng [builtins fixtures/property.pyi] [out] From 1cdc980601795c33d4fcfe02b91f76cc99487cff Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 11 Sep 2017 15:37:05 +0200 Subject: [PATCH 03/39] Implement ForwardRef type, wrap UnboundType, pass SecondPass to third one --- mypy/build.py | 2 +- mypy/semanal.py | 17 +++++++++++------ mypy/typeanal.py | 18 +++++++++++++++--- mypy/types.py | 31 +++++++++++++++++++++++++++++++ 4 files changed, 58 insertions(+), 10 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 22e5d59b7204..7648c58b9f07 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -491,7 +491,7 @@ def __init__(self, data_dir: str, self.semantic_analyzer = SemanticAnalyzer(self.modules, self.missing_modules, lib_path, self.errors, self.plugin) self.modules = self.semantic_analyzer.modules - self.semantic_analyzer_pass3 = ThirdPass(self.modules, self.errors) + self.semantic_analyzer_pass3 = ThirdPass(self.modules, self.errors, self.semantic_analyzer) self.all_types = {} # type: Dict[Expression, Type] self.indirection_detector = TypeIndirectionVisitor() self.stale_modules = set() # type: Set[str] diff --git a/mypy/semanal.py b/mypy/semanal.py index fd9b65d4e50b..7d599d3949bb 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1608,7 +1608,8 @@ def visit_block_maybe(self, b: Block) -> None: def type_analyzer(self, *, tvar_scope: Optional[TypeVarScope] = None, allow_tuple_literal: bool = False, - aliasing: bool = False) -> TypeAnalyser: + aliasing: bool = False, + third_pass: boll = False) -> TypeAnalyser: if tvar_scope is None: tvar_scope = self.tvar_scope return TypeAnalyser(self.lookup_qualified, @@ -1620,17 +1621,20 @@ def type_analyzer(self, *, self.is_typeshed_stub_file, aliasing=aliasing, allow_tuple_literal=allow_tuple_literal, - allow_unnormalized=self.is_stub_file) + allow_unnormalized=self.is_stub_file, + third_pass=third_pass) def anal_type(self, t: Type, *, tvar_scope: Optional[TypeVarScope] = None, allow_tuple_literal: bool = False, - aliasing: bool = False) -> Type: + aliasing: bool = False, + third_pass: bool = False) -> Type: if t: a = self.type_analyzer( tvar_scope=tvar_scope, aliasing=aliasing, - allow_tuple_literal=allow_tuple_literal) + allow_tuple_literal=allow_tuple_literal, + third_pass=third_pass) return t.accept(a) else: @@ -4038,9 +4042,10 @@ class ThirdPass(TraverserVisitor): straightforward type inference. """ - def __init__(self, modules: Dict[str, MypyFile], errors: Errors) -> None: + def __init__(self, modules: Dict[str, MypyFile], errors: Errors, sem) -> None: self.modules = modules self.errors = errors + self.sem = sem def visit_file(self, file_node: MypyFile, fnam: str, options: Options) -> None: self.errors.set_file(fnam, file_node.fullname()) @@ -4184,7 +4189,7 @@ def visit_type_application(self, e: TypeApplication) -> None: def analyze(self, type: Optional[Type]) -> None: if type: - analyzer = TypeAnalyserPass3(self.fail, self.options, self.is_typeshed_file) + analyzer = TypeAnalyserPass3(self.fail, self.options, self.is_typeshed_file, self.sem) type.accept(analyzer) self.check_for_omitted_generics(type) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 4cd1e780264d..4679431c33d3 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -14,7 +14,7 @@ Type, UnboundType, TypeVarType, TupleType, TypedDictType, UnionType, Instance, AnyType, CallableType, NoneTyp, DeletedType, TypeList, TypeVarDef, TypeVisitor, SyntheticTypeVisitor, StarType, PartialType, EllipsisType, UninhabitedType, TypeType, get_typ_args, set_typ_args, - CallableArgument, get_type_vars, TypeQuery, union_items, TypeOfAny + CallableArgument, get_type_vars, TypeQuery, union_items, TypeOfAny, ForwardRef ) from mypy.nodes import ( @@ -130,7 +130,8 @@ def __init__(self, is_typeshed_stub: bool, *, aliasing: bool = False, allow_tuple_literal: bool = False, - allow_unnormalized: bool = False) -> None: + allow_unnormalized: bool = False, + third_pass: bool = False) -> None: self.lookup = lookup_func self.lookup_fqn_func = lookup_fqn_func self.fail_func = fail_func @@ -143,6 +144,7 @@ def __init__(self, self.plugin = plugin self.options = options self.is_typeshed_stub = is_typeshed_stub + self.third_pass = third_pass def visit_unbound_type(self, t: UnboundType) -> Type: if t.optional: @@ -256,6 +258,8 @@ def visit_unbound_type(self, t: UnboundType) -> Type: # as a base class -- however, this will fail soon at runtime so the problem # is pretty minor. return AnyType(TypeOfAny.from_unimported_type) + if not self.third_pass: + return ForwardRef(t) # Allow unbound type variables when defining an alias if not (self.aliasing and sym.kind == TVAR and self.tvar_scope.get_binding(sym) is None): @@ -377,6 +381,9 @@ def visit_ellipsis_type(self, t: EllipsisType) -> Type: def visit_type_type(self, t: TypeType) -> Type: return TypeType.make_normalized(self.anal_type(t.item), line=t.line) + def visit_forwardref_type(self, t: ForwardRef) -> Type: + return t + def analyze_callable_type(self, t: UnboundType) -> Type: fallback = self.named_type('builtins.function') if len(t.args) == 0: @@ -580,10 +587,12 @@ class TypeAnalyserPass3(TypeVisitor[None]): def __init__(self, fail_func: Callable[[str, Context], None], options: Options, - is_typeshed_stub: bool) -> None: + is_typeshed_stub: bool, + sem) -> None: self.fail = fail_func self.options = options self.is_typeshed_stub = is_typeshed_stub + self.sem = sem def visit_instance(self, t: Instance) -> None: info = t.type @@ -710,6 +719,9 @@ def visit_partial_type(self, t: PartialType) -> None: def visit_type_type(self, t: TypeType) -> None: pass + def visit_forwardref_type(self, t: ForwardRef) -> None: + if isinstance(t, UnboundType): + t.link = self.sem.anal_type(t.link, third_pass=True) TypeVarList = List[Tuple[str, TypeVarExpr]] diff --git a/mypy/types.py b/mypy/types.py index 904694c81558..a005efed29e7 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1374,6 +1374,25 @@ def deserialize(cls, data: JsonDict) -> Type: return TypeType.make_normalized(deserialize_type(data['item'])) +class ForwardRef(Type): + """Class to wrap forward references to other types.""" + link = None # type: Type # the wrapped type + + def __init__(self, link: Type) -> None: + self.link = link + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_forwardref_type(self) + + def __hash__(self) -> int: + return hash(self.link) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, ForwardRef): + return NotImplemented + return self.link == other.link + + # # Visitor-related classes # @@ -1449,6 +1468,9 @@ def visit_partial_type(self, t: PartialType) -> T: def visit_type_type(self, t: TypeType) -> T: pass + def visit_forwardref_type(self, t: ForwardRef) -> T: + raise RuntimeError('Debugging forward ref') + class SyntheticTypeVisitor(TypeVisitor[T]): """A TypeVisitor that also knows how to visit synthetic AST constructs. @@ -1551,6 +1573,9 @@ def visit_overloaded(self, t: Overloaded) -> Type: def visit_type_type(self, t: TypeType) -> Type: return TypeType.make_normalized(t.item.accept(self), line=t.line, column=t.column) + def visit_forwardref_type(self, t: TypeType) -> Type: + return t + class TypeStrVisitor(SyntheticTypeVisitor[str]): """Visitor for pretty-printing types into strings. @@ -1706,6 +1731,9 @@ def visit_ellipsis_type(self, t: EllipsisType) -> str: def visit_type_type(self, t: TypeType) -> str: return 'Type[{}]'.format(t.item.accept(self)) + def visit_forwardref_type(self, t: ForwardRef) -> str: + return '~{}'.format(t.link.accept(self)) + def list_str(self, a: List[Type]) -> str: """Convert items of an array to strings (pretty-print types) and join the results with commas. @@ -1785,6 +1813,9 @@ def visit_overloaded(self, t: Overloaded) -> T: def visit_type_type(self, t: TypeType) -> T: return t.item.accept(self) + def visit_forwardref_type(self, t: TypeType) -> T: + return t.link.accept(self) + def visit_ellipsis_type(self, t: EllipsisType) -> T: return self.strategy([]) From 260ef028e90233cfdf622e75e11a56ae156de249 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 11 Sep 2017 16:19:12 +0200 Subject: [PATCH 04/39] Add ForwardRefRemover --- mypy/semanal.py | 83 +++++++++++++++++++++++++++++++++++++++++++++++- mypy/typeanal.py | 7 ++-- 2 files changed, 87 insertions(+), 3 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 7d599d3949bb..d69418ec68b3 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1609,7 +1609,7 @@ def type_analyzer(self, *, tvar_scope: Optional[TypeVarScope] = None, allow_tuple_literal: bool = False, aliasing: bool = False, - third_pass: boll = False) -> TypeAnalyser: + third_pass: bool = False) -> TypeAnalyser: if tvar_scope is None: tvar_scope = self.tvar_scope return TypeAnalyser(self.lookup_qualified, @@ -4617,3 +4617,84 @@ def visit_any(self, t: AnyType) -> Type: if t.type_of_any == TypeOfAny.explicit: return t.copy_modified(TypeOfAny.special_form) return t + + +def remove_forward_refs(tp: Type) -> Type: + """Returns a type with forward refs removed.""" + return tp.accept(ForwardRefRemover()) + + +class ForwardRefRemover(TypeVisitor[Type]): + def __init__(self): + self.seen = [] + + def visit_unbound_type(self, t: UnboundType) -> UnboundType: + return t + + def visit_any(self, t: AnyType) -> AnyType: + return t + + def visit_none_type(self, t: NoneTyp) -> NoneTyp: + return t + + def visit_uninhabited_type(self, t: UninhabitedType) -> UninhabitedType: + return t + + def visit_erased_type(self, t: ErasedType) -> ErasedType: + return t + + def visit_deleted_type(self, t: DeletedType) -> None: + assert False, "Internal error: deleted type in semantic analysis" + + def visit_type_var(self, t: TypeVarType) -> TypeVarType: + if t.bound: + t.bound.accept(self) + if t.constrains: + for c in t.constrains: + c.accept(self) + return t + + def visit_instance(self, t: Instance) -> Instance: + for arg in t.args: + arg.accept(self) + return t + + def visit_callable_type(self, t: CallableType) -> CallableType: + for tp in t.arg_types: + tp.accept(self) + tp.ret_type.accpet(self) + return t + + def visit_overloaded(self, t: Overloaded) -> Overloaded: + for it in t.items: + it.accept(self) + t.fallback.accept(self) + return t + + def visit_tuple_type(self, t: TupleType) -> TupleType: + for it in t.items: + it.accept(self) + t.fallback.accept(self) + return t + + def visit_typeddict_type(self, t: TypedDictType) -> TypedDictType: + t.fallback.accept(self) + return t + + def visit_union_type(self, t: UnionType) -> UnionType: + for it in t.items: + it.accept(self) + return t + + def visit_partial_type(self, t: PartialType) -> None: + assert False, "Internal error: partial type in semantic analysis" + + def visit_type_type(self, t: TypeType) -> TypeType: + t.item = t.item.accept(self) + return t + + def visit_forwardref_type(self, t: ForwardRef) -> Type: + if not any(s is t for s in self.seen): + self.seen.append(seen) + return t.link.accept(self) + return AnyType(TypeOfAny.from_error) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 4679431c33d3..a108e0bda860 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -258,11 +258,11 @@ def visit_unbound_type(self, t: UnboundType) -> Type: # as a base class -- however, this will fail soon at runtime so the problem # is pretty minor. return AnyType(TypeOfAny.from_unimported_type) - if not self.third_pass: - return ForwardRef(t) # Allow unbound type variables when defining an alias if not (self.aliasing and sym.kind == TVAR and self.tvar_scope.get_binding(sym) is None): + if not self.third_pass: + return ForwardRef(t) self.fail('Invalid type "{}"'.format(name), t) return t info = sym.node # type: TypeInfo @@ -296,6 +296,8 @@ def visit_unbound_type(self, t: UnboundType) -> Type: # Create a named TypedDictType return td.copy_modified(item_types=self.anal_array(list(td.items.values())), fallback=instance) + if not instance.type.mro and not self.third_pass: + return ForwardRef(t) return instance else: return AnyType(TypeOfAny.special_form) @@ -918,3 +920,4 @@ def make_optional_type(t: Type) -> Type: return UnionType(items + [NoneTyp()], t.line, t.column) else: return UnionType([t, NoneTyp()], t.line, t.column) + From a58a217446c39e97ab75d9a836f526b80cf59f6f Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 11 Sep 2017 18:25:46 +0200 Subject: [PATCH 05/39] Add elimination patches --- mypy/build.py | 4 ++- mypy/indirection.py | 3 ++ mypy/sametypes.py | 3 +- mypy/semanal.py | 82 ++++++++++++++++++++++++++++----------------- mypy/server/deps.py | 5 ++- mypy/subtypes.py | 2 +- mypy/typeanal.py | 7 ++-- mypy/types.py | 4 +-- 8 files changed, 69 insertions(+), 41 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 7648c58b9f07..c5a059cd5922 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -1722,10 +1722,12 @@ def semantic_analysis(self) -> None: def semantic_analysis_pass_three(self) -> None: assert self.tree is not None, "Internal error: method must be called on parsed file only" + patches = [] # type: List[Callable[[], None]] with self.wrap_context(): - self.manager.semantic_analyzer_pass3.visit_file(self.tree, self.xpath, self.options) + self.manager.semantic_analyzer_pass3.visit_file(self.tree, self.xpath, self.options, patches) if self.options.dump_type_stats: dump_type_stats(self.tree, self.xpath) + self.patches = patches + self.patches def semantic_analysis_apply_patches(self) -> None: for patch_func in self.patches: diff --git a/mypy/indirection.py b/mypy/indirection.py index 2e69c5ebd3ff..badbe38cae38 100644 --- a/mypy/indirection.py +++ b/mypy/indirection.py @@ -101,3 +101,6 @@ def visit_ellipsis_type(self, t: types.EllipsisType) -> Set[str]: def visit_type_type(self, t: types.TypeType) -> Set[str]: return self._visit(t.item) + + def visit_forwardref_type(self, t: types.ForwardRef) -> Set[str]: + return self._visit(t.link) diff --git a/mypy/sametypes.py b/mypy/sametypes.py index cba80e1ef825..256759862776 100644 --- a/mypy/sametypes.py +++ b/mypy/sametypes.py @@ -3,13 +3,12 @@ from mypy.types import ( Type, UnboundType, AnyType, NoneTyp, TupleType, TypedDictType, UnionType, CallableType, TypeVarType, Instance, TypeVisitor, ErasedType, - TypeList, Overloaded, PartialType, DeletedType, UninhabitedType, TypeType + TypeList, Overloaded, PartialType, DeletedType, UninhabitedType, TypeType, ForwardRef ) def is_same_type(left: Type, right: Type) -> bool: """Is 'left' the same type as 'right'?""" - if isinstance(right, UnboundType): # Make unbound types same as anything else to reduce the number of # generated spurious error messages. diff --git a/mypy/semanal.py b/mypy/semanal.py index d69418ec68b3..a9587fa645a8 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -80,7 +80,8 @@ from mypy.types import ( FunctionLike, UnboundType, TypeVarDef, TypeType, TupleType, UnionType, StarType, function_type, TypedDictType, NoneTyp, CallableType, Overloaded, Instance, Type, TypeVarType, AnyType, - TypeTranslator, TypeOfAny + TypeTranslator, TypeOfAny, TypeVisitor, UninhabitedType, ErasedType, DeletedType, + PartialType, ForwardRef ) from mypy.nodes import implicit_module_attrs from mypy.typeanal import ( @@ -4047,9 +4048,10 @@ def __init__(self, modules: Dict[str, MypyFile], errors: Errors, sem) -> None: self.errors = errors self.sem = sem - def visit_file(self, file_node: MypyFile, fnam: str, options: Options) -> None: + def visit_file(self, file_node: MypyFile, fnam: str, options: Options, patches) -> None: self.errors.set_file(fnam, file_node.fullname()) self.options = options + self.patches = patches self.is_typeshed_file = self.errors.is_typeshed_file(fnam) with experiments.strict_optional_set(options.strict_optional): self.accept(file_node) @@ -4080,7 +4082,7 @@ def visit_block(self, b: Block) -> None: def visit_func_def(self, fdef: FuncDef) -> None: self.errors.push_function(fdef.name()) - self.analyze(fdef.type) + self.analyze(fdef.type, fdef) super().visit_func_def(fdef) self.errors.pop_function() @@ -4089,7 +4091,7 @@ def visit_class_def(self, tdef: ClassDef) -> None: # check them again here. if not tdef.info.is_named_tuple: for type in tdef.info.bases: - self.analyze(type) + self.analyze(type, None) if tdef.info.is_protocol: if not isinstance(type, Instance) or not type.type.is_protocol: if type.type.fullname() != 'builtins.object': @@ -4104,9 +4106,9 @@ def visit_class_def(self, tdef: ClassDef) -> None: add_protocol_members(tdef.info) if tdef.analyzed is not None: if isinstance(tdef.analyzed, TypedDictExpr): - self.analyze(tdef.analyzed.info.typeddict_type) + self.analyze(tdef.analyzed.info.typeddict_type, tdef.analyzed) elif isinstance(tdef.analyzed, NamedTupleExpr): - self.analyze(tdef.analyzed.info.tuple_type) + self.analyze(tdef.analyzed.info.tuple_type, tdef.analyzed) super().visit_class_def(tdef) def visit_decorator(self, dec: Decorator) -> None: @@ -4161,20 +4163,21 @@ def visit_decorator(self, dec: Decorator) -> None: dec.var.type = sig def visit_assignment_stmt(self, s: AssignmentStmt) -> None: - self.analyze(s.type) + self.analyze(s.type, s) if isinstance(s.rvalue, IndexExpr) and isinstance(s.rvalue.analyzed, TypeAliasExpr): - self.analyze(s.rvalue.analyzed.type) + self.analyze(s.rvalue.analyzed.type, s.rvalue.analyzed) if isinstance(s.rvalue, CallExpr): - if isinstance(s.rvalue.analyzed, NewTypeExpr): - self.analyze(s.rvalue.analyzed.old_type) - if isinstance(s.rvalue.analyzed, TypedDictExpr): - self.analyze(s.rvalue.analyzed.info.typeddict_type) - if isinstance(s.rvalue.analyzed, NamedTupleExpr): - self.analyze(s.rvalue.analyzed.info.tuple_type) + analyzed = s.rvalue.analyzed + if isinstance(analyzed, NewTypeExpr): + self.analyze(analyzed.old_type, analyzed) + if isinstance(analyzed, TypedDictExpr): + self.analyze(analyzed.info.typeddict_type, analyzed) + if isinstance(analyzed, NamedTupleExpr): + self.analyze(analyzed.info.tuple_type, analyzed) super().visit_assignment_stmt(s) def visit_cast_expr(self, e: CastExpr) -> None: - self.analyze(e.type) + self.analyze(e.type, e) super().visit_cast_expr(e) def visit_reveal_type_expr(self, e: RevealTypeExpr) -> None: @@ -4182,16 +4185,38 @@ def visit_reveal_type_expr(self, e: RevealTypeExpr) -> None: def visit_type_application(self, e: TypeApplication) -> None: for type in e.types: - self.analyze(type) + self.analyze(type, e) super().visit_type_application(e) # Helpers - def analyze(self, type: Optional[Type]) -> None: + def remove_forwards_from_node(self, node: Node) -> None: + if isinstance(node, (FuncDef, CastExpr, AssignmentStmt, TypeAliasExpr)): + node.type = self.remove_forward_refs(node.type) + if isinstance(node, NewTypeExpr): + node.old_type = self.remove_forward_refs(node.old_type) + if isinstance(node, TypedDictExpr): + node.info.typeddict_type = cast(TypedDictType, + self.remove_forward_refs(node.info.typeddict_type)) + if isinstance(node, NamedTupleExpr): + node.info.tuple_type = cast(TupleType, + self.remove_forward_refs(node.info.tuple_type)) + if isinstance(node, TypeApplication): + node.types = list(self.remove_forward_refs(t) for t in node.types) + + def remove_forward_refs(self, tp: Type) -> Type: + return tp.accept(ForwardRefRemover()) + + def analyze(self, type: Optional[Type], node: Optional[Node]) -> None: + indicator = {} # type: Dict[str, bool] if type: - analyzer = TypeAnalyserPass3(self.fail, self.options, self.is_typeshed_file, self.sem) + analyzer = TypeAnalyserPass3(self.fail, self.options, self.is_typeshed_file, self.sem, indicator) type.accept(analyzer) self.check_for_omitted_generics(type) + if indicator.get('forward') and node is not None: + def patch(): + self.remove_forwards_from_node(node) + self.patches.append(patch) def check_for_omitted_generics(self, typ: Type) -> None: if 'generics' not in self.options.disallow_any or self.is_typeshed_file: @@ -4619,11 +4644,6 @@ def visit_any(self, t: AnyType) -> Type: return t -def remove_forward_refs(tp: Type) -> Type: - """Returns a type with forward refs removed.""" - return tp.accept(ForwardRefRemover()) - - class ForwardRefRemover(TypeVisitor[Type]): def __init__(self): self.seen = [] @@ -4647,10 +4667,10 @@ def visit_deleted_type(self, t: DeletedType) -> None: assert False, "Internal error: deleted type in semantic analysis" def visit_type_var(self, t: TypeVarType) -> TypeVarType: - if t.bound: - t.bound.accept(self) - if t.constrains: - for c in t.constrains: + if t.upper_bound: + t.upper_bound.accept(self) + if t.values: + for c in t.values: c.accept(self) return t @@ -4662,11 +4682,11 @@ def visit_instance(self, t: Instance) -> Instance: def visit_callable_type(self, t: CallableType) -> CallableType: for tp in t.arg_types: tp.accept(self) - tp.ret_type.accpet(self) + t.ret_type.accept(self) return t def visit_overloaded(self, t: Overloaded) -> Overloaded: - for it in t.items: + for it in t.items(): it.accept(self) t.fallback.accept(self) return t @@ -4678,6 +4698,8 @@ def visit_tuple_type(self, t: TupleType) -> TupleType: return t def visit_typeddict_type(self, t: TypedDictType) -> TypedDictType: + for tp in t.items.values(): + tp.accept(self) t.fallback.accept(self) return t @@ -4695,6 +4717,6 @@ def visit_type_type(self, t: TypeType) -> TypeType: def visit_forwardref_type(self, t: ForwardRef) -> Type: if not any(s is t for s in self.seen): - self.seen.append(seen) + self.seen.append(t) return t.link.accept(self) return AnyType(TypeOfAny.from_error) diff --git a/mypy/server/deps.py b/mypy/server/deps.py index c271fb7f51a2..0402a511a7ab 100644 --- a/mypy/server/deps.py +++ b/mypy/server/deps.py @@ -11,7 +11,7 @@ from mypy.types import ( Type, Instance, AnyType, NoneTyp, TypeVisitor, CallableType, DeletedType, PartialType, TupleType, TypeType, TypeVarType, TypedDictType, UnboundType, UninhabitedType, UnionType, - FunctionLike + FunctionLike, ForwardRef ) from mypy.server.trigger import make_trigger @@ -212,6 +212,9 @@ def visit_type_type(self, typ: TypeType) -> List[str]: # TODO: replace with actual implementation return [] + def visit_forwardref_type(self, typ: ForwardRef) -> List[str]: + return get_type_dependencies(typ.link) + def visit_type_var(self, typ: TypeVarType) -> List[str]: # TODO: replace with actual implementation return [] diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 5dfbc7c405bc..cea2df9b5010 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -5,7 +5,7 @@ Type, AnyType, UnboundType, TypeVisitor, FormalArgument, NoneTyp, function_type, Instance, TypeVarType, CallableType, TupleType, TypedDictType, UnionType, Overloaded, ErasedType, TypeList, PartialType, DeletedType, UninhabitedType, TypeType, is_named_instance, - FunctionLike, TypeOfAny + FunctionLike, TypeOfAny, ForwardRef ) import mypy.applytype import mypy.constraints diff --git a/mypy/typeanal.py b/mypy/typeanal.py index a108e0bda860..edec3e840a13 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -296,8 +296,6 @@ def visit_unbound_type(self, t: UnboundType) -> Type: # Create a named TypedDictType return td.copy_modified(item_types=self.anal_array(list(td.items.values())), fallback=instance) - if not instance.type.mro and not self.third_pass: - return ForwardRef(t) return instance else: return AnyType(TypeOfAny.special_form) @@ -590,11 +588,12 @@ def __init__(self, fail_func: Callable[[str, Context], None], options: Options, is_typeshed_stub: bool, - sem) -> None: + sem, indicator) -> None: self.fail = fail_func self.options = options self.is_typeshed_stub = is_typeshed_stub self.sem = sem + self.indicator = indicator def visit_instance(self, t: Instance) -> None: info = t.type @@ -722,6 +721,7 @@ def visit_type_type(self, t: TypeType) -> None: pass def visit_forwardref_type(self, t: ForwardRef) -> None: + self.indicator['forward'] = True if isinstance(t, UnboundType): t.link = self.sem.anal_type(t.link, third_pass=True) @@ -920,4 +920,3 @@ def make_optional_type(t: Type) -> Type: return UnionType(items + [NoneTyp()], t.line, t.column) else: return UnionType([t, NoneTyp()], t.line, t.column) - diff --git a/mypy/types.py b/mypy/types.py index a005efed29e7..364a3aef69d6 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1573,7 +1573,7 @@ def visit_overloaded(self, t: Overloaded) -> Type: def visit_type_type(self, t: TypeType) -> Type: return TypeType.make_normalized(t.item.accept(self), line=t.line, column=t.column) - def visit_forwardref_type(self, t: TypeType) -> Type: + def visit_forwardref_type(self, t: ForwardRef) -> Type: return t @@ -1813,7 +1813,7 @@ def visit_overloaded(self, t: Overloaded) -> T: def visit_type_type(self, t: TypeType) -> T: return t.item.accept(self) - def visit_forwardref_type(self, t: TypeType) -> T: + def visit_forwardref_type(self, t: ForwardRef) -> T: return t.link.accept(self) def visit_ellipsis_type(self, t: EllipsisType) -> T: From 950a02205933127419fde41f25b38e8e01f0f54f Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 11 Sep 2017 20:19:49 +0200 Subject: [PATCH 06/39] Fix replacement logic; fix newtype error formatting --- mypy/messages.py | 8 +++++-- mypy/semanal.py | 37 ++++++++++++++----------------- mypy/typeanal.py | 4 +++- mypy/types.py | 8 ------- test-data/unit/check-flags.test | 2 +- test-data/unit/check-newtype.test | 6 ++--- test-data/unit/cmdline.test | 2 +- 7 files changed, 31 insertions(+), 36 deletions(-) diff --git a/mypy/messages.py b/mypy/messages.py index d02b12a4cbde..fa3d3acbfa78 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -13,7 +13,7 @@ from mypy.types import ( Type, CallableType, Instance, TypeVarType, TupleType, TypedDictType, UnionType, NoneTyp, AnyType, Overloaded, FunctionLike, DeletedType, TypeType, - UninhabitedType, TypeOfAny + UninhabitedType, TypeOfAny, ForwardRef, UnboundType ) from mypy.nodes import ( TypeInfo, Context, MypyFile, op_methods, FuncDef, reverse_type_aliases, @@ -187,7 +187,7 @@ def quote_type_string(self, type_string: str) -> str: """Quotes a type representation for use in messages.""" no_quote_regex = r'^<(tuple|union): \d+ items>$' if (type_string in ['Module', 'overloaded function', '', ''] - or re.match(no_quote_regex, type_string) is not None): + or re.match(no_quote_regex, type_string) is not None or type_string.endswith('?')): # Messages are easier to read if these aren't quoted. We use a # regex to match strings with variable contents. return type_string @@ -302,6 +302,8 @@ def format_bare(self, typ: Type, verbosity: int = 0) -> str: return '' elif isinstance(typ, TypeType): return 'Type[{}]'.format(self.format_bare(typ.item, verbosity)) + elif isinstance(typ, ForwardRef): # may appear in semanal.py + return self.format_bare(typ.link, verbosity) elif isinstance(typ, FunctionLike): func = typ if func.is_type_obj(): @@ -343,6 +345,8 @@ def format_bare(self, typ: Type, verbosity: int = 0) -> str: # function types may result in long and difficult-to-read # error messages. return 'overloaded function' + elif isinstance(typ, UnboundType): + return str(typ) elif typ is None: raise RuntimeError('Type is None') else: diff --git a/mypy/semanal.py b/mypy/semanal.py index a9587fa645a8..a9dc1e7d683e 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1976,7 +1976,7 @@ def process_newtype_declaration(self, s: AssignmentStmt) -> None: newtype_class_info = self.build_newtype_typeinfo(name, old_type, old_type) else: message = "Argument 2 to NewType(...) must be subclassable (got {})" - self.fail(message.format(old_type), s) + self.fail(message.format(self.msg.format(old_type)), s) return check_for_explicit_any(old_type, self.options, self.is_typeshed_stub_file, self.msg, @@ -4051,6 +4051,7 @@ def __init__(self, modules: Dict[str, MypyFile], errors: Errors, sem) -> None: def visit_file(self, file_node: MypyFile, fnam: str, options: Options, patches) -> None: self.errors.set_file(fnam, file_node.fullname()) self.options = options + self.sem.options = options self.patches = patches self.is_typeshed_file = self.errors.is_typeshed_file(fnam) with experiments.strict_optional_set(options.strict_optional): @@ -4205,7 +4206,10 @@ def remove_forwards_from_node(self, node: Node) -> None: node.types = list(self.remove_forward_refs(t) for t in node.types) def remove_forward_refs(self, tp: Type) -> Type: - return tp.accept(ForwardRefRemover()) + #print("BEFORE", tp) + tp = tp.accept(ForwardRefRemover()) + #print("AFTER", tp) + return tp def analyze(self, type: Optional[Type], node: Optional[Node]) -> None: indicator = {} # type: Dict[str, bool] @@ -4668,44 +4672,37 @@ def visit_deleted_type(self, t: DeletedType) -> None: def visit_type_var(self, t: TypeVarType) -> TypeVarType: if t.upper_bound: - t.upper_bound.accept(self) + t.upper_bound = t.upper_bound.accept(self) if t.values: - for c in t.values: - c.accept(self) + t.values = [v.accept(self) for v in t.values] return t def visit_instance(self, t: Instance) -> Instance: - for arg in t.args: - arg.accept(self) + t.args = [arg.accept(self) for arg in t.args] return t def visit_callable_type(self, t: CallableType) -> CallableType: - for tp in t.arg_types: - tp.accept(self) - t.ret_type.accept(self) + t.arg_types = [tp.accept(self) for tp in t.arg_types] + t.ret_type = t.ret_type.accept(self) return t def visit_overloaded(self, t: Overloaded) -> Overloaded: - for it in t.items(): - it.accept(self) + t._items = [it.accept(self) for it in t.items()] t.fallback.accept(self) return t def visit_tuple_type(self, t: TupleType) -> TupleType: - for it in t.items: - it.accept(self) - t.fallback.accept(self) + t.items = [it.accept(self) for it in t.items] + t.fallback = t.fallback.accept(self) return t def visit_typeddict_type(self, t: TypedDictType) -> TypedDictType: - for tp in t.items.values(): - tp.accept(self) - t.fallback.accept(self) + t.items = OrderedDict([(k, tp.accept(self)) for k, tp in t.items.items()]) + t.fallback = t.fallback.accept(self) return t def visit_union_type(self, t: UnionType) -> UnionType: - for it in t.items: - it.accept(self) + t.items = [it.accept(self) for it in t.items] return t def visit_partial_type(self, t: PartialType) -> None: diff --git a/mypy/typeanal.py b/mypy/typeanal.py index edec3e840a13..a289198f8148 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -647,6 +647,8 @@ def visit_instance(self, t: Instance) -> None: else: arg_values = [arg] self.check_type_var_values(info, arg_values, tvar.name, tvar.values, i + 1, t) + if isinstance(arg, ForwardRef): + arg = arg.link if not is_subtype(arg, tvar.upper_bound): self.fail('Type argument "{}" of "{}" must be ' 'a subtype of "{}"'.format( @@ -722,7 +724,7 @@ def visit_type_type(self, t: TypeType) -> None: def visit_forwardref_type(self, t: ForwardRef) -> None: self.indicator['forward'] = True - if isinstance(t, UnboundType): + if isinstance(t.link, UnboundType): t.link = self.sem.anal_type(t.link, third_pass=True) TypeVarList = List[Tuple[str, TypeVarExpr]] diff --git a/mypy/types.py b/mypy/types.py index 364a3aef69d6..4729bfbf8a77 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1384,14 +1384,6 @@ def __init__(self, link: Type) -> None: def accept(self, visitor: 'TypeVisitor[T]') -> T: return visitor.visit_forwardref_type(self) - def __hash__(self) -> int: - return hash(self.link) - - def __eq__(self, other: object) -> bool: - if not isinstance(other, ForwardRef): - return NotImplemented - return self.link == other.link - # # Visitor-related classes diff --git a/test-data/unit/check-flags.test b/test-data/unit/check-flags.test index d5e329425b14..37c2bbebc485 100644 --- a/test-data/unit/check-flags.test +++ b/test-data/unit/check-flags.test @@ -668,7 +668,7 @@ main:5: error: Constraint 2 becomes "List[Any]" due to an unfollowed import from typing import NewType, List from missing import Unchecked -Baz = NewType('Baz', Unchecked) # E: Argument 2 to NewType(...) must be subclassable (got Any) +Baz = NewType('Baz', Unchecked) # E: Argument 2 to NewType(...) must be subclassable (got "Any") Bar = NewType('Bar', List[Unchecked]) # E: Argument 2 to NewType(...) becomes "List[Any]" due to an unfollowed import [builtins fixtures/list.pyi] diff --git a/test-data/unit/check-newtype.test b/test-data/unit/check-newtype.test index c9e01edf3ef4..037dcc402644 100644 --- a/test-data/unit/check-newtype.test +++ b/test-data/unit/check-newtype.test @@ -276,17 +276,17 @@ f = NewType('f', tp=int) # E: NewType(...) expects exactly two positional argum [case testNewTypeWithAnyFails] from typing import NewType, Any -A = NewType('A', Any) # E: Argument 2 to NewType(...) must be subclassable (got Any) +A = NewType('A', Any) # E: Argument 2 to NewType(...) must be subclassable (got "Any") [out] [case testNewTypeWithUnionsFails] from typing import NewType, Union -Foo = NewType('Foo', Union[int, float]) # E: Argument 2 to NewType(...) must be subclassable (got Union[builtins.int, builtins.float]) +Foo = NewType('Foo', Union[int, float]) # E: Argument 2 to NewType(...) must be subclassable (got "Union[builtins.int, builtins.float]") [out] [case testNewTypeWithTypeTypeFails] from typing import NewType, Type -Foo = NewType('Foo', Type[int]) # E: Argument 2 to NewType(...) must be subclassable (got Type[builtins.int]) +Foo = NewType('Foo', Type[int]) # E: Argument 2 to NewType(...) must be subclassable (got "Type[builtins.int]") a = Foo(type(3)) [builtins fixtures/args.pyi] [out] diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test index ade7f27a0338..1bd5b9c5ad59 100644 --- a/test-data/unit/cmdline.test +++ b/test-data/unit/cmdline.test @@ -806,7 +806,7 @@ Baz = NewType('Baz', Any) # this error does not come from `--disallow-any=expli Bar = NewType('Bar', List[Any]) [out] -m.py:3: error: Argument 2 to NewType(...) must be subclassable (got Any) +m.py:3: error: Argument 2 to NewType(...) must be subclassable (got "Any") m.py:4: error: Explicit "Any" is not allowed [case testDisallowAnyExplicitTypedDictSimple] From 411b24db11d495ec3b311008f4843302143baf07 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 11 Sep 2017 21:59:47 +0200 Subject: [PATCH 07/39] Fix third pass (need to go deeper) --- mypy/semanal.py | 10 ++++++++-- mypy/typeanal.py | 8 +++++--- test-data/unit/check-newtype.test | 4 ++-- test-data/unit/check-typeddict.test | 3 +-- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index a9dc1e7d683e..62f6554f5a09 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -4175,6 +4175,12 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: self.analyze(analyzed.info.typeddict_type, analyzed) if isinstance(analyzed, NamedTupleExpr): self.analyze(analyzed.info.tuple_type, analyzed) + for name in analyzed.info.names: + sym = analyzed.info.names[name] + if isinstance(sym.node, (FuncDef, Decorator)): + self.accept(sym.node) + if isinstance(sym.node, Var): + self.analyze(sym.node.type, sym.node) super().visit_assignment_stmt(s) def visit_cast_expr(self, e: CastExpr) -> None: @@ -4192,7 +4198,7 @@ def visit_type_application(self, e: TypeApplication) -> None: # Helpers def remove_forwards_from_node(self, node: Node) -> None: - if isinstance(node, (FuncDef, CastExpr, AssignmentStmt, TypeAliasExpr)): + if isinstance(node, (FuncDef, CastExpr, AssignmentStmt, TypeAliasExpr, Var)): node.type = self.remove_forward_refs(node.type) if isinstance(node, NewTypeExpr): node.old_type = self.remove_forward_refs(node.old_type) @@ -4203,7 +4209,7 @@ def remove_forwards_from_node(self, node: Node) -> None: node.info.tuple_type = cast(TupleType, self.remove_forward_refs(node.info.tuple_type)) if isinstance(node, TypeApplication): - node.types = list(self.remove_forward_refs(t) for t in node.types) + node.types = [self.remove_forward_refs(t) for t in node.types] def remove_forward_refs(self, tp: Type) -> Type: #print("BEFORE", tp) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index a289198f8148..684dd15ecca8 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -20,7 +20,7 @@ from mypy.nodes import ( TVAR, TYPE_ALIAS, UNBOUND_IMPORTED, TypeInfo, Context, SymbolTableNode, Var, Expression, IndexExpr, RefExpr, nongen_builtins, check_arg_names, check_arg_kinds, ARG_POS, ARG_NAMED, - ARG_OPT, ARG_NAMED_OPT, ARG_STAR, ARG_STAR2, TypeVarExpr + ARG_OPT, ARG_NAMED_OPT, ARG_STAR, ARG_STAR2, TypeVarExpr, FuncDef ) from mypy.tvar_scope import TypeVarScope from mypy.sametypes import is_same_type @@ -152,7 +152,7 @@ def visit_unbound_type(self, t: UnboundType) -> Type: # We don't need to worry about double-wrapping Optionals or # wrapping Anys: Union simplification will take care of that. return make_optional_type(self.visit_unbound_type(t)) - sym = self.lookup(t.name, t) + sym = self.lookup(t.name, t, suppress_errors=self.third_pass) if sym is not None: if sym.node is None: # UNBOUND_IMPORTED can happen if an unknown name was imported. @@ -261,7 +261,9 @@ def visit_unbound_type(self, t: UnboundType) -> Type: # Allow unbound type variables when defining an alias if not (self.aliasing and sym.kind == TVAR and self.tvar_scope.get_binding(sym) is None): - if not self.third_pass: + if (not self.third_pass and + not (isinstance(sym.node, FuncDef) or + isinstance(sym.node, Var) and sym.node.is_ready)): return ForwardRef(t) self.fail('Invalid type "{}"'.format(name), t) return t diff --git a/test-data/unit/check-newtype.test b/test-data/unit/check-newtype.test index 037dcc402644..badd9488adbd 100644 --- a/test-data/unit/check-newtype.test +++ b/test-data/unit/check-newtype.test @@ -281,12 +281,12 @@ A = NewType('A', Any) # E: Argument 2 to NewType(...) must be subclassable (got [case testNewTypeWithUnionsFails] from typing import NewType, Union -Foo = NewType('Foo', Union[int, float]) # E: Argument 2 to NewType(...) must be subclassable (got "Union[builtins.int, builtins.float]") +Foo = NewType('Foo', Union[int, float]) # E: Argument 2 to NewType(...) must be subclassable (got "Union[int, float]") [out] [case testNewTypeWithTypeTypeFails] from typing import NewType, Type -Foo = NewType('Foo', Type[int]) # E: Argument 2 to NewType(...) must be subclassable (got "Type[builtins.int]") +Foo = NewType('Foo', Type[int]) # E: Argument 2 to NewType(...) must be subclassable (got "Type[int]") a = Foo(type(3)) [builtins fixtures/args.pyi] [out] diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index 33b27a1ad23e..51c922316d56 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -1304,8 +1304,7 @@ m2: Mapping[str, C] = x # E: Incompatible types in assignment (expression has ty [case testForwardReferenceToTypedDictInTypedDict] from typing import Mapping from mypy_extensions import TypedDict -# Forward references don't quite work yet -X = TypedDict('X', {'a': 'A'}) # E: Invalid type "__main__.A" +X = TypedDict('X', {'a': 'A'}) A = TypedDict('A', {'b': int}) x: X reveal_type(x) # E: Revealed type is 'TypedDict('__main__.X', {'a': TypedDict('__main__.A', {'b': builtins.int})})' From b9b8528f70050672368f1a0fee2376f78868ee76 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 12 Sep 2017 00:19:01 +0200 Subject: [PATCH 08/39] Implement syntethic replacer --- mypy/build.py | 3 +- mypy/nodes.py | 2 + mypy/semanal.py | 81 ++++++++++++++++++++++++------- mypy/typeanal.py | 4 +- test-data/unit/check-classes.test | 6 ++- 5 files changed, 75 insertions(+), 21 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index c5a059cd5922..46fe028eb5d2 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -1724,7 +1724,8 @@ def semantic_analysis_pass_three(self) -> None: assert self.tree is not None, "Internal error: method must be called on parsed file only" patches = [] # type: List[Callable[[], None]] with self.wrap_context(): - self.manager.semantic_analyzer_pass3.visit_file(self.tree, self.xpath, self.options, patches) + self.manager.semantic_analyzer_pass3.visit_file(self.tree, self.xpath, + self.options, patches) if self.options.dump_type_stats: dump_type_stats(self.tree, self.xpath) self.patches = patches + self.patches diff --git a/mypy/nodes.py b/mypy/nodes.py index 8afeb3f063cd..fc1b12c02cc0 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1970,6 +1970,8 @@ class is generic then it will be a type constructor of higher kind. # Is this a newtype type? is_newtype = False + replaced = None # type: TypeInfo + FLAGS = [ 'is_abstract', 'is_enum', 'fallback_to_any', 'is_named_tuple', 'is_newtype', 'is_protocol', 'runtime_protocol' diff --git a/mypy/semanal.py b/mypy/semanal.py index 62f6554f5a09..531655497723 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -942,6 +942,7 @@ def analyze_namedtuple_classdef(self, defn: ClassDef) -> Optional[TypeInfo]: info = self.build_namedtuple_typeinfo( defn.name, items, types, default_items) node.node = info + defn.info.replaced = info defn.info = info defn.analyzed = NamedTupleExpr(info) return info @@ -1281,6 +1282,7 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> bool: # Building a new TypedDict fields, types, required_keys = self.check_typeddict_classdef(defn) info = self.build_typeddict_typeinfo(defn.name, fields, types, required_keys) + defn.info.replaced = info node.node = info defn.analyzed = TypedDictExpr(info) return True @@ -1313,6 +1315,7 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> bool: types.extend(new_types) required_keys.update(new_required_keys) info = self.build_typeddict_typeinfo(defn.name, keys, types, required_keys) + defn.info.replaced = info node.node = info defn.analyzed = TypedDictExpr(info) return True @@ -4165,6 +4168,8 @@ def visit_decorator(self, dec: Decorator) -> None: def visit_assignment_stmt(self, s: AssignmentStmt) -> None: self.analyze(s.type, s) + if isinstance(s.lvalues[0], RefExpr) and isinstance(s.lvalues[0].node, Var): + self.analyze(s.lvalues[0].node.type, s.lvalues[0].node) if isinstance(s.rvalue, IndexExpr) and isinstance(s.rvalue.analyzed, TypeAliasExpr): self.analyze(s.rvalue.analyzed.type, s.rvalue.analyzed) if isinstance(s.rvalue, CallExpr): @@ -4197,35 +4202,46 @@ def visit_type_application(self, e: TypeApplication) -> None: # Helpers - def remove_forwards_from_node(self, node: Node) -> None: + def perform_transform(self, node: Node, transform) -> None: if isinstance(node, (FuncDef, CastExpr, AssignmentStmt, TypeAliasExpr, Var)): - node.type = self.remove_forward_refs(node.type) + node.type = transform(node.type) if isinstance(node, NewTypeExpr): - node.old_type = self.remove_forward_refs(node.old_type) + node.old_type = transform(node.old_type) if isinstance(node, TypedDictExpr): node.info.typeddict_type = cast(TypedDictType, - self.remove_forward_refs(node.info.typeddict_type)) + transform(node.info.typeddict_type)) if isinstance(node, NamedTupleExpr): node.info.tuple_type = cast(TupleType, - self.remove_forward_refs(node.info.tuple_type)) + transform(node.info.tuple_type)) if isinstance(node, TypeApplication): - node.types = [self.remove_forward_refs(t) for t in node.types] + node.types = [transform(t) for t in node.types] def remove_forward_refs(self, tp: Type) -> Type: - #print("BEFORE", tp) + # print("BEFORE", tp) tp = tp.accept(ForwardRefRemover()) - #print("AFTER", tp) + # print("AFTER", tp) + return tp + + def replace_synthetic(self, tp: Type) -> Type: + print("BEFORE", tp) + tp = tp.accept(SyntheticReplacer()) + print("AFTER", tp) return tp def analyze(self, type: Optional[Type], node: Optional[Node]) -> None: indicator = {} # type: Dict[str, bool] if type: - analyzer = TypeAnalyserPass3(self.fail, self.options, self.is_typeshed_file, self.sem, indicator) + analyzer = TypeAnalyserPass3(self.fail, self.options, self.is_typeshed_file, + self.sem, indicator) type.accept(analyzer) self.check_for_omitted_generics(type) if indicator.get('forward') and node is not None: def patch(): - self.remove_forwards_from_node(node) + self.perform_transform(node, self.remove_forward_refs) + self.patches.append(patch) + if indicator.get('synthetic') and node is not None: + def patch(): + self.perform_transform(node, self.replace_synthetic) self.patches.append(patch) def check_for_omitted_generics(self, typ: Type) -> None: @@ -4654,10 +4670,7 @@ def visit_any(self, t: AnyType) -> Type: return t -class ForwardRefRemover(TypeVisitor[Type]): - def __init__(self): - self.seen = [] - +class TypeReplacer(TypeVisitor[Type]): def visit_unbound_type(self, t: UnboundType) -> UnboundType: return t @@ -4683,7 +4696,7 @@ def visit_type_var(self, t: TypeVarType) -> TypeVarType: t.values = [v.accept(self) for v in t.values] return t - def visit_instance(self, t: Instance) -> Instance: + def visit_instance(self, t: Instance) -> Type: t.args = [arg.accept(self) for arg in t.args] return t @@ -4693,18 +4706,18 @@ def visit_callable_type(self, t: CallableType) -> CallableType: return t def visit_overloaded(self, t: Overloaded) -> Overloaded: - t._items = [it.accept(self) for it in t.items()] + t._items = [cast(CallableType, it.accept(self)) for it in t.items()] t.fallback.accept(self) return t def visit_tuple_type(self, t: TupleType) -> TupleType: t.items = [it.accept(self) for it in t.items] - t.fallback = t.fallback.accept(self) + t.fallback = cast(Instance, t.fallback.accept(self)) return t def visit_typeddict_type(self, t: TypedDictType) -> TypedDictType: t.items = OrderedDict([(k, tp.accept(self)) for k, tp in t.items.items()]) - t.fallback = t.fallback.accept(self) + t.fallback = cast(Instance, t.fallback.accept(self)) return t def visit_union_type(self, t: UnionType) -> UnionType: @@ -4718,8 +4731,40 @@ def visit_type_type(self, t: TypeType) -> TypeType: t.item = t.item.accept(self) return t + +class ForwardRefRemover(TypeReplacer): + def __init__(self): + self.seen = [] + def visit_forwardref_type(self, t: ForwardRef) -> Type: if not any(s is t for s in self.seen): self.seen.append(t) return t.link.accept(self) return AnyType(TypeOfAny.from_error) + + +class SyntheticReplacer(TypeReplacer): + def __init__(self): + self.seen = [] + + def visit_tuple_type(self, t: TupleType) -> TupleType: + self.seen.append(t) + return super().visit_tuple_type(t) + + def visit_typeddict_type(self, t: TypedDictType) -> TypedDictType: + self.seen.append(t) + return super().visit_typeddict_type(t) + + def visit_instance(self, t: Instance) -> Type: + info = t.type + if info.replaced and info.replaced.tuple_type: + tp = info.replaced.tuple_type + if any(s is tp for s in self.seen): + return AnyType(TypeOfAny.from_error) + return tp.copy_modified(fallback=Instance(info.replaced, [])).accept(self) + if info.replaced and info.replaced.typeddict_type: + td = info.replaced.typeddict_type + if any(s is td for s in self.seen): + return AnyType(TypeOfAny.from_error) + return td.copy_modified(fallback=Instance(info.replaced, [])).accept(self) + return super().visit_instance(t) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 684dd15ecca8..37627659abf9 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -152,7 +152,7 @@ def visit_unbound_type(self, t: UnboundType) -> Type: # We don't need to worry about double-wrapping Optionals or # wrapping Anys: Union simplification will take care of that. return make_optional_type(self.visit_unbound_type(t)) - sym = self.lookup(t.name, t, suppress_errors=self.third_pass) + sym = self.lookup(t.name, t, suppress_errors=self.third_pass) # type: ignore if sym is not None: if sym.node is None: # UNBOUND_IMPORTED can happen if an unknown name was imported. @@ -599,6 +599,8 @@ def __init__(self, def visit_instance(self, t: Instance) -> None: info = t.type + if info.replaced: + self.indicator['synthetic'] = True # Check type argument count. if len(t.args) != len(info.type_vars): if len(t.args) == 0: diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index bea01229021b..3ea9f4a88c8e 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -3495,24 +3495,28 @@ class Process(TypedDict): [case testCorrectDoubleForwardNamedTuple] from typing import NamedTuple +x: A class A(NamedTuple): name: 'B' year: int class B(NamedTuple): director: str -x: A reveal_type(x.name.director) # E: Revealed type is 'builtins.str' +reveal_type(x) [out] [case testCrashOnDoubleForwardTypedDict] from mypy_extensions import TypedDict +x: A class A(TypedDict): name: 'B' year: int class B(TypedDict): director: str + +reveal_type(x['name']['director']) # E: Revealed type is 'builtins.str' [builtins fixtures/isinstancelist.pyi] [out] From 48d6de4405602a66b36683ec634249703aa1be64 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 12 Sep 2017 00:40:45 +0200 Subject: [PATCH 09/39] Need to go deeper (as usual) --- mypy/semanal.py | 6 ++++++ test-data/unit/check-classes.test | 3 +-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 531655497723..76561c075478 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -4113,6 +4113,12 @@ def visit_class_def(self, tdef: ClassDef) -> None: self.analyze(tdef.analyzed.info.typeddict_type, tdef.analyzed) elif isinstance(tdef.analyzed, NamedTupleExpr): self.analyze(tdef.analyzed.info.tuple_type, tdef.analyzed) + for name in tdef.analyzed.info.names: + sym = tdef.analyzed.info.names[name] + if isinstance(sym.node, (FuncDef, Decorator)): + self.accept(sym.node) + if isinstance(sym.node, Var): + self.analyze(sym.node.type, sym.node) super().visit_class_def(tdef) def visit_decorator(self, dec: Decorator) -> None: diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 3ea9f4a88c8e..37b29ccb5696 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -3428,7 +3428,7 @@ class M(TypedDict): n: N m: M lst = [n, m] -reveal_type(lst[0]) # E: Revealed type is 'TypedDict({'x': object}, fallback=typing.Mapping[builtins.str, object])' +reveal_type(lst[0]['x']) # E: Revealed type is 'TypedDict('__main__.N', {'x': Any})' [builtins fixtures/isinstancelist.pyi] [out] @@ -3503,7 +3503,6 @@ class B(NamedTuple): director: str reveal_type(x.name.director) # E: Revealed type is 'builtins.str' -reveal_type(x) [out] [case testCrashOnDoubleForwardTypedDict] From ec454415918385a94e3320ad52f8b5fff2a3e903 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 12 Sep 2017 01:23:07 +0200 Subject: [PATCH 10/39] Fix postponed fallback join --- mypy/semanal.py | 27 +++++++++++++-------------- test-data/unit/check-namedtuple.test | 2 +- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 76561c075478..04e350c4c0d1 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -2361,11 +2361,6 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], types: List[Typ # Actual signature should return OrderedDict[str, Union[types]] ordereddictype = (self.named_type_or_none('builtins.dict', [strtype, implicit_any]) or self.object_type()) - # 'builtins.tuple' has only one type parameter. - # - # TODO: The corresponding type argument in the fallback instance should be a join of - # all item types, but we can't do joins during this pass of semantic analysis - # and we are using Any as a workaround. fallback = self.named_type('__builtins__.tuple', [implicit_any]) # Note: actual signature should accept an invariant version of Iterable[UnionType[types]]. # but it can't be expressed. 'new' and 'len' should be callable types. @@ -2376,6 +2371,15 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], types: List[Typ info.is_named_tuple = True info.tuple_type = TupleType(types, fallback) + def patch() -> None: + # Calculate the correct value type for the fallback Mapping. + fallback.args[0] = join.join_type_list(list(info.tuple_type.items)) + + # We can't calculate the complete fallback type until after semantic + # analysis, since otherwise MROs might be incomplete. Postpone a callback + # function that patches the fallback. + self.patches.append(patch) + def add_field(var: Var, is_initialized_in_class: bool = False, is_property: bool = False) -> None: var.info = info @@ -2595,19 +2599,18 @@ def build_typeddict_typeinfo(self, name: str, items: List[str], fallback = (self.named_type_or_none('typing.Mapping', [self.str_type(), self.object_type()]) or self.object_type()) + info = self.basic_new_typeinfo(name, fallback) + info.typeddict_type = TypedDictType(OrderedDict(zip(items, types)), required_keys, + fallback) def patch() -> None: # Calculate the correct value type for the fallback Mapping. - fallback.args[1] = join.join_type_list(types) + fallback.args[1] = join.join_type_list(list(info.typeddict_type.items.values())) # We can't calculate the complete fallback type until after semantic # analysis, since otherwise MROs might be incomplete. Postpone a callback # function that patches the fallback. self.patches.append(patch) - - info = self.basic_new_typeinfo(name, fallback) - info.typeddict_type = TypedDictType(OrderedDict(zip(items, types)), required_keys, - fallback) return info def check_classvar(self, s: AssignmentStmt) -> None: @@ -4223,15 +4226,11 @@ def perform_transform(self, node: Node, transform) -> None: node.types = [transform(t) for t in node.types] def remove_forward_refs(self, tp: Type) -> Type: - # print("BEFORE", tp) tp = tp.accept(ForwardRefRemover()) - # print("AFTER", tp) return tp def replace_synthetic(self, tp: Type) -> Type: - print("BEFORE", tp) tp = tp.accept(SyntheticReplacer()) - print("AFTER", tp) return tp def analyze(self, type: Optional[Type], node: Optional[Node]) -> None: diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index f68279eef913..96468f32f0a1 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -415,7 +415,7 @@ b = B._make(['']) # type: B [case testNamedTupleIncompatibleRedefinition] from typing import NamedTuple class Crash(NamedTuple): - count: int # E: Incompatible types in assignment (expression has type "int", base class "tuple" defined the type as "Callable[[Tuple[Any, ...], Any], int]") + count: int # E: Incompatible types in assignment (expression has type "int", base class "tuple" defined the type as "Callable[[Tuple[int, ...], Any], int]") [builtins fixtures/tuple.pyi] [case testNamedTupleInClassNamespace] From ac32ed49e888bd7c11c462a6bd99cffd41161c5f Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 12 Sep 2017 01:52:10 +0200 Subject: [PATCH 11/39] Simplify some code and add annotations --- mypy/semanal.py | 32 +++++++++++++------------------- mypy/typeanal.py | 7 +++++-- 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 04e350c4c0d1..23517615a18f 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -4049,12 +4049,14 @@ class ThirdPass(TraverserVisitor): straightforward type inference. """ - def __init__(self, modules: Dict[str, MypyFile], errors: Errors, sem) -> None: + def __init__(self, modules: Dict[str, MypyFile], errors: Errors, + sem: SemanticAnalyzer) -> None: self.modules = modules self.errors = errors self.sem = sem - def visit_file(self, file_node: MypyFile, fnam: str, options: Options, patches) -> None: + def visit_file(self, file_node: MypyFile, fnam: str, options: Options, + patches: List[Callable[[], None]]) -> None: self.errors.set_file(fnam, file_node.fullname()) self.options = options self.sem.options = options @@ -4211,7 +4213,7 @@ def visit_type_application(self, e: TypeApplication) -> None: # Helpers - def perform_transform(self, node: Node, transform) -> None: + def perform_transform(self, node: Node, transform: Callable[[Type], Type]) -> None: if isinstance(node, (FuncDef, CastExpr, AssignmentStmt, TypeAliasExpr, Var)): node.type = transform(node.type) if isinstance(node, NewTypeExpr): @@ -4225,14 +4227,6 @@ def perform_transform(self, node: Node, transform) -> None: if isinstance(node, TypeApplication): node.types = [transform(t) for t in node.types] - def remove_forward_refs(self, tp: Type) -> Type: - tp = tp.accept(ForwardRefRemover()) - return tp - - def replace_synthetic(self, tp: Type) -> Type: - tp = tp.accept(SyntheticReplacer()) - return tp - def analyze(self, type: Optional[Type], node: Optional[Node]) -> None: indicator = {} # type: Dict[str, bool] if type: @@ -4241,12 +4235,12 @@ def analyze(self, type: Optional[Type], node: Optional[Node]) -> None: type.accept(analyzer) self.check_for_omitted_generics(type) if indicator.get('forward') and node is not None: - def patch(): - self.perform_transform(node, self.remove_forward_refs) + def patch() -> None: + self.perform_transform(node, lambda tp: tp.accept(ForwardRefRemover())) self.patches.append(patch) if indicator.get('synthetic') and node is not None: - def patch(): - self.perform_transform(node, self.replace_synthetic) + def patch() -> None: + self.perform_transform(node, lambda tp: tp.accept(SyntheticReplacer())) self.patches.append(patch) def check_for_omitted_generics(self, typ: Type) -> None: @@ -4738,8 +4732,8 @@ def visit_type_type(self, t: TypeType) -> TypeType: class ForwardRefRemover(TypeReplacer): - def __init__(self): - self.seen = [] + def __init__(self) -> None: + self.seen = [] # type: List[Type] def visit_forwardref_type(self, t: ForwardRef) -> Type: if not any(s is t for s in self.seen): @@ -4749,8 +4743,8 @@ def visit_forwardref_type(self, t: ForwardRef) -> Type: class SyntheticReplacer(TypeReplacer): - def __init__(self): - self.seen = [] + def __init__(self) -> None: + self.seen = [] # type: List[Type] def visit_tuple_type(self, t: TupleType) -> TupleType: self.seen.append(t) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 37627659abf9..bf425d038665 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -1,7 +1,7 @@ """Semantic analysis of types""" from collections import OrderedDict -from typing import Callable, List, Optional, Set, Tuple, Iterator, TypeVar, Iterable +from typing import Callable, List, Optional, Set, Tuple, Iterator, TypeVar, Iterable, Dict from itertools import chain from contextlib import contextmanager @@ -29,6 +29,9 @@ from mypy.plugin import Plugin, AnalyzerPluginInterface, AnalyzeTypeContext from mypy import nodes, messages +MYPY = False +if MYPY: + from mypy.semanal import SemanticAnalyzer T = TypeVar('T') @@ -590,7 +593,7 @@ def __init__(self, fail_func: Callable[[str, Context], None], options: Options, is_typeshed_stub: bool, - sem, indicator) -> None: + sem: 'SemanticAnalyzer', indicator: Dict[str, bool]) -> None: self.fail = fail_func self.options = options self.is_typeshed_stub = is_typeshed_stub From 3fb3019e3bab7bb0114a4c5507d8de0a6353f6ea Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 12 Sep 2017 21:05:19 +0200 Subject: [PATCH 12/39] Simplify traversal logic; add loads of tests --- mypy/semanal.py | 109 +++++++++++-------------- mypy/typeanal.py | 11 ++- mypy/types.py | 2 +- test-data/unit/check-incremental.test | 88 ++++++++++++++++++++ test-data/unit/check-isinstance.test | 14 ++-- test-data/unit/check-namedtuple.test | 96 ++++++++++++++++++++++ test-data/unit/check-type-aliases.test | 18 ++++ 7 files changed, 268 insertions(+), 70 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 23517615a18f..dc41e408963b 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -4669,26 +4669,19 @@ def visit_any(self, t: AnyType) -> Type: return t -class TypeReplacer(TypeVisitor[Type]): - def visit_unbound_type(self, t: UnboundType) -> UnboundType: - return t - - def visit_any(self, t: AnyType) -> AnyType: - return t - - def visit_none_type(self, t: NoneTyp) -> NoneTyp: - return t - - def visit_uninhabited_type(self, t: UninhabitedType) -> UninhabitedType: - return t - - def visit_erased_type(self, t: ErasedType) -> ErasedType: - return t +class TypeReplacer(TypeTranslator): + def __init__(self): + self.seen = [] # type: List[Type] - def visit_deleted_type(self, t: DeletedType) -> None: - assert False, "Internal error: deleted type in semantic analysis" + def check(self, t: Type) -> bool: + if any(t is s for s in self.seen): + return True + self.seen.append(t) + return False - def visit_type_var(self, t: TypeVarType) -> TypeVarType: + def visit_type_var(self, t: TypeVarType) -> Type: + if self.check(t): + return AnyType(TypeOfAny.from_error) if t.upper_bound: t.upper_bound = t.upper_bound.accept(self) if t.values: @@ -4696,74 +4689,70 @@ def visit_type_var(self, t: TypeVarType) -> TypeVarType: return t def visit_instance(self, t: Instance) -> Type: - t.args = [arg.accept(self) for arg in t.args] - return t + if self.check(t): + return Instance(t.type, [AnyType(TypeOfAny.from_error)] * len(t.type.defn.type_vars)) + return super().visit_instance(t) - def visit_callable_type(self, t: CallableType) -> CallableType: + def visit_callable_type(self, t: CallableType) -> Type: + # TODO: Callable and Overloaded are not copied, + # check if we could unify this with TypeTranslator. + if self.check(t): + return AnyType(TypeOfAny.from_error) t.arg_types = [tp.accept(self) for tp in t.arg_types] t.ret_type = t.ret_type.accept(self) + for v in t.variables: + if v.upper_bound: + v.upper_bound = v.upper_bound.accept(self) + if v.values: + v.values = [val.accept(self) for val in v.values] return t - def visit_overloaded(self, t: Overloaded) -> Overloaded: + def visit_overloaded(self, t: Overloaded) -> Type: + if self.check(t): + return AnyType(TypeOfAny.from_error) t._items = [cast(CallableType, it.accept(self)) for it in t.items()] t.fallback.accept(self) return t - def visit_tuple_type(self, t: TupleType) -> TupleType: - t.items = [it.accept(self) for it in t.items] - t.fallback = cast(Instance, t.fallback.accept(self)) - return t - - def visit_typeddict_type(self, t: TypedDictType) -> TypedDictType: - t.items = OrderedDict([(k, tp.accept(self)) for k, tp in t.items.items()]) - t.fallback = cast(Instance, t.fallback.accept(self)) - return t + def visit_tuple_type(self, t: TupleType) -> Type: + if self.check(t): + return AnyType(TypeOfAny.from_error) + return super().visit_tuple_type(t) - def visit_union_type(self, t: UnionType) -> UnionType: - t.items = [it.accept(self) for it in t.items] - return t + def visit_typeddict_type(self, t: TypedDictType) -> Type: + if self.check(t): + return AnyType(TypeOfAny.from_error) + return super().visit_typeddict_type(t) - def visit_partial_type(self, t: PartialType) -> None: - assert False, "Internal error: partial type in semantic analysis" + def visit_union_type(self, t: UnionType) -> Type: + if self.check(t): + return AnyType(TypeOfAny.from_error) + return super().visit_union_type(t) - def visit_type_type(self, t: TypeType) -> TypeType: - t.item = t.item.accept(self) - return t + def visit_type_type(self, t: TypeType) -> Type: + if self.check(t): + return AnyType(TypeOfAny.from_error) + return super().visit_type_type(t) class ForwardRefRemover(TypeReplacer): - def __init__(self) -> None: - self.seen = [] # type: List[Type] - def visit_forwardref_type(self, t: ForwardRef) -> Type: - if not any(s is t for s in self.seen): - self.seen.append(t) - return t.link.accept(self) - return AnyType(TypeOfAny.from_error) + return t.link.accept(self) -class SyntheticReplacer(TypeReplacer): - def __init__(self) -> None: - self.seen = [] # type: List[Type] - - def visit_tuple_type(self, t: TupleType) -> TupleType: - self.seen.append(t) - return super().visit_tuple_type(t) - - def visit_typeddict_type(self, t: TypedDictType) -> TypedDictType: - self.seen.append(t) - return super().visit_typeddict_type(t) - +class SyntheticReplacer(ForwardRefRemover): def visit_instance(self, t: Instance) -> Type: info = t.type if info.replaced and info.replaced.tuple_type: tp = info.replaced.tuple_type - if any(s is tp for s in self.seen): + if any((s is tp) or (s is t) for s in self.seen): return AnyType(TypeOfAny.from_error) + self.seen.append(t) return tp.copy_modified(fallback=Instance(info.replaced, [])).accept(self) if info.replaced and info.replaced.typeddict_type: td = info.replaced.typeddict_type - if any(s is td for s in self.seen): + if any((s is td) or (s is t) for s in self.seen): return AnyType(TypeOfAny.from_error) + self.seen.append(t) return td.copy_modified(fallback=Instance(info.replaced, [])).accept(self) return super().visit_instance(t) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index bf425d038665..e02399fe9cbd 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -303,6 +303,9 @@ def visit_unbound_type(self, t: UnboundType) -> Type: fallback=instance) return instance else: + if self.third_pass: + self.fail('Invalid type {}'.format(t), t) + return AnyType(TypeOfAny.from_error) return AnyType(TypeOfAny.special_form) def visit_any(self, t: AnyType) -> Type: @@ -721,13 +724,17 @@ def visit_type_list(self, t: TypeList) -> None: self.fail('Invalid type', t) def visit_type_var(self, t: TypeVarType) -> None: - pass + if t.upper_bound: + t.upper_bound.accept(self) + if t.values: + for v in t.values: + v.accept(self) def visit_partial_type(self, t: PartialType) -> None: pass def visit_type_type(self, t: TypeType) -> None: - pass + t.item.accept(self) def visit_forwardref_type(self, t: ForwardRef) -> None: self.indicator['forward'] = True diff --git a/mypy/types.py b/mypy/types.py index 4729bfbf8a77..64fabf93198e 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1461,7 +1461,7 @@ def visit_type_type(self, t: TypeType) -> T: pass def visit_forwardref_type(self, t: ForwardRef) -> T: - raise RuntimeError('Debugging forward ref') + raise RuntimeError('Internal error: unresolved forward reference') class SyntheticTypeVisitor(TypeVisitor[T]): diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 1ab2c2cd0e1b..58314a9b4e34 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -2951,3 +2951,91 @@ def foo(a: int, b: int) -> str: [out1] [out2] tmp/b.py:2: error: Incompatible return value type (got "int", expected "str") + +-- Some crazy selef-referential named tuples and types dicts +-- to be sure that everything can be _serialized_ (i.e. ForwardRef's are removed) + +[case testSelfRefNTIncremental1] +from typing import Tuple, NamedTuple + +Node = NamedTuple( + 'Node', + [ + ('name', str), + ('children', Tuple['Node', ...]), + ]) + +n: Node +[builtins fixtures/tuple.pyi] + + +[case testSelfRefNTIncremental2] +from typing import Tuple, NamedTuple + +Node = NamedTuple( + 'Node', + [ + ('name', str), + ('children', Tuple['Node2', ...]), + ]) + +class Node2(NamedTuple): + x: Node + y: int + +n: Node +[builtins fixtures/tuple.pyi] + +[case testSelfRefNTIncremental3] +from typing import NamedTuple, Tuple + +class Node2(NamedTuple): + x: Tuple[Node, int] + y: int + +Node = NamedTuple( + 'Node', + [ + ('name', str), + ('children', 'Node2'), + ]) + +n: Node2 +m: Node +lst = [m, n] +[builtins fixtures/tuple.pyi] + +[case testSelfRefNTIncremental4] +from typing import NamedTuple + +class Node2(NamedTuple): + x: Node + y: int + +class Node(NamedTuple): + name: str + children: Node2 + +n: Node +[builtins fixtures/tuple.pyi] + +[case testSelfRefNTIncremental5] +from typing import NamedTuple + +Node2 = NamedTuple( + 'Node2', + [ + ('x', Node), + ('y', int), + ]) + +Node = NamedTuple( + 'Node', + [ + ('name', str), + ('children', 'Node2'), + ]) + +n: Node +def f(m: Node2) -> None: pass +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-isinstance.test b/test-data/unit/check-isinstance.test index 60eef4eb4555..8279e1aeafd4 100644 --- a/test-data/unit/check-isinstance.test +++ b/test-data/unit/check-isinstance.test @@ -1446,10 +1446,10 @@ def f(x: Union[Type[int], Type[str], Type[List]]) -> None: reveal_type(x()) # E: Revealed type is 'Union[builtins.int, builtins.str]' x()[1] # E: Value of type "Union[int, str]" is not indexable else: - reveal_type(x) # E: Revealed type is 'Type[builtins.list]' + reveal_type(x) # E: Revealed type is 'Type[builtins.list[Any]]' reveal_type(x()) # E: Revealed type is 'builtins.list[]' x()[1] - reveal_type(x) # E: Revealed type is 'Union[Type[builtins.int], Type[builtins.str], Type[builtins.list]]' + reveal_type(x) # E: Revealed type is 'Union[Type[builtins.int], Type[builtins.str], Type[builtins.list[Any]]]' reveal_type(x()) # E: Revealed type is 'Union[builtins.int, builtins.str, builtins.list[]]' if issubclass(x, (str, (list,))): reveal_type(x) # E: Revealed type is 'Union[Type[builtins.str], Type[builtins.list[Any]]]' @@ -1468,10 +1468,10 @@ def f(x: Type[Union[int, str, List]]) -> None: reveal_type(x()) # E: Revealed type is 'Union[builtins.int, builtins.str]' x()[1] # E: Value of type "Union[int, str]" is not indexable else: - reveal_type(x) # E: Revealed type is 'Type[builtins.list]' + reveal_type(x) # E: Revealed type is 'Type[builtins.list[Any]]' reveal_type(x()) # E: Revealed type is 'builtins.list[]' x()[1] - reveal_type(x) # E: Revealed type is 'Union[Type[builtins.int], Type[builtins.str], Type[builtins.list]]' + reveal_type(x) # E: Revealed type is 'Union[Type[builtins.int], Type[builtins.str], Type[builtins.list[Any]]]' reveal_type(x()) # E: Revealed type is 'Union[builtins.int, builtins.str, builtins.list[]]' if issubclass(x, (str, (list,))): reveal_type(x) # E: Revealed type is 'Union[Type[builtins.str], Type[builtins.list[Any]]]' @@ -1485,17 +1485,17 @@ def f(x: Type[Union[int, str, List]]) -> None: from typing import Union, List, Tuple, Dict, Type def f(x: Type[Union[int, str, List]]) -> None: - reveal_type(x) # E: Revealed type is 'Union[Type[builtins.int], Type[builtins.str], Type[builtins.list]]' + reveal_type(x) # E: Revealed type is 'Union[Type[builtins.int], Type[builtins.str], Type[builtins.list[Any]]]' reveal_type(x()) # E: Revealed type is 'Union[builtins.int, builtins.str, builtins.list[]]' if issubclass(x, (str, (int,))): reveal_type(x) # E: Revealed type is 'Union[Type[builtins.int], Type[builtins.str]]' reveal_type(x()) # E: Revealed type is 'Union[builtins.int, builtins.str]' x()[1] # E: Value of type "Union[int, str]" is not indexable else: - reveal_type(x) # E: Revealed type is 'Type[builtins.list]' + reveal_type(x) # E: Revealed type is 'Type[builtins.list[Any]]' reveal_type(x()) # E: Revealed type is 'builtins.list[]' x()[1] - reveal_type(x) # E: Revealed type is 'Union[Type[builtins.int], Type[builtins.str], Type[builtins.list]]' + reveal_type(x) # E: Revealed type is 'Union[Type[builtins.int], Type[builtins.str], Type[builtins.list[Any]]]' reveal_type(x()) # E: Revealed type is 'Union[builtins.int, builtins.str, builtins.list[]]' if issubclass(x, (str, (list,))): reveal_type(x) # E: Revealed type is 'Union[Type[builtins.str], Type[builtins.list[Any]]]' diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 96468f32f0a1..d57eb93c411b 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -456,6 +456,102 @@ def f(x: a.X) -> None: tmp/b.py:6: error: Revealed type is 'a.X' tmp/b.py:8: error: Revealed type is 'Tuple[Any, fallback=a.X]' +-- Some crazy selef-referential named tuples and types dicts +-- to be sure that everything works + +[case testSelfRefNT1] +from typing import Tuple, NamedTuple + +Node = NamedTuple( + 'Node', + [ + ('name', str), + ('children', Tuple['Node', ...]), + ]) + +n: Node +reveal_type(n) # E: Revealed type is 'Tuple[builtins.str, builtins.tuple[Tuple[builtins.str, builtins.tuple[Any], fallback=__main__.Node]], fallback=__main__.Node]' +[builtins fixtures/tuple.pyi] + + +[case testSelfRefNT2] +from typing import Tuple, NamedTuple + +Node = NamedTuple( + 'Node', + [ + ('name', str), + ('children', Tuple['Node2', ...]), + ]) + +class Node2(NamedTuple): + x: Node + y: int + +n: Node +reveal_type(n) # E: Revealed type is 'Tuple[builtins.str, builtins.tuple[Tuple[Tuple[builtins.str, builtins.tuple[Any], fallback=__main__.Node], builtins.int, fallback=__main__.Node2]], fallback=__main__.Node]' +[builtins fixtures/tuple.pyi] + +[case testSelfRefNT3] +from typing import NamedTuple, Tuple + +class Node2(NamedTuple): + x: Tuple[Node, int] + y: int + +Node = NamedTuple( + 'Node', + [ + ('name', str), + ('children', 'Node2'), + ]) + +n: Node2 +m: Node +reveal_type(n) # E: Revealed type is 'Tuple[Tuple[Tuple[builtins.str, Tuple[Tuple[Any, builtins.int], builtins.int, fallback=__main__.Node2], fallback=__main__.Node], builtins.int], builtins.int, fallback=__main__.Node2]' +reveal_type(m) # E: Revealed type is 'Tuple[builtins.str, Tuple[Tuple[Tuple[builtins.str, Tuple[Tuple[Any, builtins.int], builtins.int, fallback=__main__.Node2], fallback=__main__.Node], builtins.int], builtins.int, fallback=__main__.Node2], fallback=__main__.Node]' +lst = [m, n] +reveal_type(lst[0]) # E: Revealed type is 'Tuple[builtins.object, builtins.object]' +[builtins fixtures/tuple.pyi] + +[case testSelfRefNT4] +from typing import NamedTuple + +class Node2(NamedTuple): + x: Node + y: int + +class Node(NamedTuple): + name: str + children: Node2 + +n: Node +reveal_type(n) # E: Revealed type is 'Tuple[builtins.str, Tuple[Tuple[builtins.str, Tuple[Any, builtins.int, fallback=__main__.Node2], fallback=__main__.Node], builtins.int, fallback=__main__.Node2], fallback=__main__.Node]' +[builtins fixtures/tuple.pyi] + +[case testSelfRefNT5] +from typing import NamedTuple + +Node2 = NamedTuple( + 'Node2', + [ + ('x', Node), + ('y', int), + ]) + +Node = NamedTuple( + 'Node', + [ + ('name', str), + ('children', 'Node2'), + ]) + +n: Node +def f(m: Node2) -> None: pass +reveal_type(n) # E: Revealed type is 'Tuple[builtins.str, Tuple[Tuple[builtins.str, Tuple[Any, builtins.int, fallback=__main__.Node2], fallback=__main__.Node], builtins.int, fallback=__main__.Node2], fallback=__main__.Node]' +reveal_type(f) # E: Revealed type is 'def (m: Tuple[Tuple[builtins.str, Tuple[Any, builtins.int, fallback=__main__.Node2], fallback=__main__.Node], builtins.int, fallback=__main__.Node2])' +[builtins fixtures/tuple.pyi] + [case testForwardReferenceInNamedTuple] from typing import NamedTuple diff --git a/test-data/unit/check-type-aliases.test b/test-data/unit/check-type-aliases.test index 71267d681d13..4ba604986d0e 100644 --- a/test-data/unit/check-type-aliases.test +++ b/test-data/unit/check-type-aliases.test @@ -112,3 +112,21 @@ EmptyTupleCallable = Callable[[Tuple[()]], None] f = None # type: EmptyTupleCallable reveal_type(f) # E: Revealed type is 'def (Tuple[])' [builtins fixtures/list.pyi] + +[case testForwardTypeAlias] +def f(p: 'Alias') -> None: + pass + +reveal_type(f) # E: Revealed type is 'def (p: builtins.int)' +Alias = int +[out] + +[case testForwardTypeAliasGeneric] +from typing import TypeVar, Tuple +def f(p: 'Alias[str]') -> None: + pass + +reveal_type(f) # E: Revealed type is 'def (p: Tuple[builtins.int, builtins.str])' +T = TypeVar('T') +Alias = Tuple[int, T] +[out] From f9b132085d9521d095ade99d44df059dfcba1d9f Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 12 Sep 2017 21:47:29 +0200 Subject: [PATCH 13/39] Take care about one more special case; add few tests and dcostrings --- mypy/semanal.py | 20 ++++++++++++++++++++ mypy/typeanal.py | 2 +- test-data/unit/check-classes.test | 10 ++++++++++ test-data/unit/check-generics.test | 6 +++--- 4 files changed, 34 insertions(+), 4 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index dc41e408963b..2198a2f8d180 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -4670,6 +4670,9 @@ def visit_any(self, t: AnyType) -> Type: class TypeReplacer(TypeTranslator): + """This is very similar TypeTranslator but tracks visited nodes to avoid + infinite recursion on potentially circular (self- or mutually-referential) types. + """ def __init__(self): self.seen = [] # type: List[Type] @@ -4736,13 +4739,30 @@ def visit_type_type(self, t: TypeType) -> Type: class ForwardRefRemover(TypeReplacer): + """This visitor tracks situations like this: + + x: A # this type is not yet known and therefore wrapped in ForwardRef + # it's content is updated in ThirdPass, now we need to unwrap this type. + A = NewType('A', int) + """ def visit_forwardref_type(self, t: ForwardRef) -> Type: return t.link.accept(self) class SyntheticReplacer(ForwardRefRemover): + """This visitor tracks situations like this: + + x: A # when analyzing this type we will get an Instance from FirstPass + # now we need to update this to actual analyzed TupleType. + class A(NamedTuple): + attr: str + """ def visit_instance(self, t: Instance) -> Type: info = t.type + # Special case, analyzed bases transformed the type into TupleType. + if info.tuple_type and not self.seen: + return info.tuple_type.copy_modified(fallback=Instance(info, [])) + # Update forward Instance's to corresponding analyzed types. if info.replaced and info.replaced.tuple_type: tp = info.replaced.tuple_type if any((s is tp) or (s is t) for s in self.seen): diff --git a/mypy/typeanal.py b/mypy/typeanal.py index e02399fe9cbd..31a5dc1d5328 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -605,7 +605,7 @@ def __init__(self, def visit_instance(self, t: Instance) -> None: info = t.type - if info.replaced: + if info.replaced or info.tuple_type: self.indicator['synthetic'] = True # Check type argument count. if len(t.args) != len(info.type_vars): diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 37b29ccb5696..7c1bb6f748c8 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -3385,6 +3385,16 @@ D = TypeVar('D') def mkdict(dict_type: Type[D]) -> D: ... reveal_type(mkdict(ExampleDict)) # E: Revealed type is '__main__.ExampleDict*[Any, Any]' +[case testTupleForwardBase] +from m import a +a[0]() # E: "int" not callable + +[file m.py] +from typing import Tuple +a = None # type: A +class A(Tuple[int, str]): pass +[builtins fixtures/tuple.pyi] + -- Synthetic types crashes -- ----------------------- diff --git a/test-data/unit/check-generics.test b/test-data/unit/check-generics.test index 6e8fcd43e6e4..59df3970a06d 100644 --- a/test-data/unit/check-generics.test +++ b/test-data/unit/check-generics.test @@ -654,14 +654,14 @@ reveal_type(f3()) # E: Revealed type is 'Union[builtins.int, __main__.Node[built from typing import TypeVar, Generic T = TypeVar('T') S = TypeVar('S') +IntNode = Node[int, S] # This is on purpose, to test basic forward references to aliases +AnyNode = Node[S, T] + class Node(Generic[T, S]): def __init__(self, x: T, y: S) -> None: self.x = x self.y = y -IntNode = Node[int, S] -AnyNode = Node[S, T] - def output() -> IntNode[str]: return Node(1, 'x') x = output() # type: IntNode # This is OK (implicit Any) From cf014b8c07e8dc112f8e540c77571584883b9db1 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 12 Sep 2017 22:09:06 +0200 Subject: [PATCH 14/39] Unify visitors --- mypy/semanal.py | 92 ++++++++++++++++++++++--------------------------- 1 file changed, 41 insertions(+), 51 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 2198a2f8d180..8b341142638f 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -4234,13 +4234,10 @@ def analyze(self, type: Optional[Type], node: Optional[Node]) -> None: self.sem, indicator) type.accept(analyzer) self.check_for_omitted_generics(type) - if indicator.get('forward') and node is not None: + if node and (indicator.get('forward') or indicator.get('synthetic')): def patch() -> None: - self.perform_transform(node, lambda tp: tp.accept(ForwardRefRemover())) - self.patches.append(patch) - if indicator.get('synthetic') and node is not None: - def patch() -> None: - self.perform_transform(node, lambda tp: tp.accept(SyntheticReplacer())) + self.perform_transform(node, + lambda tp: tp.accept(TypeReplacer())) self.patches.append(patch) def check_for_omitted_generics(self, typ: Type) -> None: @@ -4682,6 +4679,44 @@ def check(self, t: Type) -> bool: self.seen.append(t) return False + def visit_forwardref_type(self, t: ForwardRef) -> Type: + """This visitor tracks situations like this: + + x: A # this type is not yet known and therefore wrapped in ForwardRef + # it's content is updated in ThirdPass, now we need to unwrap this type. + A = NewType('A', int) + """ + return t.link.accept(self) + + def visit_instance(self, t: Instance) -> Type: + """This visitor tracks situations like this: + + x: A # when analyzing this type we will get an Instance from FirstPass + # now we need to update this to actual analyzed TupleType. + class A(NamedTuple): + attr: str + """ + info = t.type + # Special case, analyzed bases transformed the type into TupleType. + if info.tuple_type and not self.seen: + return info.tuple_type.copy_modified(fallback=Instance(info, [])) + # Update forward Instance's to corresponding analyzed types. + if info.replaced and info.replaced.tuple_type: + tp = info.replaced.tuple_type + if any((s is tp) or (s is t) for s in self.seen): + return AnyType(TypeOfAny.from_error) + self.seen.append(t) + return tp.copy_modified(fallback=Instance(info.replaced, [])).accept(self) + if info.replaced and info.replaced.typeddict_type: + td = info.replaced.typeddict_type + if any((s is td) or (s is t) for s in self.seen): + return AnyType(TypeOfAny.from_error) + self.seen.append(t) + return td.copy_modified(fallback=Instance(info.replaced, [])).accept(self) + if self.check(t): + return Instance(t.type, [AnyType(TypeOfAny.from_error)] * len(t.type.defn.type_vars)) + return super().visit_instance(t) + def visit_type_var(self, t: TypeVarType) -> Type: if self.check(t): return AnyType(TypeOfAny.from_error) @@ -4691,11 +4726,6 @@ def visit_type_var(self, t: TypeVarType) -> Type: t.values = [v.accept(self) for v in t.values] return t - def visit_instance(self, t: Instance) -> Type: - if self.check(t): - return Instance(t.type, [AnyType(TypeOfAny.from_error)] * len(t.type.defn.type_vars)) - return super().visit_instance(t) - def visit_callable_type(self, t: CallableType) -> Type: # TODO: Callable and Overloaded are not copied, # check if we could unify this with TypeTranslator. @@ -4736,43 +4766,3 @@ def visit_type_type(self, t: TypeType) -> Type: if self.check(t): return AnyType(TypeOfAny.from_error) return super().visit_type_type(t) - - -class ForwardRefRemover(TypeReplacer): - """This visitor tracks situations like this: - - x: A # this type is not yet known and therefore wrapped in ForwardRef - # it's content is updated in ThirdPass, now we need to unwrap this type. - A = NewType('A', int) - """ - def visit_forwardref_type(self, t: ForwardRef) -> Type: - return t.link.accept(self) - - -class SyntheticReplacer(ForwardRefRemover): - """This visitor tracks situations like this: - - x: A # when analyzing this type we will get an Instance from FirstPass - # now we need to update this to actual analyzed TupleType. - class A(NamedTuple): - attr: str - """ - def visit_instance(self, t: Instance) -> Type: - info = t.type - # Special case, analyzed bases transformed the type into TupleType. - if info.tuple_type and not self.seen: - return info.tuple_type.copy_modified(fallback=Instance(info, [])) - # Update forward Instance's to corresponding analyzed types. - if info.replaced and info.replaced.tuple_type: - tp = info.replaced.tuple_type - if any((s is tp) or (s is t) for s in self.seen): - return AnyType(TypeOfAny.from_error) - self.seen.append(t) - return tp.copy_modified(fallback=Instance(info.replaced, [])).accept(self) - if info.replaced and info.replaced.typeddict_type: - td = info.replaced.typeddict_type - if any((s is td) or (s is t) for s in self.seen): - return AnyType(TypeOfAny.from_error) - self.seen.append(t) - return td.copy_modified(fallback=Instance(info.replaced, [])).accept(self) - return super().visit_instance(t) From 665236bd5d5188e557e5712d31fa5c13315118fd Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 12 Sep 2017 22:32:05 +0200 Subject: [PATCH 15/39] Add some more comments and docstrings --- mypy/nodes.py | 4 ++++ mypy/semanal.py | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/mypy/nodes.py b/mypy/nodes.py index fc1b12c02cc0..dc38219be404 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1970,6 +1970,10 @@ class is generic then it will be a type constructor of higher kind. # Is this a newtype type? is_newtype = False + # If during analysis of ClassDef associated with this TypeInfo a syntethic + # type (NamedTuple or TypedDict) was generated, store the corresponding + # TypeInfo here. (This attribute does not need to be serialized, it is only + # needed during the semantic passes.) replaced = None # type: TypeInfo FLAGS = [ diff --git a/mypy/semanal.py b/mypy/semanal.py index 8b341142638f..ecb27074fe35 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -4114,6 +4114,8 @@ def visit_class_def(self, tdef: ClassDef) -> None: if tdef.info.is_protocol: add_protocol_members(tdef.info) if tdef.analyzed is not None: + # Also check synthetic types associated with this ClassDef. + # Currently these are TypedDict, and NamedTuple. if isinstance(tdef.analyzed, TypedDictExpr): self.analyze(tdef.analyzed.info.typeddict_type, tdef.analyzed) elif isinstance(tdef.analyzed, NamedTupleExpr): @@ -4178,6 +4180,10 @@ def visit_decorator(self, dec: Decorator) -> None: dec.var.type = sig def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + """Traverse the actual assignment statement and synthetic types + resulted from this assignment (if any). Currently this includes + NewType, TypedDict, and NamedTuple. + """ self.analyze(s.type, s) if isinstance(s.lvalues[0], RefExpr) and isinstance(s.lvalues[0].node, Var): self.analyze(s.lvalues[0].node.type, s.lvalues[0].node) @@ -4214,6 +4220,7 @@ def visit_type_application(self, e: TypeApplication) -> None: # Helpers def perform_transform(self, node: Node, transform: Callable[[Type], Type]) -> None: + """Apply transform to all types associated with node.""" if isinstance(node, (FuncDef, CastExpr, AssignmentStmt, TypeAliasExpr, Var)): node.type = transform(node.type) if isinstance(node, NewTypeExpr): @@ -4228,6 +4235,7 @@ def perform_transform(self, node: Node, transform: Callable[[Type], Type]) -> No node.types = [transform(t) for t in node.types] def analyze(self, type: Optional[Type], node: Optional[Node]) -> None: + # Flags appeared during analysis of type are collected in this dict. indicator = {} # type: Dict[str, bool] if type: analyzer = TypeAnalyserPass3(self.fail, self.options, self.is_typeshed_file, From 9a318aa4a0bd5c3463d55ccaa63ef4b88ad9db9e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 13 Sep 2017 00:05:21 +0200 Subject: [PATCH 16/39] Add recursive type warnings --- mypy/semanal.py | 46 ++++++++++++++++++++------- test-data/unit/check-classes.test | 14 ++++---- test-data/unit/check-incremental.test | 18 +++++------ test-data/unit/check-namedtuple.test | 26 +++++++++------ test-data/unit/pythoneval.test | 2 +- 5 files changed, 70 insertions(+), 36 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index ecb27074fe35..28c2528c8cf5 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -945,6 +945,8 @@ def analyze_namedtuple_classdef(self, defn: ClassDef) -> Optional[TypeInfo]: defn.info.replaced = info defn.info = info defn.analyzed = NamedTupleExpr(info) + defn.analyzed.line = defn.line + defn.analyzed.column = defn.column return info return None @@ -1285,6 +1287,8 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> bool: defn.info.replaced = info node.node = info defn.analyzed = TypedDictExpr(info) + defn.analyzed.line = defn.line + defn.analyzed.column = defn.column return True # Extending/merging existing TypedDicts if any(not isinstance(expr, RefExpr) or @@ -1318,6 +1322,8 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> bool: defn.info.replaced = info node.node = info defn.analyzed = TypedDictExpr(info) + defn.analyzed.line = defn.line + defn.analyzed.column = defn.column return True return False @@ -4117,9 +4123,9 @@ def visit_class_def(self, tdef: ClassDef) -> None: # Also check synthetic types associated with this ClassDef. # Currently these are TypedDict, and NamedTuple. if isinstance(tdef.analyzed, TypedDictExpr): - self.analyze(tdef.analyzed.info.typeddict_type, tdef.analyzed) + self.analyze(tdef.analyzed.info.typeddict_type, tdef.analyzed, warn=True) elif isinstance(tdef.analyzed, NamedTupleExpr): - self.analyze(tdef.analyzed.info.tuple_type, tdef.analyzed) + self.analyze(tdef.analyzed.info.tuple_type, tdef.analyzed, warn=True) for name in tdef.analyzed.info.names: sym = tdef.analyzed.info.names[name] if isinstance(sym.node, (FuncDef, Decorator)): @@ -4194,9 +4200,9 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: if isinstance(analyzed, NewTypeExpr): self.analyze(analyzed.old_type, analyzed) if isinstance(analyzed, TypedDictExpr): - self.analyze(analyzed.info.typeddict_type, analyzed) + self.analyze(analyzed.info.typeddict_type, analyzed, warn=True) if isinstance(analyzed, NamedTupleExpr): - self.analyze(analyzed.info.tuple_type, analyzed) + self.analyze(analyzed.info.tuple_type, analyzed, warn=True) for name in analyzed.info.names: sym = analyzed.info.names[name] if isinstance(sym.node, (FuncDef, Decorator)): @@ -4234,8 +4240,9 @@ def perform_transform(self, node: Node, transform: Callable[[Type], Type]) -> No if isinstance(node, TypeApplication): node.types = [transform(t) for t in node.types] - def analyze(self, type: Optional[Type], node: Optional[Node]) -> None: - # Flags appeared during analysis of type are collected in this dict. + def analyze(self, type: Optional[Type], node: Optional[Node], warn: bool = False) -> None: + # Recursive type warnings are only emitted on type definition 'node's, marked by 'warn' + # Flags appeared during analysis of 'type' are collected in this dict. indicator = {} # type: Dict[str, bool] if type: analyzer = TypeAnalyserPass3(self.fail, self.options, self.is_typeshed_file, @@ -4245,7 +4252,8 @@ def analyze(self, type: Optional[Type], node: Optional[Node]) -> None: if node and (indicator.get('forward') or indicator.get('synthetic')): def patch() -> None: self.perform_transform(node, - lambda tp: tp.accept(TypeReplacer())) + lambda tp: tp.accept(TypeReplacer(self.fail, + node, warn))) self.patches.append(patch) def check_for_omitted_generics(self, typ: Type) -> None: @@ -4678,8 +4686,16 @@ class TypeReplacer(TypeTranslator): """This is very similar TypeTranslator but tracks visited nodes to avoid infinite recursion on potentially circular (self- or mutually-referential) types. """ - def __init__(self): + def __init__(self, fail: Callable[[str, Context], None], start: Node, warn: bool) -> None: self.seen = [] # type: List[Type] + self.fail = fail + self.start = start + self.warn = warn + + def warn_recursion(self) -> None: + if self.warn: + self.fail('Recursive types not fully supported yet,' + ' nested types replaced with "Any"', self.start) def check(self, t: Type) -> bool: if any(t is s for s in self.seen): @@ -4688,7 +4704,7 @@ def check(self, t: Type) -> bool: return False def visit_forwardref_type(self, t: ForwardRef) -> Type: - """This visitor tracks situations like this: + """This visitor method tracks situations like this: x: A # this type is not yet known and therefore wrapped in ForwardRef # it's content is updated in ThirdPass, now we need to unwrap this type. @@ -4697,7 +4713,7 @@ def visit_forwardref_type(self, t: ForwardRef) -> Type: return t.link.accept(self) def visit_instance(self, t: Instance) -> Type: - """This visitor tracks situations like this: + """This visitor method tracks situations like this: x: A # when analyzing this type we will get an Instance from FirstPass # now we need to update this to actual analyzed TupleType. @@ -4708,20 +4724,26 @@ class A(NamedTuple): # Special case, analyzed bases transformed the type into TupleType. if info.tuple_type and not self.seen: return info.tuple_type.copy_modified(fallback=Instance(info, [])) - # Update forward Instance's to corresponding analyzed types. + # Update forward Instance's to corresponding analyzed NamedTuple's. if info.replaced and info.replaced.tuple_type: tp = info.replaced.tuple_type if any((s is tp) or (s is t) for s in self.seen): + self.warn_recursion() + # The key idea is that when we return to the place where + # we already was, we break the cycle and put AnyType as a leaf. return AnyType(TypeOfAny.from_error) self.seen.append(t) return tp.copy_modified(fallback=Instance(info.replaced, [])).accept(self) + # Same as above but for TypedDict's. if info.replaced and info.replaced.typeddict_type: td = info.replaced.typeddict_type if any((s is td) or (s is t) for s in self.seen): + self.warn_recursion() return AnyType(TypeOfAny.from_error) self.seen.append(t) return td.copy_modified(fallback=Instance(info.replaced, [])).accept(self) if self.check(t): + self.warn_recursion() return Instance(t.type, [AnyType(TypeOfAny.from_error)] * len(t.type.defn.type_vars)) return super().visit_instance(t) @@ -4757,11 +4779,13 @@ def visit_overloaded(self, t: Overloaded) -> Type: def visit_tuple_type(self, t: TupleType) -> Type: if self.check(t): + self.warn_recursion() return AnyType(TypeOfAny.from_error) return super().visit_tuple_type(t) def visit_typeddict_type(self, t: TypedDictType) -> Type: if self.check(t): + self.warn_recursion() return AnyType(TypeOfAny.from_error) return super().visit_typeddict_type(t) diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 7c1bb6f748c8..5e51972ced07 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -3401,14 +3401,14 @@ class A(Tuple[int, str]): pass [case testCrashOnSelfRecursiveNamedTupleVar] from typing import NamedTuple -N = NamedTuple('N', [('x', N)]) +N = NamedTuple('N', [('x', N)]) # E: Recursive types not fully supported yet, nested types replaced with "Any" n: N [out] [case testCrashOnSelfRecursiveTypedDictVar] from mypy_extensions import TypedDict -A = TypedDict('A', {'a': 'A'}) +A = TypedDict('A', {'a': 'A'}) # type: ignore a: A [builtins fixtures/isinstancelist.pyi] [out] @@ -3416,16 +3416,15 @@ a: A [case testCrashInJoinOfSelfRecursiveNamedTuples] from typing import NamedTuple -class N(NamedTuple): +class N(NamedTuple): # type: ignore x: N -class M(NamedTuple): +class M(NamedTuple): # type: ignore x: M n: N m: M lst = [n, m] [builtins fixtures/isinstancelist.pyi] -[out] [case testCorrectJoinOfSelfRecursiveTypedDicts] from mypy_extensions import TypedDict @@ -3441,6 +3440,8 @@ lst = [n, m] reveal_type(lst[0]['x']) # E: Revealed type is 'TypedDict('__main__.N', {'x': Any})' [builtins fixtures/isinstancelist.pyi] [out] +main:3: error: Recursive types not fully supported yet, nested types replaced with "Any" +main:5: error: Recursive types not fully supported yet, nested types replaced with "Any" [case testCrashInForwardRefToNamedTupleWithIsinstance] from typing import Dict, NamedTuple @@ -3511,7 +3512,8 @@ class A(NamedTuple): year: int class B(NamedTuple): director: str - +y: A +y = x reveal_type(x.name.director) # E: Revealed type is 'builtins.str' [out] diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 58314a9b4e34..49408c3c62cc 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -2958,7 +2958,7 @@ tmp/b.py:2: error: Incompatible return value type (got "int", expected "str") [case testSelfRefNTIncremental1] from typing import Tuple, NamedTuple -Node = NamedTuple( +Node = NamedTuple( # type: ignore 'Node', [ ('name', str), @@ -2972,14 +2972,14 @@ n: Node [case testSelfRefNTIncremental2] from typing import Tuple, NamedTuple -Node = NamedTuple( +Node = NamedTuple( # type: ignore 'Node', [ ('name', str), ('children', Tuple['Node2', ...]), ]) -class Node2(NamedTuple): +class Node2(NamedTuple): # type: ignore x: Node y: int @@ -2989,11 +2989,11 @@ n: Node [case testSelfRefNTIncremental3] from typing import NamedTuple, Tuple -class Node2(NamedTuple): +class Node2(NamedTuple): # type: ignore x: Tuple[Node, int] y: int -Node = NamedTuple( +Node = NamedTuple( # type: ignore 'Node', [ ('name', str), @@ -3008,11 +3008,11 @@ lst = [m, n] [case testSelfRefNTIncremental4] from typing import NamedTuple -class Node2(NamedTuple): +class Node2(NamedTuple): # type: ignore x: Node y: int -class Node(NamedTuple): +class Node(NamedTuple): # type: ignore name: str children: Node2 @@ -3022,14 +3022,14 @@ n: Node [case testSelfRefNTIncremental5] from typing import NamedTuple -Node2 = NamedTuple( +Node2 = NamedTuple( # type: ignore 'Node2', [ ('x', Node), ('y', int), ]) -Node = NamedTuple( +Node = NamedTuple( # type: ignore 'Node', [ ('name', str), diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index d57eb93c411b..9a9575a96530 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -462,7 +462,7 @@ tmp/b.py:8: error: Revealed type is 'Tuple[Any, fallback=a.X]' [case testSelfRefNT1] from typing import Tuple, NamedTuple -Node = NamedTuple( +Node = NamedTuple( # E: Recursive types not fully supported yet, nested types replaced with "Any" 'Node', [ ('name', str), @@ -477,29 +477,32 @@ reveal_type(n) # E: Revealed type is 'Tuple[builtins.str, builtins.tuple[Tuple[b [case testSelfRefNT2] from typing import Tuple, NamedTuple -Node = NamedTuple( +Node = NamedTuple( # E 'Node', [ ('name', str), ('children', Tuple['Node2', ...]), ]) -class Node2(NamedTuple): +class Node2(NamedTuple): # E x: Node y: int n: Node reveal_type(n) # E: Revealed type is 'Tuple[builtins.str, builtins.tuple[Tuple[Tuple[builtins.str, builtins.tuple[Any], fallback=__main__.Node], builtins.int, fallback=__main__.Node2]], fallback=__main__.Node]' [builtins fixtures/tuple.pyi] +[out] +main:3: error: Recursive types not fully supported yet, nested types replaced with "Any" +main:10: error: Recursive types not fully supported yet, nested types replaced with "Any" [case testSelfRefNT3] from typing import NamedTuple, Tuple -class Node2(NamedTuple): +class Node2(NamedTuple): # E x: Tuple[Node, int] y: int -Node = NamedTuple( +Node = NamedTuple( # E: Recursive types not fully supported yet, nested types replaced with "Any" 'Node', [ ('name', str), @@ -513,33 +516,38 @@ reveal_type(m) # E: Revealed type is 'Tuple[builtins.str, Tuple[Tuple[Tuple[buil lst = [m, n] reveal_type(lst[0]) # E: Revealed type is 'Tuple[builtins.object, builtins.object]' [builtins fixtures/tuple.pyi] +[out] +main:3: error: Recursive types not fully supported yet, nested types replaced with "Any" [case testSelfRefNT4] from typing import NamedTuple -class Node2(NamedTuple): +class Node2(NamedTuple): # E x: Node y: int -class Node(NamedTuple): +class Node(NamedTuple): # E name: str children: Node2 n: Node reveal_type(n) # E: Revealed type is 'Tuple[builtins.str, Tuple[Tuple[builtins.str, Tuple[Any, builtins.int, fallback=__main__.Node2], fallback=__main__.Node], builtins.int, fallback=__main__.Node2], fallback=__main__.Node]' [builtins fixtures/tuple.pyi] +[out] +main:3: error: Recursive types not fully supported yet, nested types replaced with "Any" +main:7: error: Recursive types not fully supported yet, nested types replaced with "Any" [case testSelfRefNT5] from typing import NamedTuple -Node2 = NamedTuple( +Node2 = NamedTuple( # E: Recursive types not fully supported yet, nested types replaced with "Any" 'Node2', [ ('x', Node), ('y', int), ]) -Node = NamedTuple( +Node = NamedTuple( # E: Recursive types not fully supported yet, nested types replaced with "Any" 'Node', [ ('name', str), diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index 7713bd344a0c..80bbaeeb0c94 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -1383,7 +1383,7 @@ _testTypedDictMappingMethods.py:11: error: Revealed type is 'typing.ValuesView[b [case testCrashOnComplexCheckWithNamedTupleNext] from typing import NamedTuple -MyNamedTuple = NamedTuple('MyNamedTuple', [('parent', 'MyNamedTuple')]) +MyNamedTuple = NamedTuple('MyNamedTuple', [('parent', 'MyNamedTuple')]) # type: ignore def foo(mymap) -> MyNamedTuple: return next((mymap[key] for key in mymap), None) [out] From 757fbd9831e33f7d7db802ea085c25f8c06ac176 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 13 Sep 2017 00:12:24 +0200 Subject: [PATCH 17/39] Fix lint --- mypy/typeanal.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 31a5dc1d5328..78e20e489eab 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -741,6 +741,7 @@ def visit_forwardref_type(self, t: ForwardRef) -> None: if isinstance(t.link, UnboundType): t.link = self.sem.anal_type(t.link, third_pass=True) + TypeVarList = List[Tuple[str, TypeVarExpr]] From 4502ce210ddabd4611994069e2ddb72a2cc7b571 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 13 Sep 2017 10:58:50 +0200 Subject: [PATCH 18/39] Also clean-up bases; add more tests and allow some previously skipped --- mypy/semanal.py | 21 ++++++-- test-data/unit/check-generics.test | 6 +-- test-data/unit/check-incremental.test | 77 ++++++++++++++++++++++++++- test-data/unit/check-tuples.test | 9 ++-- 4 files changed, 99 insertions(+), 14 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 28c2528c8cf5..53a5abd5e889 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -4106,7 +4106,7 @@ def visit_class_def(self, tdef: ClassDef) -> None: # check them again here. if not tdef.info.is_named_tuple: for type in tdef.info.bases: - self.analyze(type, None) + self.analyze(type, tdef.info) if tdef.info.is_protocol: if not isinstance(type, Instance) or not type.type.is_protocol: if type.type.fullname() != 'builtins.object': @@ -4239,8 +4239,21 @@ def perform_transform(self, node: Node, transform: Callable[[Type], Type]) -> No transform(node.info.tuple_type)) if isinstance(node, TypeApplication): node.types = [transform(t) for t in node.types] - - def analyze(self, type: Optional[Type], node: Optional[Node], warn: bool = False) -> None: + if isinstance(node, TypeInfo): + new_bases = [] + for base in node.bases: + new_base = transform(base) + if isinstance(new_base, Instance): + new_bases.append(new_base) + else: + # Don't fix the NamedTuple bases, they are Instance's intentionally. + # Patch the 'args' just in case, although generic tuple type are + # not supported yet. + alt_base = Instance(base.type, [transform(a) for a in base.args]) + new_bases.append(alt_base) + node.bases = new_bases + + def analyze(self, type: Optional[Type], node: Node, warn: bool = False) -> None: # Recursive type warnings are only emitted on type definition 'node's, marked by 'warn' # Flags appeared during analysis of 'type' are collected in this dict. indicator = {} # type: Dict[str, bool] @@ -4249,7 +4262,7 @@ def analyze(self, type: Optional[Type], node: Optional[Node], warn: bool = False self.sem, indicator) type.accept(analyzer) self.check_for_omitted_generics(type) - if node and (indicator.get('forward') or indicator.get('synthetic')): + if indicator.get('forward') or indicator.get('synthetic'): def patch() -> None: self.perform_transform(node, lambda tp: tp.accept(TypeReplacer(self.fail, diff --git a/test-data/unit/check-generics.test b/test-data/unit/check-generics.test index 59df3970a06d..6e8fcd43e6e4 100644 --- a/test-data/unit/check-generics.test +++ b/test-data/unit/check-generics.test @@ -654,14 +654,14 @@ reveal_type(f3()) # E: Revealed type is 'Union[builtins.int, __main__.Node[built from typing import TypeVar, Generic T = TypeVar('T') S = TypeVar('S') -IntNode = Node[int, S] # This is on purpose, to test basic forward references to aliases -AnyNode = Node[S, T] - class Node(Generic[T, S]): def __init__(self, x: T, y: S) -> None: self.x = x self.y = y +IntNode = Node[int, S] +AnyNode = Node[S, T] + def output() -> IntNode[str]: return Node(1, 'x') x = output() # type: IntNode # This is OK (implicit Any) diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 49408c3c62cc..61127b754b45 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -2952,9 +2952,83 @@ def foo(a: int, b: int) -> str: [out2] tmp/b.py:2: error: Incompatible return value type (got "int", expected "str") --- Some crazy selef-referential named tuples and types dicts +-- Some crazy selef-referential named tuples, types dicts, and aliases -- to be sure that everything can be _serialized_ (i.e. ForwardRef's are removed) +[case testForwardTypeAliasInBase1] +from typing import List +class C(List['A']): + pass + +A = List[int] +C()[0] +[builtins fixtures/list.pyi] +[out] + +[case testForwardTypeAliasInBase2] +from typing import List, Generic, TypeVar, NamedTuple +T = TypeVar('T') + +class C(A, B): #type: ignore + pass +class G(Generic[T]): pass +A = G[C] +class B(NamedTuple): + x: int + +C().x +C()[0] +[builtins fixtures/list.pyi] +[out] + +[case testGenericTypeAliasesForwardAnyIncremental1] +from typing import TypeVar, Generic +T = TypeVar('T') +S = TypeVar('S') +IntNode = Node[int, S] +AnyNode = Node[S, T] + +class Node(Generic[T, S]): + def __init__(self, x: T, y: S) -> None: + self.x = x + self.y = y + +def output() -> IntNode[str]: + return Node(1, 'x') +x = output() # type: IntNode + +y = None # type: IntNode +y.x = 1 +y.y = 1 +y.y = 'x' + +z = Node(1, 'x') # type: AnyNode +[out] + +[case testGenericTypeAliasesForwardAnyIncremental2] +from typing import TypeVar, Generic +T = TypeVar('T') +S = TypeVar('S') + +class Node(Generic[T, S]): + def __init__(self, x: T, y: S) -> None: + self.x = x + self.y = y + +def output() -> IntNode[str]: + return Node(1, 'x') +x = output() # type: IntNode + +y = None # type: IntNode +y.x = 1 +y.y = 1 +y.y = 'x' + +z = Node(1, 'x') # type: AnyNode +IntNode = Node[int, S] +AnyNode = Node[S, T] +[out] + [case testSelfRefNTIncremental1] from typing import Tuple, NamedTuple @@ -2968,7 +3042,6 @@ Node = NamedTuple( # type: ignore n: Node [builtins fixtures/tuple.pyi] - [case testSelfRefNTIncremental2] from typing import Tuple, NamedTuple diff --git a/test-data/unit/check-tuples.test b/test-data/unit/check-tuples.test index 14483d9ceea1..387eabca4764 100644 --- a/test-data/unit/check-tuples.test +++ b/test-data/unit/check-tuples.test @@ -704,18 +704,17 @@ from typing import Tuple class A(tuple): pass [out] -[case testTupleBaseClass2-skip] +[case testTupleBaseClass2] import m [file m.pyi] -# This doesn't work correctly -- no errors are reported (#867) from typing import Tuple a = None # type: A class A(Tuple[int, str]): pass x, y = a -x() # Expected: "int" not callable -y() # Expected: "str" not callable +x() # E: "int" not callable +y() # E: "str" not callable +[builtins fixtures/tuple.pyi] [out] -(should fail) [case testGenericClassWithTupleBaseClass] from typing import TypeVar, Generic, Tuple From 3b39d40e9ea9a5e642f761e21573d3ce1a352157 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 13 Sep 2017 11:07:27 +0200 Subject: [PATCH 19/39] One more TypedDict test --- test-data/unit/check-typeddict.test | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index 51c922316d56..a28d1fadac65 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -1310,3 +1310,18 @@ x: X reveal_type(x) # E: Revealed type is 'TypedDict('__main__.X', {'a': TypedDict('__main__.A', {'b': builtins.int})})' reveal_type(x['a']['b']) # E: Revealed type is 'builtins.int' [builtins fixtures/dict.pyi] + +[case testSelfRecursiveTypedDictInheriting] +from mypy_extensions import TypedDict + +class MovieBase(TypedDict): + name: str + year: int + +class Movie(MovieBase): # type: ignore # warning about recursive not fully supported + director: 'Movie' + +m: Movie +reveal_type(m['director']['name']) # E: Revealed type is 'builtins.str' +[builtins fixtures/dict.pyi] +[out] From c8b28feabbb4d06d9d93d902d07ab9dde618262e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 13 Sep 2017 11:14:13 +0200 Subject: [PATCH 20/39] Add another simple self-referrential NamedTuple test --- test-data/unit/check-namedtuple.test | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 9a9575a96530..72961322fd99 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -456,6 +456,20 @@ def f(x: a.X) -> None: tmp/b.py:6: error: Revealed type is 'a.X' tmp/b.py:8: error: Revealed type is 'Tuple[Any, fallback=a.X]' +[case testSimpleSelfReferrentialNamedTuple] +from typing import NamedTuple +class MyNamedTuple(NamedTuple): + parent: 'MyNamedTuple' + +def bar(nt: MyNamedTuple) -> MyNamedTuple: + return nt + +x: MyNamedTuple +reveal_type(x.parent) +[out] +main:2: error: Recursive types not fully supported yet, nested types replaced with "Any" +main:9: error: Revealed type is 'Tuple[Any, fallback=__main__.MyNamedTuple]' + -- Some crazy selef-referential named tuples and types dicts -- to be sure that everything works From 9f92b0fa61c915c6bacbd4c60fa543800bbb9573 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 13 Sep 2017 15:42:16 +0200 Subject: [PATCH 21/39] Fix type_override; add tests for recursive aliases; fix Callable TODO; Better error in case something is still missing --- mypy/semanal.py | 37 +++++++++++-------- mypy/types.py | 9 +++++ test-data/unit/check-incremental.test | 25 +++++++++++++ test-data/unit/check-type-aliases.test | 51 ++++++++++++++++++++++++++ 4 files changed, 106 insertions(+), 16 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 53a5abd5e889..1351923fdfc7 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -4193,8 +4193,12 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: self.analyze(s.type, s) if isinstance(s.lvalues[0], RefExpr) and isinstance(s.lvalues[0].node, Var): self.analyze(s.lvalues[0].node.type, s.lvalues[0].node) + if isinstance(s.lvalues[0], NameExpr): + node = self.sem.lookup(s.lvalues[0].name, s, suppress_errors=True) + if node: + self.analyze(node.type_override, node) if isinstance(s.rvalue, IndexExpr) and isinstance(s.rvalue.analyzed, TypeAliasExpr): - self.analyze(s.rvalue.analyzed.type, s.rvalue.analyzed) + self.analyze(s.rvalue.analyzed.type, s.rvalue.analyzed, warn=True) if isinstance(s.rvalue, CallExpr): analyzed = s.rvalue.analyzed if isinstance(analyzed, NewTypeExpr): @@ -4225,7 +4229,8 @@ def visit_type_application(self, e: TypeApplication) -> None: # Helpers - def perform_transform(self, node: Node, transform: Callable[[Type], Type]) -> None: + def perform_transform(self, node: Union[Node, SymbolTableNode], + transform: Callable[[Type], Type]) -> None: """Apply transform to all types associated with node.""" if isinstance(node, (FuncDef, CastExpr, AssignmentStmt, TypeAliasExpr, Var)): node.type = transform(node.type) @@ -4239,6 +4244,8 @@ def perform_transform(self, node: Node, transform: Callable[[Type], Type]) -> No transform(node.info.tuple_type)) if isinstance(node, TypeApplication): node.types = [transform(t) for t in node.types] + if isinstance(node, SymbolTableNode): + node.type_override = transform(node.type_override) if isinstance(node, TypeInfo): new_bases = [] for base in node.bases: @@ -4253,7 +4260,8 @@ def perform_transform(self, node: Node, transform: Callable[[Type], Type]) -> No new_bases.append(alt_base) node.bases = new_bases - def analyze(self, type: Optional[Type], node: Node, warn: bool = False) -> None: + def analyze(self, type: Optional[Type], node: Union[Node, SymbolTableNode], + warn: bool = False) -> None: # Recursive type warnings are only emitted on type definition 'node's, marked by 'warn' # Flags appeared during analysis of 'type' are collected in this dict. indicator = {} # type: Dict[str, bool] @@ -4699,7 +4707,8 @@ class TypeReplacer(TypeTranslator): """This is very similar TypeTranslator but tracks visited nodes to avoid infinite recursion on potentially circular (self- or mutually-referential) types. """ - def __init__(self, fail: Callable[[str, Context], None], start: Node, warn: bool) -> None: + def __init__(self, fail: Callable[[str, Context], None], + start: Union[Node, SymbolTableNode], warn: bool) -> None: self.seen = [] # type: List[Type] self.fail = fail self.start = start @@ -4707,11 +4716,13 @@ def __init__(self, fail: Callable[[str, Context], None], start: Node, warn: bool def warn_recursion(self) -> None: if self.warn: + assert isinstance(self.start, Node), "Internal error: invalid error context" self.fail('Recursive types not fully supported yet,' ' nested types replaced with "Any"', self.start) def check(self, t: Type) -> bool: if any(t is s for s in self.seen): + self.warn_recursion() return True self.seen.append(t) return False @@ -4756,7 +4767,6 @@ class A(NamedTuple): self.seen.append(t) return td.copy_modified(fallback=Instance(info.replaced, [])).accept(self) if self.check(t): - self.warn_recursion() return Instance(t.type, [AnyType(TypeOfAny.from_error)] * len(t.type.defn.type_vars)) return super().visit_instance(t) @@ -4770,35 +4780,30 @@ def visit_type_var(self, t: TypeVarType) -> Type: return t def visit_callable_type(self, t: CallableType) -> Type: - # TODO: Callable and Overloaded are not copied, - # check if we could unify this with TypeTranslator. if self.check(t): return AnyType(TypeOfAny.from_error) - t.arg_types = [tp.accept(self) for tp in t.arg_types] - t.ret_type = t.ret_type.accept(self) - for v in t.variables: + arg_types = [tp.accept(self) for tp in t.arg_types] + ret_type = t.ret_type.accept(self) + variables = t.variables.copy() + for v in variables: if v.upper_bound: v.upper_bound = v.upper_bound.accept(self) if v.values: v.values = [val.accept(self) for val in v.values] - return t + return t.copy_modified(arg_types=arg_types, ret_type=ret_type, variables=variables) def visit_overloaded(self, t: Overloaded) -> Type: if self.check(t): return AnyType(TypeOfAny.from_error) - t._items = [cast(CallableType, it.accept(self)) for it in t.items()] - t.fallback.accept(self) - return t + return super().visit_overloaded(t) def visit_tuple_type(self, t: TupleType) -> Type: if self.check(t): - self.warn_recursion() return AnyType(TypeOfAny.from_error) return super().visit_tuple_type(t) def visit_typeddict_type(self, t: TypedDictType) -> Type: if self.check(t): - self.warn_recursion() return AnyType(TypeOfAny.from_error) return super().visit_typeddict_type(t) diff --git a/mypy/types.py b/mypy/types.py index 64fabf93198e..d37cd2d9c42f 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1384,6 +1384,15 @@ def __init__(self, link: Type) -> None: def accept(self, visitor: 'TypeVisitor[T]') -> T: return visitor.visit_forwardref_type(self) + def serialize(self): + if isinstance(self.link, UnboundType): + name = self.link.name + if isinstance(self.link, Instance): + name = self.link.type.name() + else: + name = self.link.__class__.__name__ + assert False, "Internal error: Unresolved forward reference to {}".format(name) + # # Visitor-related classes diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 61127b754b45..4a142c90c7d9 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -2981,6 +2981,31 @@ C()[0] [builtins fixtures/list.pyi] [out] +[case testSerializeRecursiveAliases1] +from typing import Type, Callable, Union + +A = Union[A, int] # type: ignore +B = Callable[[B], int] # type: ignore +C = Type[C] # type: ignore +[out] + +[case testSerializeRecursiveAliases2] +from typing import Type, Callable, Union + +A = Union[B, int] # type: ignore +B = Callable[[C], int] # type: ignore +C = Type[A] # type: ignore +[out] + +[case testSerializeRecursiveAliases3] +from typing import Type, Callable, Union, NamedTuple + +A = Union[B, int] # type: ignore +B = Callable[[C], int] # type: ignore +class C(NamedTuple): # type: ignore + x: A +[out] + [case testGenericTypeAliasesForwardAnyIncremental1] from typing import TypeVar, Generic T = TypeVar('T') diff --git a/test-data/unit/check-type-aliases.test b/test-data/unit/check-type-aliases.test index 4ba604986d0e..4bb2f4726dd3 100644 --- a/test-data/unit/check-type-aliases.test +++ b/test-data/unit/check-type-aliases.test @@ -130,3 +130,54 @@ reveal_type(f) # E: Revealed type is 'def (p: Tuple[builtins.int, builtins.str]) T = TypeVar('T') Alias = Tuple[int, T] [out] + +[case testRecursiveAliasesErrors1] +from typing import Type, Callable, Union + +A = Union[A, int] +B = Callable[[B], int] +C = Type[C] +[out] +main:3: error: Recursive types not fully supported yet, nested types replaced with "Any" +main:4: error: Recursive types not fully supported yet, nested types replaced with "Any" +main:5: error: Recursive types not fully supported yet, nested types replaced with "Any" + +[case testRecursiveAliasesErrors2] +from typing import Type, Callable, Union + +A = Union[B, int] +B = Callable[[C], int] +C = Type[A] +[out] +main:3: error: Recursive types not fully supported yet, nested types replaced with "Any" +main:4: error: Recursive types not fully supported yet, nested types replaced with "Any" +main:5: error: Recursive types not fully supported yet, nested types replaced with "Any" + +[case testDoubleForwardAlias] +from typing import List +x: A +A = List[B] +B = List[int] +reveal_type(x) # E: Revealed type is 'builtins.list[builtins.list[builtins.int]]' +[builtins fixtures/list.pyi] +[out] + +[case testDoubleForwardAliasWithNamedTuple] +from typing import List, NamedTuple +x: A +A = List[B] +class B(NamedTuple): + x: str +reveal_type(x[0].x) # E: Revealed type is 'builtins.str' +[builtins fixtures/list.pyi] +[out] + +[case testJSONAliasApproximation] +from typing import List, Union, Dict +x: JSON +JSON = Union[int, str, List[JSON], Dict[str, JSON]] # type: ignore +reveal_type(x) # E: Revealed type is 'Union[builtins.int, builtins.str, builtins.list[Any], builtins.dict[builtins.str, Any]]' +if isinstance(x, list): + reveal_type(x) # E: Revealed type is 'builtins.list[Any]' +[builtins fixtures/isinstancelist.pyi] +[out] \ No newline at end of file From 3568fdba1579ec7f697e5acfd443b07f60025185 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 19 Sep 2017 23:22:31 +0200 Subject: [PATCH 22/39] Skip the whole ForwardRef dance in unchecked functions --- mypy/semanal.py | 8 ++++++-- mypy/typeanal.py | 9 +++++++-- test-data/unit/fixtures/dict.pyi | 1 + test-data/unit/fixtures/typing-full.pyi | 2 ++ test-data/unit/semanal-errors.test | 18 ++++++++++++++++++ 5 files changed, 34 insertions(+), 4 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 8bcdaf261185..c173a37ddf9d 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1622,7 +1622,7 @@ def type_analyzer(self, *, third_pass: bool = False) -> TypeAnalyser: if tvar_scope is None: tvar_scope = self.tvar_scope - return TypeAnalyser(self.lookup_qualified, + tpan = TypeAnalyser(self.lookup_qualified, self.lookup_fully_qualified, tvar_scope, self.fail, @@ -1633,6 +1633,8 @@ def type_analyzer(self, *, allow_tuple_literal=allow_tuple_literal, allow_unnormalized=self.is_stub_file, third_pass=third_pass) + tpan.in_dynamic_func = self.function_stack and self.function_stack[-1].is_dynamic() + return tpan def anal_type(self, t: Type, *, tvar_scope: Optional[TypeVarScope] = None, @@ -1730,6 +1732,7 @@ def analyze_alias(self, rvalue: Expression, qualified type variable names for generic aliases. If 'allow_unnormalized' is True, allow types like builtins.list[T]. """ + dynamic = self.function_stack and self.function_stack[-1].is_dynamic() res = analyze_type_alias(rvalue, self.lookup_qualified, self.lookup_fully_qualified, @@ -1738,7 +1741,8 @@ def analyze_alias(self, rvalue: Expression, self.plugin, self.options, self.is_typeshed_stub_file, - allow_unnormalized=True) + allow_unnormalized=True, + in_dynamic_func=dynamic) if res: alias_tvars = [name for (name, _) in res.accept(TypeVariableQuery(self.lookup_qualified, self.tvar_scope))] diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 78e20e489eab..bf7ae48fd17b 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -62,7 +62,8 @@ def analyze_type_alias(node: Expression, plugin: Plugin, options: Options, is_typeshed_stub: bool, - allow_unnormalized: bool = False) -> Optional[Type]: + allow_unnormalized: bool = False, + in_dynamic_func: bool = False) -> Optional[Type]: """Return type if node is valid as a type alias rvalue. Return None otherwise. 'node' must have been semantically analyzed. @@ -106,6 +107,7 @@ def analyze_type_alias(node: Expression, return None analyzer = TypeAnalyser(lookup_func, lookup_fqn_func, tvar_scope, fail_func, plugin, options, is_typeshed_stub, aliasing=True, allow_unnormalized=allow_unnormalized) + analyzer.in_dynamic_func = in_dynamic_func return type.accept(analyzer) @@ -123,6 +125,9 @@ class TypeAnalyser(SyntheticTypeVisitor[Type], AnalyzerPluginInterface): Converts unbound types into bound types. """ + # Is this called from an untyped function definition + in_dynamic_func = False # type: bool + def __init__(self, lookup_func: Callable[[str, Context], SymbolTableNode], lookup_fqn_func: Callable[[str], SymbolTableNode], @@ -264,7 +269,7 @@ def visit_unbound_type(self, t: UnboundType) -> Type: # Allow unbound type variables when defining an alias if not (self.aliasing and sym.kind == TVAR and self.tvar_scope.get_binding(sym) is None): - if (not self.third_pass and + if (not self.third_pass and not self.in_dynamic_func and not (isinstance(sym.node, FuncDef) or isinstance(sym.node, Var) and sym.node.is_ready)): return ForwardRef(t) diff --git a/test-data/unit/fixtures/dict.pyi b/test-data/unit/fixtures/dict.pyi index cf8b61f9397a..2c31f2a392cb 100644 --- a/test-data/unit/fixtures/dict.pyi +++ b/test-data/unit/fixtures/dict.pyi @@ -41,6 +41,7 @@ class tuple(Generic[T]): pass class function: pass class float: pass class bool: pass +class bytes: pass class ellipsis: pass def isinstance(x: object, t: Union[type, Tuple]) -> bool: pass diff --git a/test-data/unit/fixtures/typing-full.pyi b/test-data/unit/fixtures/typing-full.pyi index 62fac70034c0..c4baadc5638d 100644 --- a/test-data/unit/fixtures/typing-full.pyi +++ b/test-data/unit/fixtures/typing-full.pyi @@ -33,6 +33,8 @@ List = 0 Dict = 0 Set = 0 +AnyStr = TypeVar('AnyStr', str, bytes) + T = TypeVar('T') T_co = TypeVar('T_co', covariant=True) T_contra = TypeVar('T_contra', contravariant=True) diff --git a/test-data/unit/semanal-errors.test b/test-data/unit/semanal-errors.test index 0dba8f2836bf..4328da9ab9c0 100644 --- a/test-data/unit/semanal-errors.test +++ b/test-data/unit/semanal-errors.test @@ -1401,3 +1401,21 @@ class A: ... # E: Name 'A' already defined on line 2 [builtins fixtures/list.pyi] [out] + +[case testNoInvalidTypeInDynamicFunctions] +from typing import AnyStr, Dict, TypeVar +T = TypeVar('T') + +def f(): # Note no annotation + x: Dict[str, AnyStr] = {} + y: T + z: x + def nested(): pass + t: nested + +def g() -> None: + x: Dict[str, AnyStr] = {} # E: Invalid type "typing.AnyStr" + +[builtins fixtures/dict.pyi] +[typing fixtures/typing-full.pyi] +[out] From 54d93310ccbd3b3271d3d0b4e63d0563192805c1 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 19 Sep 2017 23:35:19 +0200 Subject: [PATCH 23/39] Simplify test --- test-data/unit/fixtures/dict.pyi | 1 - test-data/unit/fixtures/typing-full.pyi | 2 -- test-data/unit/semanal-errors.test | 7 +++---- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/test-data/unit/fixtures/dict.pyi b/test-data/unit/fixtures/dict.pyi index 2c31f2a392cb..cf8b61f9397a 100644 --- a/test-data/unit/fixtures/dict.pyi +++ b/test-data/unit/fixtures/dict.pyi @@ -41,7 +41,6 @@ class tuple(Generic[T]): pass class function: pass class float: pass class bool: pass -class bytes: pass class ellipsis: pass def isinstance(x: object, t: Union[type, Tuple]) -> bool: pass diff --git a/test-data/unit/fixtures/typing-full.pyi b/test-data/unit/fixtures/typing-full.pyi index c4baadc5638d..62fac70034c0 100644 --- a/test-data/unit/fixtures/typing-full.pyi +++ b/test-data/unit/fixtures/typing-full.pyi @@ -33,8 +33,6 @@ List = 0 Dict = 0 Set = 0 -AnyStr = TypeVar('AnyStr', str, bytes) - T = TypeVar('T') T_co = TypeVar('T_co', covariant=True) T_contra = TypeVar('T_contra', contravariant=True) diff --git a/test-data/unit/semanal-errors.test b/test-data/unit/semanal-errors.test index 4328da9ab9c0..ccd13f1b3028 100644 --- a/test-data/unit/semanal-errors.test +++ b/test-data/unit/semanal-errors.test @@ -1403,19 +1403,18 @@ class A: ... # E: Name 'A' already defined on line 2 [out] [case testNoInvalidTypeInDynamicFunctions] -from typing import AnyStr, Dict, TypeVar +from typing import Dict, TypeVar T = TypeVar('T') def f(): # Note no annotation - x: Dict[str, AnyStr] = {} + x: Dict[str, T] = {} y: T z: x def nested(): pass t: nested def g() -> None: - x: Dict[str, AnyStr] = {} # E: Invalid type "typing.AnyStr" + x: Dict[str, T] = {} # E: Invalid type "__main__.T" [builtins fixtures/dict.pyi] -[typing fixtures/typing-full.pyi] [out] From b9ddacce06548dceac3d2c514a9b023d792ef0e6 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 19 Sep 2017 23:49:45 +0200 Subject: [PATCH 24/39] Fix self-check --- mypy/semanal.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index c173a37ddf9d..729d3ca9a4e0 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1633,7 +1633,7 @@ def type_analyzer(self, *, allow_tuple_literal=allow_tuple_literal, allow_unnormalized=self.is_stub_file, third_pass=third_pass) - tpan.in_dynamic_func = self.function_stack and self.function_stack[-1].is_dynamic() + tpan.in_dynamic_func = bool(self.function_stack and self.function_stack[-1].is_dynamic()) return tpan def anal_type(self, t: Type, *, @@ -1732,7 +1732,7 @@ def analyze_alias(self, rvalue: Expression, qualified type variable names for generic aliases. If 'allow_unnormalized' is True, allow types like builtins.list[T]. """ - dynamic = self.function_stack and self.function_stack[-1].is_dynamic() + dynamic = bool(self.function_stack and self.function_stack[-1].is_dynamic()) res = analyze_type_alias(rvalue, self.lookup_qualified, self.lookup_fully_qualified, From 5bfe9ca01b7fa034b2f8ede94218979b9b49fd6c Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 20 Sep 2017 00:51:40 +0200 Subject: [PATCH 25/39] Fix cross-file forward references (+test) --- mypy/semanal.py | 4 +++- test-data/unit/check-namedtuple.test | 20 +++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 729d3ca9a4e0..b1d4f2b5f6fc 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -4074,6 +4074,7 @@ def visit_file(self, file_node: MypyFile, fnam: str, options: Options, self.sem.options = options self.patches = patches self.is_typeshed_file = self.errors.is_typeshed_file(fnam) + self.sem.globals = file_node.names with experiments.strict_optional_set(options.strict_optional): self.accept(file_node) @@ -4752,7 +4753,8 @@ class A(NamedTuple): """ info = t.type # Special case, analyzed bases transformed the type into TupleType. - if info.tuple_type and not self.seen: + if ((info.tuple_type or info.typeddict_type) and + all(not isinstance(s, (TupleType, TypedDictType)) for s in self.seen)): return info.tuple_type.copy_modified(fallback=Instance(info, [])) # Update forward Instance's to corresponding analyzed NamedTuple's. if info.replaced and info.replaced.tuple_type: diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 72961322fd99..793a78ea03a1 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -453,9 +453,27 @@ def f(x: a.X) -> None: x = a.X(1) reveal_type(x) [out] -tmp/b.py:6: error: Revealed type is 'a.X' +tmp/b.py:6: error: Revealed type is 'Tuple[Any, fallback=a.X]' tmp/b.py:8: error: Revealed type is 'Tuple[Any, fallback=a.X]' +[case testNamedTupleWithImportCycle2] +import a +[file a.py] +from collections import namedtuple +from b import f + +N = namedtuple('N', 'a') +[file b.py] +import a + +def f(x: a.N) -> None: + reveal_type(x) + x = a.N(1) + reveal_type(x) +[out] +tmp/b.py:4: error: Revealed type is 'Tuple[Any, fallback=a.N]' +tmp/b.py:6: error: Revealed type is 'Tuple[Any, fallback=a.N]' + [case testSimpleSelfReferrentialNamedTuple] from typing import NamedTuple class MyNamedTuple(NamedTuple): From a2912e910f0cb7c3ce16508cd9294870e253843e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 20 Sep 2017 01:50:41 +0200 Subject: [PATCH 26/39] More tests --- test-data/unit/check-classes.test | 13 +++++++++++++ test-data/unit/check-namedtuple.test | 21 +++++++++++++++++++++ test-data/unit/check-type-aliases.test | 2 +- test-data/unit/check-typeddict.test | 18 ++++++++++++++++++ 4 files changed, 53 insertions(+), 1 deletion(-) diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index c4734733d113..46ceda03e5ca 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -3720,6 +3720,19 @@ class B(object): [builtins fixtures/property.pyi] [out] +[case testCorrectIsinstanceWithForwardUnion] +from typing import Union, NamedTuple + +ForwardReferenceUnion = Union['SomeTuple', int] +class SomeTuple(NamedTuple('SomeTuple', [('elem', int)])): pass + +def f(x: ForwardReferenceUnion) -> None: + reveal_type(x) # E: Revealed type is 'Union[Tuple[builtins.int, fallback=__main__.SomeTuple], builtins.int]' + if isinstance(x, SomeTuple): + reveal_type(x) # E: Revealed type is 'Tuple[builtins.int, fallback=__main__.SomeTuple]' +[builtins fixtures/isinstance.pyi] +[out] + [case testCrashInvalidArgsSyntheticClassSyntax] from typing import List, NamedTuple from mypy_extensions import TypedDict diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 793a78ea03a1..687b60f84a56 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -491,6 +491,27 @@ main:9: error: Revealed type is 'Tuple[Any, fallback=__main__.MyNamedTuple]' -- Some crazy selef-referential named tuples and types dicts -- to be sure that everything works +[case testCrossFileNamedTupleForwardRefs] +import a +[file a.py] +import b +from typing import Any, NamedTuple + +class A: + def a(self, b: 'b.B') -> str: + return 'a' +ATuple = NamedTuple('ATuple', [('a', Any)]) + +[file b.py] +import a + +class B: + def b(self, a: 'a.A') -> str: + return 'b' + def aWithTuple(self, atuple: 'a.ATuple') -> str: + return 'a' +[out] + [case testSelfRefNT1] from typing import Tuple, NamedTuple diff --git a/test-data/unit/check-type-aliases.test b/test-data/unit/check-type-aliases.test index 4bb2f4726dd3..d076f5d82435 100644 --- a/test-data/unit/check-type-aliases.test +++ b/test-data/unit/check-type-aliases.test @@ -180,4 +180,4 @@ reveal_type(x) # E: Revealed type is 'Union[builtins.int, builtins.str, builtins if isinstance(x, list): reveal_type(x) # E: Revealed type is 'builtins.list[Any]' [builtins fixtures/isinstancelist.pyi] -[out] \ No newline at end of file +[out] diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index a28d1fadac65..a928f064da07 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -1325,3 +1325,21 @@ m: Movie reveal_type(m['director']['name']) # E: Revealed type is 'builtins.str' [builtins fixtures/dict.pyi] [out] + +[case testTypedDictWithImportCycleForward] +import a +[file a.py] +from mypy_extensions import TypedDict +from b import f + +N = TypedDict('N', {'a': str}) +[file b.py] +import a + +def f(x: a.N) -> None: + reveal_type(x) + reveal_type(x['a']) +[builtins fixtures/dict.pyi] +[out] +tmp/b.py:4: error: Revealed type is 'TypedDict('a.N', {'a': builtins.str})' +tmp/b.py:5: error: Revealed type is 'builtins.str' From 21dfbfe5ee53ac8ad15fa5bf13cb77ba928e7e58 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 20 Sep 2017 11:02:18 +0200 Subject: [PATCH 27/39] Fix situation when recursive namedtuple appears directly in base class list --- mypy/semanal.py | 30 ++++++++++++++++++---------- test-data/unit/check-namedtuple.test | 19 ++++++++++++++++++ 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index b1d4f2b5f6fc..f0ec84cc22fe 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1046,6 +1046,10 @@ def analyze_base_classes(self, defn: ClassDef) -> None: defn.has_incompatible_baseclass = True info.tuple_type = base base_types.append(base.fallback) + if isinstance(base_expr, CallExpr): + defn.analyzed = NamedTupleExpr(base.fallback.type) + defn.analyzed.line = defn.line + defn.analyzed.column = defn.column elif isinstance(base, Instance): if base.type.is_newtype: self.fail("Cannot subclass NewType", defn) @@ -4198,12 +4202,6 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: NewType, TypedDict, and NamedTuple. """ self.analyze(s.type, s) - if isinstance(s.lvalues[0], RefExpr) and isinstance(s.lvalues[0].node, Var): - self.analyze(s.lvalues[0].node.type, s.lvalues[0].node) - if isinstance(s.lvalues[0], NameExpr): - node = self.sem.lookup(s.lvalues[0].name, s, suppress_errors=True) - if node: - self.analyze(node.type_override, node) if isinstance(s.rvalue, IndexExpr) and isinstance(s.rvalue.analyzed, TypeAliasExpr): self.analyze(s.rvalue.analyzed.type, s.rvalue.analyzed, warn=True) if isinstance(s.rvalue, CallExpr): @@ -4220,6 +4218,12 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: self.accept(sym.node) if isinstance(sym.node, Var): self.analyze(sym.node.type, sym.node) + if isinstance(s.lvalues[0], RefExpr) and isinstance(s.lvalues[0].node, Var): + self.analyze(s.lvalues[0].node.type, s.lvalues[0].node) + if isinstance(s.lvalues[0], NameExpr): + node = self.sem.lookup(s.lvalues[0].name, s, suppress_errors=True) + if node: + self.analyze(node.type_override, node) super().visit_assignment_stmt(s) def visit_cast_expr(self, e: CastExpr) -> None: @@ -4743,7 +4747,7 @@ def visit_forwardref_type(self, t: ForwardRef) -> Type: """ return t.link.accept(self) - def visit_instance(self, t: Instance) -> Type: + def visit_instance(self, t: Instance, from_fallback: bool = False) -> Type: """This visitor method tracks situations like this: x: A # when analyzing this type we will get an Instance from FirstPass @@ -4753,9 +4757,10 @@ class A(NamedTuple): """ info = t.type # Special case, analyzed bases transformed the type into TupleType. - if ((info.tuple_type or info.typeddict_type) and - all(not isinstance(s, (TupleType, TypedDictType)) for s in self.seen)): - return info.tuple_type.copy_modified(fallback=Instance(info, [])) + if info.tuple_type and not from_fallback: + items = [it.accept(self) for it in info.tuple_type.items] + info.tuple_type.items = items + return TupleType(items, Instance(info, [])) # Update forward Instance's to corresponding analyzed NamedTuple's. if info.replaced and info.replaced.tuple_type: tp = info.replaced.tuple_type @@ -4808,7 +4813,10 @@ def visit_overloaded(self, t: Overloaded) -> Type: def visit_tuple_type(self, t: TupleType) -> Type: if self.check(t): return AnyType(TypeOfAny.from_error) - return super().visit_tuple_type(t) + items = [it.accept(self) for it in t.items] + fallback = self.visit_instance(t.fallback, from_fallback=True) + assert isinstance(fallback, Instance) + return TupleType(items, fallback) def visit_typeddict_type(self, t: TypedDictType) -> Type: if self.check(t): diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 687b60f84a56..653804483ca9 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -613,6 +613,25 @@ reveal_type(n) # E: Revealed type is 'Tuple[builtins.str, Tuple[Tuple[builtins.s reveal_type(f) # E: Revealed type is 'def (m: Tuple[Tuple[builtins.str, Tuple[Any, builtins.int, fallback=__main__.Node2], fallback=__main__.Node], builtins.int, fallback=__main__.Node2])' [builtins fixtures/tuple.pyi] +[case testRecursiveNamedTupleInBases] +from typing import List, NamedTuple, Union + +Exp = Union['App', 'Lit'] # E: Recursive types not fully supported yet, nested types replaced with "Any" +class App(NamedTuple('App', [('subs', List[Exp])])): pass +class Lit(NamedTuple('Lit', [('val', object)])): pass + +def my_eval(exp: Exp) -> int: + reveal_type(exp) # E: Revealed type is 'Union[Tuple[builtins.list[Any], fallback=__main__.App], Tuple[builtins.object, fallback=__main__.Lit]]' + if isinstance(exp, App): + my_eval(exp[0][0]) + return my_eval(exp.subs[0]) + if isinstance(exp, Lit): + return exp.val # E: Incompatible return value type (got "object", expected "int") + +my_eval(App([Lit(1), Lit(2)])) # OK +[builtins fixtures/isinstancelist.pyi] +[out] + [case testForwardReferenceInNamedTuple] from typing import NamedTuple From 03597ee257b003f6d519c3a91427d71f81245c62 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 20 Sep 2017 11:18:53 +0200 Subject: [PATCH 28/39] Clean-up PR: Remove unnecesary imports, outdated comment, unnecessary formatting changes --- mypy/sametypes.py | 3 ++- mypy/subtypes.py | 2 +- test-data/unit/check-namedtuple.test | 4 +--- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/mypy/sametypes.py b/mypy/sametypes.py index 256759862776..cba80e1ef825 100644 --- a/mypy/sametypes.py +++ b/mypy/sametypes.py @@ -3,12 +3,13 @@ from mypy.types import ( Type, UnboundType, AnyType, NoneTyp, TupleType, TypedDictType, UnionType, CallableType, TypeVarType, Instance, TypeVisitor, ErasedType, - TypeList, Overloaded, PartialType, DeletedType, UninhabitedType, TypeType, ForwardRef + TypeList, Overloaded, PartialType, DeletedType, UninhabitedType, TypeType ) def is_same_type(left: Type, right: Type) -> bool: """Is 'left' the same type as 'right'?""" + if isinstance(right, UnboundType): # Make unbound types same as anything else to reduce the number of # generated spurious error messages. diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 5e6c47fd27a0..049da8a07228 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -5,7 +5,7 @@ Type, AnyType, UnboundType, TypeVisitor, FormalArgument, NoneTyp, function_type, Instance, TypeVarType, CallableType, TupleType, TypedDictType, UnionType, Overloaded, ErasedType, TypeList, PartialType, DeletedType, UninhabitedType, TypeType, is_named_instance, - FunctionLike, TypeOfAny, ForwardRef + FunctionLike, TypeOfAny ) import mypy.applytype import mypy.constraints diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 653804483ca9..b356f1fa3689 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -447,14 +447,12 @@ class X(N): pass import a def f(x: a.X) -> None: - # The type of x is broken (https://github.com/python/mypy/issues/3016) but we need to - # do something reasonable here to avoid a regression. reveal_type(x) x = a.X(1) reveal_type(x) [out] +tmp/b.py:4: error: Revealed type is 'Tuple[Any, fallback=a.X]' tmp/b.py:6: error: Revealed type is 'Tuple[Any, fallback=a.X]' -tmp/b.py:8: error: Revealed type is 'Tuple[Any, fallback=a.X]' [case testNamedTupleWithImportCycle2] import a From 649ef32889cc1783a822233abdc2b8abd45cf360 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 20 Sep 2017 22:52:32 +0200 Subject: [PATCH 29/39] Add tests for generic classes, enums, with statements and for statements --- test-data/unit/check-enum.test | 18 ++++++++++++++++++ test-data/unit/check-namedtuple.test | 14 ++++++++++++++ test-data/unit/check-statements.test | 25 +++++++++++++++++++++++++ test-data/unit/check-typeddict.test | 16 ++++++++++++++++ 4 files changed, 73 insertions(+) diff --git a/test-data/unit/check-enum.test b/test-data/unit/check-enum.test index d6e7a65abfc4..30984ad07c46 100644 --- a/test-data/unit/check-enum.test +++ b/test-data/unit/check-enum.test @@ -360,6 +360,24 @@ x = y [out] main:8: error: Incompatible types in assignment (expression has type "__main__.B.E", variable has type "__main__.A.E") +[case testEnumWorkWithForward] +from enum import Enum +a: E = E.x +class E(Enum): + x = 1 + y = 2 +[out] + +[case testEnumWorkWithForward2] +from enum import Enum +b: F +F = Enum('F', {'x': 1, 'y': 2}) + +def fn(x: F) -> None: + pass +fn(b) +[out] + [case testFunctionalEnum_python2] from enum import Enum Eu = Enum(u'Eu', u'a b') diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index b356f1fa3689..0462dc8bce28 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -434,6 +434,20 @@ def f() -> None: A = NamedTuple('A', [('x', int)]) A # E: Name 'A' is not defined +[case testNamedTupleForwardAsUpperBound] +from typing import NamedTuple, TypeVar, Generic +T = TypeVar('T', bound='M') +class G(Generic[T]): + x: T + +yb: G[int] +yg: G[M] +z: int = G[M]().x.x +z = G[M]().x[0] + +M = NamedTuple('M', [('x', int)]) +[out] + [case testNamedTupleWithImportCycle] import a [file a.py] diff --git a/test-data/unit/check-statements.test b/test-data/unit/check-statements.test index 5d45c15d128d..a87c66b7ed56 100644 --- a/test-data/unit/check-statements.test +++ b/test-data/unit/check-statements.test @@ -1565,3 +1565,28 @@ reveal_type(d['weight0']) # E: Revealed type is 'builtins.float*' [builtins fixtures/floatdict.pyi] +[case testForwardRefsInForStatement] +from typing import List, NamedTuple +lst: List[N] + +for i in lst: # type: N + a: int = i.x + b: str = i[0] + +N = NamedTuple('N', [('x', int)]) +[builtins fixtures/list.pyi] +[out] + +[case testForwardRefsInWithStatement] +from typing import ContextManager +from mypy_extensions import TypedDict +cm: ContextManager[N] + +with cm as g: # type: N + a: str = g['x'] + +N = TypedDict('N', {'x': int}) +[builtins fixtures/dict.pyi] +[typing fixtures/typing-full.pyi] +[out] + diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index a928f064da07..6b1afe54a020 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -1326,6 +1326,22 @@ reveal_type(m['director']['name']) # E: Revealed type is 'builtins.str' [builtins fixtures/dict.pyi] [out] +[case testTypedDictForwardAsUpperBound] +from typing import TypeVar, Generic +from mypy_extensions import TypedDict +T = TypeVar('T', bound='M') +class G(Generic[T]): + x: T + +yb: G[int] +yg: G[M] +z: int = G().x['x'] + +class M(TypedDict): + x: int +[builtins fixtures/dict.pyi] +[out] + [case testTypedDictWithImportCycleForward] import a [file a.py] From 83f890771d5bc86af4d18f0016685f10c22c8d8a Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 20 Sep 2017 23:44:18 +0200 Subject: [PATCH 30/39] Add processing for for and with statements (+more tests) --- mypy/semanal.py | 25 +++++++++++++++++++ test-data/unit/check-statements.test | 36 ++++++++++++++++++++++++---- 2 files changed, 56 insertions(+), 5 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index a33b4d64cd0c..50196ab991f5 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -4230,6 +4230,14 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: self.analyze(node.type_override, node) super().visit_assignment_stmt(s) + def visit_for_stmt(self, s: ForStmt) -> None: + self.analyze(s.index_type, s) + super().visit_for_stmt(s) + + def visit_with_stmt(self, s: WithStmt) -> None: + self.analyze(s.target_type, s) + super().visit_with_stmt(s) + def visit_cast_expr(self, e: CastExpr) -> None: self.analyze(e.type, e) super().visit_cast_expr(e) @@ -4247,6 +4255,14 @@ def visit_type_application(self, e: TypeApplication) -> None: def perform_transform(self, node: Union[Node, SymbolTableNode], transform: Callable[[Type], Type]) -> None: """Apply transform to all types associated with node.""" + if isinstance(node, ForStmt): + node.index_type = transform(node.index_type) + self.transform_types(node.index, transform) + if isinstance(node, WithStmt): + node.target_type = transform(node.target_type) + for n in node.target: + if isinstance(n, NameExpr) and isinstance(n.node, Var) and n.node.type: + n.node.type = transform(n.node.type) if isinstance(node, (FuncDef, CastExpr, AssignmentStmt, TypeAliasExpr, Var)): node.type = transform(node.type) if isinstance(node, NewTypeExpr): @@ -4275,6 +4291,15 @@ def perform_transform(self, node: Union[Node, SymbolTableNode], new_bases.append(alt_base) node.bases = new_bases + def transform_types(self, lvalue: Lvalue, transform: Callable[[Type], Type]) -> None: + if isinstance(lvalue, RefExpr): + if isinstance(lvalue.node, Var): + var = lvalue.node + var.type = transform(var.type) + elif isinstance(lvalue, TupleExpr): + for item in lvalue.items: + self.transform_types(item, transform) + def analyze(self, type: Optional[Type], node: Union[Node, SymbolTableNode], warn: bool = False) -> None: # Recursive type warnings are only emitted on type definition 'node's, marked by 'warn' diff --git a/test-data/unit/check-statements.test b/test-data/unit/check-statements.test index a87c66b7ed56..6d8f6a4b03d2 100644 --- a/test-data/unit/check-statements.test +++ b/test-data/unit/check-statements.test @@ -1565,25 +1565,51 @@ reveal_type(d['weight0']) # E: Revealed type is 'builtins.float*' [builtins fixtures/floatdict.pyi] -[case testForwardRefsInForStatement] +[case testForwardRefsInForStatementImplicit] from typing import List, NamedTuple lst: List[N] +for i in lst: + a: int = i.x + b: str = i[0] # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +N = NamedTuple('N', [('x', int)]) +[builtins fixtures/list.pyi] +[out] + +[case testForwardRefsInForStatement] +from typing import List, NamedTuple +lst: List[M] + for i in lst: # type: N a: int = i.x - b: str = i[0] + b: str = i[0] # E: Incompatible types in assignment (expression has type "int", variable has type "str") N = NamedTuple('N', [('x', int)]) +class M(N): pass [builtins fixtures/list.pyi] [out] -[case testForwardRefsInWithStatement] -from typing import ContextManager +[case testForwardRefsInWithStatementImplicit] +from typing import ContextManager, Any from mypy_extensions import TypedDict cm: ContextManager[N] +with cm as g: + a: int = g['x'] + +N = TypedDict('N', {'x': int}) +[builtins fixtures/dict.pyi] +[typing fixtures/typing-full.pyi] +[out] + +[case testForwardRefsInWithStatement] +from typing import ContextManager, Any +from mypy_extensions import TypedDict +cm: ContextManager[Any] + with cm as g: # type: N - a: str = g['x'] + a: str = g['x'] # E: Incompatible types in assignment (expression has type "int", variable has type "str") N = TypedDict('N', {'x': int}) [builtins fixtures/dict.pyi] From 13c7176a332a1887fb68a924ac93c6752876ecee Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 21 Sep 2017 01:01:14 +0200 Subject: [PATCH 31/39] Add support for generic types with forward references --- mypy/semanal.py | 40 +++++++++++++++++++++++++++- mypy/typeanal.py | 21 +++++++++++++-- test-data/unit/check-namedtuple.test | 2 +- test-data/unit/check-typeddict.test | 4 +-- 4 files changed, 61 insertions(+), 6 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 50196ab991f5..89f0eed0d61e 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -4120,8 +4120,14 @@ def visit_class_def(self, tdef: ClassDef) -> None: # NamedTuple base classes are validated in check_namedtuple_classdef; we don't have to # check them again here. if not tdef.info.is_named_tuple: + types = list(tdef.info.bases) # type: List[Type] + for tvar in tdef.type_vars: + if tvar.upper_bound: + types.append(tvar.upper_bound) + if tvar.values: + types.extend(tvar.values) + self.analyze_types(types, tdef.info) for type in tdef.info.bases: - self.analyze(type, tdef.info) if tdef.info.is_protocol: if not isinstance(type, Instance) or not type.type.is_protocol: if type.type.fullname() != 'builtins.object': @@ -4212,6 +4218,13 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: analyzed = s.rvalue.analyzed if isinstance(analyzed, NewTypeExpr): self.analyze(analyzed.old_type, analyzed) + if isinstance(analyzed, TypeVarExpr): + types = [] + if analyzed.upper_bound: + types.append(analyzed.upper_bound) + if analyzed.values: + types.extend(analyzed.values) + self.analyze_types(types, analyzed) if isinstance(analyzed, TypedDictExpr): self.analyze(analyzed.info.typeddict_type, analyzed, warn=True) if isinstance(analyzed, NamedTupleExpr): @@ -4267,6 +4280,11 @@ def perform_transform(self, node: Union[Node, SymbolTableNode], node.type = transform(node.type) if isinstance(node, NewTypeExpr): node.old_type = transform(node.old_type) + if isinstance(node, TypeVarExpr): + if node.upper_bound: + node.upper_bound = transform(node.upper_bound) + if node.values: + node.values = [transform(v) for v in node.values] if isinstance(node, TypedDictExpr): node.info.typeddict_type = cast(TypedDictType, transform(node.info.typeddict_type)) @@ -4278,6 +4296,11 @@ def perform_transform(self, node: Union[Node, SymbolTableNode], if isinstance(node, SymbolTableNode): node.type_override = transform(node.type_override) if isinstance(node, TypeInfo): + for tvar in node.defn.type_vars: + if tvar.upper_bound: + tvar.upper_bound = transform(tvar.upper_bound) + if tvar.values: + tvar.values = [transform(v) for v in tvar.values] new_bases = [] for base in node.bases: new_base = transform(base) @@ -4317,6 +4340,21 @@ def patch() -> None: node, warn))) self.patches.append(patch) + def analyze_types(self, types: List[Type], node: Node) -> None: + # Similar to above but for nodes with multiple types. + indicator = {} # type: Dict[str, bool] + for type in types: + analyzer = TypeAnalyserPass3(self.fail, self.options, self.is_typeshed_file, + self.sem, indicator) + type.accept(analyzer) + self.check_for_omitted_generics(type) + if indicator.get('forward') or indicator.get('synthetic'): + def patch() -> None: + self.perform_transform(node, + lambda tp: tp.accept(TypeReplacer(self.fail, + node, warn=False))) + self.patches.append(patch) + def check_for_omitted_generics(self, typ: Type) -> None: if 'generics' not in self.options.disallow_any or self.is_typeshed_file: return diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 13993abc1efc..055bb6c0dc3f 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -660,6 +660,7 @@ def visit_instance(self, t: Instance) -> None: t.invalid = True elif info.defn.type_vars: # Check type argument values. + # TODO: Calling is_subtype and is_same_types in semantic analysis is a bad idea for (i, arg), tvar in zip(enumerate(t.args), info.defn.type_vars): if tvar.values: if isinstance(arg, TypeVarType): @@ -672,12 +673,28 @@ def visit_instance(self, t: Instance) -> None: else: arg_values = [arg] self.check_type_var_values(info, arg_values, tvar.name, tvar.values, i + 1, t) + # TODO: These hacks will be not necessary when this will be moved to later stage. if isinstance(arg, ForwardRef): arg = arg.link - if not is_subtype(arg, tvar.upper_bound): + bound = tvar.upper_bound + if isinstance(bound, ForwardRef): + bound = bound.link + if isinstance(bound, Instance) and bound.type.replaced: + replaced = bound.type.replaced + if replaced.tuple_type: + bound = replaced.tuple_type + if replaced.typeddict_type: + bound = replaced.typeddict_type + if isinstance(arg, Instance) and arg.type.replaced: + replaced = arg.type.replaced + if replaced.tuple_type: + arg = replaced.tuple_type + if replaced.typeddict_type: + arg = replaced.typeddict_type + if not is_subtype(arg, bound): self.fail('Type argument "{}" of "{}" must be ' 'a subtype of "{}"'.format( - arg, info.name(), tvar.upper_bound), t) + arg, info.name(), bound), t) for arg in t.args: arg.accept(self) if info.is_newtype: diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 0462dc8bce28..51ec99fb754c 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -440,7 +440,7 @@ T = TypeVar('T', bound='M') class G(Generic[T]): x: T -yb: G[int] +yb: G[int] # E: Type argument "builtins.int" of "G" must be a subtype of "Tuple[builtins.int, fallback=__main__.M]" yg: G[M] z: int = G[M]().x.x z = G[M]().x[0] diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index 6b1afe54a020..10823ce96883 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -1333,9 +1333,9 @@ T = TypeVar('T', bound='M') class G(Generic[T]): x: T -yb: G[int] +yb: G[int] # E: Type argument "builtins.int" of "G" must be a subtype of "TypedDict({'x': builtins.int}, fallback=typing.Mapping[builtins.str, builtins.object])" yg: G[M] -z: int = G().x['x'] +z: int = G[M]().x['x'] class M(TypedDict): x: int From 79b10d6cf47dc606bce3f928b42c435ed9c8a87f Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 22 Sep 2017 00:34:43 +0200 Subject: [PATCH 32/39] Prohibit forward refs to type vars and subscripted forward refs to aliases --- mypy/semanal.py | 7 +++++- mypy/typeanal.py | 31 +++++++++++++++++------ test-data/unit/check-type-aliases.test | 35 ++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 9 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 89f0eed0d61e..61e754e76138 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1630,6 +1630,7 @@ def type_analyzer(self, *, self.lookup_fully_qualified, tvar_scope, self.fail, + self.note, self.plugin, self.options, self.is_typeshed_stub_file, @@ -1638,6 +1639,7 @@ def type_analyzer(self, *, allow_unnormalized=self.is_stub_file, third_pass=third_pass) tpan.in_dynamic_func = bool(self.function_stack and self.function_stack[-1].is_dynamic()) + tpan.global_scope = not self.type and not self.function_stack return tpan def anal_type(self, t: Type, *, @@ -1737,16 +1739,19 @@ def analyze_alias(self, rvalue: Expression, If 'allow_unnormalized' is True, allow types like builtins.list[T]. """ dynamic = bool(self.function_stack and self.function_stack[-1].is_dynamic()) + global_scope = not self.type and not self.function_stack res = analyze_type_alias(rvalue, self.lookup_qualified, self.lookup_fully_qualified, self.tvar_scope, self.fail, + self.note, self.plugin, self.options, self.is_typeshed_stub_file, allow_unnormalized=True, - in_dynamic_func=dynamic) + in_dynamic_func=dynamic, + global_scope=global_scope) if res: alias_tvars = [name for (name, _) in res.accept(TypeVariableQuery(self.lookup_qualified, self.tvar_scope))] diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 055bb6c0dc3f..e7c315733951 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -20,7 +20,8 @@ from mypy.nodes import ( TVAR, TYPE_ALIAS, UNBOUND_IMPORTED, TypeInfo, Context, SymbolTableNode, Var, Expression, IndexExpr, RefExpr, nongen_builtins, check_arg_names, check_arg_kinds, ARG_POS, ARG_NAMED, - ARG_OPT, ARG_NAMED_OPT, ARG_STAR, ARG_STAR2, TypeVarExpr, FuncDef, CallExpr, NameExpr + ARG_OPT, ARG_NAMED_OPT, ARG_STAR, ARG_STAR2, TypeVarExpr, FuncDef, CallExpr, NameExpr, + Decorator ) from mypy.tvar_scope import TypeVarScope from mypy.sametypes import is_same_type @@ -59,11 +60,13 @@ def analyze_type_alias(node: Expression, lookup_fqn_func: Callable[[str], SymbolTableNode], tvar_scope: TypeVarScope, fail_func: Callable[[str, Context], None], + note_func: Callable[[str, Context], None], plugin: Plugin, options: Options, is_typeshed_stub: bool, allow_unnormalized: bool = False, - in_dynamic_func: bool = False) -> Optional[Type]: + in_dynamic_func: bool = False, + global_scope: bool = True) -> Optional[Type]: """Return type if node is valid as a type alias rvalue. Return None otherwise. 'node' must have been semantically analyzed. @@ -115,9 +118,11 @@ def analyze_type_alias(node: Expression, except TypeTranslationError: fail_func('Invalid type alias', node) return None - analyzer = TypeAnalyser(lookup_func, lookup_fqn_func, tvar_scope, fail_func, plugin, options, - is_typeshed_stub, aliasing=True, allow_unnormalized=allow_unnormalized) + analyzer = TypeAnalyser(lookup_func, lookup_fqn_func, tvar_scope, fail_func, note_func, + plugin, options, is_typeshed_stub, aliasing=True, + allow_unnormalized=allow_unnormalized) analyzer.in_dynamic_func = in_dynamic_func + analyzer.global_scope = global_scope return type.accept(analyzer) @@ -135,14 +140,17 @@ class TypeAnalyser(SyntheticTypeVisitor[Type], AnalyzerPluginInterface): Converts unbound types into bound types. """ - # Is this called from an untyped function definition + # Is this called from an untyped function definition? in_dynamic_func = False # type: bool + # Is this called from global scope? + global_scope = True # type: bool def __init__(self, lookup_func: Callable[[str, Context], SymbolTableNode], lookup_fqn_func: Callable[[str], SymbolTableNode], tvar_scope: TypeVarScope, fail_func: Callable[[str, Context], None], + note_func: Callable[[str, Context], None], plugin: Plugin, options: Options, is_typeshed_stub: bool, *, @@ -153,6 +161,7 @@ def __init__(self, self.lookup = lookup_func self.lookup_fqn_func = lookup_fqn_func self.fail_func = fail_func + self.note_func = note_func self.tvar_scope = tvar_scope self.aliasing = aliasing self.allow_tuple_literal = allow_tuple_literal @@ -280,10 +289,16 @@ def visit_unbound_type(self, t: UnboundType) -> Type: if not (self.aliasing and sym.kind == TVAR and self.tvar_scope.get_binding(sym) is None): if (not self.third_pass and not self.in_dynamic_func and - not (isinstance(sym.node, FuncDef) or - isinstance(sym.node, Var) and sym.node.is_ready)): + not (isinstance(sym.node, (FuncDef, Decorator)) or + isinstance(sym.node, Var) and sym.node.is_ready) and + not (sym.kind == TVAR and tvar_def is None)): + if t.args and not self.global_scope: + self.fail('Unsupported forward reference to "{}"'.format(t.name), t) + return AnyType(TypeOfAny.from_error) return ForwardRef(t) self.fail('Invalid type "{}"'.format(name), t) + if self.third_pass and sym.kind == TVAR: + self.note_func("Forward references to type variables are prohibited", t) return t info = sym.node # type: TypeInfo if len(t.args) > 0 and info.fullname() == 'builtins.tuple': @@ -319,7 +334,7 @@ def visit_unbound_type(self, t: UnboundType) -> Type: return instance else: if self.third_pass: - self.fail('Invalid type {}'.format(t), t) + self.fail('Invalid type "{}"'.format(t.name), t) return AnyType(TypeOfAny.from_error) return AnyType(TypeOfAny.special_form) diff --git a/test-data/unit/check-type-aliases.test b/test-data/unit/check-type-aliases.test index 35484cf669c4..400326660b30 100644 --- a/test-data/unit/check-type-aliases.test +++ b/test-data/unit/check-type-aliases.test @@ -182,6 +182,41 @@ if isinstance(x, list): [builtins fixtures/isinstancelist.pyi] [out] +[case testProhibitedForwardRefToTypeVar] +from typing import TypeVar, List + +a: List[T] + +T = TypeVar('T') +[builtins fixtures/list.pyi] +[out] +main:3: error: Invalid type "__main__.T" +main:3: note: Forward references to type variables are prohibited + +[case testUnsupportedForwardRef] +from typing import List, TypeVar + +T = TypeVar('T') + +def f(x: T) -> None: + y: A[T] # E: Unsupported forward reference to "A" + +A = List[T] +[builtins fixtures/list.pyi] +[out] + +[case testUnsupportedForwardRef2] +from typing import List, TypeVar + +def f() -> None: + X = List[int] + x: A[X] # E: Unsupported forward reference to "A" + +T = TypeVar('T') +A = List[T] +[builtins fixtures/list.pyi] +[out] + [case testNoneAlias] from typing import Union void = type(None) From 321a809cc014693f0b1554820b12785d1c6bb513 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 22 Sep 2017 01:46:27 +0200 Subject: [PATCH 33/39] Refactor code to avoid passing semantic analyzer to type analyzer, only lookup functions --- mypy/semanal.py | 20 ++++++++++++++++---- mypy/typeanal.py | 49 +++++++++++++++++++++++++++++++++++++----------- 2 files changed, 54 insertions(+), 15 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 61e754e76138..724aae1d5e26 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -4334,8 +4334,14 @@ def analyze(self, type: Optional[Type], node: Union[Node, SymbolTableNode], # Flags appeared during analysis of 'type' are collected in this dict. indicator = {} # type: Dict[str, bool] if type: - analyzer = TypeAnalyserPass3(self.fail, self.options, self.is_typeshed_file, - self.sem, indicator) + analyzer = TypeAnalyserPass3(self.sem.lookup_qualified, + self.sem.lookup_fully_qualified, + self.fail, + self.sem.note, + self.sem.plugin, + self.options, + self.is_typeshed_file, + indicator) type.accept(analyzer) self.check_for_omitted_generics(type) if indicator.get('forward') or indicator.get('synthetic'): @@ -4349,8 +4355,14 @@ def analyze_types(self, types: List[Type], node: Node) -> None: # Similar to above but for nodes with multiple types. indicator = {} # type: Dict[str, bool] for type in types: - analyzer = TypeAnalyserPass3(self.fail, self.options, self.is_typeshed_file, - self.sem, indicator) + analyzer = TypeAnalyserPass3(self.sem.lookup_qualified, + self.sem.lookup_fully_qualified, + self.fail, + self.sem.note, + self.sem.plugin, + self.options, + self.is_typeshed_file, + indicator) type.accept(analyzer) self.check_for_omitted_generics(type) if indicator.get('forward') or indicator.get('synthetic'): diff --git a/mypy/typeanal.py b/mypy/typeanal.py index e7c315733951..cbad173a4980 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -30,9 +30,6 @@ from mypy.plugin import Plugin, AnalyzerPluginInterface, AnalyzeTypeContext from mypy import nodes, messages -MYPY = False -if MYPY: - from mypy.semanal import SemanticAnalyzer T = TypeVar('T') @@ -148,7 +145,7 @@ class TypeAnalyser(SyntheticTypeVisitor[Type], AnalyzerPluginInterface): def __init__(self, lookup_func: Callable[[str, Context], SymbolTableNode], lookup_fqn_func: Callable[[str], SymbolTableNode], - tvar_scope: TypeVarScope, + tvar_scope: Optional[TypeVarScope], fail_func: Callable[[str, Context], None], note_func: Callable[[str, Context], None], plugin: Plugin, @@ -193,7 +190,10 @@ def visit_unbound_type(self, t: UnboundType) -> Type: if (fullname in nongen_builtins and t.args and not sym.normalized and not self.allow_unnormalized): self.fail(no_subscript_builtin_alias(fullname), t) - tvar_def = self.tvar_scope.get_binding(sym) + if self.tvar_scope: + tvar_def = self.tvar_scope.get_binding(sym) + else: + tvar_def = None if sym.kind == TVAR and tvar_def is not None: if len(t.args) > 0: self.fail('Type variable "{}" used with arguments'.format( @@ -287,7 +287,7 @@ def visit_unbound_type(self, t: UnboundType) -> Type: return AnyType(TypeOfAny.from_unimported_type) # Allow unbound type variables when defining an alias if not (self.aliasing and sym.kind == TVAR and - self.tvar_scope.get_binding(sym) is None): + (not self.tvar_scope or self.tvar_scope.get_binding(sym) is None)): if (not self.third_pass and not self.in_dynamic_func and not (isinstance(sym.node, (FuncDef, Decorator)) or isinstance(sym.node, Var) and sym.node.is_ready) and @@ -511,13 +511,18 @@ def fail(self, msg: str, ctx: Context) -> None: @contextmanager def tvar_scope_frame(self) -> Iterator[None]: old_scope = self.tvar_scope - self.tvar_scope = self.tvar_scope.method_frame() + if self.tvar_scope: + self.tvar_scope = self.tvar_scope.method_frame() + else: + assert self.third_pass, "Internal error: type variable scope not given" yield self.tvar_scope = old_scope def infer_type_variables(self, type: CallableType) -> List[Tuple[str, TypeVarExpr]]: """Return list of unique type variables referred to in a callable.""" + if not self.tvar_scope: + return [] # We are in third pass, nothing new here names = [] # type: List[str] tvars = [] # type: List[TypeVarExpr] for arg in type.arg_types: @@ -539,6 +544,8 @@ def infer_type_variables(self, def bind_function_type_variables(self, fun_type: CallableType, defn: Context) -> List[TypeVarDef]: """Find the type variables of the function type and bind them in our tvar_scope""" + if not self.tvar_scope: + return [] # We are in third pass, nothing new here if fun_type.variables: for var in fun_type.variables: var_expr = self.lookup(var.name, var).node @@ -561,7 +568,8 @@ def bind_function_type_variables(self, return defs def is_defined_type_var(self, tvar: str, context: Context) -> bool: - return self.tvar_scope.get_binding(self.lookup(tvar, context)) is not None + return (self.tvar_scope is not None and + self.tvar_scope.get_binding(self.lookup(tvar, context)) is not None) def anal_array(self, a: List[Type], nested: bool = True) -> List[Type]: res = [] # type: List[Type] @@ -623,14 +631,21 @@ class TypeAnalyserPass3(TypeVisitor[None]): """ def __init__(self, + lookup_func: Callable[[str, Context], SymbolTableNode], + lookup_fqn_func: Callable[[str], SymbolTableNode], fail_func: Callable[[str, Context], None], + note_func: Callable[[str, Context], None], + plugin: Plugin, options: Options, is_typeshed_stub: bool, - sem: 'SemanticAnalyzer', indicator: Dict[str, bool]) -> None: + indicator: Dict[str, bool]) -> None: + self.lookup_func = lookup_func + self.lookup_fqn_func = lookup_fqn_func self.fail = fail_func + self.note_func = note_func self.options = options + self.plugin = plugin self.is_typeshed_stub = is_typeshed_stub - self.sem = sem self.indicator = indicator def visit_instance(self, t: Instance) -> None: @@ -786,7 +801,19 @@ def visit_type_type(self, t: TypeType) -> None: def visit_forwardref_type(self, t: ForwardRef) -> None: self.indicator['forward'] = True if isinstance(t.link, UnboundType): - t.link = self.sem.anal_type(t.link, third_pass=True) + t.link = self.anal_type(t.link) + + def anal_type(self, tp: UnboundType) -> Type: + tpan = TypeAnalyser(self.lookup_func, + self.lookup_fqn_func, + None, + self.fail, + self.note_func, + self.plugin, + self.options, + self.is_typeshed_stub, + third_pass=True) + return tp.accept(tpan) TypeVarList = List[Tuple[str, TypeVarExpr]] From 076c909ae37f330a14ac82243295a77a028459fa Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 23 Sep 2017 00:47:08 +0200 Subject: [PATCH 34/39] Address the rest of the review comments --- mypy/semanal.py | 87 +++++++++++++++++-------------- mypy/types.py | 19 ++++++- test-data/unit/check-classes.test | 21 +++++--- 3 files changed, 80 insertions(+), 47 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 724aae1d5e26..0ee9f03c999c 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -4212,9 +4212,11 @@ def visit_decorator(self, dec: Decorator) -> None: dec.var.type = sig def visit_assignment_stmt(self, s: AssignmentStmt) -> None: - """Traverse the actual assignment statement and synthetic types + """Traverse the assignment statement. + + This includes the actual assignment and synthetic types resulted from this assignment (if any). Currently this includes - NewType, TypedDict, and NamedTuple. + NewType, TypedDict, NamedTuple, and TypeVar. """ self.analyze(s.type, s) if isinstance(s.rvalue, IndexExpr) and isinstance(s.rvalue.analyzed, TypeAliasExpr): @@ -4240,6 +4242,9 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: self.accept(sym.node) if isinstance(sym.node, Var): self.analyze(sym.node.type, sym.node) + # We need to pay additional attention to assignments that define a type alias. + # The resulting type is also stored in the 'type_override' attribute of + # the corresponding SymbolTableNode. if isinstance(s.lvalues[0], RefExpr) and isinstance(s.lvalues[0].node, Var): self.analyze(s.lvalues[0].node.type, s.lvalues[0].node) if isinstance(s.lvalues[0], NameExpr): @@ -4347,8 +4352,8 @@ def analyze(self, type: Optional[Type], node: Union[Node, SymbolTableNode], if indicator.get('forward') or indicator.get('synthetic'): def patch() -> None: self.perform_transform(node, - lambda tp: tp.accept(TypeReplacer(self.fail, - node, warn))) + lambda tp: tp.accept(ForwardReferenceResolver(self.fail, + node, warn))) self.patches.append(patch) def analyze_types(self, types: List[Type], node: Node) -> None: @@ -4368,8 +4373,8 @@ def analyze_types(self, types: List[Type], node: Node) -> None: if indicator.get('forward') or indicator.get('synthetic'): def patch() -> None: self.perform_transform(node, - lambda tp: tp.accept(TypeReplacer(self.fail, - node, warn=False))) + lambda tp: tp.accept(ForwardReferenceResolver(self.fail, + node, warn=False))) self.patches.append(patch) def check_for_omitted_generics(self, typ: Type) -> None: @@ -4798,9 +4803,17 @@ def visit_any(self, t: AnyType) -> Type: return t -class TypeReplacer(TypeTranslator): - """This is very similar TypeTranslator but tracks visited nodes to avoid +class ForwardReferenceResolver(TypeTranslator): + """Visitor to replace previously detected forward reference to synthetic types. + + This is similar to TypeTranslator but tracks visited nodes to avoid infinite recursion on potentially circular (self- or mutually-referential) types. + This visitor: + * Fixes forward references by unwrapping the linked type. + * Generates errors for unsupported type recursion and breaks recursion by resolving + recursive back references to Any types. + * Replaces instance types generated from unanalyzed NamedTuple and TypedDict class syntax + found in first pass with analyzed TupleType and TypedDictType. """ def __init__(self, fail: Callable[[str, Context], None], start: Union[Node, SymbolTableNode], warn: bool) -> None: @@ -4809,15 +4822,12 @@ def __init__(self, fail: Callable[[str, Context], None], self.start = start self.warn = warn - def warn_recursion(self) -> None: - if self.warn: - assert isinstance(self.start, Node), "Internal error: invalid error context" - self.fail('Recursive types not fully supported yet,' - ' nested types replaced with "Any"', self.start) - - def check(self, t: Type) -> bool: + def check_recursion(self, t: Type) -> bool: if any(t is s for s in self.seen): - self.warn_recursion() + if self.warn: + assert isinstance(self.start, Node), "Internal error: invalid error context" + self.fail('Recursive types not fully supported yet,' + ' nested types replaced with "Any"', self.start) return True self.seen.append(t) return False @@ -4825,8 +4835,8 @@ def check(self, t: Type) -> bool: def visit_forwardref_type(self, t: ForwardRef) -> Type: """This visitor method tracks situations like this: - x: A # this type is not yet known and therefore wrapped in ForwardRef - # it's content is updated in ThirdPass, now we need to unwrap this type. + x: A # This type is not yet known and therefore wrapped in ForwardRef, + # its content is updated in ThirdPass, now we need to unwrap this type. A = NewType('A', int) """ return t.link.accept(self) @@ -4834,10 +4844,13 @@ def visit_forwardref_type(self, t: ForwardRef) -> Type: def visit_instance(self, t: Instance, from_fallback: bool = False) -> Type: """This visitor method tracks situations like this: - x: A # when analyzing this type we will get an Instance from FirstPass - # now we need to update this to actual analyzed TupleType. + x: A # When analyzing this type we will get an Instance from FirstPass. + # Now we need to update this to actual analyzed TupleType. class A(NamedTuple): attr: str + + If from_fallback is True, then we always return an Instance type. This is needed + since TupleType and TypedDictType fallbacks are always instances. """ info = t.type # Special case, analyzed bases transformed the type into TupleType. @@ -4845,30 +4858,28 @@ class A(NamedTuple): items = [it.accept(self) for it in info.tuple_type.items] info.tuple_type.items = items return TupleType(items, Instance(info, [])) - # Update forward Instance's to corresponding analyzed NamedTuple's. + # Update forward Instances to corresponding analyzed NamedTuples. if info.replaced and info.replaced.tuple_type: tp = info.replaced.tuple_type - if any((s is tp) or (s is t) for s in self.seen): - self.warn_recursion() - # The key idea is that when we return to the place where - # we already was, we break the cycle and put AnyType as a leaf. + if self.check_recursion(tp): + # The key idea is that when we recursively return to a type already traversed, + # then we break the cycle and put AnyType as a leaf. return AnyType(TypeOfAny.from_error) - self.seen.append(t) return tp.copy_modified(fallback=Instance(info.replaced, [])).accept(self) - # Same as above but for TypedDict's. + # Same as above but for TypedDicts. if info.replaced and info.replaced.typeddict_type: td = info.replaced.typeddict_type - if any((s is td) or (s is t) for s in self.seen): - self.warn_recursion() + if self.check_recursion(td): + # We also break the cycles for TypedDicts as explained above for NamedTuples. return AnyType(TypeOfAny.from_error) - self.seen.append(t) return td.copy_modified(fallback=Instance(info.replaced, [])).accept(self) - if self.check(t): + if self.check_recursion(t): + # We also need to break a potential cycle with normal (non-synthetic) instance types. return Instance(t.type, [AnyType(TypeOfAny.from_error)] * len(t.type.defn.type_vars)) return super().visit_instance(t) def visit_type_var(self, t: TypeVarType) -> Type: - if self.check(t): + if self.check_recursion(t): return AnyType(TypeOfAny.from_error) if t.upper_bound: t.upper_bound = t.upper_bound.accept(self) @@ -4877,7 +4888,7 @@ def visit_type_var(self, t: TypeVarType) -> Type: return t def visit_callable_type(self, t: CallableType) -> Type: - if self.check(t): + if self.check_recursion(t): return AnyType(TypeOfAny.from_error) arg_types = [tp.accept(self) for tp in t.arg_types] ret_type = t.ret_type.accept(self) @@ -4890,12 +4901,12 @@ def visit_callable_type(self, t: CallableType) -> Type: return t.copy_modified(arg_types=arg_types, ret_type=ret_type, variables=variables) def visit_overloaded(self, t: Overloaded) -> Type: - if self.check(t): + if self.check_recursion(t): return AnyType(TypeOfAny.from_error) return super().visit_overloaded(t) def visit_tuple_type(self, t: TupleType) -> Type: - if self.check(t): + if self.check_recursion(t): return AnyType(TypeOfAny.from_error) items = [it.accept(self) for it in t.items] fallback = self.visit_instance(t.fallback, from_fallback=True) @@ -4903,16 +4914,16 @@ def visit_tuple_type(self, t: TupleType) -> Type: return TupleType(items, fallback) def visit_typeddict_type(self, t: TypedDictType) -> Type: - if self.check(t): + if self.check_recursion(t): return AnyType(TypeOfAny.from_error) return super().visit_typeddict_type(t) def visit_union_type(self, t: UnionType) -> Type: - if self.check(t): + if self.check_recursion(t): return AnyType(TypeOfAny.from_error) return super().visit_union_type(t) def visit_type_type(self, t: TypeType) -> Type: - if self.check(t): + if self.check_recursion(t): return AnyType(TypeOfAny.from_error) return super().visit_type_type(t) diff --git a/mypy/types.py b/mypy/types.py index c9484fcbe348..e1293bf96b58 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1376,8 +1376,21 @@ def deserialize(cls, data: JsonDict) -> Type: class ForwardRef(Type): - """Class to wrap forward references to other types.""" - link = None # type: Type # the wrapped type + """Class to wrap forward references to other types. + + This is used when a forward reference to an (unanalyzed) synthetic type is found, + for example: + + x: A + A = TypedDict('A', {'x': int}) + + To avoid false positives and crashes in such situations, we first wrap the second + occurrence of 'A' in ForwardRef. Then, the wrapped UnboundType is updated in the third + pass of semantic analysis and ultimately fixed in the patches after the third pass. + So that ForwardRefs are temporary and will be completely replaced with the linked types + or Any (to avoid cyclic references) before the type checking stage. + """ + link = None # type: Type # The wrapped type def __init__(self, link: Type) -> None: self.link = link @@ -1392,6 +1405,8 @@ def serialize(self): name = self.link.type.name() else: name = self.link.__class__.__name__ + # We should never get here since all forward references should be resolved + # and removed during semantic analysis. assert False, "Internal error: Unresolved forward reference to {}".format(name) diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 46ceda03e5ca..5512fd2de847 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -3611,6 +3611,10 @@ NameInfo = NewType('NameInfo', NameInfoBase) def parse_ast(name_dict: NameDict) -> None: if isinstance(name_dict[''], int): pass + x = name_dict[''] + reveal_type(x) # E: Revealed type is '__main__.NameInfo*' + x = NameInfo(NameInfoBase()) # OK + x = NameInfoBase() # E: Incompatible types in assignment (expression has type "NameInfoBase", variable has type "NameInfo") [builtins fixtures/isinstancelist.pyi] [out] @@ -3666,12 +3670,13 @@ from typing import Union, NamedTuple NodeType = Union['Foo', 'Bar'] class Foo(NamedTuple): - pass + x: int class Bar(NamedTuple): - pass + x: int -def foo(node: NodeType): +def foo(node: NodeType) -> int: x = node + return x.x [out] [case testCrashOnForwardUnionOfTypedDicts] @@ -3680,12 +3685,13 @@ from typing import Union NodeType = Union['Foo', 'Bar'] class Foo(TypedDict): - pass + x: int class Bar(TypedDict): - pass + x: int -def foo(node: NodeType): +def foo(node: NodeType) -> int: x = node + return x['x'] [builtins fixtures/isinstancelist.pyi] [out] @@ -3696,8 +3702,9 @@ NodeType = Union['Foo', 'Bar'] Foo = NewType('Foo', int) Bar = NewType('Bar', str) -def foo(node: NodeType): +def foo(node: NodeType) -> NodeType: x = node + return x [out] [case testCrashOnComplexCheckWithNamedTupleUnion] From c1a63eceeb633f59755268240c56d0d298d00877 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 23 Sep 2017 01:30:16 +0200 Subject: [PATCH 35/39] Improve two tests --- test-data/unit/check-classes.test | 2 +- test-data/unit/check-incremental.test | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 5512fd2de847..851f2d59f32d 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -3704,7 +3704,7 @@ Bar = NewType('Bar', str) def foo(node: NodeType) -> NodeType: x = node - return x + return Foo(1) [out] [case testCrashOnComplexCheckWithNamedTupleUnion] diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 3eb3c39e26da..dc1b2d0cad5a 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -2961,7 +2961,7 @@ class C(List['A']): pass A = List[int] -C()[0] +x: int = C()[0][0] [builtins fixtures/list.pyi] [out] From 97e6f47b1a16da25303525daddd995da4adc5429 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 23 Sep 2017 20:51:55 +0200 Subject: [PATCH 36/39] Add one more test as suggested in #3990 --- test-data/unit/check-incremental.test | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index dc1b2d0cad5a..f9aaf30a19e1 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -2952,6 +2952,19 @@ def foo(a: int, b: int) -> str: [out2] tmp/b.py:2: error: Incompatible return value type (got "int", expected "str") +[case testForwardNamedTupleToUnionWithOtherNamedTUple] +from typing import NamedTuple, Union + +class Person(NamedTuple): + name: Union[str, "NamePair"] + +class NamePair(NamedTuple): + first: str + last: str + +Person(name=NamePair(first="John", last="Doe")) +[out] + -- Some crazy selef-referential named tuples, types dicts, and aliases -- to be sure that everything can be _serialized_ (i.e. ForwardRef's are removed) From 8f52654aa9190cb7a265e806f1d20582f5b66ea6 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 27 Sep 2017 00:13:45 +0200 Subject: [PATCH 37/39] Address latest review comments --- mypy/semanal.py | 45 ++++++++++++++------------- mypy/typeanal.py | 35 ++++++++++----------- mypy/types.py | 2 +- test-data/unit/check-incremental.test | 12 +++++++ test-data/unit/check-namedtuple.test | 4 +-- test-data/unit/check-statements.test | 8 ++--- 6 files changed, 60 insertions(+), 46 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 0ee9f03c999c..b5005e7bac39 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -4280,7 +4280,7 @@ def perform_transform(self, node: Union[Node, SymbolTableNode], """Apply transform to all types associated with node.""" if isinstance(node, ForStmt): node.index_type = transform(node.index_type) - self.transform_types(node.index, transform) + self.transform_types_in_lvalue(node.index, transform) if isinstance(node, WithStmt): node.target_type = transform(node.target_type) for n in node.target: @@ -4324,14 +4324,15 @@ def perform_transform(self, node: Union[Node, SymbolTableNode], new_bases.append(alt_base) node.bases = new_bases - def transform_types(self, lvalue: Lvalue, transform: Callable[[Type], Type]) -> None: + def transform_types_in_lvalue(self, lvalue: Lvalue, + transform: Callable[[Type], Type]) -> None: if isinstance(lvalue, RefExpr): if isinstance(lvalue.node, Var): var = lvalue.node var.type = transform(var.type) elif isinstance(lvalue, TupleExpr): for item in lvalue.items: - self.transform_types(item, transform) + self.transform_types_in_lvalue(item, transform) def analyze(self, type: Optional[Type], node: Union[Node, SymbolTableNode], warn: bool = False) -> None: @@ -4339,14 +4340,7 @@ def analyze(self, type: Optional[Type], node: Union[Node, SymbolTableNode], # Flags appeared during analysis of 'type' are collected in this dict. indicator = {} # type: Dict[str, bool] if type: - analyzer = TypeAnalyserPass3(self.sem.lookup_qualified, - self.sem.lookup_fully_qualified, - self.fail, - self.sem.note, - self.sem.plugin, - self.options, - self.is_typeshed_file, - indicator) + analyzer = self.make_type_analyzer(indicator) type.accept(analyzer) self.check_for_omitted_generics(type) if indicator.get('forward') or indicator.get('synthetic'): @@ -4360,14 +4354,7 @@ def analyze_types(self, types: List[Type], node: Node) -> None: # Similar to above but for nodes with multiple types. indicator = {} # type: Dict[str, bool] for type in types: - analyzer = TypeAnalyserPass3(self.sem.lookup_qualified, - self.sem.lookup_fully_qualified, - self.fail, - self.sem.note, - self.sem.plugin, - self.options, - self.is_typeshed_file, - indicator) + analyzer = self.make_type_analyzer(indicator) type.accept(analyzer) self.check_for_omitted_generics(type) if indicator.get('forward') or indicator.get('synthetic'): @@ -4377,6 +4364,16 @@ def patch() -> None: node, warn=False))) self.patches.append(patch) + def make_type_analyzer(self, indicator: Dict[str, bool]) -> TypeAnalyserPass3: + return TypeAnalyserPass3(self.sem.lookup_qualified, + self.sem.lookup_fully_qualified, + self.fail, + self.sem.note, + self.sem.plugin, + self.options, + self.is_typeshed_file, + indicator) + def check_for_omitted_generics(self, typ: Type) -> None: if 'generics' not in self.options.disallow_any or self.is_typeshed_file: return @@ -4911,12 +4908,18 @@ def visit_tuple_type(self, t: TupleType) -> Type: items = [it.accept(self) for it in t.items] fallback = self.visit_instance(t.fallback, from_fallback=True) assert isinstance(fallback, Instance) - return TupleType(items, fallback) + return TupleType(items, fallback, t.line, t.column) def visit_typeddict_type(self, t: TypedDictType) -> Type: if self.check_recursion(t): return AnyType(TypeOfAny.from_error) - return super().visit_typeddict_type(t) + items = OrderedDict([ + (item_name, item_type.accept(self)) + for (item_name, item_type) in t.items.items() + ]) + fallback = self.visit_instance(t.fallback, from_fallback=True) + assert isinstance(fallback, Instance) + return TypedDictType(items, t.required_keys, fallback, t.line, t.column) def visit_union_type(self, t: UnionType) -> Type: if self.check_recursion(t): diff --git a/mypy/typeanal.py b/mypy/typeanal.py index cbad173a4980..3119c19a05e3 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -704,23 +704,8 @@ def visit_instance(self, t: Instance) -> None: arg_values = [arg] self.check_type_var_values(info, arg_values, tvar.name, tvar.values, i + 1, t) # TODO: These hacks will be not necessary when this will be moved to later stage. - if isinstance(arg, ForwardRef): - arg = arg.link - bound = tvar.upper_bound - if isinstance(bound, ForwardRef): - bound = bound.link - if isinstance(bound, Instance) and bound.type.replaced: - replaced = bound.type.replaced - if replaced.tuple_type: - bound = replaced.tuple_type - if replaced.typeddict_type: - bound = replaced.typeddict_type - if isinstance(arg, Instance) and arg.type.replaced: - replaced = arg.type.replaced - if replaced.tuple_type: - arg = replaced.tuple_type - if replaced.typeddict_type: - arg = replaced.typeddict_type + arg = self.update_type(arg) + bound = self.update_type(tvar.upper_bound) if not is_subtype(arg, bound): self.fail('Type argument "{}" of "{}" must be ' 'a subtype of "{}"'.format( @@ -734,8 +719,9 @@ def visit_instance(self, t: Instance) -> None: def check_type_var_values(self, type: TypeInfo, actuals: List[Type], arg_name: str, valids: List[Type], arg_number: int, context: Context) -> None: for actual in actuals: + actual = self.update_type(actual) if (not isinstance(actual, AnyType) and - not any(is_same_type(actual, value) for value in valids)): + not any(is_same_type(actual, self.update_type(value)) for value in valids)): if len(actuals) > 1 or not isinstance(actual, Instance): self.fail('Invalid type argument value for "{}"'.format( type.name()), context) @@ -745,6 +731,19 @@ def check_type_var_values(self, type: TypeInfo, actuals: List[Type], arg_name: s self.fail(messages.INCOMPATIBLE_TYPEVAR_VALUE.format( arg_name, class_name, actual_type_name), context) + def update_type(self, tp: Type) -> Type: + # This helper is only needed while is_subtype and is_same_type are + # called in third pass. This can be removed when TODO in visit_instance is fixed. + if isinstance(tp, ForwardRef): + tp = tp.link + if isinstance(tp, Instance) and tp.type.replaced: + replaced = tp.type.replaced + if replaced.tuple_type: + tp = replaced.tuple_type + if replaced.typeddict_type: + tp = replaced.typeddict_type + return tp + def visit_callable_type(self, t: CallableType) -> None: t.ret_type.accept(self) for arg_type in t.arg_types: diff --git a/mypy/types.py b/mypy/types.py index e1293bf96b58..dcc845922419 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1384,7 +1384,7 @@ class ForwardRef(Type): x: A A = TypedDict('A', {'x': int}) - To avoid false positives and crashes in such situations, we first wrap the second + To avoid false positives and crashes in such situations, we first wrap the first occurrence of 'A' in ForwardRef. Then, the wrapped UnboundType is updated in the third pass of semantic analysis and ultimately fixed in the patches after the third pass. So that ForwardRefs are temporary and will be completely replaced with the linked types diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index f9aaf30a19e1..f37204dad273 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -3067,6 +3067,18 @@ IntNode = Node[int, S] AnyNode = Node[S, T] [out] +[case testNamedTupleForwardAsUpperBoundSerialization] +from typing import NamedTuple, TypeVar, Generic +T = TypeVar('T', bound='M') +class G(Generic[T]): + x: T + +yg: G[M] +z: int = G[M]().x.x +z = G[M]().x[0] +M = NamedTuple('M', [('x', int)]) +[out] + [case testSelfRefNTIncremental1] from typing import Tuple, NamedTuple diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 51ec99fb754c..b03ee9c29f16 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -442,8 +442,8 @@ class G(Generic[T]): yb: G[int] # E: Type argument "builtins.int" of "G" must be a subtype of "Tuple[builtins.int, fallback=__main__.M]" yg: G[M] -z: int = G[M]().x.x -z = G[M]().x[0] +reveal_type(G[M]().x.x) # E: Revealed type is 'builtins.int' +reveal_type(G[M]().x[0]) # E: Revealed type is 'builtins.int' M = NamedTuple('M', [('x', int)]) [out] diff --git a/test-data/unit/check-statements.test b/test-data/unit/check-statements.test index 6d8f6a4b03d2..cd38fc02f3fb 100644 --- a/test-data/unit/check-statements.test +++ b/test-data/unit/check-statements.test @@ -1570,8 +1570,8 @@ from typing import List, NamedTuple lst: List[N] for i in lst: - a: int = i.x - b: str = i[0] # E: Incompatible types in assignment (expression has type "int", variable has type "str") + reveal_type(i.x) # E: Revealed type is 'builtins.int' + a: str = i[0] # E: Incompatible types in assignment (expression has type "int", variable has type "str") N = NamedTuple('N', [('x', int)]) [builtins fixtures/list.pyi] @@ -1582,8 +1582,8 @@ from typing import List, NamedTuple lst: List[M] for i in lst: # type: N - a: int = i.x - b: str = i[0] # E: Incompatible types in assignment (expression has type "int", variable has type "str") + reveal_type(i.x) # E: Revealed type is 'builtins.int' + a: str = i[0] # E: Incompatible types in assignment (expression has type "int", variable has type "str") N = NamedTuple('N', [('x', int)]) class M(N): pass From 6edd07882076e1171406804e2c70cb21ff3f42e3 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 27 Sep 2017 02:39:23 +0200 Subject: [PATCH 38/39] Improve tests; Fix one more crash on NewType MRO --- mypy/semanal.py | 10 ++- test-data/unit/check-classes.test | 91 +++++++++++++++++---------- test-data/unit/check-incremental.test | 69 ++++++++++---------- test-data/unit/check-namedtuple.test | 88 +++++++++++++------------- 4 files changed, 146 insertions(+), 112 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index b5005e7bac39..53cecba70176 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -2366,7 +2366,12 @@ def basic_new_typeinfo(self, name: str, basetype_or_fallback: Instance) -> TypeI class_def.fullname = self.qualified_name(name) info = TypeInfo(SymbolTable(), class_def, self.cur_mod_id) - info.mro = [info] + basetype_or_fallback.type.mro + class_def.info = info + mro = basetype_or_fallback.type.mro + if mro is None: + # Forward reference, MRO should be recalculated in third pass. + mro = [basetype_or_fallback.type, self.object_type().type] + info.mro = [info] + mro info.bases = [basetype_or_fallback] return info @@ -4225,6 +4230,9 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: analyzed = s.rvalue.analyzed if isinstance(analyzed, NewTypeExpr): self.analyze(analyzed.old_type, analyzed) + if analyzed.info and analyzed.info.mro: + analyzed.info.mro = [] # Force recomputation + calculate_class_mro(analyzed.info.defn, self.fail_blocker) if isinstance(analyzed, TypeVarExpr): types = [] if analyzed.upper_bound: diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 851f2d59f32d..7243de98420b 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -3583,6 +3583,7 @@ class NameInfo(NamedTuple): def parse_ast(name_dict: NameDict) -> None: if isinstance(name_dict[''], int): pass + reveal_type(name_dict['test']) # E: Revealed type is 'Tuple[builtins.bool, fallback=__main__.NameInfo]' [builtins fixtures/isinstancelist.pyi] [out] @@ -3597,6 +3598,7 @@ class NameInfo(TypedDict): def parse_ast(name_dict: NameDict) -> None: if isinstance(name_dict[''], int): pass + reveal_type(name_dict['']['ast']) # E: Revealed type is 'builtins.bool' [builtins fixtures/isinstancelist.pyi] [out] @@ -3604,22 +3606,24 @@ def parse_ast(name_dict: NameDict) -> None: from typing import Dict, NewType NameDict = Dict[str, 'NameInfo'] -class NameInfoBase: +class Base: ast: bool -NameInfo = NewType('NameInfo', NameInfoBase) +NameInfo = NewType('NameInfo', Base) def parse_ast(name_dict: NameDict) -> None: if isinstance(name_dict[''], int): pass x = name_dict[''] reveal_type(x) # E: Revealed type is '__main__.NameInfo*' - x = NameInfo(NameInfoBase()) # OK - x = NameInfoBase() # E: Incompatible types in assignment (expression has type "NameInfoBase", variable has type "NameInfo") + x = NameInfo(Base()) # OK + x = Base() # E: Incompatible types in assignment (expression has type "Base", variable has type "NameInfo") [builtins fixtures/isinstancelist.pyi] [out] [case testCorrectAttributeInForwardRefToNamedTuple] from typing import NamedTuple +proc: Process +reveal_type(proc.state) # E: Revealed type is 'builtins.int' def get_state(proc: 'Process') -> int: return proc.state @@ -3629,6 +3633,8 @@ class Process(NamedTuple): [case testCorrectItemTypeInForwardRefToTypedDict] from mypy_extensions import TypedDict +proc: Process +reveal_type(proc['state']) # E: Revealed type is 'builtins.int' def get_state(proc: 'Process') -> int: return proc['state'] @@ -3642,13 +3648,13 @@ from typing import NamedTuple x: A class A(NamedTuple): - name: 'B' - year: int + one: 'B' + other: int class B(NamedTuple): - director: str + attr: str y: A y = x -reveal_type(x.name.director) # E: Revealed type is 'builtins.str' +reveal_type(x.one.attr) # E: Revealed type is 'builtins.str' [out] [case testCrashOnDoubleForwardTypedDict] @@ -3656,26 +3662,27 @@ from mypy_extensions import TypedDict x: A class A(TypedDict): - name: 'B' - year: int + one: 'B' + other: int class B(TypedDict): - director: str + attr: str -reveal_type(x['name']['director']) # E: Revealed type is 'builtins.str' +reveal_type(x['one']['attr']) # E: Revealed type is 'builtins.str' [builtins fixtures/isinstancelist.pyi] [out] [case testCrashOnForwardUnionOfNamedTuples] from typing import Union, NamedTuple -NodeType = Union['Foo', 'Bar'] +Node = Union['Foo', 'Bar'] class Foo(NamedTuple): x: int class Bar(NamedTuple): x: int -def foo(node: NodeType) -> int: +def foo(node: Node) -> int: x = node + reveal_type(node) # E: Revealed type is 'Union[Tuple[builtins.int, fallback=__main__.Foo], Tuple[builtins.int, fallback=__main__.Bar]]' return x.x [out] @@ -3697,46 +3704,64 @@ def foo(node: NodeType) -> int: [case testSupportForwardUnionOfNewTypes] from typing import Union, NewType +x: Node +reveal_type(x.x) # E: Revealed type is 'builtins.int' -NodeType = Union['Foo', 'Bar'] -Foo = NewType('Foo', int) -Bar = NewType('Bar', str) +class A: + x: int +class B: + x: int -def foo(node: NodeType) -> NodeType: +Node = Union['Foo', 'Bar'] +Foo = NewType('Foo', A) +Bar = NewType('Bar', B) + +def foo(node: Node) -> Node: x = node - return Foo(1) + return Foo(A()) +[out] + +[case testForwardReferencesInNewTypeMRORecomputed] +from typing import NewType +x: Foo +Foo = NewType('Foo', B) +class A: + x: int +class B(A): + pass + +reveal_type(x.x) # E: Revealed type is 'builtins.int' [out] -[case testCrashOnComplexCheckWithNamedTupleUnion] +[case testCrashOnComplexNamedTupleUnionProperty] from typing import NamedTuple, Union +x: AOrB AOrB = Union['A', 'B'] class A(NamedTuple): - lat: int - lng: int + x: int class B(object): def __init__(self, a: AOrB) -> None: self.a = a @property - def lat(self) -> int: - return self.a.lat - @property - def lng(self) -> int: - return self.a.lng + def x(self) -> int: + return self.a.x + +reveal_type(x.x) # E: Revealed type is 'builtins.int' [builtins fixtures/property.pyi] [out] [case testCorrectIsinstanceWithForwardUnion] from typing import Union, NamedTuple -ForwardReferenceUnion = Union['SomeTuple', int] -class SomeTuple(NamedTuple('SomeTuple', [('elem', int)])): pass +ForwardUnion = Union['TP', int] +class TP(NamedTuple('TP', [('x', int)])): pass -def f(x: ForwardReferenceUnion) -> None: - reveal_type(x) # E: Revealed type is 'Union[Tuple[builtins.int, fallback=__main__.SomeTuple], builtins.int]' - if isinstance(x, SomeTuple): - reveal_type(x) # E: Revealed type is 'Tuple[builtins.int, fallback=__main__.SomeTuple]' +def f(x: ForwardUnion) -> None: + reveal_type(x) # E: Revealed type is 'Union[Tuple[builtins.int, fallback=__main__.TP], builtins.int]' + if isinstance(x, TP): + reveal_type(x) # E: Revealed type is 'Tuple[builtins.int, fallback=__main__.TP]' [builtins fixtures/isinstance.pyi] [out] diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index f37204dad273..e3286c3ada97 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -2956,17 +2956,18 @@ tmp/b.py:2: error: Incompatible return value type (got "int", expected "str") from typing import NamedTuple, Union class Person(NamedTuple): - name: Union[str, "NamePair"] + name: Union[str, "Pair"] -class NamePair(NamedTuple): +class Pair(NamedTuple): first: str last: str -Person(name=NamePair(first="John", last="Doe")) +Person(name=Pair(first="John", last="Doe")) [out] -- Some crazy selef-referential named tuples, types dicts, and aliases --- to be sure that everything can be _serialized_ (i.e. ForwardRef's are removed) +-- to be sure that everything can be _serialized_ (i.e. ForwardRef's are removed). +-- For this reason errors are silenced (tests with # type: ignore have equivalents in other files) [case testForwardTypeAliasInBase1] from typing import List @@ -3095,72 +3096,72 @@ n: Node [case testSelfRefNTIncremental2] from typing import Tuple, NamedTuple -Node = NamedTuple( # type: ignore - 'Node', +A = NamedTuple( # type: ignore + 'A', [ - ('name', str), - ('children', Tuple['Node2', ...]), + ('x', str), + ('y', Tuple['B', ...]), ]) -class Node2(NamedTuple): # type: ignore - x: Node +class B(NamedTuple): # type: ignore + x: A y: int -n: Node +n: A [builtins fixtures/tuple.pyi] [case testSelfRefNTIncremental3] from typing import NamedTuple, Tuple -class Node2(NamedTuple): # type: ignore - x: Tuple[Node, int] +class B(NamedTuple): # type: ignore + x: Tuple[A, int] y: int -Node = NamedTuple( # type: ignore - 'Node', +A = NamedTuple( # type: ignore + 'A', [ - ('name', str), - ('children', 'Node2'), + ('x', str), + ('y', 'B'), ]) -n: Node2 -m: Node +n: B +m: A lst = [m, n] [builtins fixtures/tuple.pyi] [case testSelfRefNTIncremental4] from typing import NamedTuple -class Node2(NamedTuple): # type: ignore - x: Node +class B(NamedTuple): # type: ignore + x: A y: int -class Node(NamedTuple): # type: ignore - name: str - children: Node2 +class A(NamedTuple): # type: ignore + x: str + y: B -n: Node +n: A [builtins fixtures/tuple.pyi] [case testSelfRefNTIncremental5] from typing import NamedTuple -Node2 = NamedTuple( # type: ignore - 'Node2', +B = NamedTuple( # type: ignore + 'B', [ - ('x', Node), + ('x', A), ('y', int), ]) -Node = NamedTuple( # type: ignore - 'Node', +A = NamedTuple( # type: ignore + 'A', [ - ('name', str), - ('children', 'Node2'), + ('x', str), + ('y', 'B'), ]) -n: Node -def f(m: Node2) -> None: pass +n: A +def f(m: B) -> None: pass [builtins fixtures/tuple.pyi] [case testCrashWithPartialGlobalAndCycle] diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index b03ee9c29f16..36dc88073560 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -542,19 +542,19 @@ reveal_type(n) # E: Revealed type is 'Tuple[builtins.str, builtins.tuple[Tuple[b [case testSelfRefNT2] from typing import Tuple, NamedTuple -Node = NamedTuple( # E - 'Node', +A = NamedTuple( # E + 'A', [ - ('name', str), - ('children', Tuple['Node2', ...]), + ('x', str), + ('y', Tuple['B', ...]), ]) -class Node2(NamedTuple): # E - x: Node +class B(NamedTuple): # E + x: A y: int -n: Node -reveal_type(n) # E: Revealed type is 'Tuple[builtins.str, builtins.tuple[Tuple[Tuple[builtins.str, builtins.tuple[Any], fallback=__main__.Node], builtins.int, fallback=__main__.Node2]], fallback=__main__.Node]' +n: A +reveal_type(n) # E: Revealed type is 'Tuple[builtins.str, builtins.tuple[Tuple[Tuple[builtins.str, builtins.tuple[Any], fallback=__main__.A], builtins.int, fallback=__main__.B]], fallback=__main__.A]' [builtins fixtures/tuple.pyi] [out] main:3: error: Recursive types not fully supported yet, nested types replaced with "Any" @@ -563,21 +563,21 @@ main:10: error: Recursive types not fully supported yet, nested types replaced w [case testSelfRefNT3] from typing import NamedTuple, Tuple -class Node2(NamedTuple): # E - x: Tuple[Node, int] +class B(NamedTuple): # E + x: Tuple[A, int] y: int -Node = NamedTuple( # E: Recursive types not fully supported yet, nested types replaced with "Any" - 'Node', +A = NamedTuple( # E: Recursive types not fully supported yet, nested types replaced with "Any" + 'A', [ - ('name', str), - ('children', 'Node2'), + ('x', str), + ('y', 'B'), ]) -n: Node2 -m: Node -reveal_type(n) # E: Revealed type is 'Tuple[Tuple[Tuple[builtins.str, Tuple[Tuple[Any, builtins.int], builtins.int, fallback=__main__.Node2], fallback=__main__.Node], builtins.int], builtins.int, fallback=__main__.Node2]' -reveal_type(m) # E: Revealed type is 'Tuple[builtins.str, Tuple[Tuple[Tuple[builtins.str, Tuple[Tuple[Any, builtins.int], builtins.int, fallback=__main__.Node2], fallback=__main__.Node], builtins.int], builtins.int, fallback=__main__.Node2], fallback=__main__.Node]' +n: B +m: A +reveal_type(n.x) # E: Revealed type is 'Tuple[Tuple[builtins.str, Tuple[Tuple[Any, builtins.int], builtins.int, fallback=__main__.B], fallback=__main__.A], builtins.int]' +reveal_type(m[0]) # E: Revealed type is 'builtins.str' lst = [m, n] reveal_type(lst[0]) # E: Revealed type is 'Tuple[builtins.object, builtins.object]' [builtins fixtures/tuple.pyi] @@ -587,16 +587,16 @@ main:3: error: Recursive types not fully supported yet, nested types replaced wi [case testSelfRefNT4] from typing import NamedTuple -class Node2(NamedTuple): # E - x: Node +class B(NamedTuple): # E + x: A y: int -class Node(NamedTuple): # E - name: str - children: Node2 +class A(NamedTuple): # E + x: str + y: B -n: Node -reveal_type(n) # E: Revealed type is 'Tuple[builtins.str, Tuple[Tuple[builtins.str, Tuple[Any, builtins.int, fallback=__main__.Node2], fallback=__main__.Node], builtins.int, fallback=__main__.Node2], fallback=__main__.Node]' +n: A +reveal_type(n.y[0]) # E: Revealed type is 'Tuple[builtins.str, Tuple[Any, builtins.int, fallback=__main__.B], fallback=__main__.A]' [builtins fixtures/tuple.pyi] [out] main:3: error: Recursive types not fully supported yet, nested types replaced with "Any" @@ -605,42 +605,42 @@ main:7: error: Recursive types not fully supported yet, nested types replaced wi [case testSelfRefNT5] from typing import NamedTuple -Node2 = NamedTuple( # E: Recursive types not fully supported yet, nested types replaced with "Any" - 'Node2', +B = NamedTuple( # E: Recursive types not fully supported yet, nested types replaced with "Any" + 'B', [ - ('x', Node), + ('x', A), ('y', int), ]) -Node = NamedTuple( # E: Recursive types not fully supported yet, nested types replaced with "Any" - 'Node', +A = NamedTuple( # E: Recursive types not fully supported yet, nested types replaced with "Any" + 'A', [ - ('name', str), - ('children', 'Node2'), + ('x', str), + ('y', 'B'), ]) -n: Node -def f(m: Node2) -> None: pass -reveal_type(n) # E: Revealed type is 'Tuple[builtins.str, Tuple[Tuple[builtins.str, Tuple[Any, builtins.int, fallback=__main__.Node2], fallback=__main__.Node], builtins.int, fallback=__main__.Node2], fallback=__main__.Node]' -reveal_type(f) # E: Revealed type is 'def (m: Tuple[Tuple[builtins.str, Tuple[Any, builtins.int, fallback=__main__.Node2], fallback=__main__.Node], builtins.int, fallback=__main__.Node2])' +n: A +def f(m: B) -> None: pass +reveal_type(n) # E: Revealed type is 'Tuple[builtins.str, Tuple[Tuple[builtins.str, Tuple[Any, builtins.int, fallback=__main__.B], fallback=__main__.A], builtins.int, fallback=__main__.B], fallback=__main__.A]' +reveal_type(f) # E: Revealed type is 'def (m: Tuple[Tuple[builtins.str, Tuple[Any, builtins.int, fallback=__main__.B], fallback=__main__.A], builtins.int, fallback=__main__.B])' [builtins fixtures/tuple.pyi] [case testRecursiveNamedTupleInBases] from typing import List, NamedTuple, Union -Exp = Union['App', 'Lit'] # E: Recursive types not fully supported yet, nested types replaced with "Any" -class App(NamedTuple('App', [('subs', List[Exp])])): pass -class Lit(NamedTuple('Lit', [('val', object)])): pass +Exp = Union['A', 'B'] # E: Recursive types not fully supported yet, nested types replaced with "Any" +class A(NamedTuple('A', [('attr', List[Exp])])): pass +class B(NamedTuple('B', [('val', object)])): pass def my_eval(exp: Exp) -> int: - reveal_type(exp) # E: Revealed type is 'Union[Tuple[builtins.list[Any], fallback=__main__.App], Tuple[builtins.object, fallback=__main__.Lit]]' - if isinstance(exp, App): + reveal_type(exp) # E: Revealed type is 'Union[Tuple[builtins.list[Any], fallback=__main__.A], Tuple[builtins.object, fallback=__main__.B]]' + if isinstance(exp, A): my_eval(exp[0][0]) - return my_eval(exp.subs[0]) - if isinstance(exp, Lit): + return my_eval(exp.attr[0]) + if isinstance(exp, B): return exp.val # E: Incompatible return value type (got "object", expected "int") -my_eval(App([Lit(1), Lit(2)])) # OK +my_eval(A([B(1), B(2)])) # OK [builtins fixtures/isinstancelist.pyi] [out] From 514b8bd75feb8349277bfb84ee3be5a9a130206f Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 27 Sep 2017 02:47:28 +0200 Subject: [PATCH 39/39] Fix formatting in tests --- test-data/unit/check-incremental.test | 27 +++++---------------------- test-data/unit/check-namedtuple.test | 27 ++++++--------------------- 2 files changed, 11 insertions(+), 43 deletions(-) diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index e3286c3ada97..1d3ee9a37436 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -3083,26 +3083,20 @@ M = NamedTuple('M', [('x', int)]) [case testSelfRefNTIncremental1] from typing import Tuple, NamedTuple -Node = NamedTuple( # type: ignore - 'Node', - [ +Node = NamedTuple('Node', [ # type: ignore ('name', str), ('children', Tuple['Node', ...]), ]) - n: Node [builtins fixtures/tuple.pyi] [case testSelfRefNTIncremental2] from typing import Tuple, NamedTuple -A = NamedTuple( # type: ignore - 'A', - [ +A = NamedTuple('A', [ # type: ignore ('x', str), ('y', Tuple['B', ...]), ]) - class B(NamedTuple): # type: ignore x: A y: int @@ -3116,14 +3110,10 @@ from typing import NamedTuple, Tuple class B(NamedTuple): # type: ignore x: Tuple[A, int] y: int - -A = NamedTuple( # type: ignore - 'A', - [ +A = NamedTuple('A', [ # type: ignore ('x', str), ('y', 'B'), ]) - n: B m: A lst = [m, n] @@ -3135,7 +3125,6 @@ from typing import NamedTuple class B(NamedTuple): # type: ignore x: A y: int - class A(NamedTuple): # type: ignore x: str y: B @@ -3146,20 +3135,14 @@ n: A [case testSelfRefNTIncremental5] from typing import NamedTuple -B = NamedTuple( # type: ignore - 'B', - [ +B = NamedTuple('B', [ # type: ignore ('x', A), ('y', int), ]) - -A = NamedTuple( # type: ignore - 'A', - [ +A = NamedTuple('A', [ # type: ignore ('x', str), ('y', 'B'), ]) - n: A def f(m: B) -> None: pass [builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 36dc88073560..9726dad27b20 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -527,13 +527,10 @@ class B: [case testSelfRefNT1] from typing import Tuple, NamedTuple -Node = NamedTuple( # E: Recursive types not fully supported yet, nested types replaced with "Any" - 'Node', - [ +Node = NamedTuple('Node', [ # E: Recursive types not fully supported yet, nested types replaced with "Any" ('name', str), ('children', Tuple['Node', ...]), ]) - n: Node reveal_type(n) # E: Revealed type is 'Tuple[builtins.str, builtins.tuple[Tuple[builtins.str, builtins.tuple[Any], fallback=__main__.Node]], fallback=__main__.Node]' [builtins fixtures/tuple.pyi] @@ -542,13 +539,10 @@ reveal_type(n) # E: Revealed type is 'Tuple[builtins.str, builtins.tuple[Tuple[b [case testSelfRefNT2] from typing import Tuple, NamedTuple -A = NamedTuple( # E - 'A', - [ +A = NamedTuple('A', [ # E ('x', str), ('y', Tuple['B', ...]), ]) - class B(NamedTuple): # E x: A y: int @@ -558,7 +552,7 @@ reveal_type(n) # E: Revealed type is 'Tuple[builtins.str, builtins.tuple[Tuple[T [builtins fixtures/tuple.pyi] [out] main:3: error: Recursive types not fully supported yet, nested types replaced with "Any" -main:10: error: Recursive types not fully supported yet, nested types replaced with "Any" +main:7: error: Recursive types not fully supported yet, nested types replaced with "Any" [case testSelfRefNT3] from typing import NamedTuple, Tuple @@ -567,13 +561,10 @@ class B(NamedTuple): # E x: Tuple[A, int] y: int -A = NamedTuple( # E: Recursive types not fully supported yet, nested types replaced with "Any" - 'A', - [ +A = NamedTuple('A', [ # E: Recursive types not fully supported yet, nested types replaced with "Any" ('x', str), ('y', 'B'), ]) - n: B m: A reveal_type(n.x) # E: Revealed type is 'Tuple[Tuple[builtins.str, Tuple[Tuple[Any, builtins.int], builtins.int, fallback=__main__.B], fallback=__main__.A], builtins.int]' @@ -605,20 +596,14 @@ main:7: error: Recursive types not fully supported yet, nested types replaced wi [case testSelfRefNT5] from typing import NamedTuple -B = NamedTuple( # E: Recursive types not fully supported yet, nested types replaced with "Any" - 'B', - [ +B = NamedTuple('B', [ # E: Recursive types not fully supported yet, nested types replaced with "Any" ('x', A), ('y', int), ]) - -A = NamedTuple( # E: Recursive types not fully supported yet, nested types replaced with "Any" - 'A', - [ +A = NamedTuple('A', [ # E: Recursive types not fully supported yet, nested types replaced with "Any" ('x', str), ('y', 'B'), ]) - n: A def f(m: B) -> None: pass reveal_type(n) # E: Revealed type is 'Tuple[builtins.str, Tuple[Tuple[builtins.str, Tuple[Any, builtins.int, fallback=__main__.B], fallback=__main__.A], builtins.int, fallback=__main__.B], fallback=__main__.A]'