Skip to content

Use tuple[object, ...] and dict[str, object] as upper bounds for ParamSpec.args and ParamSpec.kwargs #12668

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 11 commits into from
Apr 29, 2022
2 changes: 1 addition & 1 deletion mypy/checkmember.py
Original file line number Diff line number Diff line change
@@ -164,7 +164,7 @@ def _analyze_member_access(name: str,
return analyze_typeddict_access(name, typ, mx, override_info)
elif isinstance(typ, NoneType):
return analyze_none_member_access(name, typ, mx)
elif isinstance(typ, TypeVarType):
elif isinstance(typ, TypeVarLikeType):
return _analyze_member_access(name, typ.upper_bound, mx, override_info)
elif isinstance(typ, DeletedType):
mx.msg.deleted_as_rvalue(typ, mx.context)
50 changes: 47 additions & 3 deletions mypy/semanal_shared.py
Original file line number Diff line number Diff line change
@@ -2,16 +2,17 @@

from abc import abstractmethod

from typing import Optional, List, Callable
from typing_extensions import Final
from typing import Optional, List, Callable, Union
from typing_extensions import Final, Protocol
from mypy_extensions import trait

from mypy.nodes import (
Context, SymbolTableNode, FuncDef, Node, TypeInfo, Expression,
SymbolNode, SymbolTable
)
from mypy.types import (
Type, FunctionLike, Instance, TupleType, TPDICT_FB_NAMES, ProperType, get_proper_type
Type, FunctionLike, Instance, TupleType, TPDICT_FB_NAMES, ProperType, get_proper_type,
ParamSpecType, ParamSpecFlavor, Parameters, TypeVarId
)
from mypy.tvar_scope import TypeVarLikeScope
from mypy.errorcodes import ErrorCode
@@ -212,3 +213,46 @@ def calculate_tuple_fallback(typ: TupleType) -> None:
fallback = typ.partial_fallback
assert fallback.type.fullname == 'builtins.tuple'
fallback.args = (join.join_type_list(list(typ.items)),) + fallback.args[1:]


class _NamedTypeCallback(Protocol):
def __call__(
self, fully_qualified_name: str, args: Optional[List[Type]] = None
) -> Instance: ...


def paramspec_args(
name: str, fullname: str, id: Union[TypeVarId, int], *,
named_type_func: _NamedTypeCallback, line: int = -1, column: int = -1,
prefix: Optional[Parameters] = None
) -> ParamSpecType:
return ParamSpecType(
name,
fullname,
id,
flavor=ParamSpecFlavor.ARGS,
upper_bound=named_type_func('builtins.tuple', [named_type_func('builtins.object')]),
line=line,
column=column,
prefix=prefix
)


def paramspec_kwargs(
name: str, fullname: str, id: Union[TypeVarId, int], *,
named_type_func: _NamedTypeCallback, line: int = -1, column: int = -1,
prefix: Optional[Parameters] = None
) -> ParamSpecType:
return ParamSpecType(
name,
fullname,
id,
flavor=ParamSpecFlavor.KWARGS,
upper_bound=named_type_func(
'builtins.dict',
[named_type_func('builtins.str'), named_type_func('builtins.object')]
),
line=line,
column=column,
prefix=prefix
)
30 changes: 13 additions & 17 deletions mypy/typeanal.py
Original file line number Diff line number Diff line change
@@ -33,7 +33,7 @@
from mypy.tvar_scope import TypeVarLikeScope
from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError
from mypy.plugin import Plugin, TypeAnalyzerPluginInterface, AnalyzeTypeContext
from mypy.semanal_shared import SemanticAnalyzerCoreInterface
from mypy.semanal_shared import SemanticAnalyzerCoreInterface, paramspec_args, paramspec_kwargs
from mypy.errorcodes import ErrorCode
from mypy import nodes, message_registry, errorcodes as codes

@@ -711,13 +711,13 @@ def anal_star_arg_type(self, t: Type, kind: ArgKind, nested: bool) -> Type:
tvar_def = self.tvar_scope.get_binding(sym)
if isinstance(tvar_def, ParamSpecType):
if kind == ARG_STAR:
flavor = ParamSpecFlavor.ARGS
make_paramspec = paramspec_args
elif kind == ARG_STAR2:
flavor = ParamSpecFlavor.KWARGS
make_paramspec = paramspec_kwargs
else:
assert False, kind
return ParamSpecType(tvar_def.name, tvar_def.fullname, tvar_def.id, flavor,
upper_bound=self.named_type('builtins.object'),
return make_paramspec(tvar_def.name, tvar_def.fullname, tvar_def.id,
named_type_func=self.named_type,
line=t.line, column=t.column)
return self.anal_type(t, nested=nested)

@@ -855,13 +855,11 @@ def analyze_callable_args_for_paramspec(
if not isinstance(tvar_def, ParamSpecType):
return None

# TODO: Use tuple[...] or Mapping[..] instead?
obj = self.named_type('builtins.object')
return CallableType(
[ParamSpecType(tvar_def.name, tvar_def.fullname, tvar_def.id, ParamSpecFlavor.ARGS,
upper_bound=obj),
ParamSpecType(tvar_def.name, tvar_def.fullname, tvar_def.id, ParamSpecFlavor.KWARGS,
upper_bound=obj)],
[paramspec_args(tvar_def.name, tvar_def.fullname, tvar_def.id,
named_type_func=self.named_type),
paramspec_kwargs(tvar_def.name, tvar_def.fullname, tvar_def.id,
named_type_func=self.named_type)],
[nodes.ARG_STAR, nodes.ARG_STAR2],
[None, None],
ret_type=ret_type,
@@ -891,18 +889,16 @@ def analyze_callable_args_for_concatenate(
if not isinstance(tvar_def, ParamSpecType):
return None

# TODO: Use tuple[...] or Mapping[..] instead?
obj = self.named_type('builtins.object')
# ick, CallableType should take ParamSpecType
prefix = tvar_def.prefix
# we don't set the prefix here as generic arguments will get updated at some point
# in the future. CallableType.param_spec() accounts for this.
return CallableType(
[*prefix.arg_types,
ParamSpecType(tvar_def.name, tvar_def.fullname, tvar_def.id, ParamSpecFlavor.ARGS,
upper_bound=obj),
ParamSpecType(tvar_def.name, tvar_def.fullname, tvar_def.id, ParamSpecFlavor.KWARGS,
upper_bound=obj)],
paramspec_args(tvar_def.name, tvar_def.fullname, tvar_def.id,
named_type_func=self.named_type),
paramspec_kwargs(tvar_def.name, tvar_def.fullname, tvar_def.id,
named_type_func=self.named_type)],
[*prefix.arg_kinds, nodes.ARG_STAR, nodes.ARG_STAR2],
[*prefix.arg_names, None, None],
ret_type=ret_type,
75 changes: 49 additions & 26 deletions test-data/unit/check-parameter-specification.test
Original file line number Diff line number Diff line change
@@ -11,7 +11,7 @@ P2 = ParamSpec("P2", contravariant=True) # E: Only the first argument to ParamS
P3 = ParamSpec("P3", bound=int) # E: Only the first argument to ParamSpec has defined semantics
P4 = ParamSpec("P4", int, str) # E: Only the first argument to ParamSpec has defined semantics
P5 = ParamSpec("P5", covariant=True, bound=int) # E: Only the first argument to ParamSpec has defined semantics
[builtins fixtures/tuple.pyi]
[builtins fixtures/paramspec.pyi]

[case testParamSpecLocations]
from typing import Callable, List
@@ -35,7 +35,7 @@ def foo5(x: Callable[[int, str], P]) -> None: ... # E: Invalid location for Par

def foo6(x: Callable[[P], int]) -> None: ... # E: Invalid location for ParamSpec "P" \
# N: You can use ParamSpec as the first argument to Callable, e.g., 'Callable[P, int]'
[builtins fixtures/tuple.pyi]
[builtins fixtures/paramspec.pyi]

[case testParamSpecContextManagerLike]
from typing import Callable, List, Iterator, TypeVar
@@ -51,7 +51,7 @@ def whatever(x: int) -> Iterator[int]:

reveal_type(whatever) # N: Revealed type is "def (x: builtins.int) -> builtins.list[builtins.int]"
reveal_type(whatever(217)) # N: Revealed type is "builtins.list[builtins.int]"
[builtins fixtures/tuple.pyi]
[builtins fixtures/paramspec.pyi]

[case testInvalidParamSpecType]
# flags: --python-version 3.10
@@ -70,7 +70,7 @@ P = ParamSpec('P')

def f(x: Callable[P, int]) -> None: ...
reveal_type(f) # N: Revealed type is "def [P] (x: def (*P.args, **P.kwargs) -> builtins.int)"
[builtins fixtures/tuple.pyi]
[builtins fixtures/paramspec.pyi]

[case testParamSpecSimpleFunction]
from typing import Callable, TypeVar
@@ -83,7 +83,7 @@ def changes_return_type_to_str(x: Callable[P, int]) -> Callable[P, str]: ...
def returns_int(a: str, b: bool) -> int: ...

reveal_type(changes_return_type_to_str(returns_int)) # N: Revealed type is "def (a: builtins.str, b: builtins.bool) -> builtins.str"
[builtins fixtures/tuple.pyi]
[builtins fixtures/paramspec.pyi]
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just wondering, any particular reason not to use the new fixture for all ParamSpec tests? I noticed that a few where missing. Especially between L90 - L550.

Copy link
Member Author

@AlexWaygood AlexWaygood Apr 24, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I used the new fixture in all the tests that broke with the changes made in this PR ;) They mostly broke because builtins.dict is undefined in tuple.pyi, but we need it for a lot of ParamSpec-related tests if ParamSpec.kwargs is bound to dict[str, Any].

There's no particular reason not to use them in all ParamSpec tests -- but I wanted to keep the diff as small as possible, and the remaining tests don't need all the classes included in paramspec.pyi. (I think it's best to use as minimal a fixture as possible, in most situations, to avoid slowing down the test suite.)


[case testParamSpecSimpleClass]
from typing import Callable, TypeVar, Generic
@@ -199,7 +199,7 @@ g: Any
reveal_type(f(g)) # N: Revealed type is "def (*Any, **Any) -> builtins.str"

f(g)(1, 3, x=1, y=2)
[builtins fixtures/tuple.pyi]
[builtins fixtures/paramspec.pyi]

[case testParamSpecDecoratorImplementation]
from typing import Callable, Any, TypeVar, List
@@ -556,7 +556,7 @@ a: Callable[[int, bytes], str]
b: Callable[[str, bytes], str]

reveal_type(f(a, b)) # N: Revealed type is "def (builtins.bytes) -> builtins.str"
[builtins fixtures/tuple.pyi]
[builtins fixtures/paramspec.pyi]

[case testParamSpecConcatenateInReturn]
from typing_extensions import ParamSpec, Concatenate
@@ -569,7 +569,7 @@ def f(i: Callable[Concatenate[int, P], str]) -> Callable[Concatenate[int, P], st
n: Callable[[int, bytes], str]

reveal_type(f(n)) # N: Revealed type is "def (builtins.int, builtins.bytes) -> builtins.str"
[builtins fixtures/tuple.pyi]
[builtins fixtures/paramspec.pyi]

[case testParamSpecConcatenateNamedArgs]
# flags: --strict-concatenate
@@ -592,7 +592,7 @@ def f2(c: Callable[P, R]) -> Callable[Concatenate[int, P], R]:

# reason for rejection:
f2(lambda x: 42)(42, x=42)
[builtins fixtures/tuple.pyi]
[builtins fixtures/paramspec.pyi]
[out]
main:10: error: invalid syntax
[out version>=3.8]
@@ -619,7 +619,7 @@ def f2(c: Callable[P, R]) -> Callable[Concatenate[int, P], R]:

# reason for rejection:
f2(lambda x: 42)(42, x=42)
[builtins fixtures/tuple.pyi]
[builtins fixtures/paramspec.pyi]
[out]
main:9: error: invalid syntax
[out version>=3.8]
@@ -640,7 +640,7 @@ n = f(a)

reveal_type(n) # N: Revealed type is "def (builtins.int)"
reveal_type(n(42)) # N: Revealed type is "None"
[builtins fixtures/tuple.pyi]
[builtins fixtures/paramspec.pyi]

[case testCallablesAsParameters]
# credits to https://github.com/microsoft/pyright/issues/2705
@@ -658,7 +658,7 @@ def test(a: int, /, b: str) -> str: ...
abc = Foo(test)
reveal_type(abc)
bar(abc)
[builtins fixtures/tuple.pyi]
[builtins fixtures/paramspec.pyi]
[out]
main:11: error: invalid syntax
[out version>=3.8]
@@ -677,7 +677,7 @@ n: Foo[[int]]
def f(x: int) -> None: ...

n.foo(f)
[builtins fixtures/tuple.pyi]
[builtins fixtures/paramspec.pyi]

[case testParamSpecLiteralsTypeApplication]
from typing_extensions import ParamSpec
@@ -709,7 +709,7 @@ Z[bytes, str](lambda one: None) # E: Cannot infer type of lambda \
# E: Argument 1 to "Z" has incompatible type "Callable[[Any], None]"; expected "Callable[[bytes, str], None]"
Z[bytes, str](f2) # E: Argument 1 to "Z" has incompatible type "Callable[[bytes, int], None]"; expected "Callable[[bytes, str], None]"

[builtins fixtures/tuple.pyi]
[builtins fixtures/paramspec.pyi]

[case testParamSpecLiteralEllipsis]
from typing_extensions import ParamSpec
@@ -740,7 +740,7 @@ n = Z(f1)
n = Z(f2)
n = Z(f3)

[builtins fixtures/tuple.pyi]
[builtins fixtures/paramspec.pyi]

[case testParamSpecApplyConcatenateTwice]
from typing_extensions import ParamSpec, Concatenate
@@ -770,7 +770,7 @@ def f(c: C[P]) -> None:
reveal_type(p1) # N: Revealed type is "__main__.C[[builtins.str, **P`-1]]"
p2 = p1.add_str()
reveal_type(p2) # N: Revealed type is "__main__.C[[builtins.int, builtins.str, **P`-1]]"
[builtins fixtures/tuple.pyi]
[builtins fixtures/paramspec.pyi]

[case testParamSpecLiteralJoin]
from typing import Generic, Callable, Union
@@ -788,7 +788,7 @@ def func(
) -> None:
job = action if isinstance(action, Job) else Job(action)
reveal_type(job) # N: Revealed type is "__main__.Job[[builtins.int]]"
[builtins fixtures/tuple.pyi]
[builtins fixtures/paramspec.pyi]

[case testApplyParamSpecToParamSpecLiterals]
from typing import TypeVar, Generic, Callable
@@ -818,7 +818,7 @@ def func2(job: Job[..., None]) -> None:
run_job(job, "Hello", 42)
run_job(job, 42, msg="Hello")
run_job(job, x=42, msg="Hello")
[builtins fixtures/tuple.pyi]
[builtins fixtures/paramspec.pyi]

[case testExpandNonBareParamSpecAgainstCallable]
from typing import Callable, TypeVar, Any
@@ -850,7 +850,7 @@ reveal_type(A().func(f, 42)) # N: Revealed type is "builtins.int"

# TODO: this should reveal `int`
reveal_type(A().func(lambda x: x + x, 42)) # N: Revealed type is "Any"
[builtins fixtures/tuple.pyi]
[builtins fixtures/paramspec.pyi]

[case testParamSpecConstraintOnOtherParamSpec]
from typing import Callable, TypeVar, Any, Generic
@@ -880,7 +880,7 @@ reveal_type(A().func(Job(lambda x: x))) # N: Revealed type is "__main__.Job[[x:

def f(x: int, y: int) -> None: ...
reveal_type(A().func(Job(f))) # N: Revealed type is "__main__.Job[[x: builtins.int, y: builtins.int], None]"
[builtins fixtures/tuple.pyi]
[builtins fixtures/paramspec.pyi]

[case testConstraintBetweenParamSpecFunctions1]
from typing import Callable, TypeVar, Any, Generic
@@ -898,7 +898,7 @@ def func(__action: Job[_P]) -> Callable[_P, None]:
...

reveal_type(func) # N: Revealed type is "def [_P] (__main__.Job[_P`-1]) -> def (*_P.args, **_P.kwargs)"
[builtins fixtures/tuple.pyi]
[builtins fixtures/paramspec.pyi]

[case testConstraintBetweenParamSpecFunctions2]
from typing import Callable, TypeVar, Any, Generic
@@ -916,7 +916,7 @@ def func(__action: Job[_P]) -> Callable[_P, None]:
...

reveal_type(func) # N: Revealed type is "def [_P] (__main__.Job[_P`-1]) -> def (*_P.args, **_P.kwargs)"
[builtins fixtures/tuple.pyi]
[builtins fixtures/paramspec.pyi]

[case testConstraintsBetweenConcatenatePrefixes]
from typing import Any, Callable, Generic, TypeVar
@@ -937,7 +937,7 @@ def adds_await() -> Callable[
...

return decorator # we want `_T` and `_P` to refer to the same things.
[builtins fixtures/tuple.pyi]
[builtins fixtures/paramspec.pyi]

[case testParamSpecVariance]
from typing import Callable, Generic
@@ -995,7 +995,7 @@ a3: Callable[[int], None]
a3 = f3 # E: Incompatible types in assignment (expression has type "Callable[[bool], None]", variable has type "Callable[[int], None]")
a3 = f2
a3 = f1
[builtins fixtures/tuple.pyi]
[builtins fixtures/paramspec.pyi]

[case testDecoratingClassesThatUseParamSpec]
from typing import Generic, TypeVar, Callable, Any
@@ -1039,7 +1039,7 @@ reveal_type(j) # N: Revealed type is "__main__.Job[[x: _T`-1]]"
jf = j.into_callable()
reveal_type(jf) # N: Revealed type is "def [_T] (x: _T`-1)"
reveal_type(jf(1)) # N: Revealed type is "None"
[builtins fixtures/tuple.pyi]
[builtins fixtures/paramspec.pyi]

[case testStackedConcatenateIsIllegal]
from typing_extensions import Concatenate, ParamSpec
@@ -1048,7 +1048,7 @@ from typing import Callable
P = ParamSpec("P")

def x(f: Callable[Concatenate[int, Concatenate[int, P]], None]) -> None: ... # E: Nested Concatenates are invalid
[builtins fixtures/tuple.pyi]
[builtins fixtures/paramspec.pyi]

[case testPropagatedAnyConstraintsAreOK]
from typing import Any, Callable, Generic, TypeVar
@@ -1063,3 +1063,26 @@ class Job(Generic[P]): ...
@callback
def run_job(job: Job[...]) -> T: ...
[builtins fixtures/tuple.pyi]

[case testTupleAndDictOperationsOnParamSpecArgsAndKwargs]
from typing import Callable
from typing_extensions import ParamSpec

P = ParamSpec('P')

def func(callback: Callable[P, str]) -> Callable[P, str]:
def inner(*args: P.args, **kwargs: P.kwargs) -> str:
reveal_type(args[5]) # N: Revealed type is "builtins.object"
for a in args:
reveal_type(a) # N: Revealed type is "builtins.object"
b = 'foo' in args
reveal_type(b) # N: Revealed type is "builtins.bool"
reveal_type(args.count(42)) # N: Revealed type is "builtins.int"
reveal_type(len(args)) # N: Revealed type is "builtins.int"
for c, d in kwargs.items():
reveal_type(c) # N: Revealed type is "builtins.str"
reveal_type(d) # N: Revealed type is "builtins.object"
kwargs.pop('bar')
return 'baz'
return inner
[builtins fixtures/paramspec.pyi]
76 changes: 76 additions & 0 deletions test-data/unit/fixtures/paramspec.pyi
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
# builtins stub for paramspec-related test cases

from typing import (
Sequence, Generic, TypeVar, Iterable, Iterator, Tuple, Mapping, Optional, Union, Type, overload,
Protocol
)

T = TypeVar("T")
T_co = TypeVar('T_co', covariant=True)
KT = TypeVar("KT")
VT = TypeVar("VT")

class object:
def __init__(self) -> None: ...

class function: ...
class ellipsis: ...

class type:
def __init__(self, *a: object) -> None: ...
def __call__(self, *a: object) -> object: ...

class list(Sequence[T], Generic[T]):
@overload
def __getitem__(self, i: int) -> T: ...
@overload
def __getitem__(self, s: slice) -> list[T]: ...
def __contains__(self, item: object) -> bool: ...
def __iter__(self) -> Iterator[T]: ...

class int:
def __neg__(self) -> 'int': ...

class bool(int): ...
class float: ...
class slice: ...
class str: ...
class bytes: ...

class tuple(Sequence[T_co], Generic[T_co]):
def __new__(cls: Type[T], iterable: Iterable[T_co] = ...) -> T: ...
def __iter__(self) -> Iterator[T_co]: ...
def __contains__(self, item: object) -> bool: ...
def __getitem__(self, x: int) -> T_co: ...
def __mul__(self, n: int) -> Tuple[T_co, ...]: ...
def __rmul__(self, n: int) -> Tuple[T_co, ...]: ...
def __add__(self, x: Tuple[T_co, ...]) -> Tuple[T_co, ...]: ...
def __len__(self) -> int: ...
def count(self, obj: object) -> int: ...

class _ItemsView(Iterable[Tuple[KT, VT]]): ...

class dict(Mapping[KT, VT]):
@overload
def __init__(self, **kwargs: VT) -> None: ...
@overload
def __init__(self, arg: Iterable[Tuple[KT, VT]], **kwargs: VT) -> None: ...
def __getitem__(self, key: KT) -> VT: ...
def __setitem__(self, k: KT, v: VT) -> None: ...
def __iter__(self) -> Iterator[KT]: ...
def __contains__(self, item: object) -> int: ...
def update(self, a: Mapping[KT, VT]) -> None: ...
@overload
def get(self, k: KT) -> Optional[VT]: ...
@overload
def get(self, k: KT, default: Union[KT, T]) -> Union[VT, T]: ...
def __len__(self) -> int: ...
def pop(self, k: KT) -> VT: ...
def items(self) -> _ItemsView[KT, VT]: ...

def isinstance(x: object, t: type) -> bool: ...

class _Sized(Protocol):
def __len__(self) -> int: ...

def len(x: _Sized) -> int: ...