diff --git a/faust/__init__.py b/faust/__init__.py index f3ef14c75..132e784a5 100644 --- a/faust/__init__.py +++ b/faust/__init__.py @@ -56,7 +56,14 @@ class VersionInfo(NamedTuple): if _match is None: # pragma: no cover raise RuntimeError("THIS IS A BROKEN RELEASE!") _temp = _match.groups() -VERSION = version_info = VersionInfo(*_temp) +# XXX This is broken and so is the public ``faust.version_info``: the regex +# yields the strings ``(prefix, version, suffix)``, which land positionally in +# the ``(major, minor, micro)`` int fields. So ``.major`` is the ``'v'`` +# prefix or :const:`None`, ``.minor`` is the entire version string and +# ``.micro`` is the suffix -- e.g. ``VersionInfo(major=None, +# minor='0.11.5', micro='')`` instead of ``(0, 11, 5)``. +# Left as-is because fixing it changes what ``faust.VERSION`` holds. +VERSION = version_info = VersionInfo(*_temp) # type: ignore[arg-type] del _match del _temp del re diff --git a/faust/agents/agent.py b/faust/agents/agent.py index ceff3949e..6e203e1f4 100644 --- a/faust/agents/agent.py +++ b/faust/agents/agent.py @@ -185,15 +185,15 @@ def __init__( *, app: AppT, name: Optional[str] = None, - channel: Union[str, ChannelT] = None, + channel: Optional[Union[str, ChannelT]] = None, concurrency: int = 1, - sink: Iterable[SinkT] = None, - on_error: AgentErrorHandler = None, - supervisor_strategy: Type[SupervisorStrategyT] = None, + sink: Optional[Iterable[SinkT]] = None, + on_error: Optional[AgentErrorHandler] = None, + supervisor_strategy: Optional[Type[SupervisorStrategyT]] = None, help: Optional[str] = None, schema: Optional[SchemaT] = None, - key_type: ModelArg = None, - value_type: ModelArg = None, + key_type: Optional[ModelArg] = None, + value_type: Optional[ModelArg] = None, isolated_partitions: bool = False, use_reply_headers: Optional[bool] = None, **kwargs: Any, @@ -462,7 +462,7 @@ def info(self) -> Mapping: "isolated_partitions": self.isolated_partitions, } - def clone(self, *, cls: Type[AgentT] = None, **kwargs: Any) -> AgentT: + def clone(self, *, cls: Optional[Type[AgentT]] = None, **kwargs: Any) -> AgentT: """Create clone of this agent object. Keyword arguments can be passed to override any argument @@ -473,8 +473,8 @@ def clone(self, *, cls: Type[AgentT] = None, **kwargs: Any) -> AgentT: def test_context( self, channel: Optional[ChannelT] = None, - supervisor_strategy: SupervisorStrategyT = None, - on_error: AgentErrorHandler = None, + supervisor_strategy: Optional[SupervisorStrategyT] = None, + on_error: Optional[AgentErrorHandler] = None, **kwargs: Any, ) -> AgentTestWrapperT: # pragma: no cover """Create new unit-testing wrapper for this agent.""" @@ -501,11 +501,11 @@ async def on_agent_error(agent: AgentT, exc: BaseException) -> None: def _prepare_channel( self, - channel: Union[str, ChannelT] = None, + channel: Optional[Union[str, ChannelT]] = None, internal: bool = True, schema: Optional[SchemaT] = None, - key_type: ModelArg = None, - value_type: ModelArg = None, + key_type: Optional[ModelArg] = None, + value_type: Optional[ModelArg] = None, **kwargs: Any, ) -> ChannelT: app = self.app @@ -768,12 +768,12 @@ def _response_class(self, value: Any) -> Type[ReqRepResponse]: async def cast( self, - value: V = None, + value: Optional[V] = None, *, - key: K = None, + key: Optional[K] = None, partition: Optional[int] = None, timestamp: Optional[float] = None, - headers: HeadersArg = None, + headers: Optional[HeadersArg] = None, ) -> None: """RPC operation: like :meth:`ask` but do not expect reply. @@ -790,13 +790,13 @@ async def cast( async def ask( self, - value: V = None, + value: Optional[V] = None, *, - key: K = None, + key: Optional[K] = None, partition: Optional[int] = None, timestamp: Optional[float] = None, - headers: HeadersArg = None, - reply_to: ReplyToArg = None, + headers: Optional[HeadersArg] = None, + reply_to: Optional[ReplyToArg] = None, correlation_id: Optional[str] = None, ) -> Any: """RPC operation: ask agent for result of processing value. @@ -821,13 +821,13 @@ async def ask( async def ask_nowait( self, - value: V = None, + value: Optional[V] = None, *, - key: K = None, + key: Optional[K] = None, partition: Optional[int] = None, timestamp: Optional[float] = None, - headers: HeadersArg = None, - reply_to: ReplyToArg = None, + headers: Optional[HeadersArg] = None, + reply_to: Optional[ReplyToArg] = None, correlation_id: Optional[str] = None, force: bool = False, ) -> ReplyPromise: @@ -853,11 +853,11 @@ async def ask_nowait( def _create_req( self, - key: K = None, - value: V = None, - reply_to: ReplyToArg = None, + key: Optional[K] = None, + value: Optional[V] = None, + reply_to: Optional[ReplyToArg] = None, correlation_id: Optional[str] = None, - headers: HeadersArg = None, + headers: Optional[HeadersArg] = None, ) -> Tuple[V, Optional[HeadersArg]]: if reply_to is None: raise TypeError("Missing reply_to argument") @@ -890,15 +890,15 @@ def _request_class(self, value: V) -> Type[ReqRepRequest]: async def send( self, *, - key: K = None, - value: V = None, + key: Optional[K] = None, + value: Optional[V] = None, partition: Optional[int] = None, timestamp: Optional[float] = None, - headers: HeadersArg = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + headers: Optional[HeadersArg] = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, callback: Optional[MessageSentCallback] = None, - reply_to: ReplyToArg = None, + reply_to: Optional[ReplyToArg] = None, correlation_id: Optional[str] = None, force: bool = False, ) -> Awaitable[RecordMetadata]: @@ -930,8 +930,8 @@ def _get_strtopic(self, topic: Union[str, ChannelT, TopicT, AgentT]) -> str: async def map( self, values: Union[AsyncIterable, Iterable], - key: K = None, - reply_to: ReplyToArg = None, + key: Optional[K] = None, + reply_to: Optional[ReplyToArg] = None, ) -> AsyncIterator: # pragma: no cover """RPC map operation on a list of values. @@ -947,7 +947,7 @@ async def map( async def kvmap( self, items: Union[AsyncIterable[Tuple[K, V]], Iterable[Tuple[K, V]]], - reply_to: ReplyToArg = None, + reply_to: Optional[ReplyToArg] = None, ) -> AsyncIterator[str]: # pragma: no cover """RPC map operation on a list of ``(key, value)`` pairs. @@ -980,8 +980,8 @@ async def kvmap( async def join( self, values: Union[AsyncIterable[V], Iterable[V]], - key: K = None, - reply_to: ReplyToArg = None, + key: Optional[K] = None, + reply_to: Optional[ReplyToArg] = None, ) -> List[Any]: # pragma: no cover """RPC map operation on a list of values. @@ -996,7 +996,7 @@ async def join( async def kvjoin( self, items: Union[AsyncIterable[Tuple[K, V]], Iterable[Tuple[K, V]]], - reply_to: ReplyToArg = None, + reply_to: Optional[ReplyToArg] = None, ) -> List[Any]: # pragma: no cover """RPC map operation on list of ``(key, value)`` pairs. @@ -1159,15 +1159,15 @@ async def crash_test_agent(self, exc: BaseException) -> None: async def put( self, - value: V = None, - key: K = None, + value: Optional[V] = None, + key: Optional[K] = None, partition: Optional[int] = None, timestamp: Optional[float] = None, - headers: HeadersArg = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + headers: Optional[HeadersArg] = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, *, - reply_to: ReplyToArg = None, + reply_to: Optional[ReplyToArg] = None, correlation_id: Optional[str] = None, wait: bool = True, ) -> EventT: @@ -1203,7 +1203,7 @@ def to_message( offset: int = 0, timestamp: Optional[float] = None, timestamp_type: int = 0, - headers: HeadersArg = None, + headers: Optional[HeadersArg] = None, ) -> Message: try: topic_name = self._get_strtopic(self.original_channel) diff --git a/faust/app/_attached.py b/faust/app/_attached.py index 2295f6afa..d7a94b317 100644 --- a/faust/app/_attached.py +++ b/faust/app/_attached.py @@ -90,14 +90,14 @@ def enabled(self) -> bool: async def maybe_put( self, channel: Union[ChannelT, str], - key: K = None, - value: V = None, + key: Optional[K] = None, + value: Optional[V] = None, partition: Optional[int] = None, timestamp: Optional[float] = None, - headers: HeadersArg = None, + headers: Optional[HeadersArg] = None, schema: Optional[SchemaT] = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, callback: Optional[MessageSentCallback] = None, force: bool = False, ) -> Awaitable[RecordMetadata]: @@ -148,10 +148,10 @@ def put( value: V, partition: Optional[int] = None, timestamp: Optional[float] = None, - headers: HeadersArg = None, + headers: Optional[HeadersArg] = None, schema: Optional[SchemaT] = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, callback: Optional[MessageSentCallback] = None, ) -> Awaitable[RecordMetadata]: """Attach message to source topic offset.""" @@ -183,8 +183,15 @@ def put( async def commit(self, tp: TP, offset: int) -> None: """Publish all messaged attached to topic partition and offset.""" + # XXX ``publish_for_tp_offset`` is typed as returning bare awaitables, + # but ``asyncio.wait`` accepts only futures/tasks -- passing a plain + # coroutine raises TypeError on Python 3.11+. In-tree channels always + # hand back an ``asyncio.Future``, so this works today; a third-party + # ``ChannelT.publish_message`` returning a coroutine would break it. + # Wrapping in ``ensure_future`` here would change runtime behaviour, + # so the error is silenced instead. await asyncio.wait( - await self.publish_for_tp_offset(tp, offset), + await self.publish_for_tp_offset(tp, offset), # type: ignore[type-var] return_when=asyncio.ALL_COMPLETED, ) diff --git a/faust/app/base.py b/faust/app/base.py index 581253325..8e619042d 100644 --- a/faust/app/base.py +++ b/faust/app/base.py @@ -310,11 +310,17 @@ def sensors(self) -> Iterable[ServiceT]: def kafka_producer(self) -> Iterable[ServiceT]: """Return list of services required to start Kafka producer.""" - producers = [] + producers: List[ServiceT] = [] if self._should_enable_kafka_producer(): producers.append(self.app.producer) if self.app.conf.producer_threaded: - producers.append(self.app.producer.threaded_producer) + # XXX ``ProducerT.threaded_producer`` is ``Optional``: a + # producer that does not create one (only the aiokafka driver + # does) leaves it ``None``, and that ``None`` is appended to + # the list of services the worker then starts. + producers.append( + self.app.producer.threaded_producer # type: ignore[arg-type] + ) return producers def _should_enable_kafka_producer(self) -> bool: @@ -456,8 +462,8 @@ def __init__( self, id: str, *, - monitor: Monitor = None, - config_source: Any = None, + monitor: Optional[Monitor] = None, + config_source: Optional[Any] = None, loop: Optional[asyncio.AbstractEventLoop] = None, beacon: Optional[NodeT] = None, **options: Any, @@ -502,7 +508,10 @@ def __init__( self.boot_strategy = self.BootStrategy(self) - Service.__init__(self, loop=loop, beacon=beacon) + # mode declares ``beacon: NodeT = None`` -- an implicit-Optional the + # checker reads as non-optional -- but Service.__init__ handles + # ``beacon is None`` explicitly by rooting a new Node. + Service.__init__(self, loop=loop, beacon=beacon) # type: ignore[arg-type] def _init_signals(self) -> None: # Signals in Faust are the same as in Django, but asynchronous by @@ -698,7 +707,7 @@ def worker_init_post_autodiscover(self) -> None: def discover( self, *extra_modules: str, - categories: Iterable[str] = None, + categories: Optional[Iterable[str]] = None, ignore: Iterable[Any] = SCAN_IGNORE, ) -> None: """Discover decorators in packages.""" @@ -716,7 +725,11 @@ def discover( # otherwise a Django app that set e.g. # ``autodiscover=['myproj.agents']`` would still scan all of # INSTALLED_APPS (including migrations/admin). See #500. - if self.conf.autodiscover is True: + # ``Settings.autodiscover`` is a ``Param`` descriptor, but one of the + # setting's *value* types is itself a callable, so mypy reads the + # class attribute as a method and tries to bind ``self`` to it + # instead of going through ``Param.__get__``. + if self.conf.autodiscover is True: # type: ignore[misc] for fixup in self.fixups: modules |= set(fixup.autodiscover_modules()) if modules: @@ -745,7 +758,9 @@ def _on_autodiscovery_error(self, name: str) -> None: def _discovery_modules(self) -> List[str]: modules: List[str] = [] - autodiscover = self.conf.autodiscover + # See the note in ``discover()``: mypy mistakes this setting for a + # method because one of its value types is a callable. + autodiscover = self.conf.autodiscover # type: ignore[misc] if autodiscover: if isinstance(autodiscover, bool): if self.conf.origin is None: @@ -764,7 +779,9 @@ def main(self) -> NoReturn: self.finalize() self.worker_init() - if self.conf.autodiscover: + # See the note in ``discover()``: mypy mistakes this setting for a + # method because one of its value types is a callable. + if self.conf.autodiscover: # type: ignore[misc] self.discover() self.worker_init_post_autodiscover() cli(app=self) @@ -773,12 +790,12 @@ def main(self) -> NoReturn: def topic( self, *topics: str, - pattern: Union[str, Pattern] = None, + pattern: Optional[Union[str, Pattern]] = None, schema: Optional[SchemaT] = None, key_type: Optional[ModelArg] = None, value_type: Optional[ModelArg] = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, partitions: Optional[int] = None, retention: Optional[Seconds] = None, compacting: Optional[bool] = None, @@ -856,12 +873,12 @@ def channel( def agent( self, - channel: Union[str, ChannelT[_T]] = None, + channel: Optional[Union[str, ChannelT[_T]]] = None, *, name: Optional[str] = None, concurrency: int = 1, - supervisor_strategy: Type[SupervisorStrategyT] = None, - sink: Iterable[SinkT] = None, + supervisor_strategy: Optional[Type[SupervisorStrategyT]] = None, + sink: Optional[Iterable[SinkT]] = None, isolated_partitions: bool = False, use_reply_headers: bool = True, **kwargs: Any, @@ -928,7 +945,11 @@ async def _on_agent_error(self, agent: AgentT, exc: BaseException) -> None: @no_type_check def task( - self, fun: TaskArg = None, *, on_leader: bool = False, traced: bool = True + self, + fun: Optional[TaskArg] = None, + *, + on_leader: bool = False, + traced: bool = True, ) -> TaskDecoratorRet: """Define an async def function to be started with the app. @@ -1038,11 +1059,16 @@ async def around_timer(*args: Any) -> None: return _inner - def crontab( + # ``App`` inherits both ``AppT`` and ``mode.Service``, and deliberately + # shadows ``Service.crontab`` (a classmethod defining a background timer + # on a Service subclass) with the app-level ``@app.crontab(...)`` + # decorator declared by ``AppT``. Same for ``task``/``timer`` above, + # which are hidden from the checker by ``@no_type_check`` instead. + def crontab( # type: ignore[override] self, cron_format: str, *, - timezone: tzinfo = None, + timezone: Optional[tzinfo] = None, on_leader: bool = False, traced: bool = True, ) -> Callable: @@ -1146,7 +1172,7 @@ def Table( self, name: str, *, - default: Callable[[], Any] = None, + default: Optional[Callable[[], Any]] = None, window: Optional[WindowT] = None, partitions: Optional[int] = None, help: Optional[str] = None, @@ -1191,7 +1217,7 @@ def GlobalTable( self, name: str, *, - default: Callable[[], Any] = None, + default: Optional[Callable[[], Any]] = None, window: Optional[WindowT] = None, partitions: Optional[int] = None, help: Optional[str] = None, @@ -1297,7 +1323,7 @@ def page( path: str, *, base: Type[View] = View, - cors_options: Mapping[str, ResourceOptions] = None, + cors_options: Optional[Mapping[str, ResourceOptions]] = None, name: Optional[str] = None, ) -> Callable[[PageArg], Type[View]]: """Decorate view to be included in the web server.""" @@ -1369,7 +1395,7 @@ async def get( def topic_route( self, - topic: CollectionT, + topic: TopicT, shard_param: Optional[str] = None, *, query_param: Optional[str] = None, @@ -1475,7 +1501,11 @@ def traced( **context: Any, ) -> Callable: """Decorate function to be traced using the OpenTracing API.""" - assert fun + # XXX dead check: this is a truthiness test on a callable, and function + # objects are always truthy, so it can only ever fire for a falsy + # non-function callable. Left exactly as it is -- ``fun is not None`` + # would be a different runtime test. + assert fun # type: ignore[truthy-function] operation: str = name or operation_name_from_fun(fun) @wraps(fun) @@ -1502,14 +1532,14 @@ def _start_span_from_rebalancing(self, name: str) -> opentracing.Span: async def send( self, channel: Union[ChannelT, str], - key: K = None, - value: V = None, + key: Optional[K] = None, + value: Optional[V] = None, partition: Optional[int] = None, timestamp: Optional[float] = None, - headers: HeadersArg = None, + headers: Optional[HeadersArg] = None, schema: Optional[SchemaT] = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, callback: Optional[MessageSentCallback] = None, ) -> Awaitable[RecordMetadata]: """Send event to channel/topic. diff --git a/faust/app/router.py b/faust/app/router.py index e15f6efb7..8d11669ec 100644 --- a/faust/app/router.py +++ b/faust/app/router.py @@ -35,7 +35,14 @@ def external_topic_key_store(self, topic: TopicT, key: K) -> URL: """Return the URL of web server that processes the key in a topics.""" topic_name = topic.get_topic_name() k = topic.prepare_key(key, None)[0] - return self._assignor.external_key_store(topic_name, k) + # XXX PartitionAssignorT does not declare external_key_store(); only the + # concrete faust.assignor.partition_assignor.PartitionAssignor defines + # it. Settings.PartitionAssignor lets users plug in their own assignor, + # and any subclass of PartitionAssignorT that does not happen to define + # this method will raise AttributeError right here. + return self._assignor.external_key_store( # type: ignore[attr-defined] + topic_name, k + ) def table_metadata(self, table_name: str) -> HostToPartitionMap: """Return metadata stored for table in the partition assignor.""" @@ -49,7 +56,12 @@ def tables_metadata(self) -> HostToPartitionMap: def external_topics_metadata(self) -> HostToPartitionMap: """Return metadata stored for all external topics in the partition assignor.""" - return self._assignor.external_topics_metadata() + # XXX PartitionAssignorT does not declare external_topics_metadata(); + # only the concrete faust.assignor.partition_assignor.PartitionAssignor + # defines it. Settings.PartitionAssignor lets users plug in their own + # assignor, and any subclass of PartitionAssignorT that does not happen + # to define this method will raise AttributeError right here. + return self._assignor.external_topics_metadata() # type: ignore[attr-defined] @classmethod def _get_table_topic(cls, table: CollectionT) -> str: diff --git a/faust/assignor/client_assignment.py b/faust/assignor/client_assignment.py index 36069b127..6e255180a 100644 --- a/faust/assignor/client_assignment.py +++ b/faust/assignor/client_assignment.py @@ -1,7 +1,7 @@ """Client Assignment.""" import copy -from typing import List, Mapping, MutableMapping, Sequence, Set, Tuple, cast +from typing import List, Mapping, MutableMapping, Optional, Sequence, Set, Tuple, cast from faust.models import Record from faust.types import TP @@ -22,9 +22,9 @@ class CopartitionedAssignment: def __init__( self, - actives: Set[int] = None, - standbys: Set[int] = None, - topics: Set[str] = None, + actives: Optional[Set[int]] = None, + standbys: Optional[Set[int]] = None, + topics: Optional[Set[str]] = None, ) -> None: self.actives = actives or set() self.standbys = standbys or set() diff --git a/faust/auth.py b/faust/auth.py index 57e6919dd..da78d6842 100644 --- a/faust/auth.py +++ b/faust/auth.py @@ -1,7 +1,7 @@ """Authentication Credentials.""" import ssl -from typing import Any, Optional, Union +from typing import Optional, Union from aiokafka.conn import AbstractTokenProvider @@ -36,8 +36,8 @@ def __init__( *, username: Optional[str] = None, password: Optional[str] = None, - ssl_context: ssl.SSLContext = None, - mechanism: Union[str, SASLMechanism] = None, + ssl_context: Optional[ssl.SSLContext] = None, + mechanism: Optional[Union[str, SASLMechanism]] = None, ) -> None: self.username = username self.password = password @@ -93,8 +93,8 @@ def __init__( *, kerberos_service_name: str = "kafka", kerberos_domain_name: Optional[str] = None, - ssl_context: ssl.SSLContext = None, - mechanism: Union[str, SASLMechanism] = None, + ssl_context: Optional[ssl.SSLContext] = None, + mechanism: Optional[Union[str, SASLMechanism]] = None, ) -> None: self.kerberos_service_name = kerberos_service_name self.kerberos_domain_name = kerberos_domain_name @@ -122,16 +122,23 @@ class SSLCredentials(Credentials): def __init__( self, - context: ssl.SSLContext = None, + context: Optional[ssl.SSLContext] = None, *, - purpose: Any = None, + purpose: Optional[ssl.Purpose] = None, cafile: Optional[str] = None, capath: Optional[str] = None, cadata: Optional[str] = None, ) -> None: if context is None: context = ssl.create_default_context( - purpose=purpose, + # XXX ``purpose`` defaults to None here, but + # ``ssl.create_default_context`` requires an ``ssl.Purpose`` + # and raises ``TypeError`` on None -- so ``SSLCredentials()`` + # with no explicit ``purpose`` cannot build a context at all. + # Real bug, kept as-is: fixing it changes the default TLS + # purpose of a security-relevant public API, which is out of + # scope for a typing pass. + purpose=purpose, # type: ignore[arg-type] cafile=cafile, capath=capath, cadata=cadata, diff --git a/faust/channels.py b/faust/channels.py index 4ef013cfe..aab029dba 100644 --- a/faust/channels.py +++ b/faust/channels.py @@ -92,8 +92,8 @@ def __init__( app: AppT, *, schema: Optional[SchemaT] = None, - key_type: ModelArg = None, - value_type: ModelArg = None, + key_type: Optional[ModelArg] = None, + value_type: Optional[ModelArg] = None, is_iterator: bool = False, queue: Optional[ThrowableQueue] = None, maxsize: Optional[int] = None, @@ -120,7 +120,7 @@ def __init__( self.value_type = self.schema.value_type def _get_default_schema( - self, key_type: ModelArg = None, value_type: ModelArg = None + self, key_type: Optional[ModelArg] = None, value_type: Optional[ModelArg] = None ) -> SchemaT: return cast( SchemaT, @@ -201,14 +201,14 @@ def get_topic_name(self) -> str: async def send( self, *, - key: K = None, - value: V = None, + key: Optional[K] = None, + value: Optional[V] = None, partition: Optional[int] = None, timestamp: Optional[float] = None, - headers: HeadersArg = None, + headers: Optional[HeadersArg] = None, schema: Optional[SchemaT] = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, callback: Optional[MessageSentCallback] = None, force: bool = False, ) -> Awaitable[RecordMetadata]: @@ -228,14 +228,14 @@ async def send( def send_soon( self, *, - key: K = None, - value: V = None, + key: Optional[K] = None, + value: Optional[V] = None, partition: Optional[int] = None, timestamp: Optional[float] = None, - headers: HeadersArg = None, + headers: Optional[HeadersArg] = None, schema: Optional[SchemaT] = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, callback: Optional[MessageSentCallback] = None, force: bool = False, eager_partitioning: bool = False, @@ -251,14 +251,14 @@ def send_soon( def as_future_message( self, - key: K = None, - value: V = None, + key: Optional[K] = None, + value: Optional[V] = None, partition: Optional[int] = None, timestamp: Optional[float] = None, - headers: HeadersArg = None, + headers: Optional[HeadersArg] = None, schema: Optional[SchemaT] = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, callback: Optional[MessageSentCallback] = None, eager_partitioning: bool = False, ) -> FutureMessage: @@ -302,14 +302,14 @@ def prepare_headers(self, headers: Optional[HeadersArg]) -> OpenHeadersArg: async def _send_now( self, - key: K = None, - value: V = None, + key: Optional[K] = None, + value: Optional[V] = None, partition: Optional[int] = None, timestamp: Optional[float] = None, - headers: HeadersArg = None, + headers: Optional[HeadersArg] = None, schema: Optional[SchemaT] = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, callback: Optional[MessageSentCallback] = None, ) -> Awaitable[RecordMetadata]: return await self.publish_message( @@ -391,7 +391,7 @@ def prepare_key( key: K, key_serializer: CodecArg, schema: Optional[SchemaT] = None, - headers: OpenHeadersArg = None, + headers: Optional[OpenHeadersArg] = None, ) -> Tuple[Any, OpenHeadersArg]: """Prepare key before it is sent to this channel. @@ -405,7 +405,7 @@ def prepare_value( value: V, value_serializer: CodecArg, schema: Optional[SchemaT] = None, - headers: OpenHeadersArg = None, + headers: Optional[OpenHeadersArg] = None, ) -> Tuple[Any, OpenHeadersArg]: """Prepare value before it is sent to this channel. @@ -454,7 +454,9 @@ async def put(self, value: EventT[T_contra]) -> None: async def get(self, *, timeout: Optional[Seconds] = None) -> EventT[T]: """Get the next :class:`~faust.Event` received on this channel.""" - timeout_: float = want_seconds(timeout) + timeout_: Optional[float] = ( + want_seconds(timeout) if timeout is not None else None + ) if timeout_: return await asyncio.wait_for(self.queue.get(), timeout=timeout_) return await self.queue.get() @@ -594,10 +596,10 @@ def __init__( app: AppT, *, schema: Optional[SchemaT] = None, - key_type: ModelArg = None, - value_type: ModelArg = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_type: Optional[ModelArg] = None, + value_type: Optional[ModelArg] = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, allow_empty: Optional[bool] = None, **kwargs: Any, ) -> None: @@ -631,10 +633,10 @@ def _contribute_to_schema( self, schema: SchemaT, *, - key_type: ModelArg = None, - value_type: ModelArg = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_type: Optional[ModelArg] = None, + value_type: Optional[ModelArg] = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, allow_empty: Optional[bool] = None, ) -> None: # Update schema and take compat attributes @@ -649,10 +651,10 @@ def _contribute_to_schema( def _get_default_schema( self, - key_type: ModelArg = None, - value_type: ModelArg = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_type: Optional[ModelArg] = None, + value_type: Optional[ModelArg] = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, allow_empty: Optional[bool] = None, ) -> SchemaT: return cast( @@ -681,7 +683,7 @@ def prepare_key( key: K, key_serializer: CodecArg, schema: Optional[SchemaT] = None, - headers: OpenHeadersArg = None, + headers: Optional[OpenHeadersArg] = None, ) -> Tuple[Any, OpenHeadersArg]: """Serialize key to format suitable for transport.""" if key is not None: @@ -697,7 +699,7 @@ def prepare_value( value: V, value_serializer: CodecArg, schema: Optional[SchemaT] = None, - headers: OpenHeadersArg = None, + headers: Optional[OpenHeadersArg] = None, ) -> Tuple[Any, OpenHeadersArg]: """Serialize value to format suitable for transport.""" schema = schema or self.schema diff --git a/faust/cli/base.py b/faust/cli/base.py index 8e57b6485..0b1c0c3db 100644 --- a/faust/cli/base.py +++ b/faust/cli/base.py @@ -145,7 +145,10 @@ def __repr__(self) -> str: # implement through decorators." # [from https://github.com/pallets/click/issues/108] class State: - app: Optional[AppT] = None + #: The ``-A``/``--app`` option: a string like ``proj.app``, never an + #: already-instantiated app (that one is stashed on the root + #: :class:`click.Context` instead, see ``_Group.make_context``). + app: Optional[str] = None quiet: bool = False debug: bool = False workdir: Optional[str] = None @@ -153,7 +156,7 @@ class State: json: bool = False loop: Optional[str] = None logfile: Optional[str] = None - loglevel: Optional[int] = None + loglevel: Optional[str] = None blocking_timeout: Optional[float] = None console_port: Optional[int] = None @@ -161,7 +164,7 @@ class State: def compat_option( *args: Any, state_key: str, - callback: Callable[[click.Context, click.Parameter, Any], Any] = None, + callback: Optional[Callable[[click.Context, click.Parameter, Any], Any]] = None, expose_value: bool = False, **kwargs: Any, ) -> Callable[[Any], click.Parameter]: @@ -329,7 +332,12 @@ def prepare_app(app: AppT, name: Optional[str]) -> AppT: if app.conf._origin is None: app.conf._origin = name app.worker_init() - if app.conf.autodiscover: + # ``Settings.autodiscover`` is a ``Param`` descriptor, but one of the + # setting's *value* types is itself a callable, so mypy reads the class + # attribute as a method and tries to bind ``self`` to it instead of going + # through ``Param.__get__``. Same workaround as + # ``Producer.__init__``/``producer_partitioner``. + if app.conf.autodiscover: # type: ignore[misc] app.discover() app.worker_init_post_autodiscover() @@ -413,9 +421,9 @@ def make_context( info_name: str, args: str, app: Optional[AppT] = None, - parent: click.Context = None, - stdout: IO = None, - stderr: IO = None, + parent: Optional[click.Context] = None, + stdout: Optional[IO] = None, + stderr: Optional[IO] = None, side_effects: bool = True, **extra: Any, ) -> click.Context: @@ -449,7 +457,7 @@ def cli(*args: Any, **kwargs: Any) -> None: # pragma: no cover def _prepare_cli( ctx: click.Context, - app: Union[AppT, str], + app: Optional[str], quiet: bool, debug: bool, workdir: str, @@ -502,10 +510,10 @@ class Command(abc.ABC): # noqa: B024 debug: bool quiet: bool - workdir: str - datadir: str + workdir: Optional[str] + datadir: Optional[str] json: bool - logfile: str + logfile: Optional[str] _loglevel: Optional[str] _blocking_timeout: Optional[float] _console_port: Optional[int] @@ -715,7 +723,13 @@ def _table_wrap(self, table: terminal.Table, text: str) -> str: max_width = max(table.column_max_width(1), 10) return "\n".join(wrap(text, max_width)) - def say(self, message: str, file: IO = None, err: IO = None, **kwargs: Any) -> None: + def say( + self, + message: str, + file: Optional[IO] = None, + err: Optional[IO] = None, + **kwargs: Any, + ) -> None: """Print something to stdout (or use ``file=stderr`` kwarg). Note: @@ -819,8 +833,8 @@ def __init__( self, ctx: click.Context, *args: Any, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, **kwargs: Any, ) -> None: super().__init__(ctx) @@ -831,11 +845,19 @@ def __init__( self.key_serializer = key_serializer or self.app.conf.key_serializer self.value_serializer = value_serializer or self.app.conf.value_serializer - def _finalize_app(self, app: AppT) -> AppT: + def _finalize_app(self, app: Optional[AppT]) -> AppT: if app is not None: return self._finalize_concrete_app(app) else: - return self._app_from_str(self.state.app) + # XXX ``_app_from_str`` returns None for ``require_app = False`` + # commands (see faust/cli/completion.py) that were invoked without + # ``-A``, so this really can hand back None -- and + # ``AppCommand.__init__`` dereferences ``self.app.conf`` + # unconditionally right after, raising ``AttributeError: 'NoneType' + # object has no attribute 'conf'``. Real bug; fixing it means + # deciding what such commands should do without an app, which is + # out of scope for a typing pass. + return self._app_from_str(self.state.app) # type: ignore[return-value] def _app_from_str(self, appstr: Optional[str] = None) -> Optional[AppT]: if appstr: @@ -996,13 +1018,16 @@ def blocking_timeout(self, timeout: float) -> None: def call_command( command: str, - args: List[str] = None, - stdout: IO = None, - stderr: IO = None, + args: Optional[List[str]] = None, + stdout: Optional[IO] = None, + stderr: Optional[IO] = None, side_effects: bool = False, **kwargs: Any, -) -> Tuple[int, IO, IO]: - exitcode: int = 0 +) -> Tuple[Union[int, str, None], IO, IO]: + # The exit code is whatever ``SystemExit`` carried, and that is not + # necessarily an int: ``sys.exit()`` also accepts None (success) or a + # string (message printed to stderr, exit status 1). + exitcode: Union[int, str, None] = 0 if stdout is None: stdout = io.StringIO() if stderr is None: diff --git a/faust/cli/worker.py b/faust/cli/worker.py index 9c6ad1e96..c712a4ee1 100644 --- a/faust/cli/worker.py +++ b/faust/cli/worker.py @@ -88,7 +88,7 @@ def _init_worker_options( web_port: Optional[int], web_bind: Optional[str], web_host: Optional[str], - web_transport: URL, + web_transport: Optional[URL], **kwargs: Any, ) -> None: self.app.conf.web_enabled = with_web @@ -101,9 +101,13 @@ def _init_worker_options( if web_transport is not None: self.app.conf.web_transport = web_transport if web_port is not None or web_host is not None: - self.app.conf.canonical_url = ( - f"http://{self.app.conf.web_host}:{self.app.conf.web_port}" - ) + # XXX ``canonical_url`` accepts ``URLArg`` (str or URL) at runtime + # -- ``params.URL.to_python`` wraps it -- but the setting + # descriptor is declared with only its *output* type + # (``def canonical_url(self) -> URL``), so mypy sees the setter as + # accepting URL alone. The defect is in the settings descriptor + # typing, not here; assigning a str is supported behaviour. + self.app.conf.canonical_url = f"http://{self.app.conf.web_host}:{self.app.conf.web_port}" # type: ignore[assignment] # noqa: E501 @property def _Worker(self) -> Type[Worker]: diff --git a/faust/contrib/sentry.py b/faust/contrib/sentry.py index 6a527ac29..fd6644aeb 100644 --- a/faust/contrib/sentry.py +++ b/faust/contrib/sentry.py @@ -92,7 +92,7 @@ def carp(self, obj: Any) -> None: def handler_from_dsn( dsn: Optional[str] = None, workers: int = 5, - include_paths: Iterable[str] = None, + include_paths: Optional[Iterable[str]] = None, loglevel: Optional[int] = None, qsize: int = 1000, **kwargs: Any, @@ -129,7 +129,7 @@ def setup( workers: int = 4, max_queue_size: int = 1000, loglevel: Optional[int] = None, - **kwargs, + **kwargs: Any, ) -> None: sentry_handler = handler_from_dsn( dsn=dsn, workers=workers, qsize=max_queue_size, loglevel=loglevel, **kwargs diff --git a/faust/events.py b/faust/events.py index e45dd4fe1..410053036 100644 --- a/faust/events.py +++ b/faust/events.py @@ -138,8 +138,8 @@ async def send( timestamp: Optional[float] = None, headers: Any = USE_EXISTING_HEADERS, schema: Optional[SchemaT] = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, callback: Optional[MessageSentCallback] = None, force: bool = False, ) -> Awaitable[RecordMetadata]: @@ -173,8 +173,8 @@ async def forward( timestamp: Optional[float] = None, headers: Any = USE_EXISTING_HEADERS, schema: Optional[SchemaT] = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, callback: Optional[MessageSentCallback] = None, force: bool = False, ) -> Awaitable[RecordMetadata]: @@ -204,14 +204,14 @@ async def forward( async def _send( self, channel: Union[str, ChannelT], - key: K = None, - value: V = None, + key: Optional[K] = None, + value: Optional[V] = None, partition: Optional[int] = None, timestamp: Optional[float] = None, - headers: HeadersArg = None, + headers: Optional[HeadersArg] = None, schema: Optional[SchemaT] = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, callback: Optional[MessageSentCallback] = None, force: bool = False, ) -> Awaitable[RecordMetadata]: @@ -232,14 +232,14 @@ async def _send( def _attach( self, channel: Union[ChannelT, str], - key: K = None, - value: V = None, + key: Optional[K] = None, + value: Optional[V] = None, partition: Optional[int] = None, timestamp: Optional[float] = None, - headers: HeadersArg = None, + headers: Optional[HeadersArg] = None, schema: Optional[SchemaT] = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, callback: Optional[MessageSentCallback] = None, ) -> Awaitable[RecordMetadata]: return cast(_App, self.app)._attachments.put( @@ -273,9 +273,9 @@ async def __aenter__(self) -> EventT: async def __aexit__( self, - _exc_type: Type[BaseException] = None, - _exc_val: BaseException = None, - _exc_tb: TracebackType = None, + _exc_type: Optional[Type[BaseException]] = None, + _exc_val: Optional[BaseException] = None, + _exc_tb: Optional[TracebackType] = None, ) -> Optional[bool]: self.ack() return None diff --git a/faust/livecheck/app.py b/faust/livecheck/app.py index b2fab3ae8..b6eac1ecd 100644 --- a/faust/livecheck/app.py +++ b/faust/livecheck/app.py @@ -56,7 +56,12 @@ def on_stream_event_in( return None def on_stream_event_out( - self, tp: TP, offset: int, stream: StreamT, event: EventT, state: Dict = None + self, + tp: TP, + offset: int, + stream: StreamT, + event: EventT, + state: Optional[Dict] = None, ) -> None: """Call when stream is finished handling event.""" has_active_test = getattr(stream, "current_test", None) @@ -186,18 +191,22 @@ def _apply_monkeypatches(self) -> None: patches.patch_all() def _connect_signals(self) -> None: + # mode's ``SignalHandlerT`` is ``(sender, /, *args, signal, **kwargs)`` + # -- the sender has to be positional-only for a handler to match it. + # This handler accepts it by name as well, which mode never does but + # which the public signature has always allowed. AppT.on_produce_message.connect( - self.on_produce_attach_test_headers - ) # type: ignore + self.on_produce_attach_test_headers # type: ignore[arg-type] + ) def on_produce_attach_test_headers( self, sender: AppT, - key: bytes = None, - value: bytes = None, + key: Optional[bytes] = None, + value: Optional[bytes] = None, partition: Optional[int] = None, timestamp: Optional[float] = None, - headers: List[Tuple[str, bytes]] = None, + headers: Optional[List[Tuple[str, bytes]]] = None, signal: Optional[BaseSignalT] = None, **kwargs: Any, ) -> None: diff --git a/faust/livecheck/case.py b/faust/livecheck/case.py index 771d89050..5b75c554d 100644 --- a/faust/livecheck/case.py +++ b/faust/livecheck/case.py @@ -125,7 +125,7 @@ def __init__( probability: Optional[float] = None, warn_stalled_after: Optional[Seconds] = None, active: Optional[bool] = None, - signals: Iterable[BaseSignal] = None, + signals: Optional[Iterable[BaseSignal]] = None, test_expires: Optional[Seconds] = None, frequency: Optional[Seconds] = None, realtime_logs: Optional[bool] = None, diff --git a/faust/livecheck/patches/aiohttp.py b/faust/livecheck/patches/aiohttp.py index 29aa3b39f..dda48c409 100644 --- a/faust/livecheck/patches/aiohttp.py +++ b/faust/livecheck/patches/aiohttp.py @@ -50,7 +50,7 @@ def __init__( @no_type_check def _faust_trace_configs( - self, configs: List[TraceConfig] = None + self, configs: Optional[List[TraceConfig]] = None ) -> List[TraceConfig]: if configs is None: configs = [] diff --git a/faust/livecheck/runners.py b/faust/livecheck/runners.py index 2e6594e1a..e9f847e29 100644 --- a/faust/livecheck/runners.py +++ b/faust/livecheck/runners.py @@ -138,7 +138,9 @@ async def on_start(self) -> None: ) await self.case.on_test_start(self) - async def on_signal_wait(self, signal: BaseSignal, timeout: float) -> None: + async def on_signal_wait( + self, signal: BaseSignal, timeout: Optional[float] + ) -> None: """Call when the test is waiting for a signal.""" self.log_info( "∆ %r/%r %s (%rs)...", diff --git a/faust/livecheck/signals.py b/faust/livecheck/signals.py index 020767da0..4d03b9953 100644 --- a/faust/livecheck/signals.py +++ b/faust/livecheck/signals.py @@ -32,18 +32,26 @@ class BaseSignal(Generic[VT]): case: _Case index: int - def __init__(self, name: str = "", case: _Case = None, index: int = -1) -> None: + def __init__( + self, name: str = "", case: Optional[_Case] = None, index: int = -1 + ) -> None: self.name = name self.case = cast(_Case, case) self.index = index async def send( - self, value: VT = None, *, key: Any = None, force: bool = False + self, + value: Optional[VT] = None, + *, + key: Optional[Any] = None, + force: bool = False, ) -> None: """Notify test that this signal is now complete.""" raise NotImplementedError() - async def wait(self, *, key: Any = None, timeout: Optional[Seconds] = None) -> VT: + async def wait( + self, *, key: Optional[Any] = None, timeout: Optional[Seconds] = None + ) -> VT: """Wait for signal to be completed.""" raise NotImplementedError() @@ -62,7 +70,12 @@ def _wakeup_resolvers(self) -> None: async def _wait_for_resolved(self, *, timeout: Optional[float] = None) -> None: app = self.case.app app._can_resolve.clear() - await app.wait(app._can_resolve, timeout=timeout) + # mode's ``WaitArgT`` only lists ``mode.utils.locks.Event``, but + # ``Service.wait_first`` calls ``.wait()`` on anything that is not + # already awaitable, so an :class:`asyncio.Event` works identically. + # ``timeout`` is declared ``Seconds`` even though mode's own default + # for it is ``None`` and it explicitly handles ``None``. + await app.wait(app._can_resolve, timeout=timeout) # type: ignore[arg-type] def _get_current_value(self, key: Any) -> SignalEvent: return self.case.app._resolved_signals[self._index_key(key)] @@ -96,7 +109,11 @@ class Signal(BaseSignal[VT]): # topic for each test app. async def send( - self, value: VT = None, *, key: Any = None, force: bool = False + self, + value: Optional[VT] = None, + *, + key: Optional[Any] = None, + force: bool = False, ) -> None: """Notify test that this signal is now complete.""" current_test = current_test_stack.top @@ -116,7 +133,9 @@ async def send( ), ) - async def wait(self, *, key: Any = None, timeout: Optional[Seconds] = None) -> VT: + async def wait( + self, *, key: Optional[Any] = None, timeout: Optional[Seconds] = None + ) -> VT: """Wait for signal to be completed.""" # wait for key to arrive in consumer runner = self.case.current_execution @@ -125,7 +144,13 @@ async def wait(self, *, key: Any = None, timeout: Optional[Seconds] = None) -> V test = runner.test assert test k: Any = test.id if key is None else key - timeout_s = want_seconds(timeout) + # ``timeout`` is genuinely optional here: ``want_seconds(None)`` returns + # :const:`None` (its ``singledispatch`` fallback returns the argument + # unchanged), so this stays equivalent while keeping the optionality + # visible to callees that have to handle "no timeout". + timeout_s: Optional[float] = ( + want_seconds(timeout) if timeout is not None else None + ) await runner.on_signal_wait(self, timeout=timeout_s) time_start = monotonic() event = await self._wait_for_message_by_key(key=k, timeout=timeout_s) diff --git a/faust/models/base.py b/faust/models/base.py index 6b7c552a6..ff0472e49 100644 --- a/faust/models/base.py +++ b/faust/models/base.py @@ -147,7 +147,9 @@ class Model(ModelT): #: Set to True if this is an abstract base class. __is_abstract__: ClassVar[bool] = True - __validation_errors__ = None + #: Cache of validation errors, populated on first call to + #: :meth:`validate`. :const:`None` means "not validated yet". + __validation_errors__: Optional[List[ValidationError]] = None _pending_finalizers: ClassVar[Optional[List[Callable]]] = None @@ -175,7 +177,7 @@ def _maybe_namespace( cls, data: Any, *, - preferred_type: Type[ModelT] = None, + preferred_type: Optional[Type[ModelT]] = None, fast_types: Tuple[Type, ...] = (bytes, str), isinstance: Callable = isinstance, ) -> Optional[Type[ModelT]]: @@ -232,8 +234,8 @@ def loads( cls, s: bytes, *, - default_serializer: CodecArg = None, # XXX use serializer - serializer: CodecArg = None, + default_serializer: Optional[CodecArg] = None, # XXX use serializer + serializer: Optional[CodecArg] = None, ) -> ModelT: """Deserialize model object from bytes. @@ -260,10 +262,10 @@ def __init_subclass__( allow_blessed_key: Optional[bool] = None, decimals: Optional[bool] = None, coerce: Optional[bool] = None, - coercions: CoercionMapping = None, + coercions: Optional[CoercionMapping] = None, polymorphic_fields: Optional[bool] = None, validation: Optional[bool] = None, - date_parser: Callable[[Any], datetime] = None, + date_parser: Optional[Callable[[Any], datetime]] = None, lazy_creation: bool = False, **kwargs: Any, ) -> None: @@ -315,10 +317,10 @@ def _init_subclass( allow_blessed_key: Optional[bool] = None, decimals: Optional[bool] = None, coerce: Optional[bool] = None, - coercions: CoercionMapping = None, + coercions: Optional[CoercionMapping] = None, polymorphic_fields: Optional[bool] = None, validation: Optional[bool] = None, - date_parser: Callable[[Any], datetime] = None, + date_parser: Optional[Callable[[Any], datetime]] = None, ) -> None: # Can set serializer/namespace/etc. using: # class X(Record, serializer='json', namespace='com.vandelay.X'): @@ -484,7 +486,7 @@ def derive(self, *objects: ModelT, **fields: Any) -> ModelT: def _derive(self, *objects: ModelT, **fields: Any) -> ModelT: raise NotImplementedError() - def dumps(self, *, serializer: CodecArg = None) -> bytes: + def dumps(self, *, serializer: Optional[CodecArg] = None) -> bytes: """Serialize object to the target serialization format.""" return dumps(serializer or self._options.serializer, self.to_representation()) diff --git a/faust/models/fields.py b/faust/models/fields.py index e34b22a7d..2a7e61d36 100644 --- a/faust/models/fields.py +++ b/faust/models/fields.py @@ -44,7 +44,7 @@ CharacterType = TypeVar("CharacterType", str, bytes) -def _is_concrete_model(typ: Type = None) -> bool: +def _is_concrete_model(typ: Optional[Type] = None) -> bool: return ( typ is not None and inspect.isclass(typ) @@ -135,15 +135,15 @@ def __init__( field: Optional[str] = None, input_name: Optional[str] = None, output_name: Optional[str] = None, - type: Type[T] = None, - model: Type[ModelT] = None, + type: Optional[Type[T]] = None, + model: Optional[Type[ModelT]] = None, required: bool = True, - default: T = None, + default: Optional[T] = None, parent: Optional[FieldDescriptorT] = None, coerce: Optional[bool] = None, exclude: Optional[bool] = None, - date_parser: Callable[[Any], datetime] = None, - tag: Type[Tag] = None, + date_parser: Optional[Callable[[Any], datetime]] = None, + tag: Optional[Type[Tag]] = None, **options: Any, ) -> None: self.field = cast(str, field) @@ -244,7 +244,7 @@ def prepare_value( ) -> Optional[T]: return cast(T, value) - def _copy_descriptors(self, typ: Type = None) -> None: + def _copy_descriptors(self, typ: Optional[Type] = None) -> None: if typ is not None and _is_concrete_model(typ): typ._contribute_field_descriptors(self, typ._options, parent=self) @@ -346,10 +346,8 @@ def __init__( super().__init__( **kwargs, - **{ - "max_value": max_value, - "min_value": min_value, - }, + max_value=max_value, + min_value=min_value, ) def validate(self, value: T) -> Iterable[ValidationError]: @@ -394,10 +392,8 @@ def __init__( super().__init__( **kwargs, - **{ - "max_digits": max_digits, - "max_decimal_places": max_decimal_places, - }, + max_digits=max_digits, + max_decimal_places=max_decimal_places, ) def to_python(self, value: Any) -> Any: @@ -422,7 +418,11 @@ def validate(self, value: Decimal) -> Iterable[ValidationError]: mdp = self.max_decimal_places if mdp: decimal_tuple = value.as_tuple() - if abs(decimal_tuple.exponent) > mdp: + # XXX Known bug: execution does not stop after the non-finite check + # above, so an Inf/NaN Decimal reaches here with a *str* exponent + # ('n'/'N'/'F') and abs() raises TypeError instead of yielding a + # ValidationError. cast() only silences mypy; it does not fix this. + if abs(cast(int, decimal_tuple.exponent)) > mdp: yield self.validation_error( f"{self.field} must have less than {mdp} decimal places." ) @@ -430,7 +430,10 @@ def validate(self, value: Decimal) -> Iterable[ValidationError]: if max_digits: if decimal_tuple is None: decimal_tuple = value.as_tuple() - digits = len(decimal_tuple.digits[: decimal_tuple.exponent]) + # XXX Same known bug as above: for a non-finite Decimal the exponent + # is a str, so this slice raises TypeError rather than reporting a + # validation error. cast() only silences mypy; it does not fix this. + digits = len(decimal_tuple.digits[: cast(int, decimal_tuple.exponent)]) if digits > max_digits: yield self.validation_error( f"{self.field} must have less than {max_digits} digits." @@ -458,12 +461,10 @@ def __init__( self.allow_blank = allow_blank super().__init__( **kwargs, - **{ - "max_length": max_length, - "min_length": min_length, - "trim_whitespace": trim_whitespace, - "allow_blank": allow_blank, - }, + max_length=max_length, + min_length=min_length, + trim_whitespace=trim_whitespace, + allow_blank=allow_blank, ) def validate(self, value: CharacterType) -> Iterable[ValidationError]: diff --git a/faust/models/record.py b/faust/models/record.py index 032abd607..dc1f213f1 100644 --- a/faust/models/record.py +++ b/faust/models/record.py @@ -102,10 +102,10 @@ def __init_subclass__( allow_blessed_key: Optional[bool] = None, decimals: Optional[bool] = None, coerce: Optional[bool] = None, - coercions: CoercionMapping = None, + coercions: Optional[CoercionMapping] = None, polymorphic_fields: Optional[bool] = None, validation: Optional[bool] = None, - date_parser: Callable[[Any], datetime] = None, + date_parser: Optional[Callable[[Any], datetime]] = None, lazy_creation: bool = False, **kwargs: Any, ) -> None: @@ -224,7 +224,7 @@ def add_to_tagged_indices(field: str, tag: Type[Tag]) -> None: tagged_fields.add(field) def add_related_to_tagged_indices( - field: str, related_model: Type = None + field: str, related_model: Optional[Type] = None ) -> None: if related_model is None: return @@ -298,7 +298,7 @@ def add_related_to_tagged_indices( @classmethod def from_data( - cls, data: Mapping, *, preferred_type: Type[ModelT] = None + cls, data: Mapping, *, preferred_type: Optional[Type[ModelT]] = None ) -> "Record": """Create model object from Python dictionary.""" # check for blessed key to see if another model should be used. @@ -310,7 +310,11 @@ def from_data( return (self_cls or cls)(**data, __strict__=False) def __init__( - self, *args: Any, __strict__: bool = True, __faust: Any = None, **kwargs: Any + self, + *args: Any, + __strict__: bool = True, + __faust: Optional[Any] = None, + **kwargs: Any, ) -> None: # pragma: no cover ... # overridden by _BUILD_init @@ -596,7 +600,10 @@ def to_representation(self) -> Mapping[str, Any]: payload[self._blessed_key] = {"ns": options.namespace} return payload - def asdict(self) -> Dict[str, Any]: # pragma: no cover + # The body of this method is code generated by ``_BUILD_asdict`` and + # installed on every concrete Record subclass, so the empty body here is + # deliberate: it only documents the signature. + def asdict(self) -> Dict[str, Any]: # type: ignore[empty-body] # pragma: no cover """Convert record to Python dictionary.""" ... # generated by _BUILD_asdict diff --git a/faust/models/typing.py b/faust/models/typing.py index 70f27990c..3a96717fd 100644 --- a/faust/models/typing.py +++ b/faust/models/typing.py @@ -168,7 +168,7 @@ def _TypeInfo_from_type(typ: Type, *, optional: bool = False) -> TypeInfo: class Variable: - def __init__(self, name: str, *, getitem: Any = None) -> None: + def __init__(self, name: str, *, getitem: Optional[Any] = None) -> None: self.name = name self.getitem = getitem @@ -261,7 +261,7 @@ def inspect_type(cls, typ: Type) -> TypeInfo: return _TypeInfo_from_type(typ, optional=True) return _TypeInfo_from_type(typ, optional=False) - def __init__(self, expr: Type, root: "RootNode" = None) -> None: + def __init__(self, expr: Type, root: Optional["RootNode"] = None) -> None: assert root is not None assert root.type is NodeType.ROOT self.expr: Type = expr @@ -366,7 +366,7 @@ def build(self, var: Variable, *args: Type) -> str: return f"_Decimal_({var})" @staticmethod - def _maybe_coerce(value: Union[str, Decimal] = None) -> Optional[Decimal]: + def _maybe_coerce(value: Optional[Union[str, Decimal]] = None) -> Optional[Decimal]: if value is not None: if not isinstance(value, Decimal): return str_to_decimal(value) @@ -387,7 +387,9 @@ def build(self, var: Variable, *args: Type) -> str: ) return f"_iso8601_parse_({var})" - def _maybe_coerce(self, value: Union[str, datetime] = None) -> Optional[datetime]: + def _maybe_coerce( + self, value: Optional[Union[str, datetime]] = None + ) -> Optional[datetime]: if value is not None: if isinstance(value, str): return self.root.date_parser(value) @@ -577,9 +579,9 @@ class UserNode(Node): def __init__( self, expr: Type, - root: "RootNode" = None, + root: Optional["RootNode"] = None, *, - user_types: CoercionMapping = None, + user_types: Optional[CoercionMapping] = None, handler: CoercionHandler, ) -> None: super().__init__(expr, root) @@ -625,10 +627,10 @@ def add_closure(self, local_name: str, global_name: str, obj: Any) -> None: def __init__( self, expr: Type, - root: "RootNode" = None, + root: Optional["RootNode"] = None, *, - user_types: CoercionMapping = None, - date_parser: Callable[[Any], datetime] = None, + user_types: Optional[CoercionMapping] = None, + date_parser: Optional[Callable[[Any], datetime]] = None, ) -> None: assert self.type == NodeType.ROOT self.type_stats = Counter() @@ -681,8 +683,8 @@ def as_function( name: str = "expr", argument_name: str = "a", stacklevel: int = 1, - locals: Dict[str, Any] = None, - globals: Dict[str, Any] = None, + locals: Optional[Dict[str, Any]] = None, + globals: Optional[Dict[str, Any]] = None, ) -> Callable[[T], T]: sourcecode = self.as_string(name=name, argument_name=argument_name) if locals is None or globals is None and stacklevel: diff --git a/faust/sensors/base.py b/faust/sensors/base.py index 46f679e67..b41daa8c2 100644 --- a/faust/sensors/base.py +++ b/faust/sensors/base.py @@ -33,7 +33,12 @@ def on_stream_event_in( return None def on_stream_event_out( - self, tp: TP, offset: int, stream: StreamT, event: EventT, state: Dict = None + self, + tp: TP, + offset: int, + stream: StreamT, + event: EventT, + state: Optional[Dict] = None, ) -> None: """Event was acknowledged by stream. @@ -127,7 +132,7 @@ def on_rebalance_end(self, app: AppT, state: Dict) -> None: ... def on_web_request_start( - self, app: AppT, request: web.Request, *, view: web.View = None + self, app: AppT, request: web.Request, *, view: Optional[web.View] = None ) -> Dict: """Web server started working on request.""" return {"time_start": monotonic()} @@ -139,7 +144,7 @@ def on_web_request_end( response: Optional[web.Response], state: Dict, *, - view: web.View = None, + view: Optional[web.View] = None, ) -> None: """Web server finished working on request.""" ... @@ -188,7 +193,12 @@ def on_stream_event_in( } def on_stream_event_out( - self, tp: TP, offset: int, stream: StreamT, event: EventT, state: Dict = None + self, + tp: TP, + offset: int, + stream: StreamT, + event: EventT, + state: Optional[Dict] = None, ) -> None: """Call when stream is done processing an event.""" sensor_state = state or {} @@ -299,7 +309,7 @@ def on_rebalance_end(self, app: AppT, state: Dict) -> None: sensor.on_rebalance_end(app, state[sensor]) def on_web_request_start( - self, app: AppT, request: web.Request, *, view: web.View = None + self, app: AppT, request: web.Request, *, view: Optional[web.View] = None ) -> Dict: """Web server started working on request.""" return { @@ -314,7 +324,7 @@ def on_web_request_end( response: Optional[web.Response], state: Dict, *, - view: web.View = None, + view: Optional[web.View] = None, ) -> None: """Web server finished working on request.""" for sensor in self._sensors: diff --git a/faust/sensors/datadog.py b/faust/sensors/datadog.py index 5db905957..731f79610 100644 --- a/faust/sensors/datadog.py +++ b/faust/sensors/datadog.py @@ -49,7 +49,7 @@ def __init__( self.sanitize_re = re.compile(r"[^0-9a-zA-Z_]") self.re_substitution = "_" - def gauge(self, metric: str, value: float, labels: Dict = None) -> None: + def gauge(self, metric: str, value: float, labels: Optional[Dict] = None) -> None: self.client.gauge( metric, value=value, @@ -57,7 +57,9 @@ def gauge(self, metric: str, value: float, labels: Dict = None) -> None: sample_rate=self.rate, ) - def increment(self, metric: str, value: float = 1.0, labels: Dict = None) -> None: + def increment( + self, metric: str, value: float = 1.0, labels: Optional[Dict] = None + ) -> None: self.client.increment( metric, value=value, @@ -69,7 +71,9 @@ def incr(self, metric: str, count: int = 1) -> None: """Statsd compatibility.""" self.increment(metric, value=count) - def decrement(self, metric: str, value: float = 1.0, labels: Dict = None) -> float: + def decrement( + self, metric: str, value: float = 1.0, labels: Optional[Dict] = None + ) -> float: return self.client.decrement( # type: ignore metric, value=value, @@ -81,7 +85,7 @@ def decr(self, metric: str, count: float = 1.0) -> None: """Statsd compatibility.""" self.decrement(metric, value=count) - def timing(self, metric: str, value: float, labels: Dict = None) -> None: + def timing(self, metric: str, value: float, labels: Optional[Dict] = None) -> None: self.client.timing( # type: ignore metric, value=value, @@ -92,7 +96,7 @@ def timing(self, metric: str, value: float, labels: Dict = None) -> None: def timed( self, metric: Optional[str] = None, - labels: Dict = None, + labels: Optional[Dict] = None, use_ms: Optional[bool] = None, ) -> float: return self.client.timed( # type: ignore @@ -102,7 +106,9 @@ def timed( use_ms=use_ms, ) - def histogram(self, metric: str, value: float, labels: Dict = None) -> None: + def histogram( + self, metric: str, value: float, labels: Optional[Dict] = None + ) -> None: self.client.histogram( # type: ignore metric, value=value, @@ -178,7 +184,12 @@ def on_stream_event_in( return state def on_stream_event_out( - self, tp: TP, offset: int, stream: StreamT, event: EventT, state: Dict = None + self, + tp: TP, + offset: int, + stream: StreamT, + event: EventT, + state: Optional[Dict] = None, ) -> None: """Call when stream is done processing an event.""" super().on_stream_event_out(tp, offset, stream, event, state) @@ -331,7 +342,7 @@ def on_web_request_end( response: Optional[web.Response], state: Dict, *, - view: web.View = None, + view: Optional[web.View] = None, ) -> None: """Web server finished working on request.""" super().on_web_request_end(app, request, response, state, view=view) diff --git a/faust/sensors/distributed_tracing.py b/faust/sensors/distributed_tracing.py index febaa1be8..8ac0f2862 100644 --- a/faust/sensors/distributed_tracing.py +++ b/faust/sensors/distributed_tracing.py @@ -1,6 +1,6 @@ import traceback from functools import cached_property -from typing import Any, Dict +from typing import Any, Dict, Optional import aiohttp from mode import get_logger @@ -14,8 +14,8 @@ from faust.utils import _opentracing as opentracing # type: ignore from faust.utils._opentracing import Format, tags -from faust import App, EventT, Sensor, StreamT -from faust.types import TP, Message, PendingMessage, ProducerT, RecordMetadata +from faust import EventT, Sensor, StreamT +from faust.types import TP, AppT, Message, PendingMessage, ProducerT, RecordMetadata from faust.types.core import OpenHeadersArg, merge_headers from faust.utils.tracing import current_span, set_current_span @@ -23,7 +23,7 @@ class TracingSensor(Sensor): - aiohttp_sessions: Dict[str, aiohttp.ClientSession] = None + aiohttp_sessions: Optional[Dict[str, aiohttp.ClientSession]] = None @cached_property def app_tracer(self) -> opentracing.Tracer: @@ -41,7 +41,14 @@ def kafka_tracer(self) -> opentracing.Tracer: # Message received by a consumer. def on_message_in(self, tp: TP, offset: int, message: Message) -> None: - carrier_headers = {want_str(k): want_str(v) for k, v in message.headers} + # XXX ``Message.headers`` is typed ``Optional[HeadersArg]``, i.e. a list + # of pairs *or* a Mapping *or* None, but this only handles the list of + # pairs: a Mapping iterates its str keys (ValueError/mis-unpack) and + # None raises TypeError. Ignores below cover exactly that gap. + carrier_headers = { # type: ignore[str-unpack] + want_str(k): want_str(v) # type: ignore[type-var] + for k, v in message.headers # type: ignore[union-attr] + } if carrier_headers: parent_context = self.app_tracer.extract( @@ -76,7 +83,9 @@ def on_stream_event_in( ) stream_span.set_tag("stream-concurrency-index", stream.concurrency_index) stream_span.set_tag("stream-prefix", stream.prefix) - spans = stream_meta.get("stream_spans") + spans: Optional[Dict[StreamT, opentracing.Span]] = stream_meta.get( + "stream_spans" + ) if spans is None: spans = stream_meta["stream_spans"] = {} spans[stream] = stream_span @@ -84,7 +93,12 @@ def on_stream_event_in( # Event was acknowledged by stream. def on_stream_event_out( - self, tp: TP, offset: int, stream: StreamT, event: EventT, state: Dict = None + self, + tp: TP, + offset: int, + stream: StreamT, + event: EventT, + state: Optional[Dict] = None, ) -> None: stream_meta = getattr(event.message, "stream_meta", None) if stream_meta is None: @@ -165,5 +179,5 @@ def trace_inject_headers( logger.warning(f"Exception in trace_inject_headers {ex} ") return None - def on_threaded_producer_buffer_processed(self, app: App, size: int) -> None: + def on_threaded_producer_buffer_processed(self, app: AppT, size: int) -> None: pass diff --git a/faust/sensors/monitor.py b/faust/sensors/monitor.py index b37a0f8a4..5435285b4 100644 --- a/faust/sensors/monitor.py +++ b/faust/sensors/monitor.py @@ -229,30 +229,30 @@ def __init__( max_send_latency_history: Optional[int] = None, max_assignment_latency_history: Optional[int] = None, messages_sent: int = 0, - tables: MutableMapping[str, TableState] = None, + tables: Optional[MutableMapping[str, TableState]] = None, messages_active: int = 0, events_active: int = 0, messages_received_total: int = 0, - messages_received_by_topic: Counter[str] = None, + messages_received_by_topic: Optional[Counter[str]] = None, events_total: int = 0, - events_by_stream: Counter[StreamT] = None, - events_by_task: Counter[asyncio.Task] = None, - events_runtime: Deque[float] = None, - commit_latency: Deque[float] = None, - send_latency: Deque[float] = None, - assignment_latency: Deque[float] = None, + events_by_stream: Optional[Counter[StreamT]] = None, + events_by_task: Optional[Counter[asyncio.Task]] = None, + events_runtime: Optional[Deque[float]] = None, + commit_latency: Optional[Deque[float]] = None, + send_latency: Optional[Deque[float]] = None, + assignment_latency: Optional[Deque[float]] = None, events_s: int = 0, messages_s: int = 0, events_runtime_avg: float = 0.0, - topic_buffer_full: Counter[TP] = None, + topic_buffer_full: Optional[Counter[TP]] = None, rebalances: Optional[int] = None, - rebalance_return_latency: Deque[float] = None, - rebalance_end_latency: Deque[float] = None, + rebalance_return_latency: Optional[Deque[float]] = None, + rebalance_end_latency: Optional[Deque[float]] = None, rebalance_return_avg: float = 0.0, rebalance_end_avg: float = 0.0, time: Callable[[], float] = monotonic, - http_response_codes: Counter[HTTPStatus] = None, - http_response_latency: Deque[float] = None, + http_response_codes: Optional[Counter[HTTPStatus]] = None, + http_response_latency: Optional[Deque[float]] = None, http_response_latency_avg: float = 0.0, **kwargs: Any, ) -> None: @@ -484,7 +484,12 @@ def on_stream_event_in( } def on_stream_event_out( - self, tp: TP, offset: int, stream: StreamT, event: EventT, state: Dict = None + self, + tp: TP, + offset: int, + stream: StreamT, + event: EventT, + state: Optional[Dict] = None, ) -> None: """Call when stream is done processing an event.""" if state is not None: @@ -626,7 +631,7 @@ def on_rebalance_end(self, app: AppT, state: Dict) -> None: self._clear_topic_related_sensors() def on_web_request_start( - self, app: AppT, request: web.Request, *, view: web.View = None + self, app: AppT, request: web.Request, *, view: Optional[web.View] = None ) -> Dict: """Web server started working on request.""" return {"time_start": self.time()} @@ -638,7 +643,7 @@ def on_web_request_end( response: Optional[web.Response], state: Dict, *, - view: web.View = None, + view: Optional[web.View] = None, ) -> None: """Web server finished working on request.""" status_code = HTTPStatus(response.status if response is not None else 500) diff --git a/faust/sensors/prometheus.py b/faust/sensors/prometheus.py index 08f2510a0..82f35dcd0 100644 --- a/faust/sensors/prometheus.py +++ b/faust/sensors/prometheus.py @@ -34,7 +34,13 @@ generate_latest, ) except ImportError: # pragma: no cover - prometheus_client = None + # XXX ``prometheus_client`` doubles as a module and as a "is the optional + # extra installed?" sentinel (see ``setup_prometheus_sensors``), so the + # name is rebound to None when the import fails. mypy binds the name to + # the module type at the import statement and has no way to widen it to + # Optional afterwards; the defect is the module-as-sentinel pattern, not + # this assignment. + prometheus_client = None # type: ignore[assignment] __all__ = ["setup_prometheus_sensors"] @@ -326,9 +332,13 @@ def _clear_topic_partition_related_metrics(self) -> None: self.topic_partition_offset_commited, ] for metric in metrics: + # XXX ``MetricWrapperBase.collect`` is annotated ``Iterable[Metric]`` + # even though every implementation returns a list, so subscripting + # it is unchecked: with a driver that really returns a lazy + # iterable this would raise TypeError at runtime. topics_partitions = frozenset( (sample.labels["topic"], sample.labels["partition"]) - for sample in metric.collect()[0].samples + for sample in metric.collect()[0].samples # type: ignore[index] ) for topic, partition in topics_partitions: metric.remove(topic, partition) @@ -339,8 +349,11 @@ def _clear_topic_related_metrics(self) -> None: self.topic_messages_sent, ] for metric in metrics: + # XXX see ``_clear_topic_partition_related_metrics``: ``collect()`` + # is typed ``Iterable[Metric]`` but indexed as a sequence. topics = frozenset( - sample.labels["topic"] for sample in metric.collect()[0].samples + sample.labels["topic"] + for sample in metric.collect()[0].samples # type: ignore[index] ) for topic in topics: metric.remove(topic) @@ -410,7 +423,7 @@ def on_stream_event_out( offset: int, stream: StreamT, event: EventT, - state: typing.Dict = None, + state: Optional[typing.Dict] = None, ) -> None: """Call when stream is done processing an event.""" super().on_stream_event_out(tp, offset, stream, event, state) @@ -555,7 +568,7 @@ def on_web_request_end( response: typing.Optional[web.Response], state: typing.Dict, *, - view: web.View = None, + view: Optional[web.View] = None, ) -> None: """Web server finished working on request.""" super().on_web_request_end(app, request, response, state, view=view) diff --git a/faust/sensors/statsd.py b/faust/sensors/statsd.py index a9ec1d055..afc1b0ea0 100644 --- a/faust/sensors/statsd.py +++ b/faust/sensors/statsd.py @@ -99,7 +99,12 @@ def _stream_label(self, stream: StreamT) -> str: ) def on_stream_event_out( - self, tp: TP, offset: int, stream: StreamT, event: EventT, state: Dict = None + self, + tp: TP, + offset: int, + stream: StreamT, + event: EventT, + state: Optional[Dict] = None, ) -> None: """Call when stream is done processing an event.""" super().on_stream_event_out(tp, offset, stream, event, state) @@ -240,7 +245,7 @@ def on_web_request_end( response: Optional[web.Response], state: Dict, *, - view: web.View = None, + view: Optional[web.View] = None, ) -> None: """Web server finished working on request.""" super().on_web_request_end(app, request, response, state, view=view) diff --git a/faust/serializers/codecs.py b/faust/serializers/codecs.py index 1d0a15570..a944af5f6 100644 --- a/faust/serializers/codecs.py +++ b/faust/serializers/codecs.py @@ -163,7 +163,7 @@ def msgpack() -> codecs.Codec: import pickle as _pickle # nosec B403 from base64 import b64decode, b64encode from types import ModuleType -from typing import Any, Dict, MutableMapping, Optional, Tuple, cast +from typing import Any, Dict, MutableMapping, Optional, Tuple, Union, cast from mode.utils.compat import want_bytes, want_str from mode.utils.imports import load_extension_classes @@ -205,7 +205,9 @@ class Codec(CodecT): #: preserve keyword arguments in copies. kwargs: Dict - def __init__(self, children: Tuple[CodecT, ...] = None, **kwargs: Any) -> None: + def __init__( + self, children: Optional[Tuple[CodecT, ...]] = None, **kwargs: Any + ) -> None: self.children = children or () self.nodes = (self,) + self.children self.kwargs = kwargs @@ -343,10 +345,17 @@ def get_codec(name_or_codec: CodecArg) -> CodecT: if isinstance(name_or_codec, str): if "|" in name_or_codec: nodes = name_or_codec.split("|") - codec = None + # XXX ``codecs.get(node, node)`` falls back to the *name* when the + # codec is unknown, so ``codec`` can hold a plain ``str`` and this + # ``|=`` then blows up with ``TypeError: unsupported operand + # type(s) for |=: 'str' and 'Codec'`` instead of a KeyError naming + # the bad codec. Real bug (e.g. ``get_codec('bad|json')``), but + # the fallback also makes ``get_codec('|json')`` work, so changing + # it is a behaviour change and out of scope for a typing pass. + codec: Optional[Union[CodecT, str]] = None for node in nodes: if codec: - codec |= codecs[node] + codec |= codecs[node] # type: ignore[operator] else: codec = codecs.get(node, node) diff --git a/faust/serializers/registry.py b/faust/serializers/registry.py index 5f2f46e43..9018a2c31 100644 --- a/faust/serializers/registry.py +++ b/faust/serializers/registry.py @@ -27,7 +27,9 @@ class Registry(RegistryT): """ def __init__( - self, key_serializer: CodecArg = None, value_serializer: CodecArg = "json" + self, + key_serializer: Optional[CodecArg] = None, + value_serializer: CodecArg = "json", ) -> None: self.key_serializer = key_serializer self.value_serializer = value_serializer @@ -37,7 +39,7 @@ def loads_key( typ: Optional[ModelArg], key: Optional[bytes], *, - serializer: CodecArg = None, + serializer: Optional[CodecArg] = None, ) -> K: """Deserialize message key. @@ -81,7 +83,7 @@ def loads_value( typ: Optional[ModelArg], value: Optional[bytes], *, - serializer: CodecArg = None, + serializer: Optional[CodecArg] = None, ) -> Any: """Deserialize value. @@ -128,7 +130,7 @@ def dumps_key( typ: Optional[ModelArg], key: K, *, - serializer: CodecArg = None, + serializer: Optional[CodecArg] = None, skip: IsInstanceArg = (bytes,), ) -> Optional[bytes]: """Serialize key. @@ -157,7 +159,7 @@ def dumps_value( typ: Optional[ModelArg], value: V, *, - serializer: CodecArg = None, + serializer: Optional[CodecArg] = None, skip: IsInstanceArg = (bytes,), ) -> Optional[bytes]: """Serialize value. diff --git a/faust/serializers/schemas.py b/faust/serializers/schemas.py index aad969e2f..8f9ad0cd7 100644 --- a/faust/serializers/schemas.py +++ b/faust/serializers/schemas.py @@ -30,14 +30,14 @@ async def _noop_decode_error(exc: Exception, message: Message) -> None: ... -class Schema(SchemaT): +class Schema(SchemaT[KT, VT]): def __init__( self, *, - key_type: ModelArg = None, - value_type: ModelArg = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_type: Optional[ModelArg] = None, + value_type: Optional[ModelArg] = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, allow_empty: Optional[bool] = None, ) -> None: self.update( @@ -51,10 +51,10 @@ def __init__( def update( self, *, - key_type: ModelArg = None, - value_type: ModelArg = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_type: Optional[ModelArg] = None, + value_type: Optional[ModelArg] = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, allow_empty: Optional[bool] = None, ) -> None: if key_type is not None: @@ -78,7 +78,7 @@ def loads_key( message: Message, *, loads: Optional[Callable] = None, - serializer: CodecArg = None, + serializer: Optional[CodecArg] = None, ) -> KT: if loads is None: loads = app.serializers.loads_key @@ -97,7 +97,7 @@ def loads_value( message: Message, *, loads: Optional[Callable] = None, - serializer: CodecArg = None, + serializer: Optional[CodecArg] = None, ) -> VT: if loads is None: loads = app.serializers.loads_value @@ -108,7 +108,12 @@ def loads_value( ) def dumps_key( - self, app: AppT, key: K, *, serializer: CodecArg = None, headers: OpenHeadersArg + self, + app: AppT, + key: K, + *, + serializer: Optional[CodecArg] = None, + headers: OpenHeadersArg, ) -> Tuple[Any, OpenHeadersArg]: payload = app.serializers.dumps_key( self.key_type, @@ -122,7 +127,7 @@ def dumps_value( app: AppT, value: V, *, - serializer: CodecArg = None, + serializer: Optional[CodecArg] = None, headers: OpenHeadersArg, ) -> Tuple[Any, OpenHeadersArg]: payload = app.serializers.dumps_value( diff --git a/faust/stores/aerospike.py b/faust/stores/aerospike.py index 198f78dc7..6ca39c23d 100644 --- a/faust/stores/aerospike.py +++ b/faust/stores/aerospike.py @@ -2,7 +2,7 @@ import time import typing -from typing import Any, Dict, Iterator, Optional, Tuple, Union +from typing import Any, Callable, Dict, Iterator, Optional, Tuple, Union try: # pragma: no cover import aerospike @@ -40,8 +40,9 @@ class AeroSpikeStore(base.SerializedStore): """Aerospike table storage.""" client: Client + namespace: str ttl: int - policies: typing.Mapping[str, Any] + policies: Optional[typing.Mapping[str, Any]] BIN_KEY = "value_key" USERNAME_KEY: str = "user" HOSTS_KEY: str = "hosts" @@ -56,14 +57,20 @@ def __init__( url: Union[str, URL], app: AppT, table: CollectionT, - options: typing.Mapping[str, Any] = None, + options: Optional[typing.Mapping[str, Any]] = None, **kwargs: Any, ) -> None: try: - self.client = AeroSpikeStore.get_aerospike_client(options) - self.namespace = options.get(self.NAMESPACE_KEY, "") - self.ttl = options.get(self.TTL_KEY, aerospike.TTL_NEVER_EXPIRE) - self.policies = options.get(self.POLICIES_KEY, None) + # XXX `options` is declared Optional but is used unconditionally: + # constructing this store without options raises AttributeError + # ('NoneType' has no attribute 'get') from inside + # get_aerospike_client, re-raised by the handler below. That is + # the existing behaviour; adding a guard here would change the + # exception raised, so the errors are silenced instead. + self.client = AeroSpikeStore.get_aerospike_client(options) # type: ignore[arg-type] # noqa: E501 + self.namespace = options.get(self.NAMESPACE_KEY, "") # type: ignore[union-attr] # noqa: E501 + self.ttl = options.get(self.TTL_KEY, aerospike.TTL_NEVER_EXPIRE) # type: ignore[union-attr] # noqa: E501 + self.policies = options.get(self.POLICIES_KEY, None) # type: ignore[union-attr] # noqa: E501 table.use_partitioner = True except Exception as ex: self.logger.error(f"Error configuring aerospike client {ex}") @@ -71,7 +78,7 @@ def __init__( super().__init__(url, app, table, **kwargs) @staticmethod - def get_aerospike_client(aerospike_config: Dict[Any, Any]) -> Client: + def get_aerospike_client(aerospike_config: typing.Mapping[str, Any]) -> Client: """Try to get Aerospike client instance.""" global aerospike_client if aerospike_client: @@ -95,7 +102,15 @@ def get_aerospike_client(aerospike_config: Dict[Any, Any]) -> Client: raise e def _get(self, key: bytes) -> Optional[bytes]: - key = (self.namespace, self.table_name, key) + # XXX the ``key`` parameter is declared ``bytes`` but is reused as the + # ``(namespace, set, key)`` tuple Aerospike expects, and then rebound + # again from the client's return value so the handlers below report + # the key Aerospike echoed back rather than the one constructed here. + # Both rebinds are load-bearing for the logged/raised messages, so the + # errors are silenced rather than the code split up. ``str-bytes-safe`` + # below is a consequence of the same lie: mypy still believes ``key`` is + # ``bytes`` in the handlers, where it actually holds a tuple. + key = (self.namespace, self.table_name, key) # type: ignore[assignment] fun = self.client.get try: key, meta, bins = self.aerospike_fun_call_with_retry(fun=fun, key=key) @@ -103,18 +118,25 @@ def _get(self, key: bytes) -> Optional[bytes]: return bins[self.BIN_KEY] return None except aerospike.exception.RecordNotFound as ex: - self.log.debug(f"key not found {key} exception {ex}") - raise KeyError(f"key not found {key}") + self.log.debug(f"key not found {key} exception {ex}") # type: ignore[str-bytes-safe] # noqa: E501 + raise KeyError(f"key not found {key}") # type: ignore[str-bytes-safe] except Exception as ex: self.log.error( - f"Error in set for table {self.table_name} exception {ex} key {key}" + f"Error in set for table {self.table_name} exception {ex} key {key}" # type: ignore[str-bytes-safe] # noqa: E501 ) raise ex def _set(self, key: bytes, value: Optional[bytes]) -> None: try: fun = self.client.put - key = (self.namespace, self.table_name, key) + # XXX ``key`` is declared ``bytes`` but rebound to the + # ``(namespace, set, key)`` tuple Aerospike expects. The rebind + # must stay inside the ``try`` so a failure building the tuple is + # still logged and re-raised by the handler below, so the error is + # silenced rather than hoisted into a separate variable. The + # ``str-bytes-safe`` ignore below follows from the same lie: mypy + # still believes ``key`` is ``bytes`` where it holds a tuple. + key = (self.namespace, self.table_name, key) # type: ignore[assignment] vt = {self.BIN_KEY: value} self.aerospike_fun_call_with_retry( fun=fun, @@ -129,23 +151,28 @@ def _set(self, key: bytes, value: Optional[bytes]) -> None: except Exception as ex: self.log.error( - f"FaustAerospikeException Error in set for " - f"table {self.table_name} exception {ex} key {key}" + f"FaustAerospikeException Error in set for table {self.table_name} exception {ex} key {key}" # type: ignore[str-bytes-safe] # noqa: E501 ) raise ex def _del(self, key: bytes) -> None: try: - key = (self.namespace, self.table_name, key) + # XXX ``key`` is declared ``bytes`` but rebound to the + # ``(namespace, set, key)`` tuple Aerospike expects. The rebind + # must stay inside the ``try`` so a failure building the tuple is + # still logged and re-raised by the handler below, so the error is + # silenced rather than hoisted into a separate variable. The + # ``str-bytes-safe`` ignores below follow from the same lie: mypy + # still believes ``key`` is ``bytes`` where it holds a tuple. + key = (self.namespace, self.table_name, key) # type: ignore[assignment] self.aerospike_fun_call_with_retry(fun=self.client.remove, key=key) except aerospike.exception.RecordNotFound as ex: self.log.debug( - f"Error in delete for table {self.table_name} exception {ex} key {key}" + f"Error in delete for table {self.table_name} exception {ex} key {key}" # type: ignore[str-bytes-safe] # noqa: E501 ) except Exception as ex: self.log.error( - f"FaustAerospikeException Error in delete for " - f"table {self.table_name} exception {ex} key {key}" + f"FaustAerospikeException Error in delete for table {self.table_name} exception {ex} key {key}" # type: ignore[str-bytes-safe] # noqa: E501 ) raise ex @@ -165,7 +192,7 @@ def _iterkeys(self) -> Iterator[bytes]: ) raise ex - def _itervalues(self) -> Iterator[bytes]: + def _itervalues(self) -> Iterator[Optional[bytes]]: try: fun = self.client.scan @@ -177,6 +204,7 @@ def _itervalues(self) -> Iterator[bytes]: if bins: yield bins[self.BIN_KEY] else: + # A record without the value bin has no value. yield None except Exception as ex: self.log.error( @@ -213,7 +241,16 @@ def _size(self) -> int: def _contains(self, key: bytes) -> bool: try: if self.app.conf.store_check_exists: - key = (self.namespace, self.table_name, key) + # XXX ``key`` is declared ``bytes`` but rebound to the + # ``(namespace, set, key)`` tuple Aerospike expects, then + # rebound again from the client's return value. Both rebinds + # must stay inside this branch: the tuple is only built when + # ``store_check_exists`` is set, and the handler below logs + # whichever value ``key`` currently holds. So the error is + # silenced rather than the construction hoisted out. The + # ``str-bytes-safe`` ignore below follows from the same lie: + # mypy still believes ``key`` is ``bytes`` in that handler. + key = (self.namespace, self.table_name, key) # type: ignore[assignment] # noqa: E501 key, meta = self.aerospike_fun_call_with_retry( fun=self.client.exists, key=key ) @@ -225,9 +262,7 @@ def _contains(self, key: bytes) -> bool: return True except Exception as ex: self.log.error( - f"FaustAerospikeException Error in _contains for table " - f"{self.table_name} exception " - f"{ex} key {key}" + f"FaustAerospikeException Error in _contains for table {self.table_name} exception {ex} key {key}" # type: ignore[str-bytes-safe] # noqa: E501 ) raise ex @@ -254,7 +289,9 @@ def persisted_offset(self, tp: TP) -> Optional[int]: """ return None - def aerospike_fun_call_with_retry(self, fun, *args, **kwargs): + def aerospike_fun_call_with_retry( + self, fun: Callable[..., Any], *args: Any, **kwargs: Any + ) -> Any: """Call function and retry until Aerospike throws exception.""" f_tries = self.app.conf.aerospike_retries_on_exception f_delay = self.app.conf.aerospike_sleep_seconds_between_retries_on_exception diff --git a/faust/stores/base.py b/faust/stores/base.py index 011259825..e0a6a7df6 100644 --- a/faust/stores/base.py +++ b/faust/stores/base.py @@ -34,10 +34,10 @@ def __init__( table: CollectionT, *, table_name: str = "", - key_type: ModelArg = None, - value_type: ModelArg = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_type: Optional[ModelArg] = None, + value_type: Optional[ModelArg] = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, options: Optional[Mapping[str, Any]] = None, **kwargs: Any, ) -> None: @@ -162,7 +162,9 @@ def _iterkeys(self) -> Iterator[bytes]: # pragma: no cover ... @abc.abstractmethod - def _itervalues(self) -> Iterator[bytes]: # pragma: no cover + def _itervalues(self) -> Iterator[Optional[bytes]]: # pragma: no cover + # May yield None: `_values_decoded` feeds each value straight to + # `_decode_value`, which accepts `Optional[bytes]`. ... @abc.abstractmethod diff --git a/faust/stores/rocksdb.py b/faust/stores/rocksdb.py index d17ba750e..1bafb4145 100644 --- a/faust/stores/rocksdb.py +++ b/faust/stores/rocksdb.py @@ -23,6 +23,7 @@ Optional, Set, Tuple, + Type, Union, cast, ) @@ -47,7 +48,7 @@ DEFAULT_BLOCK_CACHE_SIZE = 2 * 1024**3 DEFAULT_BLOCK_CACHE_COMPRESSED_SIZE = 500 * 1024**2 DEFAULT_BLOOM_FILTER_SIZE = 3 -ERRORS_ROCKS_IO_ERROR = ( +ERRORS_ROCKS_IO_ERROR: Type[Exception] = ( Exception # use general exception to avoid missing exception issues ) @@ -261,9 +262,16 @@ def __init__( if not self.url.path: self.url /= self.table_name self.options = options or {} - self.read_only = self.options.pop("read_only", read_only) - - self.driver = self.options.pop("driver", driver) + # XXX `options` is typed `Mapping` (no `.pop`), but this deliberately + # aliases and mutates the caller's dict -- `Collection.options` -- so + # the keys below are gone for anyone who reads it afterwards. A second + # Store built from the same table (ChangeloggedObjectManager builds one + # besides `Collection.data`) therefore silently falls back to the + # defaults for `read_only`/`driver`. Copying instead would change that + # behaviour, so the aliasing is kept and the errors are silenced here. + self.read_only = self.options.pop("read_only", read_only) # type: ignore[attr-defined] # noqa: E501 + + self.driver = self.options.pop("driver", driver) # type: ignore[attr-defined] if self.driver == "rocksdict": self.use_rocksdict = True elif self.driver == "python-rocksdb": @@ -328,9 +336,7 @@ async def backup_partition( """ if not self.use_rocksdict and self._backup_engine: - partition = tp - if isinstance(tp, TP): - partition = tp.partition + partition = tp.partition if isinstance(tp, TP) else tp try: if flush: db = await self._try_open_db_for_partition(partition) @@ -363,9 +369,7 @@ def restore_backup( """ if not self.use_rocksdict and self._backup_engine: - partition = tp - if isinstance(tp, TP): - partition = tp.partition + partition = tp.partition if isinstance(tp, TP) else tp if latest: self._backup_engine.restore_latest_backup( str(self.partition_path(partition)), self._backup_path @@ -490,12 +494,11 @@ def _open_for_partition(self, partition: int) -> DB: def _get(self, key: bytes) -> Optional[bytes]: event = current_event() - partition_from_message = ( + if ( event is not None and not self.table.is_global and not self.table.use_partitioner - ) - if partition_from_message: + ): partition = event.message.partition db = self._db_for_partition(partition) value = db.get(key) @@ -642,12 +645,11 @@ async def _try_open_db_for_partition( def _contains(self, key: bytes) -> bool: event = current_event() - partition_from_message = ( + if ( event is not None and not self.table.is_global and not self.table.use_partitioner - ) - if partition_from_message: + ): partition = event.message.partition db = self._db_for_partition(partition) value = db.get(key) @@ -676,7 +678,11 @@ def _dbs_for_key(self, key: bytes) -> Iterable[DB]: def _dbs_for_actives(self) -> Iterator[DB]: actives = self.app.assignor.assigned_actives() - topic = self.table.changelog_topic_name + # `changelog_topic_name` is a property of the concrete + # `faust.tables.base.Collection`, but is missing from the + # `CollectionT` interface (which only declares the private + # `_changelog_topic_name()` method). + topic = self.table.changelog_topic_name # type: ignore[attr-defined] for partition, db in self._dbs.items(): tp = TP(topic=topic, partition=partition) # for global tables, keys from all diff --git a/faust/streams.py b/faust/streams.py index 784bdf376..a93b722d5 100644 --- a/faust/streams.py +++ b/faust/streams.py @@ -26,6 +26,7 @@ Sequence, Set, Tuple, + TypeVar, Union, cast, ) @@ -90,9 +91,16 @@ async def maybe_forward(value: Any, channel: ChannelT) -> Any: return value -def _tracks_buffer_agen( - fun: Callable[..., AsyncIterable], -) -> Callable[..., AsyncIterable]: +#: Bound to the decorated function's own type so ``_tracks_buffer_agen`` is +#: identity-preserving: ``take()`` and friends keep their real parameter lists +#: and precise ``AsyncGenerator[...]`` return types instead of being erased to +#: ``Callable[..., AsyncGenerator[Any, None]]``. +_BufferAgenFun = TypeVar( + "_BufferAgenFun", bound=Callable[..., AsyncGenerator[Any, None]] +) + + +def _tracks_buffer_agen(fun: _BufferAgenFun) -> _BufferAgenFun: """Register buffering generators (``take()`` and friends) on the stream. The cleanup for these generators -- acking consumed events, restoring @@ -106,12 +114,17 @@ def _tracks_buffer_agen( """ @wraps(fun) - def _create_agen(self: StreamT, *args: Any, **kwargs: Any) -> AsyncIterable: + def _create_agen( + self: StreamT, *args: Any, **kwargs: Any + ) -> AsyncGenerator[Any, None]: agen = fun(self, *args, **kwargs) cast("Stream", self)._active_agens.add(agen) return agen - return _create_agen + # ``_create_agen`` is a (*args, **kwargs) forwarder, so it cannot be + # expressed as the same type as ``fun``; the cast is what keeps the + # decorator transparent to callers. + return cast(_BufferAgenFun, _create_agen) class _LinkedListDirection(NamedTuple): @@ -144,8 +157,8 @@ def __init__( channel: AsyncIterator[T_co], *, app: AppT, - processors: Iterable[Processor[T]] = None, - combined: List[JoinableT] = None, + processors: Optional[Iterable[Processor[T]]] = None, + combined: Optional[List[JoinableT]] = None, on_start: Optional[Callable] = None, join_strategy: Optional[JoinT] = None, beacon: Optional[NodeT] = None, @@ -156,7 +169,10 @@ def __init__( prefix: str = "", loop: Optional[asyncio.AbstractEventLoop] = None, ) -> None: - Service.__init__(self, loop=loop, beacon=beacon) + # mode declares ``beacon: NodeT = None`` -- an implicit-Optional the + # checker reads as non-optional -- but Service.__init__ handles + # ``beacon is None`` explicitly by rooting a new Node. + Service.__init__(self, loop=loop, beacon=beacon) # type: ignore[arg-type] self.app = app self.channel = channel self.outbox = self.app.FlowControlQueue( @@ -334,7 +350,9 @@ async def events(self) -> AsyncIterable[EventT]: yield self.current_event @_tracks_buffer_agen - async def take(self, max_: int, within: Seconds) -> AsyncIterable[Sequence[T_co]]: + async def take( + self, max_: int, within: Seconds + ) -> AsyncGenerator[Sequence[T_co], None]: """Buffer n values at a time and yield a list of buffered values. Arguments: @@ -383,7 +401,10 @@ async def add_to_buffer(value: T) -> T: # strict wait for buffer to be consumed after buffer full. # If max is 1000, we are not allowed to return 1001 values. buffer_consumed.clear() - await self.wait(buffer_consumed) + # mode types the waitable as ``mode.utils.locks.Event``, + # but Service.wait_first accepts anything with an + # awaitable ``.wait()`` -- asyncio.Event included. + await self.wait(buffer_consumed) # type: ignore[arg-type] except CancelledError: # pragma: no cover raise except Exception as exc: @@ -399,8 +420,13 @@ async def add_to_buffer(value: T) -> T: self._enable_passive(cast(ChannelT, channel_it)) try: while not self.should_stop: - # wait until buffer full, or timeout - await self.wait_for_stopped(buffer_full, timeout=timeout) + # wait until buffer full, or timeout. + # Same mode annotation gap as in add_to_buffer above, plus + # ``timeout: Seconds`` omitting the ``None`` that is mode's + # own default and its "wait forever" value. + await self.wait_for_stopped( + buffer_full, timeout=timeout # type: ignore[arg-type] + ) if buffer: # make sure background thread does not add new items to # buffer while we read. @@ -429,7 +455,7 @@ async def add_to_buffer(value: T) -> T: @_tracks_buffer_agen async def take_events( self, max_: int, within: Seconds - ) -> AsyncIterable[Sequence[EventT]]: + ) -> AsyncGenerator[Sequence[EventT], None]: """Buffer n events at a time and yield a list of buffered events. Arguments: max_: Max number of messages to receive. When more than this @@ -477,7 +503,10 @@ async def add_to_buffer(value: T) -> T: # strict wait for buffer to be consumed after buffer full. # If max is 1000, we are not allowed to return 1001 values. buffer_consumed.clear() - await self.wait(buffer_consumed) + # mode types the waitable as ``mode.utils.locks.Event``, + # but Service.wait_first accepts anything with an + # awaitable ``.wait()`` -- asyncio.Event included. + await self.wait(buffer_consumed) # type: ignore[arg-type] except CancelledError: # pragma: no cover raise except Exception as exc: @@ -493,8 +522,13 @@ async def add_to_buffer(value: T) -> T: self._enable_passive(cast(ChannelT, channel_it)) try: while not self.should_stop: - # wait until buffer full, or timeout - await self.wait_for_stopped(buffer_full, timeout=timeout) + # wait until buffer full, or timeout. + # Same mode annotation gap as in add_to_buffer above, plus + # ``timeout: Seconds`` omitting the ``None`` that is mode's + # own default and its "wait forever" value. + await self.wait_for_stopped( + buffer_full, timeout=timeout # type: ignore[arg-type] + ) if buffer: # make sure background thread does not add new items to # buffer while we read. @@ -523,7 +557,7 @@ async def add_to_buffer(value: T) -> T: @_tracks_buffer_agen async def take_with_timestamp( self, max_: int, within: Seconds, timestamp_field_name: str - ) -> AsyncIterable[Sequence[T_co]]: + ) -> AsyncGenerator[Sequence[T_co], None]: """Buffer n values at a time and yield a list of buffered values with the timestamp when the message was added to kafka. @@ -566,8 +600,17 @@ async def add_to_buffer(value: T) -> T: buffer_consuming = None event = self.current_event if isinstance(value, dict) and timestamp_field_name: - value[timestamp_field_name] = event.message.timestamp - buffer_add(value) + # XXX ``self.current_event`` is Optional, and this reads + # ``event.message`` *before* the ``event is None`` guard + # below: with no current event a dict value dies with + # AttributeError here instead of the intended RuntimeError. + value[timestamp_field_name] = ( + event.message.timestamp # type: ignore[union-attr] + ) + # XXX ``buffer_add`` also runs before the guard, so a non-dict + # value is appended to ``buffer`` and then the RuntimeError + # fires, leaving ``buffer`` and ``events`` skewed by one entry. + buffer_add(cast(T_co, value)) if event is None: raise RuntimeError("Take buffer found current_event is None") event_add(event) @@ -577,7 +620,10 @@ async def add_to_buffer(value: T) -> T: # strict wait for buffer to be consumed after buffer full. # If max is 1000, we are not allowed to return 1001 values. buffer_consumed.clear() - await self.wait(buffer_consumed) + # mode types the waitable as ``mode.utils.locks.Event``, + # but Service.wait_first accepts anything with an + # awaitable ``.wait()`` -- asyncio.Event included. + await self.wait(buffer_consumed) # type: ignore[arg-type] except CancelledError: # pragma: no cover raise except Exception as exc: @@ -593,8 +639,13 @@ async def add_to_buffer(value: T) -> T: self._enable_passive(cast(ChannelT, channel_it)) try: while not self.should_stop: - # wait until buffer full, or timeout - await self.wait_for_stopped(buffer_full, timeout=timeout) + # wait until buffer full, or timeout. + # Same mode annotation gap as in add_to_buffer above, plus + # ``timeout: Seconds`` omitting the ``None`` that is mode's + # own default and its "wait forever" value. + await self.wait_for_stopped( + buffer_full, timeout=timeout # type: ignore[arg-type] + ) if buffer: # make sure background thread does not add new items to # buffer while we read. @@ -631,7 +682,7 @@ def enumerate(self, start: int = 0) -> AsyncIterable[Tuple[int, T_co]]: @_tracks_buffer_agen async def noack_take( self, max_: int, within: Seconds - ) -> AsyncIterable[Sequence[T_co]]: + ) -> AsyncGenerator[Sequence[T_co], None]: """ Buffer n values at a time and yield a list of buffered values. :param max_: Max number of messages to receive. When more than this @@ -685,7 +736,10 @@ async def add_to_buffer(value: T) -> T: # If max is 1000, we are not allowed to return 1001 # values. buffer_consumed.clear() - await self.wait(buffer_consumed) + # mode types the waitable as ``mode.utils.locks.Event``, + # but Service.wait_first accepts anything with an + # awaitable ``.wait()`` -- asyncio.Event included. + await self.wait(buffer_consumed) # type: ignore[arg-type] except CancelledError: # pragma: no cover raise except Exception as exc: @@ -701,8 +755,13 @@ async def add_to_buffer(value: T) -> T: self._enable_passive(cast(ChannelT, channel_it)) try: while not self.should_stop: - # wait until buffer full, or timeout - await self.wait_for_stopped(buffer_full, timeout=timeout) + # wait until buffer full, or timeout. + # Same mode annotation gap as in add_to_buffer above, plus + # ``timeout: Seconds`` omitting the ``None`` that is mode's + # own default and its "wait forever" value. + await self.wait_for_stopped( + buffer_full, timeout=timeout # type: ignore[arg-type] + ) if buffer: # make sure background thread does not add new items to # buffer while we read. @@ -969,8 +1028,8 @@ def derive_topic( name: str, *, schema: Optional[SchemaT] = None, - key_type: ModelArg = None, - value_type: ModelArg = None, + key_type: Optional[ModelArg] = None, + value_type: Optional[ModelArg] = None, prefix: str = "", suffix: str = "", ) -> TopicT: @@ -1044,7 +1103,7 @@ def outer_join(self, *fields: FieldDescriptorT) -> StreamT: def _join(self, join_strategy: JoinT) -> StreamT: return self.clone(join_strategy=join_strategy) - async def on_merge(self, value: T = None) -> Optional[T]: + async def on_merge(self, value: Optional[T] = None) -> Optional[T]: """Signal called when an event is to be joined.""" # TODO for joining streams # The join strategy.process method can return None @@ -1095,7 +1154,7 @@ async def on_stop(self) -> None: def __iter__(self) -> Any: return self - def __next__(self) -> T: + def __next__(self) -> T_co: raise NotImplementedError("Streams are asynchronous: use `async for`") def __aiter__(self) -> AsyncIterator[T_co]: # pragma: no cover @@ -1308,7 +1367,12 @@ async def _py_aiter(self) -> AsyncIterator[T_co]: # reset to allow calling .start again on next `async for` self.service_reset() - async def __anext__(self) -> T: # pragma: no cover + # Declaration only: `async for` drives the async generator returned by + # ``__aiter__``, never this method, which exists so that ``Stream`` + # registers as an ``AsyncIterator`` (``collections.abc`` looks for + # ``__anext__``). It cannot be marked abstract -- ``Stream`` is concrete + # and instantiated -- so the empty body has to be silenced here. + async def __anext__(self) -> T_co: # type: ignore[empty-body] # pragma: no cover ... async def ack(self, event: EventT) -> bool: diff --git a/faust/tables/base.py b/faust/tables/base.py index ca26c0f11..33a167d74 100644 --- a/faust/tables/base.py +++ b/faust/tables/base.py @@ -9,6 +9,7 @@ from typing import ( Any, Callable, + Dict, Iterable, Iterator, List, @@ -104,21 +105,21 @@ def __init__( app: AppT, *, name: Optional[str] = None, - default: Callable[[], Any] = None, - store: Union[str, URL] = None, + default: Optional[Callable[[], Any]] = None, + store: Optional[Union[str, URL]] = None, schema: Optional[SchemaT] = None, - key_type: ModelArg = None, - value_type: ModelArg = None, + key_type: Optional[ModelArg] = None, + value_type: Optional[ModelArg] = None, partitions: Optional[int] = None, window: Optional[WindowT] = None, changelog_topic: Optional[TopicT] = None, help: Optional[str] = None, - on_recover: RecoverCallback = None, + on_recover: Optional[RecoverCallback] = None, on_changelog_event: Optional[ChangelogEventCallback] = None, recovery_buffer_size: int = 1000, standby_buffer_size: Optional[int] = None, extra_topic_configs: Optional[Mapping[str, Any]] = None, - recover_callbacks: Set[RecoverCallback] = None, + recover_callbacks: Optional[Set[RecoverCallback]] = None, options: Optional[Mapping[str, Any]] = None, use_partitioner: bool = False, on_window_close: Optional[WindowCloseCallback] = None, @@ -263,8 +264,8 @@ def send_changelog( partition: Optional[int], key: Any, value: Any, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, ) -> FutureMessage: """Send modification event to changelog topic.""" if key_serializer is None: @@ -287,8 +288,8 @@ def _send_changelog( event: Optional[EventT], key: Any, value: Any, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, ) -> None: # XXX compat version of send_changelog that needs event argument. if event is None: @@ -383,15 +384,29 @@ async def _del_old_keys(self) -> None: while timestamps and window.stale(timestamps[0], time.time()): timestamp = heappop(timestamps) triggered_windows = [ + # XXX bug: this lookup can never hit. + # ``_partition_timestamp_keys`` is keyed by + # ``(partition, range_end)`` -- a ``(int, float)`` pair, + # written that way in ``_maybe_set_key_ttl`` and read that + # way in ``_maybe_del_key_ttl``. Here it is looked up by + # ``(partition, window_range)`` where ``window_range`` is + # the ``(start, end)`` tuple, so no key ever matches and + # ``triggered_windows`` is always ``[None, ...]``. The + # consequence is that ``window_data`` stays empty and + # ``on_window_close`` never receives the aggregated window + # data, only the raw per-key value. The correct key is + # ``(partition, window_range[1])``; that is a behaviour + # change, so it is not made here and the type error is + # only silenced. self._partition_timestamp_keys.get( - (partition, window_range) - ) # noqa + (partition, window_range) # type: ignore[arg-type] + ) for window_range in self._window_ranges(timestamp) ] keys_to_remove = self._partition_timestamp_keys.pop( (partition, timestamp), None ) - window_data = {} + window_data: Dict[Any, List[Any]] = {} if keys_to_remove: for windows in triggered_windows: if windows: diff --git a/faust/tables/manager.py b/faust/tables/manager.py index 9b4952555..5dfb194d5 100644 --- a/faust/tables/manager.py +++ b/faust/tables/manager.py @@ -144,7 +144,12 @@ async def on_start(self) -> None: async def wait_until_tables_registered(self) -> None: if not self.app.producer_only and not self.app.client_only: - await self.wait_for_stopped(self._tables_registered) + # mode types the waitable as ``mode.utils.locks.Event``, but + # Service.wait_first accepts anything with an awaitable + # ``.wait()`` -- asyncio.Event included. + await self.wait_for_stopped( + self._tables_registered # type: ignore[arg-type] + ) async def _update_channels(self) -> None: self._tables_finalized.set() @@ -207,5 +212,8 @@ async def wait_until_recovery_completed(self) -> bool: and not self.app.producer_only and not self.app.client_only ): - return await self.wait_for_stopped(self.recovery.completed) + # Same mode annotation gap as in wait_until_tables_registered. + return await self.wait_for_stopped( + self.recovery.completed # type: ignore[arg-type] + ) return False diff --git a/faust/tables/objects.py b/faust/tables/objects.py index a9c0de12d..ff5c9a1ec 100644 --- a/faust/tables/objects.py +++ b/faust/tables/objects.py @@ -15,6 +15,7 @@ Optional, Set, Type, + Union, ) from mode import Service @@ -107,7 +108,13 @@ def set_persisted_offset(self, tp: TP, offset: int) -> None: """Set the last persisted offset for changelog topic partition.""" self.storage.set_persisted_offset(tp, offset) - async def on_rebalance( + # XXX This override is signature-incompatible with StoreT.on_rebalance / + # Store.on_rebalance, which take (assigned, revoked, newly_assigned, + # generation_id=0). This class takes a leading `table` argument instead and + # forwards it on, so any caller holding a StoreT will call it wrongly. This + # is a real defect, not intended design; reconciling it changes runtime + # behaviour and is out of scope for a typing pass. + async def on_rebalance( # type: ignore[override] self, table: CollectionT, assigned: Set[TP], @@ -115,7 +122,15 @@ async def on_rebalance( newly_assigned: Set[TP], ) -> None: """Call when cluster is rebalancing.""" - await self.storage.on_rebalance(table, assigned, revoked, newly_assigned) + # XXX Same defect on the forwarding side: `table` is passed as the + # `assigned` argument of the underlying store and every later argument + # is shifted by one. + await self.storage.on_rebalance( + table, # type: ignore[arg-type] + assigned, + revoked, + newly_assigned, # type: ignore[arg-type] + ) async def on_recovery_completed( self, active_tps: Set[TP], standby_tps: Set[TP] @@ -168,13 +183,17 @@ def apply_changelog_batch( self.set_persisted_offset(tp, offset) async def backup_partition( - self, tp, flush: bool = True, purge: bool = False, keep: int = 1 + self, + tp: Union[TP, int], + flush: bool = True, + purge: bool = False, + keep: int = 1, ) -> None: raise NotImplementedError def restore_backup( self, - tp, + tp: Union[TP, int], latest: bool = True, backup_id: int = 0, ) -> None: diff --git a/faust/tables/recovery.py b/faust/tables/recovery.py index a9322d110..2ff38b736 100644 --- a/faust/tables/recovery.py +++ b/faust/tables/recovery.py @@ -27,7 +27,7 @@ from aiokafka.errors import IllegalStateError from mode import Service, get_logger from mode.services import WaitArgT -from mode.utils.times import humanize_seconds, humanize_seconds_ago +from mode.utils.times import Seconds, humanize_seconds, humanize_seconds_ago from yarl import URL from faust.exceptions import ConsistencyError @@ -348,7 +348,12 @@ async def _restart_recovery(self) -> None: active_highwaters = self.active_highwaters while not self.should_stop: self.log.dev("WAITING FOR NEXT RECOVERY TO START") - if await self.wait_for_stopped(self.signal_recovery_start): + # mode types the waitable as ``mode.utils.locks.Event``, but + # Service.wait_first accepts anything with an awaitable + # ``.wait()`` -- asyncio.Event included. + if await self.wait_for_stopped( + self.signal_recovery_start # type: ignore[arg-type] + ): self.signal_recovery_start.clear() break # service was stopped self.signal_recovery_start.clear() @@ -598,9 +603,15 @@ def _estimated_active_remaining_secs(self, remaining: float) -> Optional[float]: else: return None - async def _wait(self, coro: WaitArgT, timeout: Optional[int] = None) -> None: + async def _wait(self, coro: WaitArgT, timeout: Optional[Seconds] = None) -> None: signal = self.signal_recovery_start - wait_result = await self.wait_first(coro, signal, timeout=timeout) + # mode types the waitable as ``mode.utils.locks.Event``, but + # Service.wait_first accepts anything with an awaitable ``.wait()`` + # -- asyncio.Event included. ``timeout: Seconds`` also omits the + # ``None`` that is mode's own default and its "wait forever" value. + wait_result = await self.wait_first( + coro, signal, timeout=timeout # type: ignore[arg-type] + ) if wait_result.stopped: # service was stopped. raise ServiceStopped() @@ -774,7 +785,9 @@ async def _slurp_changelogs(self) -> None: buffer_sizes = self.buffer_sizes processing_times = self._processing_times - async def _maybe_signal_recovery_end(timeout=False, timeout_count=0) -> None: + async def _maybe_signal_recovery_end( + timeout: bool = False, timeout_count: int = 0 + ) -> None: # lets wait at least 2 consecutive cycles for the queue to be # empty to avoid race conditions between # the aiokafka consumer position and draining of the queue @@ -789,7 +802,7 @@ async def _maybe_signal_recovery_end(timeout=False, timeout_count=0) -> None: logger.debug("Setting recovery end") self.signal_recovery_end.set() - async def detect_aborted_tx(): + async def detect_aborted_tx() -> None: highwaters = self.active_highwaters offsets = self.active_offsets for tp, highwater in highwaters.items(): @@ -798,7 +811,18 @@ async def detect_aborted_tx(): and offsets[tp] is not None and offsets[tp] < highwater ): - if await self.app.consumer.position(tp) >= highwater: + # XXX bug: ConsumerT.position is declared + # ``Optional[int]`` and really does return None when the + # partition has no position yet, so this comparison raises + # TypeError on that path (swallowed by the ``except + # Exception`` in the caller's loop, which then skips the + # aborted-tx fixup). Guarding for None would change which + # events reach the caller, so the behaviour is kept and + # only the type error is silenced. + if ( + await self.app.consumer.position(tp) # type: ignore[operator] + >= highwater + ): logger.info(f"Aborted tx until highwater for {tp}") offsets[tp] = highwater @@ -852,7 +876,17 @@ async def detect_aborted_tx(): buf = buffers[table] buf.append(event) await table.on_changelog_event(event) - if len(buf) >= bufsize: + # XXX bug: the ``else`` branch above only logs a warning + # and falls through, so an event for a TP that is neither + # active nor standby reaches here with ``table``, + # ``offsets`` and ``bufsize`` still holding the PREVIOUS + # iteration's values -- the event is then applied to an + # unrelated table (or raises UnboundLocalError if it is + # the first event of the loop). That is why ``bufsize`` + # is still ``Optional[int]`` here and the comparison can + # raise TypeError. Fixing it means skipping the untracked + # TP, which is a behaviour change, so it is left as is. + if len(buf) >= bufsize: # type: ignore[operator] table.apply_changelog_batch(buf) buf.clear() self._last_flush_at = now diff --git a/faust/tables/table.py b/faust/tables/table.py index 2d2b765e1..5a340339f 100644 --- a/faust/tables/table.py +++ b/faust/tables/table.py @@ -5,6 +5,7 @@ from mode import Seconds from faust import windows +from faust.types.stores import StoreT from faust.types.tables import KT, VT, TableT, WindowWrapperT from faust.types.windows import WindowT from faust.utils.terminal.tables import dict_as_ansitable @@ -18,6 +19,12 @@ class Table(TableT[KT, VT], Collection): """Table (non-windowed).""" + # ``Collection`` provides ``data`` as a read-only property returning the + # underlying store, while ``FastUserDict`` (inherited via ``TableT``) + # declares it as a plain mapping attribute. Collection wins the MRO at + # runtime; redeclare it here so the two base declarations do not clash. + data: StoreT + WindowWrapper: ClassVar[Type[WindowWrapperT]] = wrappers.WindowWrapper def using_window( diff --git a/faust/tables/wrappers.py b/faust/tables/wrappers.py index 9454e6d76..b4f64616e 100644 --- a/faust/tables/wrappers.py +++ b/faust/tables/wrappers.py @@ -327,7 +327,7 @@ def __init__( self, table: TableT, *, - relative_to: RelativeArg = None, + relative_to: Optional[RelativeArg] = None, key_index: bool = False, key_index_table: Optional[TableT] = None, ) -> None: diff --git a/faust/topics.py b/faust/topics.py index 630014b60..5b8fa5323 100644 --- a/faust/topics.py +++ b/faust/topics.py @@ -98,10 +98,10 @@ def __init__( app: AppT, *, topics: Optional[Sequence[str]] = None, - pattern: Union[str, Pattern] = None, + pattern: Optional[Union[str, Pattern]] = None, schema: Optional[SchemaT] = None, - key_type: ModelArg = None, - value_type: ModelArg = None, + key_type: Optional[ModelArg] = None, + value_type: Optional[ModelArg] = None, is_iterator: bool = False, partitions: Optional[int] = None, retention: Optional[Seconds] = None, @@ -112,8 +112,8 @@ def __init__( internal: bool = False, config: Optional[Mapping[str, Any]] = None, queue: Optional[ThrowableQueue] = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, maxsize: Optional[int] = None, root: Optional[ChannelT] = None, active_partitions: Optional[Set[TP]] = None, @@ -159,14 +159,14 @@ def _compile_decode(self) -> None: async def send( self, *, - key: K = None, - value: V = None, + key: Optional[K] = None, + value: Optional[V] = None, partition: Optional[int] = None, timestamp: Optional[float] = None, - headers: HeadersArg = None, + headers: Optional[HeadersArg] = None, schema: Optional[SchemaT] = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, callback: Optional[MessageSentCallback] = None, force: bool = False, ) -> Awaitable[RecordMetadata]: @@ -202,14 +202,14 @@ async def send( def send_soon( self, *, - key: K = None, - value: V = None, + key: Optional[K] = None, + value: Optional[V] = None, partition: Optional[int] = None, timestamp: Optional[float] = None, - headers: HeadersArg = None, + headers: Optional[HeadersArg] = None, schema: Optional[SchemaT] = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, callback: Optional[MessageSentCallback] = None, force: bool = False, eager_partitioning: bool = False, @@ -299,9 +299,12 @@ def partitions(self) -> Optional[int]: return self._partitions @partitions.setter - def partitions(self, partitions: int) -> None: + def partitions(self, partitions: Optional[int]) -> None: """Set the number of partitions for this topic. + :const:`None` means "let the broker decide", which is what + ``__init__`` assigns when no ``partitions`` argument is given. + Only used for internal topics, see :attr:`partitions`. """ if partitions == 0: @@ -324,10 +327,10 @@ def derive_topic( *, topics: Optional[Sequence[str]] = None, schema: Optional[SchemaT] = None, - key_type: ModelArg = None, - value_type: ModelArg = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_type: Optional[ModelArg] = None, + value_type: Optional[ModelArg] = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, partitions: Optional[int] = None, retention: Optional[Seconds] = None, compacting: Optional[bool] = None, diff --git a/faust/transport/conductor.py b/faust/transport/conductor.py index 0bda03b02..853e38428 100644 --- a/faust/transport/conductor.py +++ b/faust/transport/conductor.py @@ -165,7 +165,15 @@ async def on_message(message: Message) -> None: delivered.add(chan) if full: for _, dest_chan in full: - on_topic_buffer_full(dest_chan) + # XXX wrong argument: ``SensorT.on_topic_buffer_full`` + # takes a ``TP`` (as ``on_pressure_high`` above is + # passed), but a channel is passed here, so + # ``Monitor.topic_buffer_full`` is keyed by channel + # and its per-TP counts are wrong. The Cython twin + # in ``_cython/conductor.pyx`` has the same bug; + # fixing either alone would make them disagree, so + # the defect is only recorded here, not fixed. + on_topic_buffer_full(dest_chan) # type: ignore[arg-type] await asyncio.wait( [ asyncio.ensure_future(dest_chan.put(dest_event)) diff --git a/faust/transport/consumer.py b/faust/transport/consumer.py index d5a82b1b8..7b7936bc7 100644 --- a/faust/transport/consumer.py +++ b/faust/transport/consumer.py @@ -179,7 +179,13 @@ async def _fetcher(self) -> None: if self.app.rebalancing: self.log.info("Restarting on rebalance") await self.crash(exc) - self.supervisor.wakeup() + # XXX ``ServiceT.supervisor`` is ``Optional`` and nothing here + # guarantees the fetcher was adopted by a supervisor, so this + # raises AttributeError when it was not. Left unguarded on + # purpose: ``wakeup()`` is what restarts the fetcher after a + # rebalance crash, and skipping it would silently leave the + # fetcher dead instead of failing loudly. + self.supervisor.wakeup() # type: ignore[union-attr] finally: self.set_shutdown() @@ -486,7 +492,14 @@ def __init__( self.not_waiting_next_records.set() self._reset_state() super().__init__(loop=loop, **kwargs) - self.transactions = self.transport.create_transaction_manager( + # Every concrete transport provides ``create_transaction_manager`` + # (see ``faust.transport.base.Transport``), but the ``TransportT`` + # interface in ``faust/types/transports.py`` does not declare it + # next to ``create_consumer``/``create_producer``/``create_conductor``. + create_transaction_manager = ( + self.transport.create_transaction_manager # type: ignore[attr-defined] + ) + self.transactions = create_transaction_manager( consumer=self, producer=self.app.producer, beacon=self.beacon, @@ -764,7 +777,10 @@ async def _wait_next_records( self, timeout: float ) -> Tuple[Optional[RecordMap], Optional[Set[TP]]]: if not self.flow_active: - await self.wait(self.can_resume_flow) + # mode types the waitable as ``mode.utils.locks.Event``, but + # Service.wait_first accepts anything with an awaitable + # ``.wait()`` -- asyncio.Event included. + await self.wait(self.can_resume_flow) # type: ignore[arg-type] try: # Set signal that _wait_next_records is waiting on the fetcher service. @@ -910,7 +926,7 @@ def verify_event_path(self, now: float, tp: TP) -> None: ... def verify_recovery_event_path(self, now: float, tp: TP) -> None: ... async def commit( - self, topics: TPorTopicSet = None, start_new_transaction: bool = True + self, topics: Optional[TPorTopicSet] = None, start_new_transaction: bool = True ) -> bool: """Maybe commit the offset for all or specific topics. @@ -954,7 +970,7 @@ async def maybe_wait_for_commit_to_finish(self) -> bool: @Service.transitions_to(CONSUMER_COMMITTING) async def force_commit( - self, topics: TPorTopicSet = None, start_new_transaction: bool = True + self, topics: Optional[TPorTopicSet] = None, start_new_transaction: bool = True ) -> bool: """Force offset commit.""" sensor_state = self.app.sensors.on_commit_initiated(self) @@ -1058,7 +1074,7 @@ async def _commit_offsets( return did_commit def _filter_tps_with_pending_acks( - self, topics: TPorTopicSet = None + self, topics: Optional[TPorTopicSet] = None ) -> Iterator[TP]: return ( tp @@ -1096,7 +1112,7 @@ def _new_offset(self, tp: TP) -> Optional[int]: # start without worrying about ends overlapping. sorted_candidates = sorted(candidates, key=lambda x: x.begin) if sorted_candidates: - stuff_to_add = [] + stuff_to_add: List[int] = [] for entry in sorted_candidates: stuff_to_add.extend(range(entry.begin, entry.end)) new_max_offset = max(stuff_to_add[-1], max_offset + 1) @@ -1111,8 +1127,9 @@ def _new_offset(self, tp: TP) -> Optional[int]: # ^-- gap # self._committed_offset[tp] is 31 # the return value will be None (the same as 31) - if self._committed_offset[tp]: - if min(acked) - self._committed_offset[tp] > 1: + committed_offset = self._committed_offset[tp] + if committed_offset: + if min(acked) - committed_offset > 1: return None # Note: acked is always kept sorted. diff --git a/faust/transport/drivers/aiokafka.py b/faust/transport/drivers/aiokafka.py index c900a486e..8a17c1c7b 100644 --- a/faust/transport/drivers/aiokafka.py +++ b/faust/transport/drivers/aiokafka.py @@ -81,6 +81,7 @@ ) from faust.types import ( TP, + AppT, ConsumerMessage, FutureMessage, HeadersArg, @@ -171,7 +172,7 @@ """.strip() -def __canon_host(host, default): +def __canon_host(host: Optional[str], default: str) -> str: """Ensure host is correctly formatted for aiokafka. That means IPv6 addresses must enclosed in squared brackets. """ @@ -194,8 +195,8 @@ def server_list(urls: List[URL], default_port: int) -> List[str]: class ConsumerRebalanceListener(aiokafka.abc.ConsumerRebalanceListener): # type: ignore # kafka's ridiculous class based callback interface makes this hacky. - def __init__(self, thread: ConsumerThread) -> None: - self._thread: ConsumerThread = thread + def __init__(self, thread: "AIOKafkaConsumerThread") -> None: + self._thread: "AIOKafkaConsumerThread" = thread def on_partitions_revoked(self, revoked: Iterable[_TopicPartition]) -> Awaitable: """Call when partitions are being revoked.""" @@ -229,7 +230,11 @@ class Consumer(ThreadDelegateConsumer): ConsumerStoppedError, ) - def _new_consumer_thread(self) -> ConsumerThread: + #: Narrows :attr:`ThreadDelegateConsumer._thread` to the thread type + #: this consumer actually creates in :meth:`_new_consumer_thread`. + _thread: "AIOKafkaConsumerThread" + + def _new_consumer_thread(self) -> "AIOKafkaConsumerThread": return AIOKafkaConsumerThread(self, loop=self.loop, beacon=self.beacon) async def create_topic( @@ -298,21 +303,27 @@ def verify_event_path(self, now: float, tp: TP) -> None: class ThreadedProducer(ServiceThread): _producer: Optional[aiokafka.AIOKafkaProducer] = None event_queue: Optional[asyncio.Queue] = None - _default_producer: Optional[aiokafka.AIOKafkaProducer] = None + #: The Faust producer this thread borrows its configuration from + #: (not an :class:`aiokafka.AIOKafkaProducer`) -- always set by __init__. + #: The ``= None`` class-level default is kept exactly as it was rather + #: than dropped, so ``ThreadedProducer._default_producer`` stays readable + #: on the class; the annotation is non-Optional because every instance + #: has a real Producer by the time anything uses it. + _default_producer: "Producer" = None # type: ignore[assignment] _push_events_task: Optional[asyncio.Task] = None - app: None + app: AppT stopped: bool _shutdown_initiated: bool = False def __init__( self, - default_producer, - app, + default_producer: "Producer", + app: AppT, *, - executor: Any = None, + executor: Optional[Any] = None, loop: Optional[asyncio.AbstractEventLoop] = None, thread_loop: Optional[asyncio.AbstractEventLoop] = None, - Worker: Type[WorkerThread] = None, + Worker: Optional[Type[WorkerThread]] = None, **kwargs: Any, ) -> None: super().__init__( @@ -324,7 +335,17 @@ def __init__( self._default_producer = default_producer self.app = app - def _shutdown_thread(self) -> None: + # XXX broken: this synchronous method overrides the coroutine + # ``mode.threads.ServiceThread._shutdown_thread``, breaking mode's + # contract. ``ServiceThread._serve()`` ends with + # ``finally: await self._shutdown_thread()``, so ``await None`` raises + # TypeError on every shutdown of this thread. The base implementation + # (on_thread_stop, stopping children/futures/exit stacks, set_shutdown) + # therefore never runs; the shutdown event only gets set because + # ``_start_thread`` catches that TypeError and calls ``set_shutdown()`` + # before re-raising it. Not fixed here: making it ``async`` changes + # runtime behaviour, which is out of scope for this annotation pass. + def _shutdown_thread(self) -> None: # type: ignore[override] # Ensure that the shutdown process is initiated only once if not self._shutdown_initiated: asyncio.run_coroutine_threadsafe(self.on_thread_stop(), self.thread_loop) @@ -333,7 +354,12 @@ async def flush(self) -> None: """Wait for producer to finish transmitting all buffered messages.""" while True: try: - msg = self.event_queue.get_nowait() + # ``event_queue`` is created in ``on_start``, and flushing is + # only reachable once the thread has started. cast, not + # assert: an assert would turn the AttributeError this raises + # when unset into an AssertionError, and vanishes under + # ``python -O``. + msg = cast(asyncio.Queue, self.event_queue).get_nowait() except QueueEmpty: break else: @@ -375,15 +401,22 @@ async def on_thread_stop(self) -> None: while not self._push_events_task.done(): await asyncio.sleep(0.1) - async def push_events(self): + async def push_events(self) -> None: while not self.stopped: + # ``event_queue`` is created by ``on_start`` before this task is + # scheduled. cast, not assert: this runs per message, and an + # assert would both change the exception type and disappear + # under ``python -O``. try: - event = await asyncio.wait_for(self.event_queue.get(), timeout=0.1) + event = await asyncio.wait_for( + cast(asyncio.Queue, self.event_queue).get(), timeout=0.1 + ) except asyncio.TimeoutError: continue self.app.sensors.on_threaded_producer_buffer_processed( - app=self.app, size=self.event_queue.qsize() + app=self.app, + size=cast(asyncio.Queue, self.event_queue).qsize(), ) await self.publish_message(event) @@ -407,7 +440,11 @@ async def publish_message( timestamp, partition, ) - producer = self._producer + # The underlying producer is created by ``on_start``, and messages + # are only published by tasks started after that. cast, not assert: + # this runs per message, and an assert would both change the + # exception type and disappear under ``python -O``. + producer = cast(aiokafka.AIOKafkaProducer, self._producer) state = self.app.sensors.on_send_initiated( producer, topic, @@ -428,7 +465,16 @@ async def publish_message( timestamp_ms=timestamp_ms, headers=headers, ) - fut.message.channel._on_published( + # XXX broken: ``_on_published`` is not on the ChannelT interface + # (it is implemented by faust.topics.Topic), and worse, the call + # below is missing an argument. Topic._on_published + # (faust/topics.py:463) is + # ``_on_published(self, fut, message, producer, state)`` -- ``fut`` + # is a required positional parameter holding the send future, and + # nothing is passed for it here, so this raises TypeError at + # runtime and ``publish_message(..., wait=True)`` can never + # succeed. Behaviour left untouched in this annotation-only pass. + fut.message.channel._on_published( # type: ignore[attr-defined] message=fut, state=state, producer=producer ) fut.set_result(ret) @@ -446,7 +492,11 @@ async def publish_message( ), ) callback = partial( - fut.message.channel._on_published, + # ``_on_published`` is not on the ChannelT interface; see the + # note on the ``wait`` branch above. This branch does supply + # the required positional ``fut``: add_done_callback passes + # the completed future as the first positional argument. + fut.message.channel._on_published, # type: ignore[attr-defined] message=fut, state=state, producer=producer, @@ -470,7 +520,13 @@ class AIOKafkaConsumerThread(ConsumerThread): def __post_init__(self) -> None: consumer = cast(Consumer, self.consumer) self._partitioner: PartitionerT = ( - self.app.conf.producer_partitioner or DefaultPartitioner() + # ``Settings.producer_partitioner`` is a ``Param`` descriptor, but + # because the setting's *value* type is itself a callable mypy + # reads the class attribute as a method and tries to bind ``self`` + # instead of going through ``Param.__get__`` -- which both fails + # and strips the leading ``key`` argument from the result type. + self.app.conf.producer_partitioner # type: ignore[misc,assignment] + or DefaultPartitioner() ) self._rebalance_listener = consumer.RebalanceListener(self) self._pending_rebalancing_spans = deque() @@ -954,7 +1010,7 @@ async def seek_wait(self, partitions: Mapping[TP, int]) -> None: await self.call_thread(self._seek_wait, consumer, partitions) async def _seek_wait( - self, consumer: Consumer, partitions: Mapping[TP, int] + self, consumer: aiokafka.AIOKafkaConsumer, partitions: Mapping[TP, int] ) -> None: for tp, offset in partitions.items(): self.log.dev("SEEK %r -> %r", tp, offset) @@ -1119,7 +1175,7 @@ class Producer(base.Producer): _transaction_producers: typing.Dict[str, aiokafka.AIOKafkaProducer] = {} _trn_locks: typing.Dict[str, Lock] = {} - def create_threaded_producer(self): + def create_threaded_producer(self) -> ThreadedProducer: return ThreadedProducer(default_producer=self, app=self.app) def __post_init__(self) -> None: @@ -1582,8 +1638,19 @@ async def _really_create_topic( else: raise Exception("Controller node is None") + # (partition_id, [replica broker ids]) pairs -- empty means "let the + # broker decide the replica assignment". + replica_assignment: List[Tuple[int, List[int]]] = [] create_topics_args = ( - [(topic, partitions, replication, [], list(config.items()))], + [ + ( + topic, + partitions, + replication, + replica_assignment, + list(config.items()), + ) + ], timeout, False, ) @@ -1619,7 +1686,7 @@ async def _really_create_topic( def credentials_to_aiokafka_auth( - credentials: Optional[CredentialsT] = None, ssl_context: Any = None + credentials: Optional[CredentialsT] = None, ssl_context: Optional[Any] = None ) -> Mapping: if credentials is not None: if isinstance(credentials, SSLCredentials): diff --git a/faust/transport/drivers/confluent.py b/faust/transport/drivers/confluent.py index 9535fc990..2c04913dd 100644 --- a/faust/transport/drivers/confluent.py +++ b/faust/transport/drivers/confluent.py @@ -80,7 +80,11 @@ class Consumer(ThreadDelegateConsumer): logger = logger - def _new_consumer_thread(self) -> ConsumerThread: + #: Narrows :attr:`ThreadDelegateConsumer._thread` to the thread type + #: this consumer actually creates in :meth:`_new_consumer_thread`. + _thread: "ConfluentConsumerThread" + + def _new_consumer_thread(self) -> "ConfluentConsumerThread": return ConfluentConsumerThread(self, loop=self.loop, beacon=self.beacon) async def create_topic( @@ -151,7 +155,15 @@ async def on_stop(self) -> None: await super().on_stop() def verify_event_path(self, now: float, tp: TP) -> None: - return self._thread.verify_event_path(now, tp) + # XXX broken: neither ConsumerThread nor ConfluentConsumerThread + # implements verify_event_path, so this raises AttributeError on + # every tick of the commit livelock detector + # (faust.transport.consumer.Consumer._commit_livelock_detector -> + # verify_all_partitions_active). Livelock detection is therefore + # dead for this driver. Not fixed here: adding a no-op stub would + # change runtime behaviour, and a real implementation belongs in + # ConfluentConsumerThread. + return self._thread.verify_event_path(now, tp) # type: ignore[attr-defined] class AsyncConsumer: @@ -283,7 +295,10 @@ def close(self) -> None: ... async def subscribe(self, topics: Iterable[str]) -> None: # XXX pattern does not work :/ await self.cast_thread( - self._ensure_consumer().subscribe, + # mode types cast_thread/call_thread as taking a coroutine + # function, but MethodQueue._process_enqueued runs whatever it + # gets through ``maybe_async``, so plain callables work too. + self._ensure_consumer().subscribe, # type: ignore[arg-type] topics=list(topics), on_assign=self._on_assign, on_revoke=self._on_revoke, @@ -669,7 +684,16 @@ async def flush(self) -> None: def key_partition(self, topic: str, key: bytes) -> TP: """Return topic and partition destination for key.""" # Get the partition count for the topic - metadata = self._producer_thread.producer.list_topics(topic) + # XXX broken: ``ProducerThread.producer`` is the Faust Producer + # (i.e. ``self``), not the underlying confluent_kafka.Producer -- + # that one is ``ProducerThread._producer``. Faust producers have no + # ``list_topics``, so this raises AttributeError and + # ``Producer.key_partition`` is dead on arrival for this driver. + # Behaviour left untouched in this annotation-only pass; the fix is + # to read ``self._producer_thread._producer``. + metadata = self._producer_thread.producer.list_topics( # type: ignore[attr-defined] # noqa: E501 + topic + ) partition_count = len(metadata.topics[topic].partitions) # Calculate the partition number based on the key hash diff --git a/faust/transport/producer.py b/faust/transport/producer.py index d70eb0aa6..f9967672f 100644 --- a/faust/transport/producer.py +++ b/faust/transport/producer.py @@ -23,10 +23,19 @@ class ProducerBuffer(Service, ProducerBufferT): + #: Set by :class:`Producer` right after the buffer is created, + #: so it is only ``None`` for a buffer that was never attached + #: to a producer. app: Optional[AppT] = None max_messages = 100 queue: Optional[asyncio.Queue] = None + #: The transport driver's threaded producer. Only assigned by + #: :class:`Producer` when the :setting:`producer_threaded` setting is + #: enabled -- annotation only, so that on a buffer that never got one + #: the attribute is still missing rather than ``None``. + threaded_producer: Optional[ServiceThread] + def __post_init__(self) -> None: self.pending = asyncio.Queue() self.message_sent = asyncio.Event() @@ -37,11 +46,24 @@ def put(self, fut: FutureMessage) -> None: The message will be eventually produced, you can await the future to wait for that to happen. """ - if self.app.conf.producer_threaded: + # ``app`` is Optional because ``Producer.__init__`` only fills it in + # after creating the buffer. Narrowed with ``cast`` rather than an + # ``assert``: this is the per-message hot path, and an assert would + # both swap the AttributeError raised by a half-built buffer for an + # AssertionError and disappear under ``python -O``. + app = cast(AppT, self.app) + if app.conf.producer_threaded: + # Likewise Optional: only set when :setting:`producer_threaded` + # is enabled, which is the branch we are in. + threaded_producer = cast(ServiceThread, self.threaded_producer) if not self.queue: - self.queue = self.threaded_producer.event_queue + # ``event_queue`` is not part of the ``ServiceThread`` + # interface: it belongs to the driver's threaded producer + # (e.g. aiokafka's ``ThreadedProducer``), which lives in a + # module this one cannot import without a cycle. + self.queue = threaded_producer.event_queue # type: ignore[attr-defined] asyncio.run_coroutine_threadsafe( - self.queue.put(fut), self.threaded_producer.thread_loop + self.queue.put(fut), threaded_producer.thread_loop ) else: self.pending.put_nowait(fut) @@ -106,12 +128,18 @@ async def _handle_pending(self) -> None: @property def size(self) -> int: """Current buffer size (messages waiting to be produced).""" - if self.app.conf.producer_threaded: + # See ``put()``: ``app`` is Optional only because it is assigned right + # after construction; ``cast`` keeps the original AttributeError + # behaviour instead of an assert that ``python -O`` would remove. + app = cast(AppT, self.app) + if app.conf.producer_threaded: if not self.queue: return 0 - queue_items = self.queue._queue # type: ignore + # ``Queue._queue`` is a CPython implementation detail (the + # underlying deque) that typeshed does not describe. + queue_items = self.queue._queue # type: ignore[attr-defined] else: - queue_items = self.pending._queue + queue_items = self.pending._queue # type: ignore[attr-defined] queue_items = cast(list, queue_items) return len(queue_items) @@ -142,13 +170,25 @@ def __init__( self.request_timeout = conf.producer_request_timeout self.ssl_context = conf.ssl_context self.credentials = conf.broker_credentials - self.partitioner = conf.producer_partitioner + # ``Settings.producer_partitioner`` is a ``Param`` descriptor, but + # because the setting's *value* type is itself a callable mypy reads + # the class attribute as a method and tries to bind ``self`` instead + # of going through ``Param.__get__`` -- which both fails and strips + # the leading ``key`` argument from the result type. + self.partitioner = conf.producer_partitioner # type: ignore[misc,assignment] api_version = self._api_version = conf.producer_api_version assert api_version is not None super().__init__(loop=loop, **kwargs) self.buffer = ProducerBuffer(loop=self.loop, beacon=self.beacon) if conf.producer_threaded: - self.threaded_producer = self.create_threaded_producer() + # XXX ``create_threaded_producer`` is not defined here or on + # ``ProducerT``; only the aiokafka driver implements it. With the + # confluent driver (faust/transport/drivers/confluent.py) this line + # raises AttributeError, so :setting:`producer_threaded` is simply + # broken there. + self.threaded_producer = ( + self.create_threaded_producer() # type: ignore[attr-defined] + ) self.buffer.threaded_producer = self.threaded_producer self.buffer.app = self.app diff --git a/faust/transport/utils.py b/faust/transport/utils.py index dc0b8d7d0..ae8fb4e4d 100644 --- a/faust/transport/utils.py +++ b/faust/transport/utils.py @@ -57,6 +57,11 @@ def records_iterator(self, index: TopicIndexMap) -> Iterator[Tuple[TP, Any]]: to_remove: Set[str] = set() sentinel = object() _next = next + # Declared up front so the unpacking below has a type to bind to: + # ``next(it, sentinel)`` is typed as the join of the buffer's item + # type and the sentinel's, which collapses to plain ``object``. + tp: TP + record: Any while index: for topic in to_remove: index.pop(topic, None) @@ -68,11 +73,13 @@ def records_iterator(self, index: TopicIndexMap) -> Iterator[Tuple[TP, Any]]: # so move that to the outer loop. to_remove.add(topic) continue - tp, record = item # type: ignore + # Not the sentinel, so it is a ``TopicBuffer`` item; mypy + # cannot narrow an ``is``-comparison against a plain object. + tp, record = item # type: ignore[misc] yield tp, record -class TopicBuffer(Iterator): +class TopicBuffer(Iterator[Tuple[TP, Any]]): """Data structure managing the buffer for incoming records in a topic.""" _buffers: Dict[TP, Iterator] diff --git a/faust/types/agents.py b/faust/types/agents.py index 70491c20f..2d83791cd 100644 --- a/faust/types/agents.py +++ b/faust/types/agents.py @@ -129,15 +129,15 @@ def __init__( *, name: Optional[str] = None, app: Optional[_AppT] = None, - channel: Union[str, ChannelT] = None, + channel: Optional[Union[str, ChannelT]] = None, concurrency: int = 1, - sink: Iterable[SinkT] = None, - on_error: AgentErrorHandler = None, - supervisor_strategy: Type[SupervisorStrategyT] = None, + sink: Optional[Iterable[SinkT]] = None, + on_error: Optional[AgentErrorHandler] = None, + supervisor_strategy: Optional[Type[SupervisorStrategyT]] = None, help: Optional[str] = None, schema: Optional[SchemaT] = None, - key_type: ModelArg = None, - value_type: ModelArg = None, + key_type: Optional[ModelArg] = None, + value_type: Optional[ModelArg] = None, isolated_partitions: bool = False, **kwargs: Any, ) -> None: @@ -160,7 +160,7 @@ def __call__( def test_context( self, channel: Optional[ChannelT] = None, - supervisor_strategy: SupervisorStrategyT = None, + supervisor_strategy: Optional[SupervisorStrategyT] = None, **kwargs: Any, ) -> "AgentTestWrapperT": ... @@ -179,24 +179,24 @@ async def on_partitions_revoked(self, revoked: Set[TP]) -> None: ... @abc.abstractmethod async def cast( self, - value: V = None, + value: Optional[V] = None, *, - key: K = None, + key: Optional[K] = None, partition: Optional[int] = None, timestamp: Optional[float] = None, - headers: HeadersArg = None, + headers: Optional[HeadersArg] = None, ) -> None: ... @abc.abstractmethod async def ask( self, - value: V = None, + value: Optional[V] = None, *, - key: K = None, + key: Optional[K] = None, partition: Optional[int] = None, timestamp: Optional[float] = None, - headers: HeadersArg = None, - reply_to: ReplyToArg = None, + headers: Optional[HeadersArg] = None, + reply_to: Optional[ReplyToArg] = None, correlation_id: Optional[str] = None, ) -> Any: ... @@ -204,14 +204,14 @@ async def ask( async def send( self, *, - key: K = None, - value: V = None, + key: Optional[K] = None, + value: Optional[V] = None, partition: Optional[int] = None, timestamp: Optional[float] = None, - headers: HeadersArg = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, - reply_to: ReplyToArg = None, + headers: Optional[HeadersArg] = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, + reply_to: Optional[ReplyToArg] = None, correlation_id: Optional[str] = None, ) -> Awaitable[RecordMetadata]: ... @@ -220,8 +220,8 @@ async def send( async def map( self, values: Union[AsyncIterable, Iterable], - key: K = None, - reply_to: ReplyToArg = None, + key: Optional[K] = None, + reply_to: Optional[ReplyToArg] = None, ) -> AsyncIterator: ... @abc.abstractmethod @@ -229,29 +229,31 @@ async def map( async def kvmap( self, items: Union[AsyncIterable[Tuple[K, V]], Iterable[Tuple[K, V]]], - reply_to: ReplyToArg = None, + reply_to: Optional[ReplyToArg] = None, ) -> AsyncIterator[str]: ... @abc.abstractmethod async def join( self, values: Union[AsyncIterable[V], Iterable[V]], - key: K = None, - reply_to: ReplyToArg = None, + key: Optional[K] = None, + reply_to: Optional[ReplyToArg] = None, ) -> List[Any]: ... @abc.abstractmethod async def kvjoin( self, items: Union[AsyncIterable[Tuple[K, V]], Iterable[Tuple[K, V]]], - reply_to: ReplyToArg = None, + reply_to: Optional[ReplyToArg] = None, ) -> List[Any]: ... @abc.abstractmethod def info(self) -> Mapping: ... @abc.abstractmethod - def clone(self, *, cls: Type["AgentT"] = None, **kwargs: Any) -> "AgentT": ... + def clone( + self, *, cls: Optional[Type["AgentT"]] = None, **kwargs: Any + ) -> "AgentT": ... @abc.abstractmethod def get_topic_names(self) -> Iterable[str]: ... @@ -305,15 +307,15 @@ def __init__( @abc.abstractmethod async def put( self, - value: V = None, - key: K = None, + value: Optional[V] = None, + key: Optional[K] = None, partition: Optional[int] = None, timestamp: Optional[float] = None, - headers: HeadersArg = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + headers: Optional[HeadersArg] = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, *, - reply_to: ReplyToArg = None, + reply_to: Optional[ReplyToArg] = None, correlation_id: Optional[str] = None, wait: bool = True, ) -> EventT: ... @@ -328,7 +330,7 @@ def to_message( offset: int = 0, timestamp: Optional[float] = None, timestamp_type: int = 0, - headers: HeadersArg = None, + headers: Optional[HeadersArg] = None, ) -> Message: ... @abc.abstractmethod diff --git a/faust/types/app.py b/faust/types/app.py index 5ef4c3605..79d5f7cb5 100644 --- a/faust/types/app.py +++ b/faust/types/app.py @@ -205,7 +205,12 @@ class AppT(ServiceT): @abc.abstractmethod def __init__( - self, id: str, *, monitor: _Monitor, config_source: Any = None, **options: Any + self, + id: str, + *, + monitor: _Monitor, + config_source: Optional[Any] = None, + **options: Any, ) -> None: self.on_startup_finished: Optional[Callable] = None @@ -238,12 +243,12 @@ def discover( def topic( self, *topics: str, - pattern: Union[str, Pattern] = None, + pattern: Optional[Union[str, Pattern]] = None, schema: Optional[_SchemaT] = None, - key_type: _ModelArg = None, - value_type: _ModelArg = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_type: Optional[_ModelArg] = None, + value_type: Optional[_ModelArg] = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, partitions: Optional[int] = None, retention: Optional[Seconds] = None, compacting: Optional[bool] = None, @@ -263,8 +268,8 @@ def channel( self, *, schema: Optional[_SchemaT] = None, - key_type: _ModelArg = None, - value_type: _ModelArg = None, + key_type: Optional[_ModelArg] = None, + value_type: Optional[_ModelArg] = None, maxsize: Optional[int] = None, loop: Optional[asyncio.AbstractEventLoop] = None, ) -> ChannelT: ... @@ -272,12 +277,12 @@ def channel( @abc.abstractmethod def agent( self, - channel: Union[str, ChannelT[_T]] = None, + channel: Optional[Union[str, ChannelT[_T]]] = None, *, name: Optional[str] = None, concurrency: int = 1, - supervisor_strategy: Type[SupervisorStrategyT] = None, - sink: Iterable[SinkT] = None, + supervisor_strategy: Optional[Type[SupervisorStrategyT]] = None, + sink: Optional[Iterable[SinkT]] = None, isolated_partitions: bool = False, use_reply_headers: bool = True, **kwargs: Any, @@ -304,7 +309,7 @@ def crontab( self, cron_format: str, *, - timezone: tzinfo = None, + timezone: Optional[tzinfo] = None, on_leader: bool = False, traced: bool = True, ) -> Callable: ... @@ -322,7 +327,7 @@ def Table( self, name: str, *, - default: Callable[[], Any] = None, + default: Optional[Callable[[], Any]] = None, window: Optional[WindowT] = None, partitions: Optional[int] = None, help: Optional[str] = None, @@ -334,7 +339,7 @@ def GlobalTable( self, name: str, *, - default: Callable[[], Any] = None, + default: Optional[Callable[[], Any]] = None, window: Optional[WindowT] = None, partitions: Optional[int] = None, help: Optional[str] = None, @@ -371,7 +376,7 @@ def page( path: str, *, base: Type[View] = View, - cors_options: Mapping[str, ResourceOptions] = None, + cors_options: Optional[Mapping[str, ResourceOptions]] = None, name: Optional[str] = None, ) -> Callable[[PageArg], Type[View]]: ... @@ -388,7 +393,7 @@ def table_route( @abc.abstractmethod def command( - self, *options: Any, base: Type[_AppCommand] = None, **kwargs: Any + self, *options: Any, base: Optional[Type[_AppCommand]] = None, **kwargs: Any ) -> Callable[[Callable], Type[_AppCommand]]: ... @abc.abstractmethod @@ -411,14 +416,14 @@ def trace( async def send( self, channel: Union[ChannelT, str], - key: K = None, - value: V = None, + key: Optional[K] = None, + value: Optional[V] = None, partition: Optional[int] = None, timestamp: Optional[float] = None, - headers: HeadersArg = None, + headers: Optional[HeadersArg] = None, schema: Optional[_SchemaT] = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, callback: Optional[MessageSentCallback] = None, ) -> Awaitable[RecordMetadata]: ... @@ -456,6 +461,7 @@ def on_rebalance_return(self) -> None: ... def on_rebalance_end(self) -> None: ... @property + @abc.abstractmethod def conf(self) -> _Settings: ... @conf.setter diff --git a/faust/types/auth.py b/faust/types/auth.py index 30b99a56b..c8c5b8a78 100644 --- a/faust/types/auth.py +++ b/faust/types/auth.py @@ -39,7 +39,7 @@ class CredentialsT: CredentialsArg = Union[CredentialsT, ssl.SSLContext] -def to_credentials(obj: CredentialsArg = None) -> Optional[CredentialsT]: +def to_credentials(obj: Optional[CredentialsArg] = None) -> Optional[CredentialsT]: if obj is not None: from faust.auth import SSLCredentials # XXX :( diff --git a/faust/types/channels.py b/faust/types/channels.py index b717c1a68..60a2f8ad9 100644 --- a/faust/types/channels.py +++ b/faust/types/channels.py @@ -48,12 +48,12 @@ def __init__( app: _AppT, *, schema: Optional[_SchemaT] = None, - key_type: _ModelArg = None, - value_type: _ModelArg = None, + key_type: Optional[_ModelArg] = None, + value_type: Optional[_ModelArg] = None, is_iterator: bool = False, queue: Optional[ThrowableQueue] = None, maxsize: Optional[int] = None, - root: "ChannelT" = None, + root: Optional["ChannelT"] = None, active_partitions: Optional[Set[TP]] = None, loop: Optional[asyncio.AbstractEventLoop] = None, ) -> None: ... @@ -76,14 +76,14 @@ def get_topic_name(self) -> str: ... async def send( self, *, - key: K = None, - value: V = None, + key: Optional[K] = None, + value: Optional[V] = None, partition: Optional[int] = None, timestamp: Optional[float] = None, - headers: HeadersArg = None, + headers: Optional[HeadersArg] = None, schema: Optional[_SchemaT] = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, callback: Optional[MessageSentCallback] = None, force: bool = False, ) -> Awaitable[RecordMetadata]: ... @@ -92,14 +92,14 @@ async def send( def send_soon( self, *, - key: K = None, - value: V = None, + key: Optional[K] = None, + value: Optional[V] = None, partition: Optional[int] = None, timestamp: Optional[float] = None, - headers: HeadersArg = None, + headers: Optional[HeadersArg] = None, schema: Optional[_SchemaT] = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, callback: Optional[MessageSentCallback] = None, force: bool = False, eager_partitioning: bool = False, @@ -108,14 +108,14 @@ def send_soon( @abc.abstractmethod def as_future_message( self, - key: K = None, - value: V = None, + key: Optional[K] = None, + value: Optional[V] = None, partition: Optional[int] = None, timestamp: Optional[float] = None, - headers: HeadersArg = None, + headers: Optional[HeadersArg] = None, schema: Optional[_SchemaT] = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, callback: Optional[MessageSentCallback] = None, eager_partitioning: bool = False, ) -> FutureMessage: ... diff --git a/faust/types/codecs.py b/faust/types/codecs.py index 32efe9cf2..554e8adc7 100644 --- a/faust/types/codecs.py +++ b/faust/types/codecs.py @@ -13,7 +13,7 @@ class CodecT(metaclass=abc.ABCMeta): @abc.abstractmethod def __init__( - self, children: Tuple["CodecT", ...] = None, **kwargs: Any + self, children: Optional[Tuple["CodecT", ...]] = None, **kwargs: Any ) -> None: ... @abc.abstractmethod diff --git a/faust/types/events.py b/faust/types/events.py index 9b1498609..b1deaa804 100644 --- a/faust/types/events.py +++ b/faust/types/events.py @@ -55,14 +55,14 @@ def __init__( async def send( self, channel: Union[str, _ChannelT], - key: K = None, - value: V = None, + key: Optional[K] = None, + value: Optional[V] = None, partition: Optional[int] = None, timestamp: Optional[float] = None, - headers: HeadersArg = None, + headers: Optional[HeadersArg] = None, schema: Optional[_SchemaT] = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, callback: Optional[MessageSentCallback] = None, force: bool = False, ) -> Awaitable[RecordMetadata]: ... @@ -71,14 +71,14 @@ async def send( async def forward( self, channel: Union[str, _ChannelT], - key: Any = None, - value: Any = None, + key: Optional[Any] = None, + value: Optional[Any] = None, partition: Optional[int] = None, timestamp: Optional[float] = None, - headers: HeadersArg = None, + headers: Optional[HeadersArg] = None, schema: Optional[_SchemaT] = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, callback: Optional[MessageSentCallback] = None, force: bool = False, ) -> Awaitable[RecordMetadata]: ... diff --git a/faust/types/models.py b/faust/types/models.py index f0710e42a..9abc63474 100644 --- a/faust/types/models.py +++ b/faust/types/models.py @@ -146,7 +146,7 @@ class ModelT(base): # type: ignore @classmethod @abc.abstractmethod def from_data( - cls, data: Any, *, preferred_type: Type["ModelT"] = None + cls, data: Any, *, preferred_type: Optional[Type["ModelT"]] = None ) -> "ModelT": ... @classmethod @@ -155,15 +155,15 @@ def loads( cls, s: bytes, *, - default_serializer: CodecArg = None, # XXX use serializer - serializer: CodecArg = None, + default_serializer: Optional[CodecArg] = None, # XXX use serializer + serializer: Optional[CodecArg] = None, ) -> "ModelT": ... @abc.abstractmethod def __init__(self, *args: Any, **kwargs: Any) -> None: ... @abc.abstractmethod - def dumps(self, *, serializer: CodecArg = None) -> bytes: ... + def dumps(self, *, serializer: Optional[CodecArg] = None) -> bytes: ... @abc.abstractmethod def derive(self, *objects: "ModelT", **fields: Any) -> "ModelT": ... @@ -206,10 +206,10 @@ def __init__( type: Optional[Type[T]] = None, model: Optional[Type[ModelT]] = None, required: bool = True, - default: T = None, - parent: "FieldDescriptorT" = None, + default: Optional[T] = None, + parent: Optional["FieldDescriptorT"] = None, exclude: Optional[bool] = None, - date_parser: Callable[[Any], datetime] = None, + date_parser: Optional[Callable[[Any], datetime]] = None, **kwargs: Any, ) -> None: # we have to do this in __init__ or mypy will think diff --git a/faust/types/sensors.py b/faust/types/sensors.py index a55d39e52..b3f4d1496 100644 --- a/faust/types/sensors.py +++ b/faust/types/sensors.py @@ -33,7 +33,12 @@ def on_stream_event_in( @abc.abstractmethod def on_stream_event_out( - self, tp: TP, offset: int, stream: StreamT, event: EventT, state: Dict = None + self, + tp: TP, + offset: int, + stream: StreamT, + event: EventT, + state: Optional[Dict] = None, ) -> None: ... @abc.abstractmethod @@ -101,7 +106,7 @@ def on_rebalance_end(self, app: _AppT, state: Dict) -> None: ... @abc.abstractmethod def on_web_request_start( - self, app: _AppT, request: web.Request, *, view: web.View = None + self, app: _AppT, request: web.Request, *, view: Optional[web.View] = None ) -> Dict: ... @abc.abstractmethod @@ -112,7 +117,7 @@ def on_web_request_end( response: Optional[web.Response], state: Dict, *, - view: web.View = None, + view: Optional[web.View] = None, ) -> None: ... @abc.abstractmethod diff --git a/faust/types/serializers.py b/faust/types/serializers.py index dd8b93756..c1008579a 100644 --- a/faust/types/serializers.py +++ b/faust/types/serializers.py @@ -30,7 +30,9 @@ class RegistryT(abc.ABC): @abc.abstractmethod def __init__( - self, key_serializer: CodecArg = None, value_serializer: CodecArg = "json" + self, + key_serializer: Optional[CodecArg] = None, + value_serializer: CodecArg = "json", ) -> None: ... @abc.abstractmethod @@ -39,7 +41,7 @@ def loads_key( typ: Optional[_ModelArg], key: Optional[bytes], *, - serializer: CodecArg = None, + serializer: Optional[CodecArg] = None, ) -> K: ... @abc.abstractmethod @@ -48,17 +50,21 @@ def loads_value( typ: Optional[_ModelArg], value: Optional[bytes], *, - serializer: CodecArg = None, + serializer: Optional[CodecArg] = None, ) -> Any: ... @abc.abstractmethod def dumps_key( - self, typ: Optional[_ModelArg], key: K, *, serializer: CodecArg = None + self, typ: Optional[_ModelArg], key: K, *, serializer: Optional[CodecArg] = None ) -> Optional[bytes]: ... @abc.abstractmethod def dumps_value( - self, typ: Optional[_ModelArg], value: V, *, serializer: CodecArg = None + self, + typ: Optional[_ModelArg], + value: V, + *, + serializer: Optional[CodecArg] = None, ) -> Optional[bytes]: ... @@ -74,10 +80,10 @@ class SchemaT(Generic[KT, VT]): def __init__( self, *, - key_type: _ModelArg = None, - value_type: _ModelArg = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_type: Optional[_ModelArg] = None, + value_type: Optional[_ModelArg] = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, allow_empty: Optional[bool] = None, ) -> None: ... @@ -85,10 +91,10 @@ def __init__( def update( self, *, - key_type: _ModelArg = None, - value_type: _ModelArg = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_type: Optional[_ModelArg] = None, + value_type: Optional[_ModelArg] = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, allow_empty: Optional[bool] = None, ) -> None: ... @@ -99,7 +105,7 @@ def loads_key( message: _Message, *, loads: Optional[Callable] = None, - serializer: CodecArg = None, + serializer: Optional[CodecArg] = None, ) -> KT: ... @abc.abstractmethod @@ -109,7 +115,7 @@ def loads_value( message: _Message, *, loads: Optional[Callable] = None, - serializer: CodecArg = None, + serializer: Optional[CodecArg] = None, ) -> VT: ... @abc.abstractmethod @@ -118,7 +124,7 @@ def dumps_key( app: _AppT, key: K, *, - serializer: CodecArg = None, + serializer: Optional[CodecArg] = None, headers: OpenHeadersArg, ) -> Tuple[Any, OpenHeadersArg]: ... @@ -128,7 +134,7 @@ def dumps_value( app: _AppT, value: V, *, - serializer: CodecArg = None, + serializer: Optional[CodecArg] = None, headers: OpenHeadersArg, ) -> Tuple[Any, OpenHeadersArg]: ... diff --git a/faust/types/settings/params.py b/faust/types/settings/params.py index c55267cd7..7de1420fa 100644 --- a/faust/types/settings/params.py +++ b/faust/types/settings/params.py @@ -294,19 +294,19 @@ def __init__( *, name: str, env_name: Optional[str] = None, - default: IT = None, + default: Optional[IT] = None, default_alias: Optional[str] = None, default_template: Optional[str] = None, allow_none: Optional[bool] = None, ignore_default: Optional[bool] = None, - section: _Section = None, + section: Optional[_Section] = None, version_introduced: Optional[str] = None, version_deprecated: Optional[str] = None, version_removed: Optional[str] = None, - version_changed: Mapping[str, str] = None, + version_changed: Optional[Mapping[str, str]] = None, deprecation_reason: Optional[str] = None, - related_cli_options: Mapping[str, List[str]] = None, - related_settings: List[Any] = None, + related_cli_options: Optional[Mapping[str, List[str]]] = None, + related_settings: Optional[List[Any]] = None, help: Optional[str] = None, **kwargs: Any, ) -> None: @@ -366,9 +366,19 @@ def on_set_default(self, fun: OnDefaultCallable) -> OnDefaultCallable: self._on_set_default_ = fun return fun - def __get__(self, obj: Any, type: Type = None) -> OT: + @typing.overload + def __get__(self, obj: None, type: Optional[Type] = None) -> "Param[IT, OT]": ... + + @typing.overload + def __get__(self, obj: Any, type: Optional[Type] = None) -> OT: ... + + def __get__(self, obj: Any, type: Optional[Type] = None) -> Any: + # Accessed on the class (obj is None) this returns the descriptor + # itself, and accessed on a Settings instance it returns the value: + # the same split :class:`property` declares, so the overloads above + # mirror the ones in the base class. if obj is None: - return self # type: ignore + return self if self.version_deprecated: # we use UserWarning because DeprecationWarning is silenced # by default in Python. @@ -393,7 +403,7 @@ def prepare_get(self, conf: _Settings, value: OT) -> OT: """Prepare value when accessed/retrieved.""" return value - def on_set(self, settings: Any, value: OT) -> None: + def on_set(self, settings: Any, value: Optional[OT]) -> None: """What happens when the setting is stored/set.""" settings.__dict__[self.storage_name] = value assert getattr(settings, self.storage_name) == value @@ -424,11 +434,14 @@ def on_init_set_default( ``Settings.__init__`` or :const:`None` if not set. """ if provided_value is None: - default_value = self.default + default_value: Optional[IT] = self.default if self._on_set_default_: default_value = self._on_set_default_(conf) if default_value is None and self.default_template: - default_value = self.default_template.format(conf=conf) + # ``default_template`` renders to a string that is then fed + # through ``to_python``, so only settings whose input type + # accepts :class:`str` are allowed to define one. + default_value = cast(IT, self.default_template.format(conf=conf)) setattr( conf, self.storage_name, self.prepare_init_default(conf, default_value) ) @@ -443,28 +456,31 @@ def build_deprecation_warning(self) -> str: alt_removal=alt_removal, ) - def validate_before(self, value: IT = None) -> None: + def validate_before(self, value: Optional[IT] = None) -> None: """Validate value before setting is converted to the target type.""" ... - def validate_after(self, value: OT) -> None: + def validate_after(self, value: Optional[OT]) -> None: """Validate value after it has been converted to its target type.""" ... - def prepare_set(self, conf: _Settings, value: IT) -> OT: + def prepare_set(self, conf: _Settings, value: Optional[IT]) -> Optional[OT]: """Prepare value for storage.""" skip_validate = value is None and self.allow_none if not skip_validate: self.validate_before(value) + new_value: Optional[OT] if value is not None: new_value = self.to_python(conf, value) else: - new_value = value + new_value = None if not skip_validate: self.validate_after(new_value) return new_value - def prepare_init_default(self, conf: _Settings, value: IT) -> OT: + def prepare_init_default( + self, conf: _Settings, value: Optional[IT] + ) -> Optional[OT]: """Prepare default value for storage.""" if value is not None: return self.to_python(conf, value) @@ -522,7 +538,7 @@ def _init_options( self, min_value: Optional[int] = None, max_value: Optional[int] = None, - number_aliases: Mapping[IT, OT] = None, + number_aliases: Optional[Mapping[IT, OT]] = None, **kwargs: Any, ) -> None: if min_value is not None: @@ -541,7 +557,7 @@ def to_python(self, conf: _Settings, value: IT) -> OT: except KeyError: return self.convert(conf, value) - def validate_after(self, value: OT) -> None: + def validate_after(self, value: Optional[OT]) -> None: """Validate number value.""" v = cast(int, value) min_ = self.min_value diff --git a/faust/types/settings/settings.py b/faust/types/settings/settings.py index cf080dab8..8f68b2cf4 100644 --- a/faust/types/settings/settings.py +++ b/faust/types/settings/settings.py @@ -77,35 +77,35 @@ def __init__( id: str, *, # Common settings: - autodiscover: AutodiscoverArg = None, - datadir: typing.Union[str, Path] = None, - tabledir: typing.Union[str, Path] = None, + autodiscover: Optional[AutodiscoverArg] = None, + datadir: Optional[typing.Union[str, Path]] = None, + tabledir: Optional[typing.Union[str, Path]] = None, debug: Optional[bool] = None, env_prefix: Optional[str] = None, id_format: Optional[str] = None, origin: Optional[str] = None, - timezone: typing.Union[str, tzinfo] = None, + timezone: Optional[typing.Union[str, tzinfo]] = None, version: Optional[int] = None, # Agent settings: - agent_supervisor: SymbolArg[Type[SupervisorStrategyT]] = None, + agent_supervisor: Optional[SymbolArg[Type[SupervisorStrategyT]]] = None, # Broker settings: - broker: BrokerArg = None, - broker_consumer: BrokerArg = None, - broker_producer: BrokerArg = None, + broker: Optional[BrokerArg] = None, + broker_consumer: Optional[BrokerArg] = None, + broker_producer: Optional[BrokerArg] = None, broker_api_version: Optional[str] = None, broker_check_crcs: Optional[bool] = None, broker_client_id: Optional[str] = None, broker_commit_every: Optional[int] = None, broker_commit_interval: Optional[Seconds] = None, broker_commit_livelock_soft_timeout: Optional[Seconds] = None, - broker_credentials: CredentialsArg = None, + broker_credentials: Optional[CredentialsArg] = None, broker_heartbeat_interval: Optional[Seconds] = None, broker_max_poll_interval: Optional[Seconds] = None, broker_max_poll_records: Optional[int] = None, broker_rebalance_timeout: Optional[Seconds] = None, broker_request_timeout: Optional[Seconds] = None, broker_session_timeout: Optional[Seconds] = None, - ssl_context: ssl.SSLContext = None, + ssl_context: Optional[ssl.SSLContext] = None, # Consumer settings: consumer_api_version: Optional[str] = None, consumer_max_fetch_size: Optional[int] = None, @@ -114,11 +114,11 @@ def __init__( consumer_metadata_max_age_ms: Optional[int] = None, consumer_connections_max_idle_ms: Optional[int] = None, # Topic serialization settings: - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, # Logging settings: - logging_config: Mapping = None, - loghandlers: List[logging.Handler] = None, + logging_config: Optional[Mapping] = None, + loghandlers: Optional[List[logging.Handler]] = None, # Producer settings: producer_acks: Optional[int] = None, producer_api_version: Optional[str] = None, @@ -126,7 +126,7 @@ def __init__( producer_linger_ms: Optional[int] = None, producer_max_batch_size: Optional[int] = None, producer_max_request_size: Optional[int] = None, - producer_partitioner: SymbolArg[PartitionerT] = None, + producer_partitioner: Optional[SymbolArg[PartitionerT]] = None, producer_request_timeout: Optional[Seconds] = None, producer_threaded: bool = False, producer_metadata_max_age_ms: Optional[int] = None, @@ -137,14 +137,14 @@ def __init__( reply_to: Optional[str] = None, reply_to_prefix: Optional[str] = None, # Stream settings: - processing_guarantee: Union[str, ProcessingGuarantee] = None, + processing_guarantee: Optional[Union[str, ProcessingGuarantee]] = None, stream_buffer_maxsize: Optional[int] = None, stream_processing_timeout: Optional[Seconds] = None, stream_publish_on_commit: Optional[bool] = None, stream_recovery_delay: Optional[Seconds] = None, stream_wait_empty: Optional[bool] = None, # Table settings: - store: URLArg = None, + store: Optional[URLArg] = None, table_cleanup_interval: Optional[Seconds] = None, table_key_index_size: Optional[int] = None, table_standby_replicas: Optional[int] = None, @@ -154,44 +154,44 @@ def __init__( topic_partitions: Optional[int] = None, topic_replication_factor: Optional[int] = None, # Web server settings: - cache: URLArg = None, - canonical_url: URLArg = None, - web: URLArg = None, + cache: Optional[URLArg] = None, + canonical_url: Optional[URLArg] = None, + web: Optional[URLArg] = None, web_bind: Optional[str] = None, - web_application_options: typing.Mapping[str, typing.Any] = None, - web_cors_options: typing.Mapping[str, ResourceOptions] = None, + web_application_options: Optional[typing.Mapping[str, typing.Any]] = None, + web_cors_options: Optional[typing.Mapping[str, ResourceOptions]] = None, web_enabled: Optional[bool] = None, web_host: Optional[str] = None, web_in_thread: Optional[bool] = None, web_port: Optional[int] = None, - web_ssl_context: ssl.SSLContext = None, - web_transport: URLArg = None, + web_ssl_context: Optional[ssl.SSLContext] = None, + web_transport: Optional[URLArg] = None, # Worker settings: worker_redirect_stdouts: Optional[bool] = None, - worker_redirect_stdouts_level: Severity = None, + worker_redirect_stdouts_level: Optional[Severity] = None, # Extension settings: - Agent: SymbolArg[Type[AgentT]] = None, - ConsumerScheduler: SymbolArg[Type[SchedulingStrategyT]] = None, - Event: SymbolArg[Type[EventT]] = None, - Schema: SymbolArg[Type[SchemaT]] = None, - Stream: SymbolArg[Type[StreamT]] = None, - Table: SymbolArg[Type[TableT]] = None, - SetTable: SymbolArg[Type[TableT]] = None, - GlobalTable: SymbolArg[Type[GlobalTableT]] = None, - SetGlobalTable: SymbolArg[Type[GlobalTableT]] = None, - TableManager: SymbolArg[Type[TableManagerT]] = None, - Serializers: SymbolArg[Type[RegistryT]] = None, - Worker: SymbolArg[Type[_WorkerT]] = None, - PartitionAssignor: SymbolArg[Type[PartitionAssignorT]] = None, - LeaderAssignor: SymbolArg[Type[LeaderAssignorT]] = None, - Router: SymbolArg[Type[RouterT]] = None, - Topic: SymbolArg[Type[TopicT]] = None, - HttpClient: SymbolArg[Type[HttpClientT]] = None, - Monitor: SymbolArg[Type[SensorT]] = None, + Agent: Optional[SymbolArg[Type[AgentT]]] = None, + ConsumerScheduler: Optional[SymbolArg[Type[SchedulingStrategyT]]] = None, + Event: Optional[SymbolArg[Type[EventT]]] = None, + Schema: Optional[SymbolArg[Type[SchemaT]]] = None, + Stream: Optional[SymbolArg[Type[StreamT]]] = None, + Table: Optional[SymbolArg[Type[TableT]]] = None, + SetTable: Optional[SymbolArg[Type[TableT]]] = None, + GlobalTable: Optional[SymbolArg[Type[GlobalTableT]]] = None, + SetGlobalTable: Optional[SymbolArg[Type[GlobalTableT]]] = None, + TableManager: Optional[SymbolArg[Type[TableManagerT]]] = None, + Serializers: Optional[SymbolArg[Type[RegistryT]]] = None, + Worker: Optional[SymbolArg[Type[_WorkerT]]] = None, + PartitionAssignor: Optional[SymbolArg[Type[PartitionAssignorT]]] = None, + LeaderAssignor: Optional[SymbolArg[Type[LeaderAssignorT]]] = None, + Router: Optional[SymbolArg[Type[RouterT]]] = None, + Topic: Optional[SymbolArg[Type[TopicT]]] = None, + HttpClient: Optional[SymbolArg[Type[HttpClientT]]] = None, + Monitor: Optional[SymbolArg[Type[SensorT]]] = None, # Deprecated settings: stream_ack_cancelled_tasks: Optional[bool] = None, stream_ack_exceptions: Optional[bool] = None, - url: URLArg = None, + url: Optional[URLArg] = None, **kwargs: Any, ) -> None: ... # replaced by __init_subclass__ in BaseSettings @@ -206,7 +206,7 @@ def on_init(self, id: str, **kwargs: Any) -> None: def _init_env_prefix( self, - env: Mapping[str, str] = None, + env: Optional[Mapping[str, str]] = None, env_prefix: Optional[str] = None, **kwargs: Any, ) -> None: diff --git a/faust/types/stores.py b/faust/types/stores.py index 8d89a383d..d452cf5bc 100644 --- a/faust/types/stores.py +++ b/faust/types/stores.py @@ -48,8 +48,8 @@ def __init__( table: _CollectionT, *, table_name: str = "", - key_type: _ModelArg = None, - value_type: _ModelArg = None, + key_type: Optional[_ModelArg] = None, + value_type: Optional[_ModelArg] = None, key_serializer: CodecArg = "", value_serializer: CodecArg = "", options: Optional[Mapping[str, Any]] = None, diff --git a/faust/types/streams.py b/faust/types/streams.py index b5ab4b1e6..b81159727 100644 --- a/faust/types/streams.py +++ b/faust/types/streams.py @@ -120,16 +120,16 @@ class StreamT(AsyncIterable[T_co], JoinableT, ServiceT): @abc.abstractmethod def __init__( self, - channel: AsyncIterator[T_co] = None, + channel: Optional[AsyncIterator[T_co]] = None, *, app: Optional[_AppT] = None, - processors: Iterable[Processor[T]] = None, - combined: List[JoinableT] = None, + processors: Optional[Iterable[Processor[T]]] = None, + combined: Optional[List[JoinableT]] = None, on_start: Optional[Callable] = None, - join_strategy: _JoinT = None, + join_strategy: Optional[_JoinT] = None, beacon: Optional[NodeT] = None, concurrency_index: Optional[int] = None, - prev: "StreamT" = None, + prev: Optional["StreamT"] = None, active_partitions: Optional[Set[TP]] = None, enable_acks: bool = True, prefix: str = "", @@ -186,8 +186,8 @@ def derive_topic( name: str, *, schema: Optional[_SchemaT] = None, - key_type: ModelArg = None, - value_type: ModelArg = None, + key_type: Optional[ModelArg] = None, + value_type: Optional[ModelArg] = None, prefix: str = "", suffix: str = "", ) -> TopicT: ... @@ -202,7 +202,7 @@ def __copy__(self) -> "StreamT": ... def __iter__(self) -> Any: ... @abc.abstractmethod - def __next__(self) -> T: ... + def __next__(self) -> T_co: ... @abc.abstractmethod def __aiter__(self) -> AsyncIterator[T_co]: ... diff --git a/faust/types/tables.py b/faust/types/tables.py index ffde15400..b25476ab4 100644 --- a/faust/types/tables.py +++ b/faust/types/tables.py @@ -95,6 +95,7 @@ class CollectionT(ServiceT, JoinableT): options: Optional[Mapping[str, Any]] last_closed_window: float use_partitioner: bool + synchronize_all_active_partitions: bool is_global: bool = False @@ -104,16 +105,16 @@ def __init__( app: _AppT, *, name: Optional[str] = None, - default: Callable[[], Any] = None, - store: Union[str, URL] = None, + default: Optional[Callable[[], Any]] = None, + store: Optional[Union[str, URL]] = None, schema: Optional[_SchemaT] = None, - key_type: _ModelArg = None, - value_type: _ModelArg = None, + key_type: Optional[_ModelArg] = None, + value_type: Optional[_ModelArg] = None, partitions: Optional[int] = None, window: Optional[WindowT] = None, changelog_topic: Optional[TopicT] = None, help: Optional[str] = None, - on_recover: RecoverCallback = None, + on_recover: Optional[RecoverCallback] = None, on_changelog_event: Optional[ChangelogEventCallback] = None, recovery_buffer_size: int = 1000, standby_buffer_size: Optional[int] = None, @@ -155,8 +156,8 @@ def send_changelog( partition: Optional[int], key: Any, value: Any, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, ) -> FutureMessage: ... @abc.abstractmethod @@ -392,7 +393,7 @@ def __init__( self, table: TableT, *, - relative_to: RelativeArg = None, + relative_to: Optional[RelativeArg] = None, key_index: bool = False, key_index_table: Optional[TableT] = None, ) -> None: ... diff --git a/faust/types/topics.py b/faust/types/topics.py index 09d52d563..8498c29b0 100644 --- a/faust/types/topics.py +++ b/faust/types/topics.py @@ -64,10 +64,10 @@ def __init__( app: _AppT, *, topics: Optional[Sequence[str]] = None, - pattern: Union[str, Pattern] = None, + pattern: Optional[Union[str, Pattern]] = None, schema: Optional[_SchemaT] = None, - key_type: _ModelArg = None, - value_type: _ModelArg = None, + key_type: Optional[_ModelArg] = None, + value_type: Optional[_ModelArg] = None, is_iterator: bool = False, partitions: Optional[int] = None, retention: Optional[Seconds] = None, @@ -78,8 +78,8 @@ def __init__( internal: bool = False, config: Optional[Mapping[str, Any]] = None, queue: Optional[ThrowableQueue] = None, - key_serializer: CodecArg = None, - value_serializer: CodecArg = None, + key_serializer: Optional[CodecArg] = None, + value_serializer: Optional[CodecArg] = None, maxsize: Optional[int] = None, root: Optional[ChannelT] = None, active_partitions: Optional[Set[TP]] = None, @@ -111,8 +111,8 @@ def derive_topic( *, topics: Optional[Sequence[str]] = None, schema: Optional[_SchemaT] = None, - key_type: _ModelArg = None, - value_type: _ModelArg = None, + key_type: Optional[_ModelArg] = None, + value_type: Optional[_ModelArg] = None, partitions: Optional[int] = None, retention: Optional[Seconds] = None, compacting: Optional[bool] = None, diff --git a/faust/types/transports.py b/faust/types/transports.py index 205c56e0f..12ad90d0d 100644 --- a/faust/types/transports.py +++ b/faust/types/transports.py @@ -32,6 +32,8 @@ from .tuples import TP, FutureMessage, Message, RecordMetadata if typing.TYPE_CHECKING: + from mode.threads import ServiceThread + from .app import AppT as _AppT else: @@ -114,6 +116,12 @@ class ProducerT(ServiceT): partitioner: Optional[PartitionerT] request_timeout: float + #: The driver's threaded producer, created by ``Producer.__init__`` when + #: the :setting:`producer_threaded` setting is enabled, and ``None`` + #: otherwise. Quoted so this core types module keeps its import graph: + #: ``mode.threads`` is imported for type checking only. + threaded_producer: Optional["ServiceThread"] + @abc.abstractmethod def __init__( self, @@ -357,7 +365,7 @@ async def seek_wait(self, partitions: Mapping[TP, int]) -> None: ... @abc.abstractmethod async def commit( - self, topics: TPorTopicSet = None, start_new_transaction: bool = True + self, topics: Optional[TPorTopicSet] = None, start_new_transaction: bool = True ) -> bool: ... @abc.abstractmethod diff --git a/faust/types/tuples.py b/faust/types/tuples.py index ba8fb940e..c48dfc9ae 100644 --- a/faust/types/tuples.py +++ b/faust/types/tuples.py @@ -150,7 +150,7 @@ def __init__( checksum: Optional[bytes], serialized_key_size: Optional[int] = None, serialized_value_size: Optional[int] = None, - tp: TP = None, + tp: Optional[TP] = None, time_in: Optional[float] = None, time_out: Optional[float] = None, time_total: Optional[float] = None, diff --git a/faust/types/web.py b/faust/types/web.py index 62bc32a42..e9e9ee0ac 100644 --- a/faust/types/web.py +++ b/faust/types/web.py @@ -121,7 +121,7 @@ def __init__( self, timeout: Optional[Seconds] = None, key_prefix: Optional[str] = None, - backend: Union[Type[CacheBackendT], str] = None, + backend: Optional[Union[Type[CacheBackendT], str]] = None, **kwargs: Any, ) -> None: ... @@ -145,7 +145,7 @@ def cache( timeout: Optional[Seconds] = None, include_headers: bool = False, key_prefix: Optional[str] = None, - backend: Union[Type[CacheBackendT], str] = None, + backend: Optional[Union[Type[CacheBackendT], str]] = None, ) -> CacheT: ... @abc.abstractmethod diff --git a/faust/utils/_opentracing.py b/faust/utils/_opentracing.py index 03493061a..16c617bc8 100644 --- a/faust/utils/_opentracing.py +++ b/faust/utils/_opentracing.py @@ -9,7 +9,7 @@ This intentionally implements only the small surface Faust touches. """ -from typing import Any +from typing import Any, Literal class Span: @@ -24,7 +24,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: def __enter__(self) -> "Span": return self - def __exit__(self, *exc_info: Any) -> bool: + def __exit__(self, *exc_info: Any) -> Literal[False]: return False def finish(self, *args: Any, **kwargs: Any) -> None: ... diff --git a/faust/utils/agent_stopper.py b/faust/utils/agent_stopper.py index e5db7322b..f7314b1aa 100644 --- a/faust/utils/agent_stopper.py +++ b/faust/utils/agent_stopper.py @@ -3,10 +3,12 @@ import logging import traceback +from faust.types import AppT + log = logging.getLogger(__name__) -async def agent_stopper(app) -> None: +async def agent_stopper(app: AppT) -> None: """ Raise exception and crash app """ @@ -15,4 +17,8 @@ async def agent_stopper(app) -> None: # force the exit code of the application not to be 0 # and prevent offsets from progressing - app._crash(RuntimeError) + # ``Service._crash`` is annotated as taking an exception instance, but the + # reason is only stored and later re-raised, and ``raise`` accepts an + # exception class just as well. Passing the class (not an instance) is + # deliberate here and is pinned by tests/unit/utils/test_agent_stopper.py. + app._crash(RuntimeError) # type: ignore[arg-type] diff --git a/faust/utils/codegen.py b/faust/utils/codegen.py index ef3c9f93f..95f33b7a8 100644 --- a/faust/utils/codegen.py +++ b/faust/utils/codegen.py @@ -1,6 +1,6 @@ """Utilities for generating code at runtime.""" -from typing import Any, Callable, Dict, List, Mapping, Tuple, cast +from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple, cast __all__ = [ "Function", @@ -28,8 +28,8 @@ def Function( args: List[str], body: List[str], *, - globals: Dict[str, Any] = None, - locals: Dict[str, Any] = None, + globals: Optional[Dict[str, Any]] = None, + locals: Optional[Dict[str, Any]] = None, return_type: Any = MISSING, argsep: str = ", ", ) -> Callable: @@ -55,7 +55,7 @@ def build_closure_source( body: List[str], *, outer_name: str = "__outer__", - outer_args: List[str] = None, + outer_args: Optional[List[str]] = None, closures: Dict[str, str], return_type: Any = MISSING, indentlevel: int = 0, @@ -91,8 +91,8 @@ def build_closure( source: str, *args: Any, return_type: Any = MISSING, - globals: Dict[str, Any] = None, - locals: Dict[str, Any] = None, + globals: Optional[Dict[str, Any]] = None, + locals: Optional[Dict[str, Any]] = None, ) -> Callable: assert locals is not None if return_type is not MISSING: @@ -108,8 +108,8 @@ def build_function( source: str, *, return_type: Any = MISSING, - globals: Dict[str, Any] = None, - locals: Dict[str, Any] = None, + globals: Optional[Dict[str, Any]] = None, + locals: Optional[Dict[str, Any]] = None, ) -> Callable: """Generate function from Python from source code string.""" assert locals is not None diff --git a/faust/utils/cron.py b/faust/utils/cron.py index ed96191f2..597674865 100644 --- a/faust/utils/cron.py +++ b/faust/utils/cron.py @@ -2,11 +2,12 @@ import time from datetime import datetime, tzinfo +from typing import Optional from croniter.croniter import croniter -def secs_for_next(cron_format: str, tz: tzinfo = None) -> float: +def secs_for_next(cron_format: str, tz: Optional[tzinfo] = None) -> float: """Return seconds until next execution given Crontab style format.""" now_ts = time.time() # If we have a tz object we'll make now timezone aware, and diff --git a/faust/utils/terminal/tables.py b/faust/utils/terminal/tables.py index dc0dacf26..ba82c905a 100644 --- a/faust/utils/terminal/tables.py +++ b/faust/utils/terminal/tables.py @@ -29,7 +29,7 @@ def table( data: TableDataT, *, title: str, - target: IO = None, + target: Optional[IO] = None, tty: Optional[bool] = None, **kwargs: Any, ) -> Table: @@ -57,7 +57,7 @@ def logtable( data: TableDataT, *, title: str, - target: IO = None, + target: Optional[IO] = None, tty: Optional[bool] = None, headers: Optional[Sequence[str]] = None, **kwargs: Any, diff --git a/faust/utils/tracing.py b/faust/utils/tracing.py index fba366cba..faa2bc3dd 100644 --- a/faust/utils/tracing.py +++ b/faust/utils/tracing.py @@ -44,7 +44,7 @@ def noop_span() -> opentracing.Span: def finish_span( - span: Optional[opentracing.Span], *, error: BaseException = None + span: Optional[opentracing.Span], *, error: Optional[BaseException] = None ) -> None: """Finish span, and optionally set error tag.""" if span is not None: @@ -69,7 +69,7 @@ def operation_name_from_fun(fun: Any) -> str: def traced_from_parent_span( - parent_span: opentracing.Span = None, + parent_span: Optional[opentracing.Span] = None, callback: Optional[Callable] = None, **extra_context: Any, ) -> Callable: diff --git a/faust/utils/venusian.py b/faust/utils/venusian.py index 38397c4a8..5caf265c2 100644 --- a/faust/utils/venusian.py +++ b/faust/utils/venusian.py @@ -4,7 +4,7 @@ callback argument. """ -from typing import Any, Callable +from typing import Any, Callable, Optional import venusian from venusian import Scanner, attach as _attach @@ -16,7 +16,7 @@ def attach( fun: Callable, category: str, *, - callback: Callable[[Scanner, str, Any], None] = None, + callback: Optional[Callable[[Scanner, str, Any], None]] = None, **kwargs: Any, ) -> None: """Shortcut for :func:`venusian.attach`. diff --git a/faust/web/base.py b/faust/web/base.py index f62fb2955..13287372a 100644 --- a/faust/web/base.py +++ b/faust/web/base.py @@ -132,7 +132,7 @@ class BlueprintManager: _enabled: List[Tuple[str, _BPArg]] _active: MutableMapping[str, BlueprintT] - def __init__(self, initial: _BPList = None) -> None: + def __init__(self, initial: Optional[_BPList] = None) -> None: self.applied = False self._enabled = list(initial) if initial else [] self._active = {} @@ -197,7 +197,7 @@ def text( content_type: Optional[str] = None, status: int = 200, reason: Optional[str] = None, - headers: MutableMapping = None, + headers: Optional[MutableMapping] = None, ) -> Response: """Create text response, using "text/plain" content-type.""" ... @@ -210,7 +210,7 @@ def html( content_type: Optional[str] = None, status: int = 200, reason: Optional[str] = None, - headers: MutableMapping = None, + headers: Optional[MutableMapping] = None, ) -> Response: """Create HTML response from string, ``text/html`` content-type.""" ... @@ -223,7 +223,7 @@ def json( content_type: Optional[str] = None, status: int = 200, reason: Optional[str] = None, - headers: MutableMapping = None, + headers: Optional[MutableMapping] = None, ) -> Response: """Create new JSON response. @@ -242,7 +242,7 @@ def bytes( content_type: Optional[str] = None, status: int = 200, reason: Optional[str] = None, - headers: MutableMapping = None, + headers: Optional[MutableMapping] = None, ) -> Response: """Create new ``bytes`` response - for binary data.""" ... @@ -300,7 +300,7 @@ def route( self, pattern: str, handler: Callable, - cors_options: Mapping[str, ResourceOptions] = None, + cors_options: Optional[Mapping[str, ResourceOptions]] = None, ) -> None: """Add route for handler.""" ... @@ -325,7 +325,7 @@ def add_view( view_cls: Type[View], *, prefix: str = "", - cors_options: Mapping[str, ResourceOptions] = None, + cors_options: Optional[Mapping[str, ResourceOptions]] = None, ) -> View: """Add route for view.""" view: View = view_cls(self.app, self) diff --git a/faust/web/blueprints.py b/faust/web/blueprints.py index 7b69c58ec..84c75333f 100644 --- a/faust/web/blueprints.py +++ b/faust/web/blueprints.py @@ -111,7 +111,7 @@ def cache( timeout: Optional[Seconds] = None, include_headers: bool = False, key_prefix: Optional[str] = None, - backend: Union[Type[CacheBackendT], str] = None, + backend: Optional[Union[Type[CacheBackendT], str]] = None, ) -> CacheT: """Cache API.""" if key_prefix is None: @@ -123,7 +123,7 @@ def route( uri: str, *, name: Optional[str] = None, - cors_options: Mapping[str, ResourceOptions] = None, + cors_options: Optional[Mapping[str, ResourceOptions]] = None, base: Type[View] = View, ) -> RouteDecoratorRet: """Create route by decorating handler or view class.""" diff --git a/faust/web/cache/backends/redis.py b/faust/web/cache/backends/redis.py index 13773523b..a1fc71d31 100644 --- a/faust/web/cache/backends/redis.py +++ b/faust/web/cache/backends/redis.py @@ -23,11 +23,18 @@ except ImportError: # pragma: no cover # ``on_start`` (and the module-level ``if redis is None`` guards) key off # ``redis`` being ``None`` when the library is missing; bind both names so - # the guard fires instead of raising ``NameError``. - redis = aredis = None # noqa + # the guard fires instead of raising ``NameError``. mypy has no way to + # express "this module name may be ``None`` when the import failed", so the + # sentinel assignment has to be excused here. + redis = aredis = None # type: ignore[assignment] # noqa if typing.TYPE_CHECKING: # pragma: no cover - from redis import StrictRedis as _RedisClientT + from redis.asyncio import Redis as _AsyncRedis, RedisCluster as _AsyncRedisCluster + + # The backend awaits every command, so the clients it builds are the + # asyncio ones. ``RedisCluster`` is not a subclass of ``Redis``, they only + # share the (command-less) ``AbstractRedis`` base, hence the union. + _RedisClientT = Union[_AsyncRedis, _AsyncRedisCluster] else: class _RedisClientT: ... # noqa @@ -100,10 +107,12 @@ def _init_schemes(self) -> Mapping[str, Type[_RedisClientT]]: } async def _get(self, key: str) -> Optional[bytes]: - value: Optional[bytes] = await self.client.get(key) - if value is not None: + # Clients created with ``decode_responses`` hand back ``str``, so the + # reply is not necessarily ``bytes``; ``want_bytes`` normalises it. + value: Union[bytes, str, None] = await self.client.get(key) + if isinstance(value, str): return want_bytes(value) - return None + return value async def _set( self, key: str, value: bytes, timeout: Optional[float] = None diff --git a/faust/web/cache/cache.py b/faust/web/cache/cache.py index c2de50b75..2a36861df 100644 --- a/faust/web/cache/cache.py +++ b/faust/web/cache/cache.py @@ -27,7 +27,7 @@ def __init__( timeout: Optional[Seconds] = None, include_headers: bool = False, key_prefix: Optional[str] = None, - backend: Union[Type[CacheBackendT], str] = None, + backend: Optional[Union[Type[CacheBackendT], str]] = None, **kwargs: Any, ) -> None: self.timeout = timeout diff --git a/faust/web/drivers/aiohttp.py b/faust/web/drivers/aiohttp.py index f514b4749..802d63958 100644 --- a/faust/web/drivers/aiohttp.py +++ b/faust/web/drivers/aiohttp.py @@ -20,7 +20,7 @@ from mode.threads import ServiceThread from faust.types import AppT -from faust.types.web import ResourceOptions as _ResourceOptions +from faust.types.web import ResourceOptions as _ResourceOptions, View from faust.utils import json as _json from faust.web import base @@ -31,11 +31,18 @@ NON_OPTIONS_METHODS = frozenset({"GET", "PUT", "POST", "DELETE"}) -def _prepare_cors_options(opts: Mapping[str, Any]) -> Mapping[str, ResourceOptions]: +#: Callers may pass either Faust's own :class:`faust.types.web.ResourceOptions` +#: namedtuple or an already-converted :mod:`aiohttp_cors` one. +AnyResourceOptions = Union[_ResourceOptions, ResourceOptions] + + +def _prepare_cors_options( + opts: Mapping[str, AnyResourceOptions], +) -> Mapping[str, ResourceOptions]: return {k: _faust_to_aiohttp_options(v) for k, v in opts.items()} -def _faust_to_aiohttp_options(opts: ResourceOptions) -> ResourceOptions: +def _faust_to_aiohttp_options(opts: AnyResourceOptions) -> ResourceOptions: if isinstance(opts, _ResourceOptions): return ResourceOptions(**opts._asdict()) return opts @@ -134,7 +141,7 @@ def text( content_type: Optional[str] = None, status: int = 200, reason: Optional[str] = None, - headers: MutableMapping = None, + headers: Optional[MutableMapping] = None, ) -> base.Response: """Create text response, using "text/plain" content-type.""" response = Response( @@ -153,7 +160,7 @@ def html( content_type: Optional[str] = None, status: int = 200, reason: Optional[str] = None, - headers: MutableMapping = None, + headers: Optional[MutableMapping] = None, ) -> base.Response: """Create HTML response from string, ``text/html`` content-type.""" return self.text( @@ -171,7 +178,7 @@ def json( content_type: Optional[str] = None, status: int = 200, reason: Optional[str] = None, - headers: MutableMapping = None, + headers: Optional[MutableMapping] = None, ) -> Any: """Create new JSON response. @@ -214,7 +221,7 @@ def bytes( content_type: Optional[str] = None, status: int = 200, reason: Optional[str] = None, - headers: MutableMapping = None, + headers: Optional[MutableMapping] = None, ) -> base.Response: """Create new ``bytes`` response - for binary data.""" response = Response( @@ -234,16 +241,25 @@ def route( self, pattern: str, handler: Callable, - cors_options: Mapping[str, ResourceOptions] = None, + cors_options: Optional[Mapping[str, AnyResourceOptions]] = None, ) -> None: """Add route for web view or handler.""" async_handler = self._wrap_into_asyncdef(handler) + # ``base.Web.route`` types the handler as a bare ``Callable``, but this + # driver needs the set of HTTP methods it implements, and ``get_methods`` + # only exists on :class:`~faust.web.views.View`. + # XXX this cast is not guaranteed: ``base.Web.add_view`` does pass a + # ``View``, but the public :meth:`faust.web.views.View.route` forwards + # any callable straight to here, and such a handler blows up with + # ``AttributeError: 'function' object has no attribute 'get_methods'`` + # on the lines below. + view = cast(View, handler) if cors_options or self.cors_options: - for method in NON_OPTIONS_METHODS & handler.get_methods(): + for method in NON_OPTIONS_METHODS & view.get_methods(): r = self.web_app.router.add_route(method, pattern, async_handler) self.cors.add(r, _prepare_cors_options(cors_options or {})) else: - for method in handler.get_methods(): + for method in view.get_methods(): self.web_app.router.add_route(method, pattern, async_handler) def _wrap_into_asyncdef(self, handler: Callable) -> Callable: @@ -284,7 +300,11 @@ def response_to_bytes(self, response: base.Response) -> _bytes: elif isinstance(resp.body, Payload): raise NotImplementedError("Does not support Payload") else: - body = resp.body + # aiohttp declares the body as ``bytes | bytearray | Payload | + # None``, so ``bytearray`` survives the branches above. The cast + # does not convert it: ``_response_to_bytes`` only feeds the value + # to ``bytes.join``, which takes any bytes-like object. + body = cast(_bytes, resp.body) return self._response_to_bytes( resp.status, resp.headers, diff --git a/faust/web/views.py b/faust/web/views.py index c4651a492..0bcd62dfa 100644 --- a/faust/web/views.py +++ b/faust/web/views.py @@ -135,7 +135,7 @@ def path_for(self, view_name: str, **kwargs: Any) -> str: return self.web.url_for(view_name, **kwargs) def url_for( - self, view_name: str, _base_url: Union[str, URL] = None, **kwargs: Any + self, view_name: str, _base_url: Optional[Union[str, URL]] = None, **kwargs: Any ) -> URL: """Return the canonical URL for view by name. @@ -201,7 +201,7 @@ def text( content_type: Optional[str] = None, status: int = 200, reason: Optional[str] = None, - headers: MutableMapping = None, + headers: Optional[MutableMapping] = None, ) -> Response: """Create text response, using "text/plain" content-type.""" return self.web.text( @@ -219,7 +219,7 @@ def html( content_type: Optional[str] = None, status: int = 200, reason: Optional[str] = None, - headers: MutableMapping = None, + headers: Optional[MutableMapping] = None, ) -> Response: """Create HTML response from string, ``text/html`` content-type.""" return self.web.html( @@ -237,7 +237,7 @@ def json( content_type: Optional[str] = None, status: int = 200, reason: Optional[str] = None, - headers: MutableMapping = None, + headers: Optional[MutableMapping] = None, ) -> Response: """Create new JSON response. @@ -261,7 +261,7 @@ def bytes( content_type: Optional[str] = None, status: int = 200, reason: Optional[str] = None, - headers: MutableMapping = None, + headers: Optional[MutableMapping] = None, ) -> Response: """Create new ``bytes`` response - for binary data.""" return self.web.bytes( @@ -301,7 +301,11 @@ def notfound(self, reason: str = "Not Found", **kwargs: Any) -> Response: return self.error(404, reason, **kwargs) def error( - self, status: int, reason: str, headers: MutableMapping = None, **kwargs: Any + self, + status: int, + reason: str, + headers: Optional[MutableMapping] = None, + **kwargs: Any, ) -> Response: """Create error JSON response.""" return self.json({"error": reason, **kwargs}, status=status, headers=headers) diff --git a/faust/worker.py b/faust/worker.py index d17f1491f..4cbf96e2d 100644 --- a/faust/worker.py +++ b/faust/worker.py @@ -336,7 +336,11 @@ def change_workdir(self, path: Path) -> None: def autodiscover(self) -> None: """Autodiscover modules and files to find @agent decorators, etc.""" - if self.app.conf.autodiscover: + # ``Settings.autodiscover`` is a class-level setting descriptor typed as + # ``AutodiscoverArg``, and that union contains ``Callable[[], ...]``. + # mypy therefore tries to bind ``self`` to it on attribute access; + # the setting is a plain value, not a method. + if self.app.conf.autodiscover: # type: ignore[misc] self.app.discover() def _setproctitle(self, info: str, *, ident: str = PSIDENT) -> None: diff --git a/pyproject.toml b/pyproject.toml index c7f3126a9..ee1c27d56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -98,102 +98,30 @@ disallow_untyped_defs = true # it can surface new errors, so bump it deliberately. python_version = "3.12" -# Type-checking ratchet. +# The type-checking ratchet is gone: `mypy -p faust` (run by `scripts/check`) +# now checks all 164 modules in the package with nothing silenced wholesale. # -# `scripts/check` runs `mypy -p faust`, so it has to exit zero. The package is -# not clean yet, so the modules that still have errors are silenced here. -# Everything NOT listed is checked -- today that is 86 of faust's 164 modules, -# plus every module added from here on. +# Keep it that way. If a change makes mypy fail, fix the annotations rather +# than re-introducing an `ignore_errors` list -- that list was only ever a +# migration aid, and re-adding one would hide every other error in the module +# too. For a single line the checker genuinely gets wrong, use a specific +# `# type: ignore[code]` with a comment saying why. # -# This list may only shrink. Do not add a module to it to turn a red build -# green; fix the annotations instead. When you clean one up, delete its line -# in the same pull request so it cannot regress. To see what is outstanding, -# comment this section out and run `mypy -p faust`. +# mypy is pinned in requirements/typecheck.txt because a different version +# reports a different set of errors; bump it deliberately. + +# Every setting in `faust.types.settings.settings` is a method with a docstring and +# no body, decorated with `@sections.
.setting(...)`. The decorator throws +# the function away and returns a `params.Param` descriptor built from its name, +# docstring and return annotation, so the body is never executed -- but mypy sees a +# function declaring a non-None return type and never returning, and reports +# `empty-body` for all ~100 of them. The check does not apply to this pattern. # -# Regenerating it requires the pinned mypy from requirements/typecheck.txt; a -# different version reports a different set. +# Note this disables one error code, not the module: every OTHER error code is +# still reported here, so new mistakes in it are caught. [[tool.mypy.overrides]] -module = [ - "faust", - "faust.agents.agent", - "faust.app._attached", - "faust.app.base", - "faust.app.router", - "faust.assignor.client_assignment", - "faust.assignor.partition_assignor", - "faust.auth", - "faust.channels", - "faust.cli.base", - "faust.cli.worker", - "faust.contrib.sentry", - "faust.events", - "faust.livecheck.app", - "faust.livecheck.case", - "faust.livecheck.signals", - "faust.models.base", - "faust.models.fields", - "faust.models.record", - "faust.models.typing", - "faust.sensors.base", - "faust.sensors.datadog", - "faust.sensors.distributed_tracing", - "faust.sensors.monitor", - "faust.sensors.prometheus", - "faust.sensors.statsd", - "faust.serializers.codecs", - "faust.serializers.schemas", - "faust.stores.aerospike", - "faust.stores.base", - "faust.stores.rocksdb", - "faust.streams", - "faust.tables.base", - "faust.tables.globaltable", - "faust.tables.manager", - "faust.tables.objects", - "faust.tables.recovery", - "faust.tables.sets", - "faust.tables.table", - "faust.topics", - "faust.transport.conductor", - "faust.transport.consumer", - "faust.transport.drivers.aiokafka", - "faust.transport.drivers.confluent", - "faust.transport.producer", - "faust.transport.utils", - "faust.types.agents", - "faust.types.app", - "faust.types.auth", - "faust.types.channels", - "faust.types.codecs", - "faust.types.models", - "faust.types.sensors", - "faust.types.serializers", - "faust.types.settings.params", - "faust.types.settings.settings", - "faust.types.stores", - "faust.types.streams", - "faust.types.tables", - "faust.types.topics", - "faust.types.transports", - "faust.types.tuples", - "faust.types.web", - "faust.utils._opentracing", - "faust.utils.agent_stopper", - "faust.utils.codegen", - "faust.utils.cron", - "faust.utils.terminal.tables", - "faust.utils.tracing", - "faust.utils.venusian", - "faust.web.base", - "faust.web.blueprints", - "faust.web.cache.backends.redis", - "faust.web.cache.cache", - "faust.web.drivers.aiohttp", - "faust.web.views", - "faust.windows", - "faust.worker", -] -ignore_errors = true +module = ["faust.types.settings.settings"] +disable_error_code = ["empty-body"] [tool.pytest.ini_options] asyncio_mode = "auto" diff --git a/requirements/requirements.txt b/requirements/requirements.txt index b64d8e1c7..62c54daf9 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -2,7 +2,15 @@ aiohttp>=3.8.0,<4.0 aiohttp_cors>=0.7,<2.0 aiokafka>=0.10.0 click>=6.7,<8.2 -mode-streaming>=0.4.0 +# >=0.6.0, not >=0.4.0: mode ships PEP 561 type information, so its annotations +# are part of faust's own type-check surface. Older releases annotate several +# helpers less precisely -- `want_seconds(float)` rather than +# `want_seconds(Seconds)`, `level_name(int)` rather than `level_name(str | int)` +# -- and `mypy -p faust` reports 38 errors in 14 files against 0.4.1 on a tree +# that is clean against 0.6.0. Pinning the floor to the version CI resolves +# keeps `scripts/check` reproducible, for the same reason mypy itself is pinned +# in typecheck.txt. +mode-streaming>=0.6.0 terminaltables>=3.1,<4.0 yarl>=1.0,<2.0 croniter>=0.3.16