From 92188ddfa87e27b623c19f1655324030bc12a45b Mon Sep 17 00:00:00 2001 From: Julius Park Date: Tue, 11 Oct 2022 12:03:53 -0700 Subject: [PATCH 01/20] initial commit --- pymongo/collection.py | 8 ++++---- test/test_encryption.py | 2 -- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/pymongo/collection.py b/pymongo/collection.py index 8f1afc575d..f7150c80f4 100644 --- a/pymongo/collection.py +++ b/pymongo/collection.py @@ -566,7 +566,7 @@ def _insert_command(session, sock_info, retryable_write): def insert_one( self, - document: _DocumentIn, + document: Union[_DocumentType, RawBSONDocument], bypass_document_validation: bool = False, session: Optional["ClientSession"] = None, comment: Optional[Any] = None, @@ -614,7 +614,7 @@ def insert_one( """ common.validate_is_document_type("document", document) if not (isinstance(document, RawBSONDocument) or "_id" in document): - document["_id"] = ObjectId() + document["_id"] = ObjectId() # type: ignore[index] write_concern = self._write_concern_for(session) return InsertOneResult( @@ -633,7 +633,7 @@ def insert_one( @_csot.apply def insert_many( self, - documents: Iterable[_DocumentIn], + documents: Iterable[Union[_DocumentType, RawBSONDocument]], ordered: bool = True, bypass_document_validation: bool = False, session: Optional["ClientSession"] = None, @@ -697,7 +697,7 @@ def gen(): common.validate_is_document_type("document", document) if not isinstance(document, RawBSONDocument): if "_id" not in document: - document["_id"] = ObjectId() + document["_id"] = ObjectId() # type: ignore[index] inserted_ids.append(document["_id"]) yield (message._INSERT, document) diff --git a/test/test_encryption.py b/test/test_encryption.py index 567d606893..159f1c7331 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -82,8 +82,6 @@ from pymongo.operations import InsertOne, ReplaceOne, UpdateOne from pymongo.write_concern import WriteConcern -KMS_PROVIDERS = {"local": {"key": b"\x00" * 96}} - def get_client_opts(client): return client._MongoClient__options From ea414327ebf07eedac5cf433ed0fb94a0d170b14 Mon Sep 17 00:00:00 2001 From: Julius Park Date: Tue, 18 Oct 2022 13:20:03 -0700 Subject: [PATCH 02/20] add documentation --- doc/examples/type_hints.rst | 5 ++++- test/test_encryption.py | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/examples/type_hints.rst b/doc/examples/type_hints.rst index 6858e95290..7413288929 100644 --- a/doc/examples/type_hints.rst +++ b/doc/examples/type_hints.rst @@ -92,7 +92,9 @@ Note that when using :class:`~bson.son.SON`, the key and value types must be giv Typed Collection ---------------- -You can use :py:class:`~typing.TypedDict` (Python 3.8+) when using a well-defined schema for the data in a :class:`~pymongo.collection.Collection`: +You can use :py:class:`~typing.TypedDict` (Python 3.8+) when using a well-defined schema for the data in a :class:`~pymongo.collection.Collection`. +Note that all `schema_validation`_ for inserts and updates is done on the server. Do not rely on type hints for schema validation, only for providing +a more ergonomic interface: .. doctest:: @@ -243,3 +245,4 @@ Another example is trying to set a value on a :class:`~bson.raw_bson.RawBSONDocu .. _limitations in mypy: https://github.com/python/mypy/issues/3737 .. _mypy config: https://mypy.readthedocs.io/en/stable/config_file.html .. _test_mypy module: https://github.com/mongodb/mongo-python-driver/blob/master/test/test_mypy.py +.. _schema validation: https://www.mongodb.com/docs/manual/core/schema-validation/#when-to-use-schema-validation diff --git a/test/test_encryption.py b/test/test_encryption.py index 159f1c7331..567d606893 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -82,6 +82,8 @@ from pymongo.operations import InsertOne, ReplaceOne, UpdateOne from pymongo.write_concern import WriteConcern +KMS_PROVIDERS = {"local": {"key": b"\x00" * 96}} + def get_client_opts(client): return client._MongoClient__options From 6e21760599380c75512835103f5cfefe88caeffb Mon Sep 17 00:00:00 2001 From: Julius Park Date: Tue, 18 Oct 2022 14:18:02 -0700 Subject: [PATCH 03/20] add mypy test --- doc/examples/type_hints.rst | 4 ++-- test/test_collection.py | 2 +- test/test_mypy.py | 23 +++++++++++++++++++++-- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/doc/examples/type_hints.rst b/doc/examples/type_hints.rst index 7413288929..af3e61034d 100644 --- a/doc/examples/type_hints.rst +++ b/doc/examples/type_hints.rst @@ -93,8 +93,8 @@ Typed Collection ---------------- You can use :py:class:`~typing.TypedDict` (Python 3.8+) when using a well-defined schema for the data in a :class:`~pymongo.collection.Collection`. -Note that all `schema_validation`_ for inserts and updates is done on the server. Do not rely on type hints for schema validation, only for providing -a more ergonomic interface: +Note that all `schema_validation`_ for inserts and updates is done on the server. This is due to the fact that these methods automatically add +an "_id" field. Do not rely on TypedDicts for schema validation, only for providing a more ergonomic interface: .. doctest:: diff --git a/test/test_collection.py b/test/test_collection.py index 37f1b1eae2..e7ac248124 100644 --- a/test/test_collection.py +++ b/test/test_collection.py @@ -785,7 +785,7 @@ def test_insert_many_invalid(self): db.test.insert_many(1) # type: ignore[arg-type] with self.assertRaisesRegex(TypeError, "documents must be a non-empty list"): - db.test.insert_many(RawBSONDocument(encode({"_id": 2}))) # type: ignore[arg-type] + db.test.insert_many(RawBSONDocument(encode({"_id": 2}))) def test_delete_one(self): self.db.test.drop() diff --git a/test/test_mypy.py b/test/test_mypy.py index c692c70789..831fed1f4d 100644 --- a/test/test_mypy.py +++ b/test/test_mypy.py @@ -18,7 +18,7 @@ import os import tempfile import unittest -from typing import TYPE_CHECKING, Any, Dict, Iterable, Iterator, List +from typing import TYPE_CHECKING, Any, Dict, Iterable, Iterator, List, Optional try: from typing import TypedDict # type: ignore[attr-defined] @@ -40,7 +40,15 @@ class Movie(TypedDict): # type: ignore[misc] from test import IntegrationTest from test.utils import rs_or_single_client -from bson import CodecOptions, decode, decode_all, decode_file_iter, decode_iter, encode +from bson import ( + CodecOptions, + ObjectId, + decode, + decode_all, + decode_file_iter, + decode_iter, + encode, +) from bson.raw_bson import RawBSONDocument from bson.son import SON from pymongo import ASCENDING, MongoClient @@ -304,6 +312,17 @@ def test_typeddict_document_type(self) -> None: assert retreived["year"] == 1 assert retreived["name"] == "a" + @only_type_check + def test_typeddict_document_type_insertion(self) -> None: + client: MongoClient[Movie] = MongoClient() + coll: Collection[Movie] = client.test.test + insert = coll.insert_one(Movie(name="THX-1138", year=1971)) + out: Optional[Movie] = coll.find_one({"name": "THX-1138"}) + assert out is not None + assert out.name == "THX-1138" + assert out.year == "1971" + assert out.id == ObjectId() + @only_type_check def test_raw_bson_document_type(self) -> None: client = MongoClient(document_class=RawBSONDocument) From 2a86839dc8d8ebefcc463f41f279a8473df251f3 Mon Sep 17 00:00:00 2001 From: Julius Park Date: Thu, 20 Oct 2022 10:37:05 -0700 Subject: [PATCH 04/20] fix test --- doc/examples/type_hints.rst | 12 +++++++++--- test/test_mypy.py | 18 ++++++++++++------ 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/doc/examples/type_hints.rst b/doc/examples/type_hints.rst index af3e61034d..02cd18c0b6 100644 --- a/doc/examples/type_hints.rst +++ b/doc/examples/type_hints.rst @@ -92,16 +92,20 @@ Note that when using :class:`~bson.son.SON`, the key and value types must be giv Typed Collection ---------------- -You can use :py:class:`~typing.TypedDict` (Python 3.8+) when using a well-defined schema for the data in a :class:`~pymongo.collection.Collection`. +You can use :py:class:`~typing_extensions.TypedDict` (Python 3.8+) when using a well-defined schema for the data in a :class:`~pymongo.collection.Collection`. Note that all `schema_validation`_ for inserts and updates is done on the server. This is due to the fact that these methods automatically add -an "_id" field. Do not rely on TypedDicts for schema validation, only for providing a more ergonomic interface: +an "_id" field. The "_id" field is decorated by :py:class:`~typing_extensions.NotRequired` decorator to allow it to be accessed when reading +from `result` (albeit without type-checking for that specific field, hence why it should not be used for schema validation). +Another option would be to generate the "_id" field yourself, and make it a required field, which would give the expected behavior. .. doctest:: - >>> from typing import TypedDict + >>> from typing_extensions import TypedDict, NotRequired >>> from pymongo import MongoClient >>> from pymongo.collection import Collection + >>> from bson import ObjectId >>> class Movie(TypedDict): + ... _id: NotRequired[ObjectId] ... name: str ... year: int ... @@ -111,6 +115,8 @@ an "_id" field. Do not rely on TypedDicts for schema validation, only for provid >>> result = collection.find_one({"name": "Jurassic Park"}) >>> assert result is not None >>> assert result["year"] == 1993 + >>> # Mypy will not check this because it is NotRequired + >>> assert result["_id"] == tuple() Typed Database -------------- diff --git a/test/test_mypy.py b/test/test_mypy.py index 831fed1f4d..119f9a6dbd 100644 --- a/test/test_mypy.py +++ b/test/test_mypy.py @@ -21,10 +21,14 @@ from typing import TYPE_CHECKING, Any, Dict, Iterable, Iterator, List, Optional try: - from typing import TypedDict # type: ignore[attr-defined] + from typing_extensions import NotRequired, TypedDict + + from bson import ObjectId # Not available in Python 3.7 class Movie(TypedDict): # type: ignore[misc] + _id: ObjectId + idiot: ObjectId name: str year: int @@ -312,16 +316,18 @@ def test_typeddict_document_type(self) -> None: assert retreived["year"] == 1 assert retreived["name"] == "a" + # TODO: mypy --install-types --non-interactive test/test_mypy.py + # run just this file in CI @only_type_check def test_typeddict_document_type_insertion(self) -> None: client: MongoClient[Movie] = MongoClient() coll: Collection[Movie] = client.test.test - insert = coll.insert_one(Movie(name="THX-1138", year=1971)) - out: Optional[Movie] = coll.find_one({"name": "THX-1138"}) + insert = coll.insert_one(Movie(_id=ObjectId(), name="THX-1138", year=1971)) + out = coll.find_one({"name": "THX-1138"}) assert out is not None - assert out.name == "THX-1138" - assert out.year == "1971" - assert out.id == ObjectId() + # This should fail because the output is a Movie. + assert out["foo"] # type:ignore[typeddict-item] + assert type(out["_id"]) == ObjectId @only_type_check def test_raw_bson_document_type(self) -> None: From d4ef6b8819213b6fe165d662f9ed2651b936dcf9 Mon Sep 17 00:00:00 2001 From: Julius Park Date: Thu, 20 Oct 2022 10:54:46 -0700 Subject: [PATCH 05/20] fix the tests --- .github/workflows/test-python.yml | 1 + pymongo/client_session.py | 2 +- pymongo/helpers.py | 4 ++-- pymongo/monitoring.py | 2 +- test/__init__.py | 23 +++++++++++--------- test/test_mypy.py | 35 ++++++++++++++++++++++++++++++- test/utils.py | 5 +++-- 7 files changed, 55 insertions(+), 17 deletions(-) diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index d451197e4e..0bbc21ff67 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -67,6 +67,7 @@ jobs: # Test overshadowed codec_options.py file mypy --install-types --non-interactive bson/codec_options.py mypy --install-types --non-interactive --disable-error-code var-annotated --disable-error-code attr-defined --disable-error-code union-attr --disable-error-code assignment --disable-error-code no-redef --disable-error-code index --allow-redefinition --allow-untyped-globals --exclude "test/mypy_fails/*.*" test + mypy --install-types --non-interactive test/test_mypy.py linkcheck: name: Check Links diff --git a/pymongo/client_session.py b/pymongo/client_session.py index 3ff98a579f..d2479942e4 100644 --- a/pymongo/client_session.py +++ b/pymongo/client_session.py @@ -435,7 +435,7 @@ def _max_time_expired_error(exc): # From the transactions spec, all the retryable writes errors plus # WriteConcernFailed. -_UNKNOWN_COMMIT_ERROR_CODES = _RETRYABLE_ERROR_CODES | frozenset( +_UNKNOWN_COMMIT_ERROR_CODES: frozenset = _RETRYABLE_ERROR_CODES | frozenset( [ 64, # WriteConcernFailed 50, # MaxTimeMSExpired diff --git a/pymongo/helpers.py b/pymongo/helpers.py index 4df8ab8e7a..dd210db188 100644 --- a/pymongo/helpers.py +++ b/pymongo/helpers.py @@ -44,7 +44,7 @@ # From the SDAM spec, the "not primary" error codes are combined with the # "node is recovering" error codes (of which the "node is shutting down" # errors are a subset). -_NOT_PRIMARY_CODES = ( +_NOT_PRIMARY_CODES: frozenset = ( frozenset( [ 10058, # LegacyNotPrimary <=3.2 "not primary" error code @@ -58,7 +58,7 @@ | _SHUTDOWN_CODES ) # From the retryable writes spec. -_RETRYABLE_ERROR_CODES = _NOT_PRIMARY_CODES | frozenset( +_RETRYABLE_ERROR_CODES: frozenset = _NOT_PRIMARY_CODES | frozenset( [ 7, # HostNotFound 6, # HostUnreachable diff --git a/pymongo/monitoring.py b/pymongo/monitoring.py index c53e7e5727..5b729652ad 100644 --- a/pymongo/monitoring.py +++ b/pymongo/monitoring.py @@ -528,7 +528,7 @@ def register(listener: _EventListener) -> None: # Note - to avoid bugs from forgetting which if these is all lowercase and # which are camelCase, and at the same time avoid having to add a test for # every command, use all lowercase here and test against command_name.lower(). -_SENSITIVE_COMMANDS = set( +_SENSITIVE_COMMANDS: set = set( [ "authenticate", "saslstart", diff --git a/test/__init__.py b/test/__init__.py index b89cd88d26..32e4cabdd4 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -43,11 +43,10 @@ HAVE_IPADDRESS = True except ImportError: HAVE_IPADDRESS = False - from contextlib import contextmanager from functools import wraps from test.version import Version -from typing import Callable, Dict, Generator, no_type_check +from typing import Any, Callable, Dict, Generator, no_type_check from unittest import SkipTest from urllib.parse import quote_plus @@ -331,7 +330,9 @@ def hello(self): def _connect(self, host, port, **kwargs): kwargs.update(self.default_client_options) - client = pymongo.MongoClient(host, port, serverSelectionTimeoutMS=5000, **kwargs) + client: MongoClient = pymongo.MongoClient( + host, port, serverSelectionTimeoutMS=5000, **kwargs + ) try: try: client.admin.command(HelloCompat.LEGACY_CMD) # Can we connect? @@ -356,7 +357,7 @@ def _init_client(self): if self.client is not None: # Return early when connected to dataLake as mongohoused does not # support the getCmdLineOpts command and is tested without TLS. - build_info = self.client.admin.command("buildInfo") + build_info: Any = self.client.admin.command("buildInfo") if "dataLake" in build_info: self.is_data_lake = True self.auth_enabled = True @@ -521,14 +522,16 @@ def has_secondaries(self): @property def storage_engine(self): try: - return self.server_status.get("storageEngine", {}).get("name") + return self.server_status.get("storageEngine", {}).get( + "name" + ) # type:ignore[union-attr] except AttributeError: # Raised if self.server_status is None. return None def _check_user_provided(self): """Return True if db_user/db_password is already an admin user.""" - client = pymongo.MongoClient( + client: MongoClient = pymongo.MongoClient( host, port, username=db_user, @@ -694,7 +697,7 @@ def supports_secondary_read_pref(self): if self.has_secondaries: return True if self.is_mongos: - shard = self.client.config.shards.find_one()["host"] + shard = self.client.config.shards.find_one()["host"] # type:ignore[index] num_members = shard.count(",") + 1 return num_members > 1 return False @@ -1015,12 +1018,12 @@ def fork( """ def _print_threads(*args: object) -> None: - if _print_threads.called: + if _print_threads.called: # type:ignore[attr-defined] return - _print_threads.called = True + _print_threads.called = True # type:ignore[attr-defined] print_thread_tracebacks() - _print_threads.called = False + _print_threads.called = False # type:ignore[attr-defined] def _target() -> None: signal.signal(signal.SIGUSR1, _print_threads) diff --git a/test/test_mypy.py b/test/test_mypy.py index 119f9a6dbd..390c13ca4c 100644 --- a/test/test_mypy.py +++ b/test/test_mypy.py @@ -28,7 +28,15 @@ # Not available in Python 3.7 class Movie(TypedDict): # type: ignore[misc] _id: ObjectId - idiot: ObjectId + name: str + year: int + + class ImplicitMovie(TypedDict): # type: ignore[misc] + _id: NotRequired[ObjectId] + name: str + year: int + + class EmptyMovie(TypedDict): # type: ignore[misc] name: str year: int @@ -329,6 +337,31 @@ def test_typeddict_document_type_insertion(self) -> None: assert out["foo"] # type:ignore[typeddict-item] assert type(out["_id"]) == ObjectId + # This should work the same as the test above, but this time using NotRequired to allow + # automatic insertion of the _id field by insert_one. + @only_type_check + def test_typeddict_document_type_not_required(self) -> None: + client: MongoClient[ImplicitMovie] = MongoClient() + coll: Collection[ImplicitMovie] = client.test.test + insert = coll.insert_one(ImplicitMovie(name="THX-1138", year=1971)) + out = coll.find_one({"name": "THX-1138"}) + assert out is not None + # This should fail because the output is a Movie. + assert out["foo"] # type:ignore[typeddict-item] + assert type(out["_id"]) == ObjectId + + @only_type_check + def test_typeddict_document_type_empty(self) -> None: + client: MongoClient[EmptyMovie] = MongoClient() + coll: Collection[EmptyMovie] = client.test.test + insert = coll.insert_one(EmptyMovie(name="THX-1138", year=1971)) + out = coll.find_one({"name": "THX-1138"}) + assert out is not None + # This should fail because the output is a Movie. + assert out["foo"] # type:ignore[typeddict-item] + # This should fail because _id is not included in our TypedDict definition. + assert type(out["_id"]) == ObjectId # type:ignore[typeddict-item] + @only_type_check def test_raw_bson_document_type(self) -> None: client = MongoClient(document_class=RawBSONDocument) diff --git a/test/utils.py b/test/utils.py index 6f35b48538..c8fae1f32d 100644 --- a/test/utils.py +++ b/test/utils.py @@ -29,6 +29,7 @@ from collections import abc, defaultdict from functools import partial from test import client_context, db_pwd, db_user +from typing import Any from bson import json_util from bson.objectid import ObjectId @@ -601,7 +602,7 @@ def ensure_all_connected(client: MongoClient) -> None: Depending on the use-case, the caller may need to clear any event listeners that are configured on the client. """ - hello = client.admin.command(HelloCompat.LEGACY_CMD) + hello: Any = client.admin.command(HelloCompat.LEGACY_CMD) if "setName" not in hello: raise ConfigurationError("cluster is not a replica set") @@ -612,7 +613,7 @@ def ensure_all_connected(client: MongoClient) -> None: def discover(): i = 0 while i < 100 and connected_host_list != target_host_list: - hello = client.admin.command( + hello: Any = client.admin.command( HelloCompat.LEGACY_CMD, read_preference=ReadPreference.SECONDARY ) connected_host_list.update([hello["me"]]) From 3a063b86d5aa873fa56f6247bfa1d11a07e24d06 Mon Sep 17 00:00:00 2001 From: Julius Park Date: Thu, 20 Oct 2022 11:09:45 -0700 Subject: [PATCH 06/20] fix documentation, etc --- doc/examples/type_hints.rst | 15 +++++++++------ test/__init__.py | 4 ++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/doc/examples/type_hints.rst b/doc/examples/type_hints.rst index 02cd18c0b6..3c264f92f6 100644 --- a/doc/examples/type_hints.rst +++ b/doc/examples/type_hints.rst @@ -94,9 +94,10 @@ Typed Collection You can use :py:class:`~typing_extensions.TypedDict` (Python 3.8+) when using a well-defined schema for the data in a :class:`~pymongo.collection.Collection`. Note that all `schema_validation`_ for inserts and updates is done on the server. This is due to the fact that these methods automatically add -an "_id" field. The "_id" field is decorated by :py:class:`~typing_extensions.NotRequired` decorator to allow it to be accessed when reading -from `result` (albeit without type-checking for that specific field, hence why it should not be used for schema validation). -Another option would be to generate the "_id" field yourself, and make it a required field, which would give the expected behavior. +an "_id" field. In the example below the "_id" field is marked by the :py:class:`~typing_extensions.NotRequired` notation to allow it to be accessed when reading +from `result`. If it is simply not included in the definition, then it will be automatically added, but it will raise a type-checking error if you attempt to access it. +Another option would be to generate the "_id" field yourself, and make it a required field. This would give the expected behavior, but would then also prevent you from +relying on PyMongo to insert the "_id" field. .. doctest:: @@ -111,12 +112,14 @@ Another option would be to generate the "_id" field yourself, and make it a requ ... >>> client: MongoClient = MongoClient() >>> collection: Collection[Movie] = client.test.test - >>> inserted = collection.insert_one({"name": "Jurassic Park", "year": 1993 }) + >>> # If NotRequired was not specified above, then you would be required to specify _id + >>> # when you construct the Movie object. + >>> inserted = collection.insert_one(Movie(name="Jurassic Park", year=1993)) >>> result = collection.find_one({"name": "Jurassic Park"}) >>> assert result is not None >>> assert result["year"] == 1993 - >>> # Mypy will not check this because it is NotRequired - >>> assert result["_id"] == tuple() + >>> # This will be type checked, despite being not originally present + >>> assert type(result["_id"]) == ObjectId Typed Database -------------- diff --git a/test/__init__.py b/test/__init__.py index 32e4cabdd4..eb66e45667 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -522,9 +522,9 @@ def has_secondaries(self): @property def storage_engine(self): try: - return self.server_status.get("storageEngine", {}).get( + return self.server_status.get("storageEngine", {}).get( # type:ignore[union-attr] "name" - ) # type:ignore[union-attr] + ) except AttributeError: # Raised if self.server_status is None. return None From 8322387df81852307ca0b73a06cee92bc5f482dd Mon Sep 17 00:00:00 2001 From: Julius Park Date: Thu, 20 Oct 2022 16:14:43 -0700 Subject: [PATCH 07/20] fix tests so they test all the methods --- .github/workflows/test-python.yml | 2 +- doc/examples/type_hints.rst | 14 ++++--- pymongo/operations.py | 4 +- test/test_mypy.py | 66 ++++++++++++++++++++----------- test/utils.py | 4 +- 5 files changed, 55 insertions(+), 35 deletions(-) diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 0bbc21ff67..3c20e7fb23 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -59,7 +59,7 @@ jobs: cache-dependency-path: 'setup.py' - name: Install dependencies run: | - python -m pip install -U pip mypy + python -m pip install -U pip mypy typing_extensions pip install -e ".[zstd, encryption, ocsp]" - name: Run mypy run: | diff --git a/doc/examples/type_hints.rst b/doc/examples/type_hints.rst index 3c264f92f6..5839667eb0 100644 --- a/doc/examples/type_hints.rst +++ b/doc/examples/type_hints.rst @@ -92,12 +92,14 @@ Note that when using :class:`~bson.son.SON`, the key and value types must be giv Typed Collection ---------------- -You can use :py:class:`~typing_extensions.TypedDict` (Python 3.8+) when using a well-defined schema for the data in a :class:`~pymongo.collection.Collection`. -Note that all `schema_validation`_ for inserts and updates is done on the server. This is due to the fact that these methods automatically add -an "_id" field. In the example below the "_id" field is marked by the :py:class:`~typing_extensions.NotRequired` notation to allow it to be accessed when reading -from `result`. If it is simply not included in the definition, then it will be automatically added, but it will raise a type-checking error if you attempt to access it. -Another option would be to generate the "_id" field yourself, and make it a required field. This would give the expected behavior, but would then also prevent you from -relying on PyMongo to insert the "_id" field. +You can use :py:class:`~typing_extensions.TypedDict` (Python 3.8+) when using a well-defined schema for the data in a +:class:`~pymongo.collection.Collection`. Note that all `schema validation`_ for inserts and updates is done on the server. +These methods automatically add an "_id" field. In the example below the "_id" field is +marked by the :py:class:`~typing_extensions.NotRequired` notation to allow it to be accessed when reading from +``result``. If it is simply not included in the definition, then it will be automatically added, but it will raise a +type-checking error if you attempt to access it. Another option would be to generate the "_id" field yourself, and make +it a required field. This would give the expected behavior, but would then also prevent you from relying on PyMongo to +insert the "_id" field. .. doctest:: diff --git a/pymongo/operations.py b/pymongo/operations.py index 84e8bf4d35..1ec1aa783b 100644 --- a/pymongo/operations.py +++ b/pymongo/operations.py @@ -19,7 +19,7 @@ from pymongo.collation import validate_collation_or_none from pymongo.common import validate_boolean, validate_is_mapping, validate_list from pymongo.helpers import _gen_index_name, _index_document, _index_list -from pymongo.typings import _CollationIn, _DocumentIn, _Pipeline +from pymongo.typings import _CollationIn, _DocumentIn, _DocumentType, _Pipeline class InsertOne(object): @@ -27,7 +27,7 @@ class InsertOne(object): __slots__ = ("_doc",) - def __init__(self, document: _DocumentIn) -> None: + def __init__(self, document: Union[_DocumentIn, _DocumentType]) -> None: """Create an InsertOne instance. For use with :meth:`~pymongo.collection.Collection.bulk_write`. diff --git a/test/test_mypy.py b/test/test_mypy.py index 390c13ca4c..58b56acb20 100644 --- a/test/test_mypy.py +++ b/test/test_mypy.py @@ -36,7 +36,7 @@ class ImplicitMovie(TypedDict): # type: ignore[misc] name: str year: int - class EmptyMovie(TypedDict): # type: ignore[misc] + class MovieWithoutId(TypedDict): # type: ignore[misc] name: str year: int @@ -324,18 +324,23 @@ def test_typeddict_document_type(self) -> None: assert retreived["year"] == 1 assert retreived["name"] == "a" - # TODO: mypy --install-types --non-interactive test/test_mypy.py - # run just this file in CI @only_type_check def test_typeddict_document_type_insertion(self) -> None: client: MongoClient[Movie] = MongoClient() coll: Collection[Movie] = client.test.test - insert = coll.insert_one(Movie(_id=ObjectId(), name="THX-1138", year=1971)) - out = coll.find_one({"name": "THX-1138"}) - assert out is not None - # This should fail because the output is a Movie. - assert out["foo"] # type:ignore[typeddict-item] - assert type(out["_id"]) == ObjectId + mov = Movie(_id=ObjectId(), name="THX-1138", year=1971) + for meth, arg in [ + (coll.insert_many, [mov]), + (coll.insert_one, mov), + (coll.bulk_write, [InsertOne(mov)]), + ]: + meth(arg) # type:ignore[operator] + out = coll.find_one({"name": "THX-1138"}) + assert out is not None + # This should fail because the output is a Movie. + assert out["foo"] # type:ignore[typeddict-item] + assert type(out["_id"]) == ObjectId + coll.drop() # This should work the same as the test above, but this time using NotRequired to allow # automatic insertion of the _id field by insert_one. @@ -343,24 +348,37 @@ def test_typeddict_document_type_insertion(self) -> None: def test_typeddict_document_type_not_required(self) -> None: client: MongoClient[ImplicitMovie] = MongoClient() coll: Collection[ImplicitMovie] = client.test.test - insert = coll.insert_one(ImplicitMovie(name="THX-1138", year=1971)) - out = coll.find_one({"name": "THX-1138"}) - assert out is not None - # This should fail because the output is a Movie. - assert out["foo"] # type:ignore[typeddict-item] - assert type(out["_id"]) == ObjectId + mov = ImplicitMovie(name="THX-1138", year=1971) + for meth, arg in [ + (coll.insert_many, [mov]), + (coll.insert_one, mov), + (coll.bulk_write, [InsertOne(mov)]), + ]: + meth(arg) # type:ignore[operator] + out = coll.find_one({"name": "THX-1138"}) + assert out is not None + # This should fail because the output is a Movie. + assert out["foo"] # type:ignore[typeddict-item] + assert type(out["_id"]) == ObjectId + coll.drop() @only_type_check def test_typeddict_document_type_empty(self) -> None: - client: MongoClient[EmptyMovie] = MongoClient() - coll: Collection[EmptyMovie] = client.test.test - insert = coll.insert_one(EmptyMovie(name="THX-1138", year=1971)) - out = coll.find_one({"name": "THX-1138"}) - assert out is not None - # This should fail because the output is a Movie. - assert out["foo"] # type:ignore[typeddict-item] - # This should fail because _id is not included in our TypedDict definition. - assert type(out["_id"]) == ObjectId # type:ignore[typeddict-item] + client: MongoClient[MovieWithoutId] = MongoClient() + coll: Collection[MovieWithoutId] = client.test.test + mov = MovieWithoutId(name="THX-1138", year=1971) + for meth, arg in [ + (coll.insert_many, [mov]), + (coll.insert_one, mov), + (coll.bulk_write, [InsertOne(mov)]), + ]: + meth(arg) # type:ignore[operator] + out = coll.find_one({"name": "THX-1138"}) + assert out is not None + # This should fail because the output is a Movie. + assert out["foo"] # type:ignore[typeddict-item] + # This should fail because _id is not included in our TypedDict definition. + assert type(out["_id"]) == ObjectId # type:ignore[typeddict-item] @only_type_check def test_raw_bson_document_type(self) -> None: diff --git a/test/utils.py b/test/utils.py index c8fae1f32d..5027f73a6b 100644 --- a/test/utils.py +++ b/test/utils.py @@ -602,7 +602,7 @@ def ensure_all_connected(client: MongoClient) -> None: Depending on the use-case, the caller may need to clear any event listeners that are configured on the client. """ - hello: Any = client.admin.command(HelloCompat.LEGACY_CMD) + hello: dict = client.admin.command(HelloCompat.LEGACY_CMD) if "setName" not in hello: raise ConfigurationError("cluster is not a replica set") @@ -613,7 +613,7 @@ def ensure_all_connected(client: MongoClient) -> None: def discover(): i = 0 while i < 100 and connected_host_list != target_host_list: - hello: Any = client.admin.command( + hello: dict = client.admin.command( HelloCompat.LEGACY_CMD, read_preference=ReadPreference.SECONDARY ) connected_host_list.update([hello["me"]]) From f9ba0c7ab02de2f32392e69fc7fb9fa5fc21371f Mon Sep 17 00:00:00 2001 From: Julius Park Date: Tue, 25 Oct 2022 10:09:17 -0700 Subject: [PATCH 08/20] simplify tests --- test/test_mypy.py | 73 ++++++++++++----------------------------------- 1 file changed, 19 insertions(+), 54 deletions(-) diff --git a/test/test_mypy.py b/test/test_mypy.py index 58b56acb20..e8af4ffe17 100644 --- a/test/test_mypy.py +++ b/test/test_mypy.py @@ -17,8 +17,20 @@ import os import tempfile +import typing import unittest -from typing import TYPE_CHECKING, Any, Dict, Iterable, Iterator, List, Optional +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Iterable, + Iterator, + List, + Optional, + Sequence, + Tuple, +) try: from typing_extensions import NotRequired, TypedDict @@ -67,6 +79,7 @@ class MovieWithoutId(TypedDict): # type: ignore[misc] from pymongo.collection import Collection from pymongo.operations import InsertOne from pymongo.read_preferences import ReadPreference +from pymongo.results import BulkWriteResult, InsertManyResult, InsertOneResult TEST_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "mypy_fails") @@ -326,59 +339,11 @@ def test_typeddict_document_type(self) -> None: @only_type_check def test_typeddict_document_type_insertion(self) -> None: - client: MongoClient[Movie] = MongoClient() - coll: Collection[Movie] = client.test.test - mov = Movie(_id=ObjectId(), name="THX-1138", year=1971) - for meth, arg in [ - (coll.insert_many, [mov]), - (coll.insert_one, mov), - (coll.bulk_write, [InsertOne(mov)]), - ]: - meth(arg) # type:ignore[operator] - out = coll.find_one({"name": "THX-1138"}) - assert out is not None - # This should fail because the output is a Movie. - assert out["foo"] # type:ignore[typeddict-item] - assert type(out["_id"]) == ObjectId - coll.drop() - - # This should work the same as the test above, but this time using NotRequired to allow - # automatic insertion of the _id field by insert_one. - @only_type_check - def test_typeddict_document_type_not_required(self) -> None: - client: MongoClient[ImplicitMovie] = MongoClient() - coll: Collection[ImplicitMovie] = client.test.test - mov = ImplicitMovie(name="THX-1138", year=1971) - for meth, arg in [ - (coll.insert_many, [mov]), - (coll.insert_one, mov), - (coll.bulk_write, [InsertOne(mov)]), - ]: - meth(arg) # type:ignore[operator] - out = coll.find_one({"name": "THX-1138"}) - assert out is not None - # This should fail because the output is a Movie. - assert out["foo"] # type:ignore[typeddict-item] - assert type(out["_id"]) == ObjectId - coll.drop() - - @only_type_check - def test_typeddict_document_type_empty(self) -> None: - client: MongoClient[MovieWithoutId] = MongoClient() - coll: Collection[MovieWithoutId] = client.test.test - mov = MovieWithoutId(name="THX-1138", year=1971) - for meth, arg in [ - (coll.insert_many, [mov]), - (coll.insert_one, mov), - (coll.bulk_write, [InsertOne(mov)]), - ]: - meth(arg) # type:ignore[operator] - out = coll.find_one({"name": "THX-1138"}) - assert out is not None - # This should fail because the output is a Movie. - assert out["foo"] # type:ignore[typeddict-item] - # This should fail because _id is not included in our TypedDict definition. - assert type(out["_id"]) == ObjectId # type:ignore[typeddict-item] + coll: Collection[Movie] = self.db.test + mov = Movie(name="THX-1138", year=1971) + coll.insert_one(mov) + coll.insert_many([mov]) + coll.bulk_write([InsertOne(mov)]) @only_type_check def test_raw_bson_document_type(self) -> None: From 1bb46a59181328bee107e4e284888aa42d67f90d Mon Sep 17 00:00:00 2001 From: Julius Park Date: Tue, 25 Oct 2022 10:26:05 -0700 Subject: [PATCH 09/20] not sure why this is not working --- test/test_mypy.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/test/test_mypy.py b/test/test_mypy.py index e8af4ffe17..8d4706dcbc 100644 --- a/test/test_mypy.py +++ b/test/test_mypy.py @@ -14,7 +14,6 @@ """Test that each file in mypy_fails/ actually fails mypy, and test some sample client code that uses PyMongo typings.""" - import os import tempfile import typing @@ -30,6 +29,7 @@ Optional, Sequence, Tuple, + Union, ) try: @@ -39,16 +39,6 @@ # Not available in Python 3.7 class Movie(TypedDict): # type: ignore[misc] - _id: ObjectId - name: str - year: int - - class ImplicitMovie(TypedDict): # type: ignore[misc] - _id: NotRequired[ObjectId] - name: str - year: int - - class MovieWithoutId(TypedDict): # type: ignore[misc] name: str year: int @@ -339,11 +329,20 @@ def test_typeddict_document_type(self) -> None: @only_type_check def test_typeddict_document_type_insertion(self) -> None: - coll: Collection[Movie] = self.db.test - mov = Movie(name="THX-1138", year=1971) + client: MongoClient[Movie] = MongoClient() + coll = client.test.test + mov = {"name": "THX-1138", "year": 1971} + movie = Movie(name="THX-1138", year=1971) coll.insert_one(mov) + coll.insert_one(movie) coll.insert_many([mov]) - coll.bulk_write([InsertOne(mov)]) + coll.insert_many([movie]) + bad_mov = {"name": "THX-1138", "year": "WRONG TYPE"} # type:ignore[typeddict-item] + bad_movie = Movie(name="THX-1138", year="WRONG TYPE") # type:ignore[typeddict-item] + coll.insert_one(bad_mov) + coll.insert_one(bad_movie) + coll.insert_many([bad_mov]) + coll.insert_many([bad_movie]) @only_type_check def test_raw_bson_document_type(self) -> None: From 813972456757af40cc9a21832844a1fd5b48bef4 Mon Sep 17 00:00:00 2001 From: Julius Park Date: Fri, 28 Oct 2022 12:20:56 -0700 Subject: [PATCH 10/20] update tests --- test/test_mypy.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/test/test_mypy.py b/test/test_mypy.py index 8d4706dcbc..7ec741400e 100644 --- a/test/test_mypy.py +++ b/test/test_mypy.py @@ -333,15 +333,20 @@ def test_typeddict_document_type_insertion(self) -> None: coll = client.test.test mov = {"name": "THX-1138", "year": 1971} movie = Movie(name="THX-1138", year=1971) - coll.insert_one(mov) + coll.insert_one(mov) # type:ignore[arg-type] + coll.insert_one({"name": "THX-1138", "year": 1971}) # This will work because it is in-line. coll.insert_one(movie) - coll.insert_many([mov]) + coll.insert_many([mov]) # type:ignore[list-item] coll.insert_many([movie]) - bad_mov = {"name": "THX-1138", "year": "WRONG TYPE"} # type:ignore[typeddict-item] + bad_mov = {"name": "THX-1138", "year": "WRONG TYPE"} bad_movie = Movie(name="THX-1138", year="WRONG TYPE") # type:ignore[typeddict-item] - coll.insert_one(bad_mov) + coll.insert_one(bad_mov) # type:ignore[arg-type] + coll.insert_one({"name": "THX-1138", "year": "WRONG TYPE"}) # type:ignore[typeddict-item] coll.insert_one(bad_movie) - coll.insert_many([bad_mov]) + coll.insert_many([bad_mov]) # type:ignore[list-item] + coll.insert_many( + [{"name": "THX-1138", "year": "WRONG TYPE"}] + ) # type:ignore[typeddict-item] coll.insert_many([bad_movie]) @only_type_check From 91f366635b07b45146c9036fa16a830c043bc558 Mon Sep 17 00:00:00 2001 From: Julius Park Date: Fri, 28 Oct 2022 12:26:25 -0700 Subject: [PATCH 11/20] clean up docs --- doc/examples/type_hints.rst | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/doc/examples/type_hints.rst b/doc/examples/type_hints.rst index 5839667eb0..0c6e8e4eb2 100644 --- a/doc/examples/type_hints.rst +++ b/doc/examples/type_hints.rst @@ -94,33 +94,24 @@ Typed Collection You can use :py:class:`~typing_extensions.TypedDict` (Python 3.8+) when using a well-defined schema for the data in a :class:`~pymongo.collection.Collection`. Note that all `schema validation`_ for inserts and updates is done on the server. -These methods automatically add an "_id" field. In the example below the "_id" field is -marked by the :py:class:`~typing_extensions.NotRequired` notation to allow it to be accessed when reading from -``result``. If it is simply not included in the definition, then it will be automatically added, but it will raise a -type-checking error if you attempt to access it. Another option would be to generate the "_id" field yourself, and make -it a required field. This would give the expected behavior, but would then also prevent you from relying on PyMongo to -insert the "_id" field. +These methods automatically add an "_id" field. .. doctest:: - >>> from typing_extensions import TypedDict, NotRequired + >>> from typing_extensions import TypedDict >>> from pymongo import MongoClient >>> from pymongo.collection import Collection - >>> from bson import ObjectId >>> class Movie(TypedDict): - ... _id: NotRequired[ObjectId] ... name: str ... year: int ... >>> client: MongoClient = MongoClient() >>> collection: Collection[Movie] = client.test.test - >>> # If NotRequired was not specified above, then you would be required to specify _id - >>> # when you construct the Movie object. >>> inserted = collection.insert_one(Movie(name="Jurassic Park", year=1993)) >>> result = collection.find_one({"name": "Jurassic Park"}) >>> assert result is not None >>> assert result["year"] == 1993 - >>> # This will be type checked, despite being not originally present + >>> # This will not be type checked, despite being present, because it is added by PyMongo. >>> assert type(result["_id"]) == ObjectId Typed Database From 6cfe733d0187b5eecb9557fe6e53911e7656e001 Mon Sep 17 00:00:00 2001 From: Julius Park Date: Fri, 28 Oct 2022 12:50:06 -0700 Subject: [PATCH 12/20] fix autoformatter 'fix' --- test/test_mypy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_mypy.py b/test/test_mypy.py index 7ec741400e..e0cd40eda9 100644 --- a/test/test_mypy.py +++ b/test/test_mypy.py @@ -345,8 +345,8 @@ def test_typeddict_document_type_insertion(self) -> None: coll.insert_one(bad_movie) coll.insert_many([bad_mov]) # type:ignore[list-item] coll.insert_many( - [{"name": "THX-1138", "year": "WRONG TYPE"}] - ) # type:ignore[typeddict-item] + [{"name": "THX-1138", "year": "WRONG TYPE"}] # type:ignore[typeddict-item] + ) coll.insert_many([bad_movie]) @only_type_check From 671eea7daf465157f01ff86e393cc286783bd688 Mon Sep 17 00:00:00 2001 From: Julius Park Date: Fri, 28 Oct 2022 13:06:36 -0700 Subject: [PATCH 13/20] remove unused imports --- pymongo/collection.py | 2 +- test/test_mypy.py | 30 ++++-------------------------- test/utils.py | 1 - 3 files changed, 5 insertions(+), 28 deletions(-) diff --git a/pymongo/collection.py b/pymongo/collection.py index f7150c80f4..23efe8fd35 100644 --- a/pymongo/collection.py +++ b/pymongo/collection.py @@ -71,7 +71,7 @@ InsertOneResult, UpdateResult, ) -from pymongo.typings import _CollationIn, _DocumentIn, _DocumentType, _Pipeline +from pymongo.typings import _CollationIn, _DocumentType, _Pipeline from pymongo.write_concern import WriteConcern _FIND_AND_MODIFY_DOC_FIELDS = {"value": 1} diff --git a/test/test_mypy.py b/test/test_mypy.py index e0cd40eda9..49580fff5a 100644 --- a/test/test_mypy.py +++ b/test/test_mypy.py @@ -16,24 +16,11 @@ sample client code that uses PyMongo typings.""" import os import tempfile -import typing import unittest -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Dict, - Iterable, - Iterator, - List, - Optional, - Sequence, - Tuple, - Union, -) +from typing import TYPE_CHECKING, Any, Dict, Iterable, Iterator, List try: - from typing_extensions import NotRequired, TypedDict + from typing_extensions import TypedDict from bson import ObjectId @@ -43,7 +30,7 @@ class Movie(TypedDict): # type: ignore[misc] year: int except ImportError: - TypeDict = None + TypedDict = None try: @@ -54,22 +41,13 @@ class Movie(TypedDict): # type: ignore[misc] from test import IntegrationTest from test.utils import rs_or_single_client -from bson import ( - CodecOptions, - ObjectId, - decode, - decode_all, - decode_file_iter, - decode_iter, - encode, -) +from bson import CodecOptions, decode, decode_all, decode_file_iter, decode_iter, encode from bson.raw_bson import RawBSONDocument from bson.son import SON from pymongo import ASCENDING, MongoClient from pymongo.collection import Collection from pymongo.operations import InsertOne from pymongo.read_preferences import ReadPreference -from pymongo.results import BulkWriteResult, InsertManyResult, InsertOneResult TEST_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "mypy_fails") diff --git a/test/utils.py b/test/utils.py index 5027f73a6b..59349f4fdc 100644 --- a/test/utils.py +++ b/test/utils.py @@ -29,7 +29,6 @@ from collections import abc, defaultdict from functools import partial from test import client_context, db_pwd, db_user -from typing import Any from bson import json_util from bson.objectid import ObjectId From 3ab850edcb75087a15946ccd108575eb94d17ab2 Mon Sep 17 00:00:00 2001 From: Julius Park Date: Fri, 28 Oct 2022 13:33:24 -0700 Subject: [PATCH 14/20] one final unused import --- test/test_mypy.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/test_mypy.py b/test/test_mypy.py index 49580fff5a..1d1f41f89d 100644 --- a/test/test_mypy.py +++ b/test/test_mypy.py @@ -20,11 +20,9 @@ from typing import TYPE_CHECKING, Any, Dict, Iterable, Iterator, List try: + # Not available in Python 3.7 from typing_extensions import TypedDict - from bson import ObjectId - - # Not available in Python 3.7 class Movie(TypedDict): # type: ignore[misc] name: str year: int From 7c1dab216cb597bfe81611f168bafbe67307d70b Mon Sep 17 00:00:00 2001 From: Julius Park Date: Fri, 28 Oct 2022 13:57:56 -0700 Subject: [PATCH 15/20] remove typing_extensions, defer bulk write change --- .github/workflows/test-python.yml | 2 +- pymongo/operations.py | 2 +- test/test_mypy.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 3c20e7fb23..0bbc21ff67 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -59,7 +59,7 @@ jobs: cache-dependency-path: 'setup.py' - name: Install dependencies run: | - python -m pip install -U pip mypy typing_extensions + python -m pip install -U pip mypy pip install -e ".[zstd, encryption, ocsp]" - name: Run mypy run: | diff --git a/pymongo/operations.py b/pymongo/operations.py index 1ec1aa783b..dd7d73f0df 100644 --- a/pymongo/operations.py +++ b/pymongo/operations.py @@ -27,7 +27,7 @@ class InsertOne(object): __slots__ = ("_doc",) - def __init__(self, document: Union[_DocumentIn, _DocumentType]) -> None: + def __init__(self, document: _DocumentIn) -> None: """Create an InsertOne instance. For use with :meth:`~pymongo.collection.Collection.bulk_write`. diff --git a/test/test_mypy.py b/test/test_mypy.py index 1d1f41f89d..5aa7219f1f 100644 --- a/test/test_mypy.py +++ b/test/test_mypy.py @@ -21,7 +21,7 @@ try: # Not available in Python 3.7 - from typing_extensions import TypedDict + from typing import TypedDict class Movie(TypedDict): # type: ignore[misc] name: str From 3869d8d9c8bd8df79963443443f6b6d3a149c17e Mon Sep 17 00:00:00 2001 From: Julius Park Date: Fri, 28 Oct 2022 13:57:56 -0700 Subject: [PATCH 16/20] remove typing_extensions, defer bulk write change --- doc/examples/type_hints.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/examples/type_hints.rst b/doc/examples/type_hints.rst index 0c6e8e4eb2..782fbf2fde 100644 --- a/doc/examples/type_hints.rst +++ b/doc/examples/type_hints.rst @@ -98,7 +98,7 @@ These methods automatically add an "_id" field. .. doctest:: - >>> from typing_extensions import TypedDict + >>> from typing import TypedDict >>> from pymongo import MongoClient >>> from pymongo.collection import Collection >>> class Movie(TypedDict): From 327fe90d009179bf5e100b435abaaa5913ab9f75 Mon Sep 17 00:00:00 2001 From: Julius Park Date: Fri, 28 Oct 2022 14:01:31 -0700 Subject: [PATCH 17/20] one final reference to typing_extensions --- doc/examples/type_hints.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/examples/type_hints.rst b/doc/examples/type_hints.rst index 782fbf2fde..e829441976 100644 --- a/doc/examples/type_hints.rst +++ b/doc/examples/type_hints.rst @@ -92,7 +92,7 @@ Note that when using :class:`~bson.son.SON`, the key and value types must be giv Typed Collection ---------------- -You can use :py:class:`~typing_extensions.TypedDict` (Python 3.8+) when using a well-defined schema for the data in a +You can use :py:class:`~typing.TypedDict` (Python 3.8+) when using a well-defined schema for the data in a :class:`~pymongo.collection.Collection`. Note that all `schema validation`_ for inserts and updates is done on the server. These methods automatically add an "_id" field. From 2d58a215d619cb2f1530c65503deac685e76ca35 Mon Sep 17 00:00:00 2001 From: Julius Park Date: Fri, 28 Oct 2022 14:13:04 -0700 Subject: [PATCH 18/20] my ignores are wrong? --- pymongo/operations.py | 2 +- test/test_mypy.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pymongo/operations.py b/pymongo/operations.py index dd7d73f0df..84e8bf4d35 100644 --- a/pymongo/operations.py +++ b/pymongo/operations.py @@ -19,7 +19,7 @@ from pymongo.collation import validate_collation_or_none from pymongo.common import validate_boolean, validate_is_mapping, validate_list from pymongo.helpers import _gen_index_name, _index_document, _index_list -from pymongo.typings import _CollationIn, _DocumentIn, _DocumentType, _Pipeline +from pymongo.typings import _CollationIn, _DocumentIn, _Pipeline class InsertOne(object): diff --git a/test/test_mypy.py b/test/test_mypy.py index 5aa7219f1f..4ee31a069e 100644 --- a/test/test_mypy.py +++ b/test/test_mypy.py @@ -309,19 +309,19 @@ def test_typeddict_document_type_insertion(self) -> None: coll = client.test.test mov = {"name": "THX-1138", "year": 1971} movie = Movie(name="THX-1138", year=1971) - coll.insert_one(mov) # type:ignore[arg-type] + coll.insert_one(mov) # type: ignore[arg-type] coll.insert_one({"name": "THX-1138", "year": 1971}) # This will work because it is in-line. coll.insert_one(movie) - coll.insert_many([mov]) # type:ignore[list-item] + coll.insert_many([mov]) # type: ignore[list-item] coll.insert_many([movie]) bad_mov = {"name": "THX-1138", "year": "WRONG TYPE"} - bad_movie = Movie(name="THX-1138", year="WRONG TYPE") # type:ignore[typeddict-item] + bad_movie = Movie(name="THX-1138", year="WRONG TYPE") # type: ignore[typeddict-item] coll.insert_one(bad_mov) # type:ignore[arg-type] - coll.insert_one({"name": "THX-1138", "year": "WRONG TYPE"}) # type:ignore[typeddict-item] + coll.insert_one({"name": "THX-1138", "year": "WRONG TYPE"}) # type: ignore[typeddict-item] coll.insert_one(bad_movie) - coll.insert_many([bad_mov]) # type:ignore[list-item] + coll.insert_many([bad_mov]) # type: ignore[list-item] coll.insert_many( - [{"name": "THX-1138", "year": "WRONG TYPE"}] # type:ignore[typeddict-item] + [{"name": "THX-1138", "year": "WRONG TYPE"}] # type: ignore[typeddict-item] ) coll.insert_many([bad_movie]) From 75e0dc558033bbc0222ac2b065741c32cd50ae12 Mon Sep 17 00:00:00 2001 From: Julius Park Date: Mon, 31 Oct 2022 10:11:52 -0700 Subject: [PATCH 19/20] add back typing_extensions --- .github/workflows/test-python.yml | 1 + test/test_mypy.py | 12 ++++-------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 0bbc21ff67..cbebc94e6f 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -67,6 +67,7 @@ jobs: # Test overshadowed codec_options.py file mypy --install-types --non-interactive bson/codec_options.py mypy --install-types --non-interactive --disable-error-code var-annotated --disable-error-code attr-defined --disable-error-code union-attr --disable-error-code assignment --disable-error-code no-redef --disable-error-code index --allow-redefinition --allow-untyped-globals --exclude "test/mypy_fails/*.*" test + python -m pip install -U typing_extensions mypy --install-types --non-interactive test/test_mypy.py linkcheck: diff --git a/test/test_mypy.py b/test/test_mypy.py index 4ee31a069e..0553bf98d1 100644 --- a/test/test_mypy.py +++ b/test/test_mypy.py @@ -19,16 +19,12 @@ import unittest from typing import TYPE_CHECKING, Any, Dict, Iterable, Iterator, List -try: - # Not available in Python 3.7 - from typing import TypedDict +from typing_extensions import TypedDict - class Movie(TypedDict): # type: ignore[misc] - name: str - year: int -except ImportError: - TypedDict = None +class Movie(TypedDict): # type: ignore[misc] + name: str + year: int try: From e0d1d98ca847b42fc27cc228a3b519aca1a0674d Mon Sep 17 00:00:00 2001 From: Julius Park Date: Mon, 31 Oct 2022 10:48:58 -0700 Subject: [PATCH 20/20] add back in try except --- test/test_mypy.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/test/test_mypy.py b/test/test_mypy.py index 0553bf98d1..a1e94937b2 100644 --- a/test/test_mypy.py +++ b/test/test_mypy.py @@ -19,12 +19,15 @@ import unittest from typing import TYPE_CHECKING, Any, Dict, Iterable, Iterator, List -from typing_extensions import TypedDict +try: + from typing_extensions import TypedDict + class Movie(TypedDict): # type: ignore[misc] + name: str + year: int -class Movie(TypedDict): # type: ignore[misc] - name: str - year: int +except ImportError: + TypedDict = None try: