From f4b4954a999b5500802646e7fca48036d3acb414 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Mon, 19 Jan 2026 12:45:49 -0500 Subject: [PATCH 1/8] rename attributes file --- obstore/python/obstore/__init__.py | 2 ++ obstore/python/obstore/{_attributes.pyi => _attributes.py} | 0 2 files changed, 2 insertions(+) rename obstore/python/obstore/{_attributes.pyi => _attributes.py} (100%) diff --git a/obstore/python/obstore/__init__.py b/obstore/python/obstore/__init__.py index 6a28793c..eae709ba 100644 --- a/obstore/python/obstore/__init__.py +++ b/obstore/python/obstore/__init__.py @@ -1,6 +1,7 @@ from typing import TYPE_CHECKING from . import _obstore, store # pyright:ignore[reportMissingModuleSource] +from ._attributes import Attribute, Attributes from ._obstore import * # noqa: F403 # pyright:ignore[reportMissingModuleSource] if TYPE_CHECKING: @@ -9,3 +10,4 @@ __all__ = ["exceptions", "store"] __all__ += _obstore.__all__ +__all__ += ["Attribute", "Attributes"] diff --git a/obstore/python/obstore/_attributes.pyi b/obstore/python/obstore/_attributes.py similarity index 100% rename from obstore/python/obstore/_attributes.pyi rename to obstore/python/obstore/_attributes.py From 487f899f845c01666ef4ace809a70f08fc57bba0 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Mon, 19 Jan 2026 12:49:39 -0500 Subject: [PATCH 2/8] remove comments --- obstore/python/obstore/_attributes.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/obstore/python/obstore/_attributes.py b/obstore/python/obstore/_attributes.py index 27a1fa88..9db1dc06 100644 --- a/obstore/python/obstore/_attributes.py +++ b/obstore/python/obstore/_attributes.py @@ -41,17 +41,6 @@ See [Cache-Control](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control). Any other string key specifies a user-defined metadata field for the object. - -!!! warning "Not importable at runtime" - - To use this type hint in your code, import it within a `TYPE_CHECKING` block: - - ```py - from __future__ import annotations - from typing import TYPE_CHECKING - if TYPE_CHECKING: - from obstore import Attribute - ``` """ Attributes: TypeAlias = dict[Attribute, str] @@ -61,15 +50,4 @@ retrieved from [`get`][obstore.get]/[`get_async`][obstore.get_async]. Unlike ObjectMeta, Attributes are not returned by listing APIs - -!!! warning "Not importable at runtime" - - To use this type hint in your code, import it within a `TYPE_CHECKING` block: - - ```py - from __future__ import annotations - from typing import TYPE_CHECKING - if TYPE_CHECKING: - from obstore import Attributes - ``` """ From c49289f57b8aedd0ea60e0c82c2e9cc02faaed80 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Mon, 19 Jan 2026 13:06:03 -0500 Subject: [PATCH 3/8] feat: Importable typed dicts --- obstore/python/obstore/__init__.py | 23 ++++- obstore/python/obstore/_buffered.pyi | 2 +- obstore/python/obstore/_get.pyi | 129 +------------------------- obstore/python/obstore/_get_types.py | 99 ++++++++++++++++++++ obstore/python/obstore/_head.pyi | 2 +- obstore/python/obstore/_list.pyi | 86 +---------------- obstore/python/obstore/_list_types.py | 58 ++++++++++++ obstore/python/obstore/_obstore.pyi | 24 +---- obstore/python/obstore/_put.pyi | 87 +---------------- obstore/python/obstore/_put_types.py | 64 +++++++++++++ obstore/python/obstore/_sign.pyi | 28 +----- obstore/python/obstore/_sign_types.py | 29 ++++++ 12 files changed, 284 insertions(+), 347 deletions(-) create mode 100644 obstore/python/obstore/_get_types.py create mode 100644 obstore/python/obstore/_list_types.py create mode 100644 obstore/python/obstore/_put_types.py create mode 100644 obstore/python/obstore/_sign_types.py diff --git a/obstore/python/obstore/__init__.py b/obstore/python/obstore/__init__.py index eae709ba..d7c4d04c 100644 --- a/obstore/python/obstore/__init__.py +++ b/obstore/python/obstore/__init__.py @@ -2,12 +2,31 @@ from . import _obstore, store # pyright:ignore[reportMissingModuleSource] from ._attributes import Attribute, Attributes +from ._get_types import GetOptions, OffsetRange, SuffixRange +from ._list_types import ListChunkType, ListResult, ObjectMeta from ._obstore import * # noqa: F403 # pyright:ignore[reportMissingModuleSource] +from ._put_types import PutMode, PutResult, UpdateVersion +from ._sign_types import HTTP_METHOD, SignCapableStore if TYPE_CHECKING: from . import exceptions # noqa: TC004 -__all__ = ["exceptions", "store"] +__all__ = [ + "HTTP_METHOD", + "Attribute", + "Attributes", + "GetOptions", + "ListChunkType", + "ListResult", + "ObjectMeta", + "OffsetRange", + "PutMode", + "PutResult", + "SignCapableStore", + "SuffixRange", + "UpdateVersion", + "exceptions", + "store", +] __all__ += _obstore.__all__ -__all__ += ["Attribute", "Attributes"] diff --git a/obstore/python/obstore/_buffered.pyi b/obstore/python/obstore/_buffered.pyi index bd62fc94..94f265f7 100644 --- a/obstore/python/obstore/_buffered.pyi +++ b/obstore/python/obstore/_buffered.pyi @@ -3,7 +3,7 @@ from contextlib import AbstractAsyncContextManager, AbstractContextManager from ._attributes import Attributes from ._bytes import Bytes -from ._list import ObjectMeta +from ._list_types import ObjectMeta from ._store import ObjectStore if sys.version_info >= (3, 11): diff --git a/obstore/python/obstore/_get.pyi b/obstore/python/obstore/_get.pyi index 65189148..bc76591b 100644 --- a/obstore/python/obstore/_get.pyi +++ b/obstore/python/obstore/_get.pyi @@ -1,136 +1,11 @@ from collections.abc import Sequence -from datetime import datetime -from typing import TypedDict from ._attributes import Attributes from ._bytes import Bytes -from ._list import ObjectMeta +from ._get_types import GetOptions +from ._list_types import ObjectMeta from .store import ObjectStore -class OffsetRange(TypedDict): - """Request all bytes starting from a given byte offset. - - !!! warning "Not importable at runtime" - - To use this type hint in your code, import it within a `TYPE_CHECKING` block: - - ```py - from __future__ import annotations - from typing import TYPE_CHECKING - if TYPE_CHECKING: - from obstore import OffsetRange - ``` - """ - - offset: int - """The byte offset for the offset range request.""" - -class SuffixRange(TypedDict): - """Request up to the last `n` bytes. - - !!! warning "Not importable at runtime" - - To use this type hint in your code, import it within a `TYPE_CHECKING` block: - - ```py - from __future__ import annotations - from typing import TYPE_CHECKING - if TYPE_CHECKING: - from obstore import SuffixRange - ``` - """ - - suffix: int - """The number of bytes from the suffix to request.""" - -class GetOptions(TypedDict, total=False): - """Options for a get request. - - All options are optional. - - !!! warning "Not importable at runtime" - - To use this type hint in your code, import it within a `TYPE_CHECKING` block: - - ```py - from __future__ import annotations - from typing import TYPE_CHECKING - if TYPE_CHECKING: - from obstore import GetOptions - ``` - """ - - if_match: str | None - """ - Request will succeed if the `ObjectMeta::e_tag` matches - otherwise returning [`PreconditionError`][obstore.exceptions.PreconditionError]. - See - Examples: - ```text - If-Match: "xyzzy" - If-Match: "xyzzy", "r2d2xxxx", "c3piozzzz" - If-Match: * - ``` - """ - - if_none_match: str | None - """ - Request will succeed if the `ObjectMeta::e_tag` does not match - otherwise returning [`NotModifiedError`][obstore.exceptions.NotModifiedError]. - See - Examples: - ```text - If-None-Match: "xyzzy" - If-None-Match: "xyzzy", "r2d2xxxx", "c3piozzzz" - If-None-Match: * - ``` - """ - - if_unmodified_since: datetime | None - """ - Request will succeed if the object has been modified since - - """ - - if_modified_since: datetime | None - """ - Request will succeed if the object has not been modified since - otherwise returning [`PreconditionError`][obstore.exceptions.PreconditionError]. - Some stores, such as S3, will only return `NotModified` for exact - timestamp matches, instead of for any timestamp greater than or equal. - - """ - - range: tuple[int, int] | Sequence[int] | OffsetRange | SuffixRange - """ - Request transfer of only the specified range of bytes - otherwise returning [`NotModifiedError`][obstore.exceptions.NotModifiedError]. - The semantics of this tuple are: - - `(int, int)`: Request a specific range of bytes `(start, end)`. - If the given range is zero-length or starts after the end of the object, an - error will be returned. Additionally, if the range ends after the end of the - object, the entire remainder of the object will be returned. Otherwise, the - exact requested range will be returned. - The `end` offset is _exclusive_. - - `{"offset": int}`: Request all bytes starting from a given byte offset. - This is equivalent to `bytes={int}-` as an HTTP header. - - `{"suffix": int}`: Request the last `int` bytes. Note that here, `int` is _the - size of the request_, not the byte offset. This is equivalent to `bytes=-{int}` - as an HTTP header. - - """ - - version: str | None - """ - Request a particular object version - """ - - head: bool - """ - Request transfer of no content - - """ - class GetResult: """Result for a get request. diff --git a/obstore/python/obstore/_get_types.py b/obstore/python/obstore/_get_types.py new file mode 100644 index 00000000..91f338fe --- /dev/null +++ b/obstore/python/obstore/_get_types.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, TypedDict + +if TYPE_CHECKING: + from collections.abc import Sequence + from datetime import datetime + + +class OffsetRange(TypedDict): + """Request all bytes starting from a given byte offset.""" + + offset: int + """The byte offset for the offset range request.""" + + +class SuffixRange(TypedDict): + """Request up to the last `n` bytes.""" + + suffix: int + """The number of bytes from the suffix to request.""" + + +class GetOptions(TypedDict, total=False): + """Options for a get request. + + All options are optional. + """ + + if_match: str | None + """ + Request will succeed if the `ObjectMeta::e_tag` matches + otherwise returning [`PreconditionError`][obstore.exceptions.PreconditionError]. + See + Examples: + ```text + If-Match: "xyzzy" + If-Match: "xyzzy", "r2d2xxxx", "c3piozzzz" + If-Match: * + ``` + """ + + if_none_match: str | None + """ + Request will succeed if the `ObjectMeta::e_tag` does not match + otherwise returning [`NotModifiedError`][obstore.exceptions.NotModifiedError]. + See + Examples: + ```text + If-None-Match: "xyzzy" + If-None-Match: "xyzzy", "r2d2xxxx", "c3piozzzz" + If-None-Match: * + ``` + """ + + if_unmodified_since: datetime | None + """ + Request will succeed if the object has been modified since + + """ + + if_modified_since: datetime | None + """ + Request will succeed if the object has not been modified since + otherwise returning [`PreconditionError`][obstore.exceptions.PreconditionError]. + Some stores, such as S3, will only return `NotModified` for exact + timestamp matches, instead of for any timestamp greater than or equal. + + """ + + range: tuple[int, int] | Sequence[int] | OffsetRange | SuffixRange + """ + Request transfer of only the specified range of bytes + otherwise returning [`NotModifiedError`][obstore.exceptions.NotModifiedError]. + The semantics of this tuple are: + - `(int, int)`: Request a specific range of bytes `(start, end)`. + If the given range is zero-length or starts after the end of the object, an + error will be returned. Additionally, if the range ends after the end of the + object, the entire remainder of the object will be returned. Otherwise, the + exact requested range will be returned. + The `end` offset is _exclusive_. + - `{"offset": int}`: Request all bytes starting from a given byte offset. + This is equivalent to `bytes={int}-` as an HTTP header. + - `{"suffix": int}`: Request the last `int` bytes. Note that here, `int` is _the + size of the request_, not the byte offset. This is equivalent to `bytes=-{int}` + as an HTTP header. + + """ + + version: str | None + """ + Request a particular object version + """ + + head: bool + """ + Request transfer of no content + + """ diff --git a/obstore/python/obstore/_head.pyi b/obstore/python/obstore/_head.pyi index eb96f0bc..6fedd92c 100644 --- a/obstore/python/obstore/_head.pyi +++ b/obstore/python/obstore/_head.pyi @@ -1,4 +1,4 @@ -from ._list import ObjectMeta +from ._list_types import ObjectMeta from .store import ObjectStore def head(store: ObjectStore, path: str) -> ObjectMeta: diff --git a/obstore/python/obstore/_list.pyi b/obstore/python/obstore/_list.pyi index 06732447..f5bb7892 100644 --- a/obstore/python/obstore/_list.pyi +++ b/obstore/python/obstore/_list.pyi @@ -1,11 +1,11 @@ # ruff: noqa: A001, UP035 import sys -from datetime import datetime -from typing import Generic, Literal, Sequence, TypedDict, TypeVar, overload +from typing import Generic, Literal, Sequence, overload from arro3.core import RecordBatch, Table +from ._list_types import ListChunkType, ListResult, ObjectMeta from ._store import ObjectStore if sys.version_info >= (3, 11): @@ -13,88 +13,6 @@ if sys.version_info >= (3, 11): else: from typing_extensions import Self -class ObjectMeta(TypedDict): - """The metadata that describes an object. - - !!! warning "Not importable at runtime" - - To use this type hint in your code, import it within a `TYPE_CHECKING` block: - - ```py - from __future__ import annotations - from typing import TYPE_CHECKING - if TYPE_CHECKING: - from obstore import ObjectMeta - ``` - """ - - path: str - """The full path to the object""" - - last_modified: datetime - """The last modified time""" - - size: int - """The size in bytes of the object""" - - e_tag: str | None - """The unique identifier for the object - - """ - - version: str | None - """A version indicator for this object""" - -# We removed constraints here so that ListStream types work even when arro3-core is not -# installed. https://github.com/developmentseed/obstore/issues/572 -ListChunkType = TypeVar("ListChunkType", covariant=True) # noqa: PYI001, PLC0105 -"""The data structure used for holding list results. - -By default, listing APIs return a `list` of [`ObjectMeta`][obstore.ObjectMeta]. However -for improved performance when listing large buckets, you can pass `return_arrow=True`. -Then an [Arrow `RecordBatch`][arro3.core.RecordBatch] will be returned instead, with -columns containing the same information as would be contained in the Python -[`ObjectMeta`][obstore.ObjectMeta]. - -!!! warning "Not importable at runtime" - - To use this type hint in your code, import it within a `TYPE_CHECKING` block: - - ```py - from __future__ import annotations - from typing import TYPE_CHECKING - if TYPE_CHECKING: - from obstore import ListChunkType - ``` -""" - -class ListResult(TypedDict, Generic[ListChunkType]): - """Result of a list call. - - Includes objects, prefixes (directories) and a token for the next set of results. - Individual result sets may be limited to 1,000 objects based on the underlying - object storage's limitations. - - This implements [`obstore.ListResult`][]. - - !!! warning "Not importable at runtime" - - To use this type hint in your code, import it within a `TYPE_CHECKING` block: - - ```py - from __future__ import annotations - from typing import TYPE_CHECKING - if TYPE_CHECKING: - from obstore import ListResult - ``` - """ - - common_prefixes: Sequence[str] - """Prefixes that are common (like directories)""" - - objects: ListChunkType - """Object metadata for the listing""" - class ListStream(Generic[ListChunkType]): """A stream of [ObjectMeta][obstore.ObjectMeta] that can be polled in a sync or async fashion. diff --git a/obstore/python/obstore/_list_types.py b/obstore/python/obstore/_list_types.py new file mode 100644 index 00000000..e6f474ad --- /dev/null +++ b/obstore/python/obstore/_list_types.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +# ruff: noqa: UP035 +from typing import TYPE_CHECKING, Generic, Sequence, TypedDict, TypeVar + +if TYPE_CHECKING: + from datetime import datetime + + +class ObjectMeta(TypedDict): + """The metadata that describes an object.""" + + path: str + """The full path to the object""" + + last_modified: datetime + """The last modified time""" + + size: int + """The size in bytes of the object""" + + e_tag: str | None + """The unique identifier for the object + + """ + + version: str | None + """A version indicator for this object""" + + +# We removed constraints here so that ListStream types work even when arro3-core is not +# installed. https://github.com/developmentseed/obstore/issues/572 +ListChunkType = TypeVar("ListChunkType", covariant=True) # noqa: PLC0105 +"""The data structure used for holding list results. + +By default, listing APIs return a `list` of [`ObjectMeta`][obstore.ObjectMeta]. However +for improved performance when listing large buckets, you can pass `return_arrow=True`. +Then an [Arrow `RecordBatch`][arro3.core.RecordBatch] will be returned instead, with +columns containing the same information as would be contained in the Python +[`ObjectMeta`][obstore.ObjectMeta]. +""" + + +class ListResult(TypedDict, Generic[ListChunkType]): + """Result of a list call. + + Includes objects, prefixes (directories) and a token for the next set of results. + Individual result sets may be limited to 1,000 objects based on the underlying + object storage's limitations. + + This implements [`obstore.ListResult`][]. + """ + + common_prefixes: Sequence[str] + """Prefixes that are common (like directories)""" + + objects: ListChunkType + """Object metadata for the listing""" diff --git a/obstore/python/obstore/_obstore.pyi b/obstore/python/obstore/_obstore.pyi index c9b1f925..1708298f 100644 --- a/obstore/python/obstore/_obstore.pyi +++ b/obstore/python/obstore/_obstore.pyi @@ -1,5 +1,4 @@ from . import _store -from ._attributes import Attribute, Attributes from ._buffered import ( AsyncReadableFile, AsyncWritableFile, @@ -15,10 +14,7 @@ from ._copy import copy, copy_async from ._delete import delete, delete_async from ._get import ( BytesStream, - GetOptions, GetResult, - OffsetRange, - SuffixRange, get, get_async, get_range, @@ -28,44 +24,28 @@ from ._get import ( ) from ._head import head, head_async from ._list import ( - ListChunkType, - ListResult, ListStream, - ObjectMeta, list, # noqa: A004 list_with_delimiter, list_with_delimiter_async, ) -from ._put import PutMode, PutResult, UpdateVersion, put, put_async +from ._put import put, put_async from ._rename import rename, rename_async from ._scheme import parse_scheme -from ._sign import HTTP_METHOD, SignCapableStore, sign, sign_async +from ._sign import sign, sign_async __version__: str _object_store_version: str _object_store_source: str __all__ = [ - "HTTP_METHOD", "AsyncReadableFile", "AsyncWritableFile", - "Attribute", - "Attributes", "Bytes", "BytesStream", - "GetOptions", "GetResult", - "ListChunkType", - "ListResult", "ListStream", - "ObjectMeta", - "OffsetRange", - "PutMode", - "PutResult", "ReadableFile", - "SignCapableStore", - "SuffixRange", - "UpdateVersion", "WritableFile", "__version__", "_object_store_source", diff --git a/obstore/python/obstore/_put.pyi b/obstore/python/obstore/_put.pyi index 9174b8f8..40995404 100644 --- a/obstore/python/obstore/_put.pyi +++ b/obstore/python/obstore/_put.pyi @@ -1,100 +1,17 @@ import sys from collections.abc import AsyncIterable, AsyncIterator, Iterable, Iterator from pathlib import Path -from typing import IO, Literal, TypedDict +from typing import IO from ._attributes import Attributes +from ._put_types import PutMode, PutResult from .store import ObjectStore -if sys.version_info >= (3, 10): - from typing import TypeAlias -else: - from typing_extensions import TypeAlias - if sys.version_info >= (3, 12): from collections.abc import Buffer else: from typing_extensions import Buffer -class UpdateVersion(TypedDict, total=False): - """Uniquely identifies a version of an object to update. - - Stores will use differing combinations of `e_tag` and `version` to provide - conditional updates, and it is therefore recommended applications preserve both - - !!! warning "Not importable at runtime" - - To use this type hint in your code, import it within a `TYPE_CHECKING` block: - - ```py - from __future__ import annotations - from typing import TYPE_CHECKING - if TYPE_CHECKING: - from obstore import UpdateVersion - ``` - """ - - e_tag: str | None - """The unique identifier for the newly created object. - - """ - - version: str | None - """A version indicator for the newly created object.""" - -PutMode: TypeAlias = Literal["create", "overwrite"] | UpdateVersion -"""Configure preconditions for the put operation -There are three modes: -- Overwrite: Perform an atomic write operation, overwriting any object present at the - provided path. -- Create: Perform an atomic write operation, returning - [`AlreadyExistsError`][obstore.exceptions.AlreadyExistsError] if an object already - exists at the provided path. -- Update: Perform an atomic write operation if the current version of the object matches - the provided [`UpdateVersion`][obstore.UpdateVersion], returning - [`PreconditionError`][obstore.exceptions.PreconditionError] otherwise. -If a string is provided, it must be one of: -- `"overwrite"` -- `"create"` -If a `dict` is provided, it must meet the criteria of -[`UpdateVersion`][obstore.UpdateVersion]. - -!!! warning "Not importable at runtime" - - To use this type hint in your code, import it within a `TYPE_CHECKING` block: - - ```py - from __future__ import annotations - from typing import TYPE_CHECKING - if TYPE_CHECKING: - from obstore import PutMode - ``` -""" - -class PutResult(TypedDict): - """Result for a put request. - - !!! warning "Not importable at runtime" - - To use this type hint in your code, import it within a `TYPE_CHECKING` block: - - ```py - from __future__ import annotations - from typing import TYPE_CHECKING - if TYPE_CHECKING: - from obstore import PutResult - ``` - """ - - e_tag: str | None - """ - The unique identifier for the newly created object - - """ - - version: str | None - """A version indicator for the newly created object.""" - def put( store: ObjectStore, path: str, diff --git a/obstore/python/obstore/_put_types.py b/obstore/python/obstore/_put_types.py new file mode 100644 index 00000000..6bc31056 --- /dev/null +++ b/obstore/python/obstore/_put_types.py @@ -0,0 +1,64 @@ +"""Importable type hints for _put.""" + +from __future__ import annotations + +import sys +from typing import Literal, TypedDict + +if sys.version_info >= (3, 10): + from typing import TypeAlias +else: + from typing_extensions import TypeAlias + + +class UpdateVersion(TypedDict, total=False): + """Uniquely identifies a version of an object to update. + + Stores will use differing combinations of `e_tag` and `version` to provide + conditional updates, and it is therefore recommended applications preserve both + """ + + e_tag: str | None + """The unique identifier for the newly created object. + + """ + + version: str | None + """A version indicator for the newly created object.""" + + +PutMode: TypeAlias = Literal["create", "overwrite"] | UpdateVersion +"""Configure preconditions for the put operation. + +There are three modes: + +- Overwrite: Perform an atomic write operation, overwriting any object present at the + provided path. +- Create: Perform an atomic write operation, returning + [`AlreadyExistsError`][obstore.exceptions.AlreadyExistsError] if an object already + exists at the provided path. +- Update: Perform an atomic write operation if the current version of the object matches + the provided [`UpdateVersion`][obstore.UpdateVersion], returning + [`PreconditionError`][obstore.exceptions.PreconditionError] otherwise. + +If a string is provided, it must be one of: + +- `"overwrite"` +- `"create"` + +If a `dict` is provided, it must meet the criteria of +[`UpdateVersion`][obstore.UpdateVersion]. +""" + + +class PutResult(TypedDict): + """Result for a put request.""" + + e_tag: str | None + """ + The unique identifier for the newly created object + + """ + + version: str | None + """A version indicator for the newly created object.""" diff --git a/obstore/python/obstore/_sign.pyi b/obstore/python/obstore/_sign.pyi index 8bbbf4b8..fa3a89cc 100644 --- a/obstore/python/obstore/_sign.pyi +++ b/obstore/python/obstore/_sign.pyi @@ -1,30 +1,8 @@ -import sys from collections.abc import Sequence from datetime import timedelta -from typing import Literal, overload +from typing import overload -from .store import AzureStore, GCSStore, S3Store - -if sys.version_info >= (3, 10): - from typing import TypeAlias -else: - from typing_extensions import TypeAlias - -HTTP_METHOD: TypeAlias = Literal[ - "GET", - "PUT", - "POST", - "HEAD", - "PATCH", - "TRACE", - "DELETE", - "OPTIONS", - "CONNECT", -] -"""Allowed HTTP Methods for signing.""" - -SignCapableStore: TypeAlias = AzureStore | GCSStore | S3Store -"""ObjectStore instances that are capable of signing.""" +from ._sign_types import HTTP_METHOD, SignCapableStore @overload def sign( @@ -61,7 +39,7 @@ def sign( # type: ignore[misc] # docstring in pyi file expires_in: How long the signed URL(s) should be valid. Returns: - _description_ + Signed URL """ diff --git a/obstore/python/obstore/_sign_types.py b/obstore/python/obstore/_sign_types.py new file mode 100644 index 00000000..22831777 --- /dev/null +++ b/obstore/python/obstore/_sign_types.py @@ -0,0 +1,29 @@ +"""Importable type hints for _sign.""" + +from __future__ import annotations + +import sys +from typing import Literal + +from .store import AzureStore, GCSStore, S3Store + +if sys.version_info >= (3, 10): + from typing import TypeAlias +else: + from typing_extensions import TypeAlias + +HTTP_METHOD: TypeAlias = Literal[ + "GET", + "PUT", + "POST", + "HEAD", + "PATCH", + "TRACE", + "DELETE", + "OPTIONS", + "CONNECT", +] +"""Allowed HTTP Methods for signing.""" + +SignCapableStore: TypeAlias = AzureStore | GCSStore | S3Store +"""ObjectStore instances that are capable of signing.""" From b36881b42348980a580f044b8da858c12075fe5a Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Mon, 19 Jan 2026 13:07:24 -0500 Subject: [PATCH 4/8] Add test --- tests/test_importable_type_hints.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 tests/test_importable_type_hints.py diff --git a/tests/test_importable_type_hints.py b/tests/test_importable_type_hints.py new file mode 100644 index 00000000..1260d10d --- /dev/null +++ b/tests/test_importable_type_hints.py @@ -0,0 +1,11 @@ +def test_typed_dicts_should_be_importable(): + from obstore import ( + Attribute, # noqa: F401 + Attributes, # noqa: F401 + GetOptions, # noqa: F401 + ListChunkType, # noqa: F401 + ListResult, # noqa: F401 + ObjectMeta, # noqa: F401 + OffsetRange, # noqa: F401 + SuffixRange, # noqa: F401 + ) From 19a3dcb40358b22e2f1efb920d403ffe2bbbd128 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Mon, 19 Jan 2026 13:37:44 -0500 Subject: [PATCH 5/8] Move GetResult and BytesStream to be importable protocols --- obstore/python/obstore/__init__.py | 4 +- obstore/python/obstore/_get.pyi | 160 +------------------------ obstore/python/obstore/_get_types.py | 171 ++++++++++++++++++++++++++- obstore/python/obstore/store.py | 2 +- tests/test_importable_type_hints.py | 2 + 5 files changed, 177 insertions(+), 162 deletions(-) diff --git a/obstore/python/obstore/__init__.py b/obstore/python/obstore/__init__.py index d7c4d04c..4d8a6a93 100644 --- a/obstore/python/obstore/__init__.py +++ b/obstore/python/obstore/__init__.py @@ -2,7 +2,7 @@ from . import _obstore, store # pyright:ignore[reportMissingModuleSource] from ._attributes import Attribute, Attributes -from ._get_types import GetOptions, OffsetRange, SuffixRange +from ._get_types import BytesStream, GetOptions, GetResult, OffsetRange, SuffixRange from ._list_types import ListChunkType, ListResult, ObjectMeta from ._obstore import * # noqa: F403 # pyright:ignore[reportMissingModuleSource] from ._put_types import PutMode, PutResult, UpdateVersion @@ -16,7 +16,9 @@ "HTTP_METHOD", "Attribute", "Attributes", + "BytesStream", "GetOptions", + "GetResult", "ListChunkType", "ListResult", "ObjectMeta", diff --git a/obstore/python/obstore/_get.pyi b/obstore/python/obstore/_get.pyi index bc76591b..4f36d665 100644 --- a/obstore/python/obstore/_get.pyi +++ b/obstore/python/obstore/_get.pyi @@ -1,167 +1,9 @@ from collections.abc import Sequence -from ._attributes import Attributes from ._bytes import Bytes -from ._get_types import GetOptions -from ._list_types import ObjectMeta +from ._get_types import GetOptions, GetResult from .store import ObjectStore -class GetResult: - """Result for a get request. - - You can materialize the entire buffer by using either `bytes` or `bytes_async`, or - you can stream the result using `stream`. `__iter__` and `__aiter__` are implemented - as aliases to `stream`, so you can alternatively call `iter()` or `aiter()` on - `GetResult` to start an iterator. - - Using as an async iterator: - ```py - resp = await obs.get_async(store, path) - # 5MB chunk size in stream - stream = resp.stream(min_chunk_size=5 * 1024 * 1024) - async for buf in stream: - print(len(buf)) - ``` - - Using as a sync iterator: - ```py - resp = obs.get(store, path) - # 20MB chunk size in stream - stream = resp.stream(min_chunk_size=20 * 1024 * 1024) - for buf in stream: - print(len(buf)) - ``` - - !!! warning "Not importable at runtime" - - To use this type hint in your code, import it within a `TYPE_CHECKING` block: - - ```py - from __future__ import annotations - from typing import TYPE_CHECKING - if TYPE_CHECKING: - from obstore import GetResult - ``` - """ - - @property - def attributes(self) -> Attributes: - """Additional object attributes.""" - - def bytes(self) -> Bytes: - """Collect the data into a `Bytes` object. - - This implements the Python buffer protocol. You can copy the buffer to Python - memory by passing to [`bytes`][]. - """ - - async def bytes_async(self) -> Bytes: - """Collect the data into a `Bytes` object. - - This implements the Python buffer protocol. You can copy the buffer to Python - memory by passing to [`bytes`][]. - """ - - def buffer(self) -> Bytes: - """Collect the data into a `Bytes` object. - - This is an alias of the [`bytes()`][obstore.GetResult.bytes] method to comply - with the [`obspec.Get`][] protocol. - """ - - async def buffer_async(self) -> Bytes: - """Collect the data into a `Bytes` object. - - This is an alias of the [`bytes_async()`][obstore.GetResult.bytes_async] method - to comply with the [`obspec.GetAsync`][] protocol. - """ - - @property - def meta(self) -> ObjectMeta: - """The ObjectMeta for this object.""" - - @property - def range(self) -> tuple[int, int]: - """The range of bytes returned by this request. - - Note that this is `(start, stop)` **not** `(start, length)`. - """ - - def stream(self, min_chunk_size: int = 10 * 1024 * 1024) -> BytesStream: - r"""Return a chunked stream over the result's bytes. - - Args: - min_chunk_size: The minimum size in bytes for each chunk in the returned - `BytesStream`. All chunks except for the last chunk will be at least - this size. Defaults to 10\*1024\*1024 (10MB). - - Returns: - A chunked stream - - """ - - def __aiter__(self) -> BytesStream: - """Return a chunked stream over the result's bytes. - - Uses the default (10MB) chunk size. - """ - - def __iter__(self) -> BytesStream: - """Return a chunked stream over the result's bytes. - - Uses the default (10MB) chunk size. - """ - -class BytesStream: - """An async stream of bytes. - - !!! note "Request timeouts" - The underlying stream needs to stay alive until the last chunk is polled. If the - file is large, it may exceed the default timeout of 30 seconds. In this case, - you may see an error like: - - ``` - GenericError: Generic { - store: "HTTP", - source: reqwest::Error { - kind: Decode, - source: reqwest::Error { - kind: Body, - source: TimedOut, - }, - }, - } - ``` - - To fix this, set the `timeout` parameter in the - [`client_options`][obstore.store.ClientConfig] passed when creating the store. - - !!! warning "Not importable at runtime" - - To use this type hint in your code, import it within a `TYPE_CHECKING` block: - - ```py - from __future__ import annotations - from typing import TYPE_CHECKING - if TYPE_CHECKING: - from obstore import BytesStream - ``` - """ - - def __aiter__(self) -> BytesStream: - """Return `Self` as an async iterator.""" - - def __iter__(self) -> BytesStream: - """Return `Self` as an async iterator.""" - - # Note: this returns bytes, not Bytes - async def __anext__(self) -> bytes: - """Return the next chunk of bytes in the stream.""" - - # Note: this returns bytes, not Bytes - def __next__(self) -> bytes: - """Return the next chunk of bytes in the stream.""" - def get( store: ObjectStore, path: str, diff --git a/obstore/python/obstore/_get_types.py b/obstore/python/obstore/_get_types.py index 91f338fe..62723392 100644 --- a/obstore/python/obstore/_get_types.py +++ b/obstore/python/obstore/_get_types.py @@ -1,11 +1,15 @@ from __future__ import annotations -from typing import TYPE_CHECKING, TypedDict +from typing import TYPE_CHECKING, Protocol, TypedDict if TYPE_CHECKING: from collections.abc import Sequence from datetime import datetime + from ._attributes import Attributes + from ._bytes import Bytes + from ._list_types import ObjectMeta + class OffsetRange(TypedDict): """Request all bytes starting from a given byte offset.""" @@ -97,3 +101,168 @@ class GetOptions(TypedDict, total=False): Request transfer of no content """ + + +# Note: the public API exposes a Protocol, not the literal GetResult class exported from +# Rust because we don't want users to rely on nominal subtyping. +class GetResult(Protocol): + """Result for a get request. + + You can materialize the entire buffer by using either `bytes` or `bytes_async`, or + you can stream the result using `stream`. `__iter__` and `__aiter__` are implemented + as aliases to `stream`, so you can alternatively call `iter()` or `aiter()` on + `GetResult` to start an iterator. + + Using as an async iterator: + ```py + resp = await obs.get_async(store, path) + # 5MB chunk size in stream + stream = resp.stream(min_chunk_size=5 * 1024 * 1024) + async for buf in stream: + print(len(buf)) + ``` + + Using as a sync iterator: + ```py + resp = obs.get(store, path) + # 20MB chunk size in stream + stream = resp.stream(min_chunk_size=20 * 1024 * 1024) + for buf in stream: + print(len(buf)) + ``` + """ + + @property + def attributes(self) -> Attributes: + """Additional object attributes.""" + ... + + def bytes(self) -> Bytes: + """Collect the data into a `Bytes` object. + + This implements the Python buffer protocol. You can copy the buffer to Python + memory by passing to [`bytes`][]. + """ + ... + + async def bytes_async(self) -> Bytes: + """Collect the data into a `Bytes` object. + + This implements the Python buffer protocol. You can copy the buffer to Python + memory by passing to [`bytes`][]. + """ + ... + + def buffer(self) -> Bytes: + """Collect the data into a `Bytes` object. + + This is an alias of the [`bytes()`][obstore.GetResult.bytes] method to comply + with the [`obspec.Get`][] protocol. + """ + ... + + async def buffer_async(self) -> Bytes: + """Collect the data into a `Bytes` object. + + This is an alias of the [`bytes_async()`][obstore.GetResult.bytes_async] method + to comply with the [`obspec.GetAsync`][] protocol. + """ + ... + + @property + def meta(self) -> ObjectMeta: + """The ObjectMeta for this object.""" + ... + + @property + def range(self) -> tuple[int, int]: + """The range of bytes returned by this request. + + Note that this is `(start, stop)` **not** `(start, length)`. + """ + ... + + def stream(self, min_chunk_size: int = 10 * 1024 * 1024) -> BytesStream: + r"""Return a chunked stream over the result's bytes. + + Args: + min_chunk_size: The minimum size in bytes for each chunk in the returned + `BytesStream`. All chunks except for the last chunk will be at least + this size. Defaults to 10\*1024\*1024 (10MB). + + Returns: + A chunked stream + + """ + ... + + def __aiter__(self) -> BytesStream: + """Return a chunked stream over the result's bytes. + + Uses the default (10MB) chunk size. + """ + ... + + def __iter__(self) -> BytesStream: + """Return a chunked stream over the result's bytes. + + Uses the default (10MB) chunk size. + """ + ... + + +# Note: the public API exposes a Protocol, not the literal GetResult class exported from +# Rust because we don't want users to rely on nominal subtyping. +class BytesStream(Protocol): + """An async stream of bytes. + + !!! note "Request timeouts" + The underlying stream needs to stay alive until the last chunk is polled. If the + file is large, it may exceed the default timeout of 30 seconds. In this case, + you may see an error like: + + ``` + GenericError: Generic { + store: "HTTP", + source: reqwest::Error { + kind: Decode, + source: reqwest::Error { + kind: Body, + source: TimedOut, + }, + }, + } + ``` + + To fix this, set the `timeout` parameter in the + [`client_options`][obstore.store.ClientConfig] passed when creating the store. + + !!! warning "Not importable at runtime" + + To use this type hint in your code, import it within a `TYPE_CHECKING` block: + + ```py + from __future__ import annotations + from typing import TYPE_CHECKING + if TYPE_CHECKING: + from obstore import BytesStream + ``` + """ + + def __aiter__(self) -> BytesStream: + """Return `Self` as an async iterator.""" + ... + + def __iter__(self) -> BytesStream: + """Return `Self` as an async iterator.""" + ... + + # Note: this returns bytes, not Bytes + async def __anext__(self) -> bytes: + """Return the next chunk of bytes in the stream.""" + ... + + # Note: this returns bytes, not Bytes + def __next__(self) -> bytes: + """Return the next chunk of bytes in the stream.""" + ... diff --git a/obstore/python/obstore/store.py b/obstore/python/obstore/store.py index 4d6823ec..f3ee2332 100644 --- a/obstore/python/obstore/store.py +++ b/obstore/python/obstore/store.py @@ -28,6 +28,7 @@ from obstore import ( Attributes, GetOptions, + GetResult, ListResult, ListStream, ObjectMeta, @@ -36,7 +37,6 @@ ) from obstore._obstore import ( # pyright:ignore[reportMissingModuleSource] Bytes, - GetResult, ) from obstore._store import ( AzureAccessKey, # noqa: TC004 diff --git a/tests/test_importable_type_hints.py b/tests/test_importable_type_hints.py index 1260d10d..1108052b 100644 --- a/tests/test_importable_type_hints.py +++ b/tests/test_importable_type_hints.py @@ -2,7 +2,9 @@ def test_typed_dicts_should_be_importable(): from obstore import ( Attribute, # noqa: F401 Attributes, # noqa: F401 + BytesStream, # noqa: F401 GetOptions, # noqa: F401 + GetResult, # noqa: F401 ListChunkType, # noqa: F401 ListResult, # noqa: F401 ObjectMeta, # noqa: F401 From f143b7b366aa5c5136616173560f9198ac844f78 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Mon, 19 Jan 2026 14:01:22 -0500 Subject: [PATCH 6/8] Export protocol type classes --- obstore/python/obstore/__init__.py | 15 +- obstore/python/obstore/_buffered.pyi | 252 +--------------------- obstore/python/obstore/_buffered_types.py | 249 +++++++++++++++++++++ obstore/python/obstore/_bytes.py | 1 + obstore/python/obstore/_bytes.pyi | 1 - obstore/python/obstore/_get_types.py | 13 +- obstore/python/obstore/_list.pyi | 54 +---- obstore/python/obstore/_list_types.py | 50 ++++- obstore/python/obstore/_obstore.pyi | 16 -- obstore/python/obstore/store.py | 4 +- pyo3-bytes/{bytes.pyi => bytes.py} | 69 ++++-- tests/test_importable_type_hints.py | 13 +- 12 files changed, 383 insertions(+), 354 deletions(-) create mode 100644 obstore/python/obstore/_buffered_types.py create mode 120000 obstore/python/obstore/_bytes.py delete mode 120000 obstore/python/obstore/_bytes.pyi rename pyo3-bytes/{bytes.pyi => bytes.py} (76%) diff --git a/obstore/python/obstore/__init__.py b/obstore/python/obstore/__init__.py index 4d8a6a93..7e4456a0 100644 --- a/obstore/python/obstore/__init__.py +++ b/obstore/python/obstore/__init__.py @@ -2,8 +2,15 @@ from . import _obstore, store # pyright:ignore[reportMissingModuleSource] from ._attributes import Attribute, Attributes +from ._buffered_types import ( + AsyncReadableFile, + AsyncWritableFile, + ReadableFile, + WritableFile, +) +from ._bytes import Bytes from ._get_types import BytesStream, GetOptions, GetResult, OffsetRange, SuffixRange -from ._list_types import ListChunkType, ListResult, ObjectMeta +from ._list_types import ListChunkType, ListResult, ListStream, ObjectMeta from ._obstore import * # noqa: F403 # pyright:ignore[reportMissingModuleSource] from ._put_types import PutMode, PutResult, UpdateVersion from ._sign_types import HTTP_METHOD, SignCapableStore @@ -14,20 +21,26 @@ __all__ = [ "HTTP_METHOD", + "AsyncReadableFile", + "AsyncWritableFile", "Attribute", "Attributes", + "Bytes", "BytesStream", "GetOptions", "GetResult", "ListChunkType", "ListResult", + "ListStream", "ObjectMeta", "OffsetRange", "PutMode", "PutResult", + "ReadableFile", "SignCapableStore", "SuffixRange", "UpdateVersion", + "WritableFile", "exceptions", "store", ] diff --git a/obstore/python/obstore/_buffered.pyi b/obstore/python/obstore/_buffered.pyi index 94f265f7..11b7f38a 100644 --- a/obstore/python/obstore/_buffered.pyi +++ b/obstore/python/obstore/_buffered.pyi @@ -1,21 +1,12 @@ -import sys -from contextlib import AbstractAsyncContextManager, AbstractContextManager - from ._attributes import Attributes -from ._bytes import Bytes -from ._list_types import ObjectMeta +from ._buffered_types import ( + AsyncReadableFile, + AsyncWritableFile, + ReadableFile, + WritableFile, +) from ._store import ObjectStore -if sys.version_info >= (3, 11): - from typing import Self -else: - from typing_extensions import Self - -if sys.version_info >= (3, 12): - from collections.abc import Buffer -else: - from typing_extensions import Buffer - def open_reader( store: ObjectStore, path: str, @@ -47,170 +38,6 @@ async def open_reader_async( Refer to the documentation for [open_reader][obstore.open_reader]. """ -class ReadableFile: - """A synchronous-buffered reader that implements a similar interface as a Python - [`BufferedReader`][io.BufferedReader]. - - Internally this maintains a buffer of the requested size, and uses - [`get_range`][obstore.get_range] to populate its internal buffer once depleted. This - buffer is cleared on seek. - - Whilst simple, this interface will typically be outperformed by the native `obstore` - methods that better map to the network APIs. This is because most object stores have - very [high first-byte latencies], on the order of 100-200ms, and so avoiding - unnecessary round-trips is critical to throughput. - - Systems looking to sequentially scan a file should instead consider using - [`get`][obstore.get], or [`get_range`][obstore.get_range] to read a particular - range. - - Systems looking to read multiple ranges of a file should instead consider using - [`get_ranges`][obstore.get_ranges], which will optimise the vectored IO. - - [high first-byte latencies]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/optimizing-performance.html - - !!! warning "Not importable at runtime" - - To use this type hint in your code, import it within a `TYPE_CHECKING` block: - - ```py - from __future__ import annotations - from typing import TYPE_CHECKING - if TYPE_CHECKING: - from obstore import ReadableFile - ``` - """ # noqa: D205 - - def close(self) -> None: - """Close the current file. - - This is currently a no-op. - """ - - @property - def meta(self) -> ObjectMeta: - """Access the metadata of the underlying file.""" - - def read(self, size: int | None = None, /) -> Bytes: - """Read up to `size` bytes from the object and return them. - - As a convenience, if size is unspecified or `None`, all bytes until EOF are - returned. - """ - - def readall(self) -> Bytes: - """Read and return all the bytes from the stream until EOF.""" - - def readline(self) -> Bytes: - """Read a single line of the file, up until the next newline character.""" - - def readlines(self, hint: int = -1, /) -> list[Bytes]: - """Read all remaining lines into a list of buffers.""" - - def seek(self, offset: int, whence: int = ..., /) -> int: - """Change the stream position. - - Change the stream position to the given byte `offset`, interpreted relative to - the position indicated by `whence`, and return the new absolute position. Values - for `whence` are: - - - [`os.SEEK_SET`][] or 0: start of the stream (the default); `offset` should be zero or positive - - [`os.SEEK_CUR`][] or 1: current stream position; `offset` may be negative - - [`os.SEEK_END`][] or 2: end of the stream; `offset` is usually negative - """ - - def seekable(self) -> bool: - """Return True if the stream supports random access.""" - - @property - def size(self) -> int: - """The size in bytes of the object.""" - - def tell(self) -> int: - """Return the current stream position.""" - -class AsyncReadableFile: - """An async-buffered reader that implements a similar interface as a Python - [`BufferedReader`][io.BufferedReader]. - - Internally this maintains a buffer of the requested size, and uses - [`get_range`][obstore.get_range] to populate its internal buffer once depleted. This - buffer is cleared on seek. - - Whilst simple, this interface will typically be outperformed by the native `obstore` - methods that better map to the network APIs. This is because most object stores have - very [high first-byte latencies], on the order of 100-200ms, and so avoiding - unnecessary round-trips is critical to throughput. - - Systems looking to sequentially scan a file should instead consider using - [`get`][obstore.get], or [`get_range`][obstore.get_range] to read a particular - range. - - Systems looking to read multiple ranges of a file should instead consider using - [`get_ranges`][obstore.get_ranges], which will optimise the vectored IO. - - [high first-byte latencies]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/optimizing-performance.html - - !!! warning "Not importable at runtime" - - To use this type hint in your code, import it within a `TYPE_CHECKING` block: - - ```py - from __future__ import annotations - from typing import TYPE_CHECKING - if TYPE_CHECKING: - from obstore import AsyncReadableFile - ``` - """ # noqa: D205 - - def close(self) -> None: - """Close the current file. - - This is currently a no-op. - """ - - @property - def meta(self) -> ObjectMeta: - """Access the metadata of the underlying file.""" - - async def read(self, size: int | None = None, /) -> Bytes: - """Read up to `size` bytes from the object and return them. - - As a convenience, if size is unspecified or `None`, all bytes until EOF are - returned. - """ - - async def readall(self) -> Bytes: - """Read and return all the bytes from the stream until EOF.""" - - async def readline(self) -> Bytes: - """Read a single line of the file, up until the next newline character.""" - - async def readlines(self, hint: int = -1, /) -> list[Bytes]: - """Read all remaining lines into a list of buffers.""" - - async def seek(self, offset: int, whence: int = ..., /) -> int: - """Change the stream position. - - Change the stream position to the given byte `offset`, interpreted relative to - the position indicated by `whence`, and return the new absolute position. Values - for `whence` are: - - - [`os.SEEK_SET`][] or 0: start of the stream (the default); `offset` should be zero or positive - - [`os.SEEK_CUR`][] or 1: current stream position; `offset` may be negative - - [`os.SEEK_END`][] or 2: end of the stream; `offset` is usually negative - """ - - def seekable(self) -> bool: - """Return True if the stream supports random access.""" - - @property - def size(self) -> int: - """The size in bytes of the object.""" - - async def tell(self) -> int: - """Return the current stream position.""" - def open_writer( store: ObjectStore, path: str, @@ -250,70 +77,3 @@ def open_writer_async( Refer to the documentation for [open_writer][obstore.open_writer]. """ - -class WritableFile(AbstractContextManager): - """A buffered writable file object with synchronous operations. - - This implements a similar interface as a Python - [`BufferedWriter`][io.BufferedWriter]. - - !!! warning "Not importable at runtime" - - To use this type hint in your code, import it within a `TYPE_CHECKING` block: - - ```py - from __future__ import annotations - from typing import TYPE_CHECKING - if TYPE_CHECKING: - from obstore import WritableFile - ``` - """ - - def __enter__(self) -> Self: ... - def __exit__(self, exc_type, exc_value, traceback) -> None: ... # noqa: ANN001 - def close(self) -> None: - """Close the current file.""" - - def closed(self) -> bool: - """Check whether this file has been closed. - - Note that this is a method, not an attribute. - """ - - def flush(self) -> None: - """Flushes this output stream, ensuring that all intermediately buffered contents reach their destination.""" - - def write(self, buffer: bytes | Buffer, /) -> int: - """Write the [bytes-like object](https://docs.python.org/3/glossary.html#term-bytes-like-object), `buffer`, and return the number of bytes written.""" - -class AsyncWritableFile(AbstractAsyncContextManager): - """A buffered writable file object with **asynchronous** operations. - - !!! warning "Not importable at runtime" - - To use this type hint in your code, import it within a `TYPE_CHECKING` block: - - ```py - from __future__ import annotations - from typing import TYPE_CHECKING - if TYPE_CHECKING: - from obstore import AsyncWritableFile - ``` - """ - - async def __aenter__(self) -> Self: ... - async def __aexit__(self, exc_type, exc_value, traceback) -> None: ... # noqa: ANN001 - async def close(self) -> None: - """Close the current file.""" - - async def closed(self) -> bool: - """Check whether this file has been closed. - - Note that this is an async method, not an attribute. - """ - - async def flush(self) -> None: - """Flushes this output stream, ensuring that all intermediately buffered contents reach their destination.""" - - async def write(self, buffer: bytes | Buffer, /) -> int: - """Write the [bytes-like object](https://docs.python.org/3/glossary.html#term-bytes-like-object), `buffer`, and return the number of bytes written.""" diff --git a/obstore/python/obstore/_buffered_types.py b/obstore/python/obstore/_buffered_types.py new file mode 100644 index 00000000..0d70c8f7 --- /dev/null +++ b/obstore/python/obstore/_buffered_types.py @@ -0,0 +1,249 @@ +# ruff: noqa: E501 + +from __future__ import annotations + +import sys +from contextlib import AbstractAsyncContextManager, AbstractContextManager +from typing import TYPE_CHECKING, Protocol + +if TYPE_CHECKING: + from ._bytes import Bytes + from ._list_types import ObjectMeta + + +if sys.version_info >= (3, 11): + from typing import Self +else: + from typing_extensions import Self + +if sys.version_info >= (3, 12): + from collections.abc import Buffer +else: + from typing_extensions import Buffer + + +# Note: the public API exposes a Protocol, not the literal class exported from +# Rust because we don't want users to rely on nominal subtyping. +class ReadableFile(Protocol): + """A synchronous-buffered reader that implements a similar interface as a Python + [`BufferedReader`][io.BufferedReader]. + + Internally this maintains a buffer of the requested size, and uses + [`get_range`][obstore.get_range] to populate its internal buffer once depleted. This + buffer is cleared on seek. + + Whilst simple, this interface will typically be outperformed by the native `obstore` + methods that better map to the network APIs. This is because most object stores have + very [high first-byte latencies], on the order of 100-200ms, and so avoiding + unnecessary round-trips is critical to throughput. + + Systems looking to sequentially scan a file should instead consider using + [`get`][obstore.get], or [`get_range`][obstore.get_range] to read a particular + range. + + Systems looking to read multiple ranges of a file should instead consider using + [`get_ranges`][obstore.get_ranges], which will optimise the vectored IO. + + [high first-byte latencies]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/optimizing-performance.html + """ # noqa: D205 + + def close(self) -> None: + """Close the current file. + + This is currently a no-op. + """ + + @property + def meta(self) -> ObjectMeta: + """Access the metadata of the underlying file.""" + ... + + def read(self, size: int | None = None, /) -> Bytes: + """Read up to `size` bytes from the object and return them. + + As a convenience, if size is unspecified or `None`, all bytes until EOF are + returned. + """ + ... + + def readall(self) -> Bytes: + """Read and return all the bytes from the stream until EOF.""" + ... + + def readline(self) -> Bytes: + """Read a single line of the file, up until the next newline character.""" + ... + + def readlines(self, hint: int = -1, /) -> list[Bytes]: + """Read all remaining lines into a list of buffers.""" + ... + + def seek(self, offset: int, whence: int = ..., /) -> int: + """Change the stream position. + + Change the stream position to the given byte `offset`, interpreted relative to + the position indicated by `whence`, and return the new absolute position. Values + for `whence` are: + + - [`os.SEEK_SET`][] or 0: start of the stream (the default); `offset` should be + zero or positive + - [`os.SEEK_CUR`][] or 1: current stream position; `offset` may be negative + - [`os.SEEK_END`][] or 2: end of the stream; `offset` is usually negative + """ + ... + + def seekable(self) -> bool: + """Return True if the stream supports random access.""" + ... + + @property + def size(self) -> int: + """The size in bytes of the object.""" + ... + + def tell(self) -> int: + """Return the current stream position.""" + ... + + +# Note: the public API exposes a Protocol, not the literal class exported from +# Rust because we don't want users to rely on nominal subtyping. +class AsyncReadableFile(Protocol): + """An async-buffered reader that implements a similar interface as a Python + [`BufferedReader`][io.BufferedReader]. + + Internally this maintains a buffer of the requested size, and uses + [`get_range`][obstore.get_range] to populate its internal buffer once depleted. This + buffer is cleared on seek. + + Whilst simple, this interface will typically be outperformed by the native `obstore` + methods that better map to the network APIs. This is because most object stores have + very [high first-byte latencies], on the order of 100-200ms, and so avoiding + unnecessary round-trips is critical to throughput. + + Systems looking to sequentially scan a file should instead consider using + [`get`][obstore.get], or [`get_range`][obstore.get_range] to read a particular + range. + + Systems looking to read multiple ranges of a file should instead consider using + [`get_ranges`][obstore.get_ranges], which will optimise the vectored IO. + + [high first-byte latencies]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/optimizing-performance.html + """ # noqa: D205 + + def close(self) -> None: + """Close the current file. + + This is currently a no-op. + """ + + @property + def meta(self) -> ObjectMeta: + """Access the metadata of the underlying file.""" + ... + + async def read(self, size: int | None = None, /) -> Bytes: + """Read up to `size` bytes from the object and return them. + + As a convenience, if size is unspecified or `None`, all bytes until EOF are + returned. + """ + ... + + async def readall(self) -> Bytes: + """Read and return all the bytes from the stream until EOF.""" + ... + + async def readline(self) -> Bytes: + """Read a single line of the file, up until the next newline character.""" + ... + + async def readlines(self, hint: int = -1, /) -> list[Bytes]: + """Read all remaining lines into a list of buffers.""" + ... + + async def seek(self, offset: int, whence: int = ..., /) -> int: + """Change the stream position. + + Change the stream position to the given byte `offset`, interpreted relative to + the position indicated by `whence`, and return the new absolute position. Values + for `whence` are: + + - [`os.SEEK_SET`][] or 0: start of the stream (the default); `offset` should be + zero or positive + - [`os.SEEK_CUR`][] or 1: current stream position; `offset` may be negative + - [`os.SEEK_END`][] or 2: end of the stream; `offset` is usually negative + """ + ... + + def seekable(self) -> bool: + """Return True if the stream supports random access.""" + ... + + @property + def size(self) -> int: + """The size in bytes of the object.""" + ... + + async def tell(self) -> int: + """Return the current stream position.""" + ... + + +# Note: the public API exposes a Protocol, not the literal class exported from +# Rust because we don't want users to rely on nominal subtyping. +class WritableFile(AbstractContextManager, Protocol): + """A buffered writable file object with synchronous operations. + + This implements a similar interface as a Python + [`BufferedWriter`][io.BufferedWriter]. + """ + + def __enter__(self) -> Self: ... + def __exit__(self, exc_type, exc_value, traceback) -> None: ... # noqa: ANN001 + def close(self) -> None: + """Close the current file.""" + + def closed(self) -> bool: + """Check whether this file has been closed. + + Note that this is a method, not an attribute. + """ + ... + + def flush(self) -> None: + """Flushes this output stream, ensuring that all intermediately buffered contents reach their destination.""" + ... + + def write(self, buffer: bytes | Buffer, /) -> int: + """Write the [bytes-like object] `buffer` and return the number of bytes written. + + [bytes-like object]: https://docs.python.org/3/glossary.html#term-bytes-like-object + """ + ... + + +# Note: the public API exposes a Protocol, not the literal class exported from +# Rust because we don't want users to rely on nominal subtyping. +class AsyncWritableFile(AbstractAsyncContextManager, Protocol): + """A buffered writable file object with **asynchronous** operations.""" + + async def __aenter__(self) -> Self: ... + async def __aexit__(self, exc_type, exc_value, traceback) -> None: ... # noqa: ANN001 + async def close(self) -> None: + """Close the current file.""" + + async def closed(self) -> bool: + """Check whether this file has been closed. + + Note that this is an async method, not an attribute. + """ + ... + + async def flush(self) -> None: + """Flushes this output stream, ensuring that all intermediately buffered contents reach their destination.""" + ... + + async def write(self, buffer: bytes | Buffer, /) -> int: + """Write the [bytes-like object](https://docs.python.org/3/glossary.html#term-bytes-like-object), `buffer`, and return the number of bytes written.""" + ... diff --git a/obstore/python/obstore/_bytes.py b/obstore/python/obstore/_bytes.py new file mode 120000 index 00000000..94fdc716 --- /dev/null +++ b/obstore/python/obstore/_bytes.py @@ -0,0 +1 @@ +../../../pyo3-bytes/bytes.py \ No newline at end of file diff --git a/obstore/python/obstore/_bytes.pyi b/obstore/python/obstore/_bytes.pyi deleted file mode 120000 index 4b8c6de8..00000000 --- a/obstore/python/obstore/_bytes.pyi +++ /dev/null @@ -1 +0,0 @@ -../../../pyo3-bytes/bytes.pyi \ No newline at end of file diff --git a/obstore/python/obstore/_get_types.py b/obstore/python/obstore/_get_types.py index 62723392..6cbd9a0a 100644 --- a/obstore/python/obstore/_get_types.py +++ b/obstore/python/obstore/_get_types.py @@ -211,7 +211,7 @@ def __iter__(self) -> BytesStream: ... -# Note: the public API exposes a Protocol, not the literal GetResult class exported from +# Note: the public API exposes a Protocol, not the literal class exported from # Rust because we don't want users to rely on nominal subtyping. class BytesStream(Protocol): """An async stream of bytes. @@ -236,17 +236,6 @@ class BytesStream(Protocol): To fix this, set the `timeout` parameter in the [`client_options`][obstore.store.ClientConfig] passed when creating the store. - - !!! warning "Not importable at runtime" - - To use this type hint in your code, import it within a `TYPE_CHECKING` block: - - ```py - from __future__ import annotations - from typing import TYPE_CHECKING - if TYPE_CHECKING: - from obstore import BytesStream - ``` """ def __aiter__(self) -> BytesStream: diff --git a/obstore/python/obstore/_list.pyi b/obstore/python/obstore/_list.pyi index f5bb7892..b7aaf168 100644 --- a/obstore/python/obstore/_list.pyi +++ b/obstore/python/obstore/_list.pyi @@ -1,62 +1,12 @@ # ruff: noqa: A001, UP035 -import sys -from typing import Generic, Literal, Sequence, overload +from typing import Literal, Sequence, overload from arro3.core import RecordBatch, Table -from ._list_types import ListChunkType, ListResult, ObjectMeta +from ._list_types import ListResult, ListStream, ObjectMeta from ._store import ObjectStore -if sys.version_info >= (3, 11): - from typing import Self -else: - from typing_extensions import Self - -class ListStream(Generic[ListChunkType]): - """A stream of [ObjectMeta][obstore.ObjectMeta] that can be polled in a sync or - async fashion. - - This implements [`obstore.ListStream`][]. - - !!! warning "Not importable at runtime" - - To use this type hint in your code, import it within a `TYPE_CHECKING` block: - - ```py - from __future__ import annotations - from typing import TYPE_CHECKING - if TYPE_CHECKING: - from obstore import ListStream - ``` - """ # noqa: D205 - - def __aiter__(self) -> Self: - """Return `Self` as an async iterator.""" - - def __iter__(self) -> Self: - """Return `Self` as an async iterator.""" - - async def collect_async(self) -> ListChunkType: - """Collect all remaining ObjectMeta objects in the stream. - - This ignores the `chunk_size` parameter from the `list` call and collects all - remaining data into a single chunk. - """ - - def collect(self) -> ListChunkType: - """Collect all remaining ObjectMeta objects in the stream. - - This ignores the `chunk_size` parameter from the `list` call and collects all - remaining data into a single chunk. - """ - - async def __anext__(self) -> ListChunkType: - """Return the next chunk of ObjectMeta in the stream.""" - - def __next__(self) -> ListChunkType: - """Return the next chunk of ObjectMeta in the stream.""" - @overload def list( store: ObjectStore, diff --git a/obstore/python/obstore/_list_types.py b/obstore/python/obstore/_list_types.py index e6f474ad..a081f0e1 100644 --- a/obstore/python/obstore/_list_types.py +++ b/obstore/python/obstore/_list_types.py @@ -1,11 +1,17 @@ from __future__ import annotations # ruff: noqa: UP035 -from typing import TYPE_CHECKING, Generic, Sequence, TypedDict, TypeVar +import sys +from typing import TYPE_CHECKING, Generic, Protocol, Sequence, TypedDict, TypeVar if TYPE_CHECKING: from datetime import datetime +if sys.version_info >= (3, 11): + from typing import Self +else: + from typing_extensions import Self + class ObjectMeta(TypedDict): """The metadata that describes an object.""" @@ -56,3 +62,45 @@ class ListResult(TypedDict, Generic[ListChunkType]): objects: ListChunkType """Object metadata for the listing""" + + +# Note: the public API exposes a Protocol, not the literal class exported from +# Rust because we don't want users to rely on nominal subtyping. +class ListStream(Protocol[ListChunkType]): + """A stream of [ObjectMeta][obstore.ObjectMeta] that can be polled in a sync or + async fashion. + + This implements [`obstore.ListStream`][]. + """ # noqa: D205 + + def __aiter__(self) -> Self: + """Return `Self` as an async iterator.""" + ... + + def __iter__(self) -> Self: + """Return `Self` as an async iterator.""" + ... + + async def collect_async(self) -> ListChunkType: + """Collect all remaining ObjectMeta objects in the stream. + + This ignores the `chunk_size` parameter from the `list` call and collects all + remaining data into a single chunk. + """ + ... + + def collect(self) -> ListChunkType: + """Collect all remaining ObjectMeta objects in the stream. + + This ignores the `chunk_size` parameter from the `list` call and collects all + remaining data into a single chunk. + """ + ... + + async def __anext__(self) -> ListChunkType: + """Return the next chunk of ObjectMeta in the stream.""" + ... + + def __next__(self) -> ListChunkType: + """Return the next chunk of ObjectMeta in the stream.""" + ... diff --git a/obstore/python/obstore/_obstore.pyi b/obstore/python/obstore/_obstore.pyi index 1708298f..8e6344bc 100644 --- a/obstore/python/obstore/_obstore.pyi +++ b/obstore/python/obstore/_obstore.pyi @@ -1,20 +1,13 @@ from . import _store from ._buffered import ( - AsyncReadableFile, - AsyncWritableFile, - ReadableFile, - WritableFile, open_reader, open_reader_async, open_writer, open_writer_async, ) -from ._bytes import Bytes from ._copy import copy, copy_async from ._delete import delete, delete_async from ._get import ( - BytesStream, - GetResult, get, get_async, get_range, @@ -24,7 +17,6 @@ from ._get import ( ) from ._head import head, head_async from ._list import ( - ListStream, list, # noqa: A004 list_with_delimiter, list_with_delimiter_async, @@ -39,14 +31,6 @@ _object_store_version: str _object_store_source: str __all__ = [ - "AsyncReadableFile", - "AsyncWritableFile", - "Bytes", - "BytesStream", - "GetResult", - "ListStream", - "ReadableFile", - "WritableFile", "__version__", "_object_store_source", "_object_store_version", diff --git a/obstore/python/obstore/store.py b/obstore/python/obstore/store.py index f3ee2332..9e8139fd 100644 --- a/obstore/python/obstore/store.py +++ b/obstore/python/obstore/store.py @@ -27,6 +27,7 @@ from obstore import ( Attributes, + Bytes, GetOptions, GetResult, ListResult, @@ -35,9 +36,6 @@ PutMode, PutResult, ) - from obstore._obstore import ( # pyright:ignore[reportMissingModuleSource] - Bytes, - ) from obstore._store import ( AzureAccessKey, # noqa: TC004 AzureBearerToken, # noqa: TC004 diff --git a/pyo3-bytes/bytes.pyi b/pyo3-bytes/bytes.py similarity index 76% rename from pyo3-bytes/bytes.pyi rename to pyo3-bytes/bytes.py index c6aad76d..c32d8664 100644 --- a/pyo3-bytes/bytes.pyi +++ b/pyo3-bytes/bytes.py @@ -1,14 +1,18 @@ -# ruff: noqa: D205 +"""Type hint for `pyo3_bytes.Bytes` objects.""" + +# ruff: noqa: D205, D105 +from __future__ import annotations import sys -from typing import overload +from typing import Protocol, overload if sys.version_info >= (3, 12): from collections.abc import Buffer else: from typing_extensions import Buffer -class Bytes(Buffer): + +class Bytes(Buffer, Protocol): # noqa: PLW1641 """A `bytes`-like buffer. This implements the Python buffer protocol, allowing zero-copy access @@ -17,7 +21,7 @@ class Bytes(Buffer): You can pass this to `memoryview` for a zero-copy view into the underlying data or to `bytes` to copy the underlying data into a Python `bytes`. - Many methods from the Python `bytes` class are implemented on this, + Many methods from the Python `bytes` class are implemented on this. """ def __init__(self, buf: Buffer = b"") -> None: @@ -25,6 +29,7 @@ def __init__(self, buf: Buffer = b"") -> None: This will be a zero-copy view on the Python byte slice. """ + def __add__(self, other: Buffer) -> Bytes: ... def __buffer__(self, flags: int) -> memoryview[int]: ... def __contains__(self, other: Buffer) -> bool: ... @@ -37,66 +42,88 @@ def __getitem__(self, key: int | slice, /) -> int | Bytes: ... # type: ignore[m def __mul__(self, other: Buffer) -> int: ... def __len__(self) -> int: ... def removeprefix(self, prefix: Buffer, /) -> Bytes: - """If the binary data starts with the prefix string, return `bytes[len(prefix):]`. + """If the binary data starts with the prefix string, return + `bytes[len(prefix):]`. + Otherwise, return the original binary data. """ + ... + def removesuffix(self, suffix: Buffer, /) -> Bytes: """If the binary data ends with the suffix string and that suffix is not empty, return `bytes[:-len(suffix)]`. Otherwise, return the original binary data. """ + ... + def isalnum(self) -> bool: - """Return `True` if all bytes in the sequence are alphabetical ASCII characters or - ASCII decimal digits and the sequence is not empty, `False` otherwise. + """Return `True` if all bytes in the sequence are alphabetical ASCII characters + or ASCII decimal digits and the sequence is not empty, `False` otherwise. Alphabetic ASCII characters are those byte values in the sequence `b'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'`. ASCII decimal digits are those byte values in the sequence `b'0123456789'`. """ + ... + def isalpha(self) -> bool: - """Return `True` if all bytes in the sequence are alphabetic ASCII characters and - the sequence is not empty, `False` otherwise. + """Return `True` if all bytes in the sequence are alphabetic ASCII characters + and the sequence is not empty, `False` otherwise. Alphabetic ASCII characters are those byte values in the sequence `b'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'`. """ + ... + def isascii(self) -> bool: - """Return `True` if the sequence is empty or all bytes in the sequence are ASCII, - `False` otherwise. + """Return `True` if the sequence is empty or all bytes in the sequence are + ASCII, `False` otherwise. ASCII bytes are in the range `0-0x7F`. """ + ... + def isdigit(self) -> bool: """Return `True` if all bytes in the sequence are ASCII decimal digits and the sequence is not empty, `False` otherwise. ASCII decimal digits are those byte values in the sequence `b'0123456789'`. """ + ... + def islower(self) -> bool: - """Return `True` if there is at least one lowercase ASCII character in the sequence - and no uppercase ASCII characters, `False` otherwise. + """Return `True` if there is at least one lowercase ASCII character in the + sequence and no uppercase ASCII characters, `False` otherwise. """ + ... + def isspace(self) -> bool: - r"""Return `True` if all bytes in the sequence are ASCII whitespace and the sequence - is not empty, `False` otherwise. + r"""Return `True` if all bytes in the sequence are ASCII whitespace and the + sequence is not empty, `False` otherwise. ASCII whitespace characters are those byte values in the sequence `b' \t\n\r\x0b\f'` (space, tab, newline, carriage return, vertical tab, form feed). """ + ... + def isupper(self) -> bool: - """Return `True` if there is at least one uppercase alphabetic ASCII character in - the sequence and no lowercase ASCII characters, `False` otherwise. + """Return `True` if there is at least one uppercase alphabetic ASCII character + in the sequence and no lowercase ASCII characters, `False` otherwise. """ + ... def lower(self) -> Bytes: - """Return a copy of the sequence with all the uppercase ASCII characters converted - to their corresponding lowercase counterpart. + """Return a copy of the sequence with all the uppercase ASCII characters + converted to their corresponding lowercase counterpart. """ + ... def upper(self) -> Bytes: - """Return a copy of the sequence with all the lowercase ASCII characters converted - to their corresponding uppercase counterpart. + """Return a copy of the sequence with all the lowercase ASCII characters + converted to their corresponding uppercase counterpart. """ + ... def to_bytes(self) -> bytes: """Copy this buffer's contents into a Python `bytes` object.""" + ... diff --git a/tests/test_importable_type_hints.py b/tests/test_importable_type_hints.py index 1108052b..baefe1a8 100644 --- a/tests/test_importable_type_hints.py +++ b/tests/test_importable_type_hints.py @@ -1,13 +1,24 @@ -def test_typed_dicts_should_be_importable(): +def test_types_should_be_importable(): from obstore import ( + HTTP_METHOD, # noqa: F401 + AsyncReadableFile, # noqa: F401 + AsyncWritableFile, # noqa: F401 Attribute, # noqa: F401 Attributes, # noqa: F401 + Bytes, # noqa: F401 BytesStream, # noqa: F401 GetOptions, # noqa: F401 GetResult, # noqa: F401 ListChunkType, # noqa: F401 ListResult, # noqa: F401 + ListStream, # noqa: F401 ObjectMeta, # noqa: F401 OffsetRange, # noqa: F401 + PutMode, # noqa: F401 + PutResult, # noqa: F401 + ReadableFile, # noqa: F401 + SignCapableStore, # noqa: F401 SuffixRange, # noqa: F401 + UpdateVersion, # noqa: F401 + WritableFile, # noqa: F401 ) From 09ea63c2c2eba2a319c8601d7ab6f22cc1161a4f Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Mon, 19 Jan 2026 14:04:40 -0500 Subject: [PATCH 7/8] allow unused noqa directive, because we alias the bytes.py as _bytes.py --- pyo3-bytes/bytes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyo3-bytes/bytes.py b/pyo3-bytes/bytes.py index c32d8664..4f6d7d89 100644 --- a/pyo3-bytes/bytes.py +++ b/pyo3-bytes/bytes.py @@ -1,6 +1,6 @@ """Type hint for `pyo3_bytes.Bytes` objects.""" -# ruff: noqa: D205, D105 +# ruff: noqa: D205, D105 # noqa: RUF100 from __future__ import annotations import sys From 40eb70262a0c2ca8f3ae3726057ef39314b0757c Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Mon, 19 Jan 2026 14:17:59 -0500 Subject: [PATCH 8/8] switch bytes.py back to bytes.pyi --- obstore/python/obstore/__init__.py | 2 -- obstore/python/obstore/_bytes.py | 1 - obstore/python/obstore/_bytes.pyi | 1 + obstore/python/obstore/_obstore.pyi | 2 ++ obstore/python/obstore/store.py | 5 ++++- pyo3-bytes/{bytes.py => bytes.pyi} | 18 ++---------------- tests/test_importable_type_hints.py | 1 - 7 files changed, 9 insertions(+), 21 deletions(-) delete mode 120000 obstore/python/obstore/_bytes.py create mode 120000 obstore/python/obstore/_bytes.pyi rename pyo3-bytes/{bytes.py => bytes.pyi} (94%) diff --git a/obstore/python/obstore/__init__.py b/obstore/python/obstore/__init__.py index 7e4456a0..c793f10e 100644 --- a/obstore/python/obstore/__init__.py +++ b/obstore/python/obstore/__init__.py @@ -8,7 +8,6 @@ ReadableFile, WritableFile, ) -from ._bytes import Bytes from ._get_types import BytesStream, GetOptions, GetResult, OffsetRange, SuffixRange from ._list_types import ListChunkType, ListResult, ListStream, ObjectMeta from ._obstore import * # noqa: F403 # pyright:ignore[reportMissingModuleSource] @@ -25,7 +24,6 @@ "AsyncWritableFile", "Attribute", "Attributes", - "Bytes", "BytesStream", "GetOptions", "GetResult", diff --git a/obstore/python/obstore/_bytes.py b/obstore/python/obstore/_bytes.py deleted file mode 120000 index 94fdc716..00000000 --- a/obstore/python/obstore/_bytes.py +++ /dev/null @@ -1 +0,0 @@ -../../../pyo3-bytes/bytes.py \ No newline at end of file diff --git a/obstore/python/obstore/_bytes.pyi b/obstore/python/obstore/_bytes.pyi new file mode 120000 index 00000000..4b8c6de8 --- /dev/null +++ b/obstore/python/obstore/_bytes.pyi @@ -0,0 +1 @@ +../../../pyo3-bytes/bytes.pyi \ No newline at end of file diff --git a/obstore/python/obstore/_obstore.pyi b/obstore/python/obstore/_obstore.pyi index 8e6344bc..12f2d7e6 100644 --- a/obstore/python/obstore/_obstore.pyi +++ b/obstore/python/obstore/_obstore.pyi @@ -5,6 +5,7 @@ from ._buffered import ( open_writer, open_writer_async, ) +from ._bytes import Bytes from ._copy import copy, copy_async from ._delete import delete, delete_async from ._get import ( @@ -31,6 +32,7 @@ _object_store_version: str _object_store_source: str __all__ = [ + "Bytes", "__version__", "_object_store_source", "_object_store_version", diff --git a/obstore/python/obstore/store.py b/obstore/python/obstore/store.py index 9e8139fd..652f28aa 100644 --- a/obstore/python/obstore/store.py +++ b/obstore/python/obstore/store.py @@ -27,7 +27,6 @@ from obstore import ( Attributes, - Bytes, GetOptions, GetResult, ListResult, @@ -36,6 +35,9 @@ PutMode, PutResult, ) + from obstore._obstore import ( # pyright:ignore[reportMissingModuleSource] + Bytes, # noqa: TC004 + ) from obstore._store import ( AzureAccessKey, # noqa: TC004 AzureBearerToken, # noqa: TC004 @@ -79,6 +81,7 @@ "AzureSASToken", "AzureStore", "BackoffConfig", + "Bytes", "ClientConfig", "GCSConfig", "GCSCredential", diff --git a/pyo3-bytes/bytes.py b/pyo3-bytes/bytes.pyi similarity index 94% rename from pyo3-bytes/bytes.py rename to pyo3-bytes/bytes.pyi index 4f6d7d89..1da2163d 100644 --- a/pyo3-bytes/bytes.py +++ b/pyo3-bytes/bytes.pyi @@ -1,7 +1,6 @@ """Type hint for `pyo3_bytes.Bytes` objects.""" -# ruff: noqa: D205, D105 # noqa: RUF100 -from __future__ import annotations +# ruff: noqa: D205 import sys from typing import Protocol, overload @@ -11,8 +10,7 @@ else: from typing_extensions import Buffer - -class Bytes(Buffer, Protocol): # noqa: PLW1641 +class Bytes(Buffer, Protocol): """A `bytes`-like buffer. This implements the Python buffer protocol, allowing zero-copy access @@ -47,13 +45,11 @@ def removeprefix(self, prefix: Buffer, /) -> Bytes: Otherwise, return the original binary data. """ - ... def removesuffix(self, suffix: Buffer, /) -> Bytes: """If the binary data ends with the suffix string and that suffix is not empty, return `bytes[:-len(suffix)]`. Otherwise, return the original binary data. """ - ... def isalnum(self) -> bool: """Return `True` if all bytes in the sequence are alphabetical ASCII characters @@ -63,7 +59,6 @@ def isalnum(self) -> bool: `b'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'`. ASCII decimal digits are those byte values in the sequence `b'0123456789'`. """ - ... def isalpha(self) -> bool: """Return `True` if all bytes in the sequence are alphabetic ASCII characters @@ -72,7 +67,6 @@ def isalpha(self) -> bool: Alphabetic ASCII characters are those byte values in the sequence `b'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'`. """ - ... def isascii(self) -> bool: """Return `True` if the sequence is empty or all bytes in the sequence are @@ -80,7 +74,6 @@ def isascii(self) -> bool: ASCII bytes are in the range `0-0x7F`. """ - ... def isdigit(self) -> bool: """Return `True` if all bytes in the sequence are ASCII decimal digits and the @@ -88,13 +81,11 @@ def isdigit(self) -> bool: ASCII decimal digits are those byte values in the sequence `b'0123456789'`. """ - ... def islower(self) -> bool: """Return `True` if there is at least one lowercase ASCII character in the sequence and no uppercase ASCII characters, `False` otherwise. """ - ... def isspace(self) -> bool: r"""Return `True` if all bytes in the sequence are ASCII whitespace and the @@ -104,26 +95,21 @@ def isspace(self) -> bool: in the sequence `b' \t\n\r\x0b\f'` (space, tab, newline, carriage return, vertical tab, form feed). """ - ... def isupper(self) -> bool: """Return `True` if there is at least one uppercase alphabetic ASCII character in the sequence and no lowercase ASCII characters, `False` otherwise. """ - ... def lower(self) -> Bytes: """Return a copy of the sequence with all the uppercase ASCII characters converted to their corresponding lowercase counterpart. """ - ... def upper(self) -> Bytes: """Return a copy of the sequence with all the lowercase ASCII characters converted to their corresponding uppercase counterpart. """ - ... def to_bytes(self) -> bytes: """Copy this buffer's contents into a Python `bytes` object.""" - ... diff --git a/tests/test_importable_type_hints.py b/tests/test_importable_type_hints.py index baefe1a8..5a12f1e9 100644 --- a/tests/test_importable_type_hints.py +++ b/tests/test_importable_type_hints.py @@ -5,7 +5,6 @@ def test_types_should_be_importable(): AsyncWritableFile, # noqa: F401 Attribute, # noqa: F401 Attributes, # noqa: F401 - Bytes, # noqa: F401 BytesStream, # noqa: F401 GetOptions, # noqa: F401 GetResult, # noqa: F401