DM-39410: Removed Ellipsis workaround from lsst.utils

doc/changes/DM-39410.misc.md

@@ -0,0 +1 @@
+Replaced the use of `lsst.utils.ellipsis` mypy workaround with the native type `type.EllipsisType` available since Python 3.10.

pyproject.toml

@@ -16,6 +16,7 @@ classifiers = [
     "Operating System :: OS Independent",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
     "Topic :: Scientific/Engineering :: Astronomy",
 ]
 keywords = ["lsst"]

python/lsst/daf/butler/core/serverModels.py

@@ -98,7 +98,6 @@ def expression(self) -> Any:
             if isinstance(regexes, list):
                 expression.extend(regexes)
             else:
-                # This avoids mypy needing to import Ellipsis type
                 if self._allow_ellipsis:
                     return ...
                 raise ValueError("Expression matches everything but that is not allowed.")

python/lsst/daf/butler/core/utils.py

@@ -29,14 +29,11 @@
 import functools
 import logging
 import re
-from typing import TYPE_CHECKING, Any, Callable, List, Optional, Pattern, TypeVar, Union
+from types import EllipsisType
+from typing import Any, Callable, List, Optional, Pattern, TypeVar, Union
 
 from lsst.utils.iteration import ensure_iterable
 
-if TYPE_CHECKING:
-    from lsst.utils.ellipsis import Ellipsis, EllipsisType
-
-
 _LOG = logging.getLogger(__name__)
 
 
@@ -97,11 +94,11 @@
         A list of regex Patterns or simple strings. Returns ``...`` if
         the provided expressions would match everything.
     """
-    if expressions is Ellipsis or expressions is None:
-        return Ellipsis
+    if expressions is ... or expressions is None:
+        return ...
     expressions = list(ensure_iterable(expressions))
     if not expressions or "*" in expressions:
-        return Ellipsis
+        return ...
 
     # List of special glob characters supported by fnmatch.
     # See: https://docs.python.org/3/library/fnmatch.html

python/lsst/daf/butler/registries/remote.py

@@ -30,7 +30,6 @@
 import httpx
 from lsst.daf.butler import __version__
 from lsst.resources import ResourcePath, ResourcePathExpression
-from lsst.utils.ellipsis import Ellipsis
 from lsst.utils.introspection import get_full_type_name
 from lsst.utils.iteration import ensure_iterable
 
@@ -666,7 +665,7 @@ def queryDimensionRecords(  # type: ignore
     def queryDatasetAssociations(
         self,
         datasetType: str | DatasetType,
-        collections: CollectionArgType | None = Ellipsis,
+        collections: CollectionArgType | None = ...,
         *,
         collectionTypes: Iterable[CollectionType] = CollectionType.all(),
         flattenChains: bool = False,

python/lsst/daf/butler/registries/sql.py

@@ -45,7 +45,6 @@
 import sqlalchemy
 from lsst.daf.relation import LeafRelation, Relation
 from lsst.resources import ResourcePathExpression
-from lsst.utils.ellipsis import Ellipsis
 from lsst.utils.iteration import ensure_iterable
 
 from ..core import (
@@ -1285,7 +1284,7 @@ def queryDimensionRecords(
     def queryDatasetAssociations(
         self,
         datasetType: Union[str, DatasetType],
-        collections: CollectionArgType | None = Ellipsis,
+        collections: CollectionArgType | None = ...,
         *,
         collectionTypes: Iterable[CollectionType] = CollectionType.all(),
         flattenChains: bool = False,

python/lsst/daf/butler/registry/_registry.py

@@ -27,6 +27,7 @@
 import logging
 import re
 from abc import ABC, abstractmethod
+from types import EllipsisType
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -45,7 +46,6 @@
 
 from lsst.resources import ResourcePathExpression
 from lsst.utils import doImportType
-from lsst.utils.ellipsis import Ellipsis, EllipsisType
 
 from ..core import (
     Config,
@@ -1616,7 +1616,7 @@ def queryDimensionRecords(
     def queryDatasetAssociations(
         self,
         datasetType: Union[str, DatasetType],
-        collections: CollectionArgType | None = Ellipsis,
+        collections: CollectionArgType | None = ...,
         *,
         collectionTypes: Iterable[CollectionType] = CollectionType.all(),
         flattenChains: bool = False,

python/lsst/daf/butler/registry/collections/_base.py

@@ -29,7 +29,6 @@
 from typing import TYPE_CHECKING, Any, Generic, TypeVar, cast
 
 import sqlalchemy
-from lsst.utils.ellipsis import Ellipsis
 
 from ...core import DimensionUniverse, Timespan, TimespanDatabaseRepresentation, ddl
 from .._collectionType import CollectionType
@@ -489,7 +488,7 @@ def resolve_nested(record: CollectionRecord, done: set[str]) -> Iterator[Collect
 
         result: list[CollectionRecord] = []
 
-        if wildcard.patterns is Ellipsis:
+        if wildcard.patterns is ...:
             for record in self._records.values():
                 result.extend(resolve_nested(record, done))
             del resolve_nested

python/lsst/daf/butler/registry/datasets/byDimensions/_manager.py

@@ -8,7 +8,6 @@
 from typing import TYPE_CHECKING, Any
 
 import sqlalchemy
-from lsst.utils.ellipsis import Ellipsis
 
 from ....core import DatasetId, DatasetIdGenEnum, DatasetRef, DatasetType, DimensionUniverse, ddl
 from ..._collection_summary import CollectionSummary
@@ -396,7 +395,7 @@ def resolve_wildcard(
             elif missing is not None:
                 missing.append(name)
         already_warned = False
-        if wildcard.patterns is Ellipsis:
+        if wildcard.patterns is ...:
             if explicit_only:
                 raise TypeError(
                     "Universal wildcard '...' is not permitted for dataset types in this context."

python/lsst/daf/butler/registry/wildcards.py

@@ -30,10 +30,10 @@
 import dataclasses
 import re
 from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
+from types import EllipsisType
 from typing import Any
 
 from deprecated.sphinx import deprecated
-from lsst.utils.ellipsis import Ellipsis, EllipsisType
 from lsst.utils.iteration import ensure_iterable
 from pydantic import BaseModel
 
@@ -117,10 +117,10 @@
         """
         assert expression is not None
         # See if we were given ...; just return that if we were.
-        if expression is Ellipsis:
+        if expression is ...:
             if not allowAny:
                 raise TypeError("This expression may not be unconstrained.")
-            return Ellipsis
+            return ...
         if isinstance(expression, cls):
             # This is already a CategorizedWildcard.  Make sure it meets the
             # reqs. implied by the kwargs we got.
@@ -185,8 +185,8 @@
                     # This returns a list but we know we only passed in
                     # single value.
                     converted = globToRegex(element)
-                    if converted is Ellipsis:
-                        return Ellipsis
+                    if converted is ...:
+                        return ...
                     element = converted[0]
                     # Let regex and ... go through to the next check
                     if isinstance(element, str):
@@ -235,11 +235,11 @@
 
         for element in ensure_iterable(expression):
             retval = process(element)
-            if retval is Ellipsis:
+            if retval is ...:
                 # One of the globs matched everything
                 if not allowAny:
                     raise TypeError("This expression may not be unconstrained.")
-                return Ellipsis
+                return ...
         del process
         return self
 
@@ -331,7 +331,7 @@
             )
         except TypeError as err:
             raise CollectionExpressionError(str(err)) from None
-        assert wildcard is not Ellipsis
+        assert wildcard is not ...
         assert not wildcard.patterns
         assert not wildcard.items
         deduplicated = []
@@ -387,15 +387,15 @@
     """An an ordered list of explicitly-named collections. (`tuple` [ `str` ]).
     """
 
-    patterns: tuple[re.Pattern, ...] | EllipsisType = Ellipsis
+    patterns: tuple[re.Pattern, ...] | EllipsisType = ...
     """Regular expression patterns to match against collection names, or the
     special value ``...`` indicating all collections.
 
     `...` must be accompanied by ``strings=()``.
     """
 
     def __post_init__(self) -> None:
-        if self.patterns is Ellipsis and self.strings:
+        if self.patterns is ... and self.strings:
             raise ValueError(
                 f"Collection wildcard matches any string, but still has explicit strings {self.strings}."
             )
@@ -435,14 +435,14 @@
         """
         if isinstance(expression, cls):
             return expression
-        if expression is Ellipsis:
+        if expression is ...:
             return cls()
         wildcard = CategorizedWildcard.fromExpression(
             expression,
             allowAny=True,
             allowPatterns=True,
         )
-        if wildcard is Ellipsis:
+        if wildcard is ...:
             return cls()
         result = cls(
             strings=tuple(wildcard.strings),
@@ -496,7 +496,7 @@
         return not self.strings and not self.patterns
 
     def __str__(self) -> str:
-        if self.patterns is Ellipsis:
+        if self.patterns is ...:
             return "..."
         else:
             terms = list(self.strings)
@@ -518,7 +518,7 @@
     instances.
     """
 
-    patterns: tuple[re.Pattern, ...] | EllipsisType = Ellipsis
+    patterns: tuple[re.Pattern, ...] | EllipsisType = ...
     """Regular expressions to be matched against dataset type names, or the
     special value ``...`` indicating all dataset types.
 
@@ -564,7 +564,7 @@
             )
         except TypeError as err:
             raise DatasetTypeExpressionError(f"Invalid dataset type expression: {expression!r}.") from err
-        if wildcard is Ellipsis:
+        if wildcard is ...:
             return cls()
         values: dict[str, DatasetType | None] = {}
         for name in wildcard.strings:
@@ -579,7 +579,7 @@
         return cls(values, patterns=tuple(wildcard.patterns))
 
     def __str__(self) -> str:
-        if self.patterns is Ellipsis:
+        if self.patterns is ...:
             return "..."
         else:
             terms = list(self.values.keys())

python/lsst/daf/butler/script/queryDataIds.py

@@ -22,11 +22,11 @@
 
 import logging
 from collections.abc import Iterable
+from types import EllipsisType
 from typing import TYPE_CHECKING
 
 import numpy as np
 from astropy.table import Table as AstropyTable
-from lsst.utils.ellipsis import Ellipsis, EllipsisType
 
 from .._butler import Butler, DataCoordinate
 from ..cli.utils import sortAstropyTable
@@ -131,7 +131,7 @@ def queryDataIds(
 
     query_collections: Iterable[str] | EllipsisType | None = None
     if datasets:
-        query_collections = collections if collections else Ellipsis
+        query_collections = collections if collections else ...
     results = butler.registry.queryDataIds(
         dimensions, datasets=datasets, where=where, collections=query_collections
     )

python/lsst/daf/butler/script/queryDatasets.py

@@ -23,11 +23,11 @@
 import dataclasses
 from collections import defaultdict
 from collections.abc import Iterable
+from types import EllipsisType
 from typing import TYPE_CHECKING
 
 import numpy as np
 from astropy.table import Table as AstropyTable
-from lsst.utils.ellipsis import Ellipsis, EllipsisType
 
 from .._butler import Butler
 from ..cli.utils import sortAstropyTable
@@ -175,7 +175,7 @@ def _getDatasets(
         self, glob: Iterable[str], collections: Iterable[str], where: str, find_first: bool
     ) -> None:
         datasetTypes = glob if glob else ...
-        query_collections: Iterable[str] | EllipsisType = collections if collections else Ellipsis
+        query_collections: Iterable[str] | EllipsisType = collections if collections else ...
 
         self.datasets = self.butler.registry.queryDatasets(
             datasetType=datasetTypes, collections=query_collections, where=where, findFirst=find_first

python/lsst/daf/butler/script/queryDimensionRecords.py

@@ -22,10 +22,10 @@
 from __future__ import annotations
 
 from collections.abc import Iterable
+from types import EllipsisType
 from typing import Any
 
 from astropy.table import Table
-from lsst.utils.ellipsis import Ellipsis, EllipsisType
 
 from .._butler import Butler
 from ..core import Timespan
@@ -50,7 +50,7 @@ def queryDimensionRecords(
 
     query_collections: Iterable[str] | EllipsisType | None = None
     if datasets:
-        query_collections = collections if collections else Ellipsis
+        query_collections = collections if collections else ...
     query_results = butler.registry.queryDimensionRecords(
         element, datasets=datasets, collections=query_collections, where=where, check=not no_check
     )

python/lsst/daf/butler/script/retrieveArtifacts.py

@@ -24,10 +24,9 @@
 __all__ = ("retrieveArtifacts",)
 
 import logging
+from types import EllipsisType
 from typing import TYPE_CHECKING
 
-from lsst.utils.ellipsis import Ellipsis, EllipsisType
-
 from .._butler import Butler
 
 if TYPE_CHECKING:
@@ -79,7 +78,7 @@ def retrieveArtifacts(
         The destination URIs of every transferred artifact.
     """
     query_types = dataset_type if dataset_type else ...
-    query_collections: tuple[str, ...] | EllipsisType = collections if collections else Ellipsis
+    query_collections: tuple[str, ...] | EllipsisType = collections if collections else ...
 
     butler = Butler(repo, writeable=False)
 

python/lsst/daf/butler/script/transferDatasets.py

@@ -23,8 +23,7 @@
 __all__ = ("transferDatasets",)
 
 import logging
-
-from lsst.utils.ellipsis import Ellipsis, EllipsisType
+from types import EllipsisType
 
 from .._butler import Butler
 from ..registry.queries import DatasetQueryResults
@@ -73,7 +72,7 @@
     dest_butler = Butler(dest, writeable=True)
 
     dataset_type_expr = ... if not dataset_type else dataset_type
-    collections_expr: tuple[str, ...] | EllipsisType = Ellipsis if not collections else collections
+    collections_expr: tuple[str, ...] | EllipsisType = ... if not collections else collections
 
     source_refs = source_butler.registry.queryDatasets(
         datasetType=dataset_type_expr, collections=collections_expr, where=where, findFirst=find_first