DM-43121: Set up server to client error propagation

.github/workflows/mypy.yaml

@@ -9,3 +9,6 @@ on:
 jobs:
   call-workflow:
     uses: lsst/rubin_workflows/.github/workflows/mypy.yaml@main
+    with:
+      # https://github.com/python/mypy/issues/17002
+      mypy_package: mypy!=1.9.0

python/lsst/daf/butler/__init__.py

@@ -86,7 +86,6 @@
 from .registry import (
     CollectionType,
     MissingCollectionError,
-    MissingDatasetTypeError,
     NoDefaultCollectionError,
     Registry,
     RegistryConfig,

python/lsst/daf/butler/_exceptions.py

@@ -26,7 +26,60 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """Specialized Butler exceptions."""
-__all__ = ("DatasetTypeNotSupportedError", "EmptyQueryResultError", "ValidationError")
+__all__ = (
+    "DatasetNotFoundError",
+    "ButlerUserError",
+    "DatasetTypeNotSupportedError",
+    "EmptyQueryResultError",
+    "MissingDatasetTypeError",
+    "ValidationError",
+)
+
+from ._exceptions_legacy import DatasetTypeError
+
+
+class ButlerUserError(Exception):
+    """Base class for Butler exceptions that contain a user-facing error
+    message.
+
+    Parameters
+    ----------
+    detail : `str`
+        Details about the error that occurred.
+    """
+
+    # When used with Butler server, exceptions inheriting from
+    # this class will be sent to the client side and re-raised by RemoteButler
+    # there.  Be careful that error messages do not contain security-sensitive
+    # information.
+    #
+    # This should only be used for "expected" errors that occur because of
+    # errors in user-supplied data passed to Butler methods.  It should not be
+    # used for any issues caused by the Butler configuration file, errors in
+    # the library code itself or the underlying databases.
+    #
+    # When you create a new subclass of this type, add it to the list in
+    # _USER_ERROR_TYPES below.
+
+    error_type: str
+    """Unique name for this error type, used to identify it when sending
+    information about the error to the client.
+    """
+
+    def __init__(self, detail: str):
+        return super().__init__(detail)
+
+
+class DatasetNotFoundError(LookupError, ButlerUserError):
+    """The requested dataset could not be found."""
+
+    error_type = "dataset_not_found"
+
+
+class MissingDatasetTypeError(DatasetTypeError, KeyError, ButlerUserError):
+    """Exception raised when a dataset type does not exist."""
+
+    error_type = "missing_dataset_type"
 
 
 class DatasetTypeNotSupportedError(RuntimeError):
@@ -61,3 +114,41 @@ def __init__(self, reasons: list[str]):
     def __str__(self) -> str:
         # There may be multiple reasons, format them into multiple lines.
         return "Possible reasons for empty result:\n" + "\n".join(self.reasons)
+
+
+class UnknownButlerUserError(ButlerUserError):
+    """Raised when the server sends an ``error_type`` for which we don't know
+    the corresponding exception type.  (This may happen if an old version of
+    the Butler client library connects to a new server).
+    """
+
+    error_type = "unknown"
+
+
+_USER_ERROR_TYPES: tuple[type[ButlerUserError], ...] = (
+    DatasetNotFoundError,
+    MissingDatasetTypeError,
+    UnknownButlerUserError,
+)
+_USER_ERROR_MAPPING = {e.error_type: e for e in _USER_ERROR_TYPES}
+assert len(_USER_ERROR_MAPPING) == len(
+    _USER_ERROR_TYPES
+), "Subclasses of ButlerUserError must have unique 'error_type' property"
+
+
+def create_butler_user_error(error_type: str, message: str) -> ButlerUserError:
+    """Instantiate one of the subclasses of `ButlerUserError` based on its
+    ``error_type`` string.
+
+    Parameters
+    ----------
+    error_type : `str`
+        The value from the ``error_type`` class attribute on the exception
+        subclass you wish to instantiate.
+    message : `str`
+        Detailed error message passed to the exception constructor.
+    """
+    cls = _USER_ERROR_MAPPING.get(error_type)
+    if cls is None:
+        raise UnknownButlerUserError(f"Unknown exception type '{error_type}': {message}")
+    return cls(message)

python/lsst/daf/butler/remote_butler/server/_exceptions.py → python/lsst/daf/butler/_exceptions_legacy.py

@@ -25,17 +25,27 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from fastapi import HTTPException
+__all__ = ("DatasetTypeError", "RegistryError")
 
+# The classes in this file exist only for backwards compatibility.  New
+# exception types should not inherit from these.
 
-class NotFoundException(HTTPException):
-    """Exception corresponding to a 404 server error.
 
-    Parameters
-    ----------
-    message : `str`, optional
-        Message to include in the exception.
+class RegistryError(Exception):
+    """Base class for many exception classes produced by Registry methods.
+
+    Notes
+    -----
+    The client code that needs to handle exceptions generated by the Registry
+    methods can catch this class or one of its many subclasses as described by
+    the particular method documentation. While most of the Registry methods
+    should only raise the exceptions of this type, it is hard to guarantee
+    that they will never raise other exception types. If the client needs to
+    handle all possible exceptions, then it should also catch a standard
+    `Exception` type as well. Additionally, some Registry methods can be
+    explicitly documented to raise exceptions outside this class hierarchy.
     """
 
-    def __init__(self, message: str = "Not found"):
-        super().__init__(status_code=404, detail=message)
+
+class DatasetTypeError(RegistryError):
+    """Exception raised for problems with dataset types."""

python/lsst/daf/butler/direct_butler.py

@@ -58,7 +58,7 @@
 from ._dataset_ref import DatasetRef
 from ._dataset_type import DatasetType
 from ._deferredDatasetHandle import DeferredDatasetHandle
-from ._exceptions import ValidationError
+from ._exceptions import DatasetNotFoundError, ValidationError
 from ._limited_butler import LimitedButler
 from ._registry_shim import RegistryShim
 from ._storage_class import StorageClass, StorageClassFactory
@@ -881,7 +881,7 @@ def _findDatasetRef(
             else:
                 if collections is None:
                     collections = self._registry.defaults.collections
-                raise LookupError(
+                raise DatasetNotFoundError(
                     f"Dataset {datasetType.name} with data ID {dataId} "
                     f"could not be found in collections {collections}."
                 )

python/lsst/daf/butler/registry/__init__.py

@@ -25,6 +25,12 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+# Re-export some top-level exception types for backwards compatibility -- these
+# used to be part of registry.
+from .._exceptions import MissingDatasetTypeError
+from .._exceptions_legacy import DatasetTypeError, RegistryError
+
+# Registry imports.
 from . import interfaces, managers, queries, wildcards
 from ._collection_summary import *
 from ._collection_type import *

python/lsst/daf/butler/registry/_exceptions.py

@@ -34,51 +34,26 @@
     "ConflictingDefinitionError",
     "DataIdError",
     "DataIdValueError",
-    "DatasetTypeError",
     "DatasetTypeExpressionError",
     "DimensionNameError",
     "InconsistentDataIdError",
     "MissingCollectionError",
-    "MissingDatasetTypeError",
     "MissingSpatialOverlapError",
     "NoDefaultCollectionError",
     "OrphanedRecordError",
     "RegistryConsistencyError",
-    "RegistryError",
     "UnsupportedIdGeneratorError",
     "UserExpressionError",
     "UserExpressionSyntaxError",
 )
 
-
-class RegistryError(Exception):
-    """Base class for many exception classes produced by Registry methods.
-
-    Notes
-    -----
-    The client code that needs to handle exceptions generated by the Registry
-    methods can catch this class or one of its many subclasses as described by
-    the particular method documentation. While most of the Registry methods
-    should only raise the exceptions of this type, it is hard to guarantee
-    that they will never raise other exception types. If the client needs to
-    handle all possible exceptions, then it should also catch a standard
-    `Exception` type as well. Additionally, some Registry methods can be
-    explicitly documented to raise exceptions outside this class hierarchy.
-    """
+from .._exceptions_legacy import RegistryError
 
 
 class ArgumentError(RegistryError):
     """Exception raised when method arguments are invalid or inconsistent."""
 
 
-class DatasetTypeError(RegistryError):
-    """Exception raised for problems with dataset types."""
-
-
-class MissingDatasetTypeError(DatasetTypeError, KeyError):
-    """Exception raised when a dataset type does not exist."""
-
-
 class DatasetTypeExpressionError(RegistryError):
     """Exception raised for an incorrect dataset type expression."""
 

python/lsst/daf/butler/registry/datasets/byDimensions/_manager.py

@@ -12,14 +12,10 @@
 import sqlalchemy
 
 from ...._dataset_ref import DatasetId, DatasetIdGenEnum, DatasetRef, DatasetType
+from ...._exceptions_legacy import DatasetTypeError
 from ....dimensions import DimensionUniverse
 from ..._collection_summary import CollectionSummary
-from ..._exceptions import (
-    ConflictingDefinitionError,
-    DatasetTypeError,
-    DatasetTypeExpressionError,
-    OrphanedRecordError,
-)
+from ..._exceptions import ConflictingDefinitionError, DatasetTypeExpressionError, OrphanedRecordError
 from ...interfaces import DatasetRecordStorage, DatasetRecordStorageManager, VersionTuple
 from ...wildcards import DatasetTypeWildcard
 from ._storage import ByDimensionsDatasetRecordStorage, ByDimensionsDatasetRecordStorageUUID

python/lsst/daf/butler/registry/interfaces/_datasets.py

@@ -39,9 +39,9 @@
 
 from ..._dataset_ref import DatasetId, DatasetIdGenEnum, DatasetRef
 from ..._dataset_type import DatasetType
+from ..._exceptions import MissingDatasetTypeError
 from ..._timespan import Timespan
 from ...dimensions import DataCoordinate
-from .._exceptions import MissingDatasetTypeError
 from ._versioning import VersionedExtension, VersionTuple
 
 if TYPE_CHECKING:

python/lsst/daf/butler/registry/queries/_query_backend.py

@@ -48,9 +48,10 @@
 
 from ..._column_tags import DatasetColumnTag, DimensionKeyColumnTag
 from ..._dataset_type import DatasetType
+from ..._exceptions import MissingDatasetTypeError
+from ..._exceptions_legacy import DatasetTypeError
 from ...dimensions import DimensionGroup, DimensionRecordSet, DimensionUniverse
 from .._collection_type import CollectionType
-from .._exceptions import DatasetTypeError, MissingDatasetTypeError
 from ..wildcards import CollectionWildcard
 from ._query_context import QueryContext
 from .find_first_dataset import FindFirstDataset

python/lsst/daf/butler/registry/queries/_results.py

@@ -45,6 +45,7 @@
 
 from ..._dataset_ref import DatasetRef
 from ..._dataset_type import DatasetType
+from ..._exceptions_legacy import DatasetTypeError
 from ...dimensions import (
     DataCoordinate,
     DataCoordinateIterable,
@@ -53,7 +54,6 @@
     DimensionGroup,
     DimensionRecord,
 )
-from .._exceptions import DatasetTypeError
 from ._query import Query
 from ._structs import OrderByClause
 

python/lsst/daf/butler/registry/tests/_registry.py

@@ -56,6 +56,8 @@
 from ..._dataset_association import DatasetAssociation
 from ..._dataset_ref import DatasetIdFactory, DatasetIdGenEnum, DatasetRef
 from ..._dataset_type import DatasetType
+from ..._exceptions import MissingDatasetTypeError
+from ..._exceptions_legacy import DatasetTypeError
 from ..._storage_class import StorageClass
 from ..._timespan import Timespan
 from ...dimensions import DataCoordinate, DataCoordinateSet, SkyPixDimension
@@ -68,11 +70,9 @@
     CollectionTypeError,
     ConflictingDefinitionError,
     DataIdValueError,
-    DatasetTypeError,
     DatasetTypeExpressionError,
     InconsistentDataIdError,
     MissingCollectionError,
-    MissingDatasetTypeError,
     NoDefaultCollectionError,
     OrphanedRecordError,
 )