Merge pull request #777 from lsst/tickets/DM-37808

DM-37808: Update pre-commit dependencies

.pre-commit-config.yaml

@@ -1,14 +1,14 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v4.4.0
     hooks:
       - id: check-yaml
         args:
           - "--unsafe"
       - id: end-of-file-fixer
       - id: trailing-whitespace
   - repo: https://github.com/psf/black
-    rev: 22.3.0
+    rev: 23.1.0
     hooks:
       - id: black
         # It is recommended to specify the latest version of Python
@@ -17,11 +17,11 @@ repos:
         # https://pre-commit.com/#top_level-default_language_version
         language_version: python3.10
   - repo: https://github.com/pycqa/isort
-    rev: 5.10.1
+    rev: 5.12.0
     hooks:
       - id: isort
         name: isort (python)
   - repo: https://github.com/PyCQA/flake8
-    rev: 4.0.1
+    rev: 6.0.0
     hooks:
       - id: flake8

doc/changes/DM-36775.bugfix.md

@@ -1,2 +1 @@
 Fix bug in pandas dataframe to arrow conversion that would crash with some pandas object datatypes.
-

doc/conf.py

@@ -6,7 +6,7 @@
 from documenteer.conf.pipelinespkg import *  # noqa: F403, import *
 
 project = "daf_butler"
-html_theme_options["logotext"] = project     # noqa: F405, unknown name
+html_theme_options["logotext"] = project  # noqa: F405, unknown name
 html_title = project
 html_short_title = project
 doxylink = {}

doc/lsst.daf.butler/organizing.rst

@@ -102,4 +102,4 @@ Querying a ``CHAINED`` collection simply queries its child collections in order,
 ``CHAINED`` collections may contain other chained collections, as long as they do not contain cycles, and they can also include restrictions on the dataset types to search for within each child collection (see :ref:`daf_butler_collection_expressions`).
 
 The usual constraint on dataset type and data ID uniqueness within a collection is only lazily enforced for chained collections: operations that query them either deduplicate results themselves or terminate single-dataset searches after the first match in a child collection is found.
-In some methods, like `Registry.queryDatasets`, this behavior is optional: passing ``findFirst=True`` will enforce the constraint, while ``findFirst=False`` will not.
+In some methods, like `Registry.queryDatasets`, this behavior is optional: passing ``findFirst=True`` will enforce the constraint, while ``findFirst=False`` will not.

doc/lsst.daf.butler/scripts/options-file.rst

@@ -25,5 +25,3 @@ would only report the datastore section of the config.
 .. note::
 
   Options explicitly given on the command-line always take precedence over those specified from an external options file.
-
-

python/lsst/daf/butler/_butler.py

@@ -1954,7 +1954,6 @@ def ingest(
                 groupedData[ref.datasetType][ref.dataId] = (dataset, resolvedRefs)
 
             if existingRefs:
-
                 if len(dataset.refs) != len(existingRefs):
                     # Keeping track of partially pre-existing datasets is hard
                     # and should generally never happen. For now don't allow

python/lsst/daf/butler/_butlerConfig.py

@@ -68,7 +68,6 @@ def __init__(
         other: Optional[Union[ResourcePathExpression, Config]] = None,
         searchPaths: Sequence[ResourcePathExpression] | None = None,
     ):
-
         self.configDir: Optional[ResourcePath] = None
 
         # If this is already a ButlerConfig we assume that defaults

python/lsst/daf/butler/cli/butler.py

@@ -318,7 +318,6 @@ def _raiseIfDuplicateCommands(commands: defaultdict[str, list[str]]) -> None:
 
 
 class ButlerCLI(LoaderCLI):
-
     localCmdPkg = "lsst.daf.butler.cli.cmd"
 
     pluginEnvVar = "DAF_BUTLER_PLUGINS"

python/lsst/daf/butler/cli/cliLog.py

@@ -172,7 +172,6 @@ def initLog(
         if not log_tty:
             logging.basicConfig(force=True, handlers=[logging.NullHandler()])
         elif longlog:
-
             # Want to create our own Formatter so that we can get high
             # precision timestamps. This requires we attach our own
             # default stream handler.

python/lsst/daf/butler/cli/cmd/commands.py

@@ -172,8 +172,7 @@ def create(*args: Any, **kwargs: Any) -> None:
     "outfile",
     type=click.File(mode="w"),
     default="-",
-    help="Print the (possibly-expanded) configuration for a repository to a file, or to stdout "
-    "by default.",
+    help="Print the (possibly-expanded) configuration for a repository to a file, or to stdout by default.",
 )
 @options_file_option()
 def config_dump(*args: Any, **kwargs: Any) -> None:
@@ -223,7 +222,7 @@ def config_validate(*args: Any, **kwargs: Any) -> None:
 )
 @click.option(
     "--unstore",
-    help=("""Remove all datasets in the collection from all datastores in which they appear."""),
+    help="Remove all datasets in the collection from all datastores in which they appear.",
     is_flag=True,
 )
 @click.option(

python/lsst/daf/butler/cli/opt/options.py

@@ -60,7 +60,6 @@
 
 
 class CollectionTypeCallback:
-
     collectionTypes = tuple(collectionType.name for collectionType in CollectionType.all())
 
     @staticmethod

python/lsst/daf/butler/cli/utils.py

@@ -937,7 +937,6 @@ class MWCtxObj:
     """
 
     def __init__(self) -> None:
-
         self.args = None
 
     @staticmethod

python/lsst/daf/butler/core/_column_tags.py

@@ -39,7 +39,6 @@
 
 
 class _BaseColumnTag:
-
     __slots__ = ()
 
     @classmethod

python/lsst/daf/butler/core/config.py

@@ -429,7 +429,6 @@ def _processExplicitIncludes(self):
         names = self.nameTuples()
         for path in names:
             if path[-1] == self.includeKey:
-
                 log.debug("Processing file include directive at %s", self._D + self._D.join(path))
                 basePath = path[:-1]
 
@@ -1123,7 +1122,6 @@ class ConfigSubset(Config):
     """
 
     def __init__(self, other=None, validate=True, mergeDefaults=True, searchPaths=None):
-
         # Create a blank object to receive the defaults
         # Once we have the defaults we then update with the external values
         super().__init__()
@@ -1151,7 +1149,6 @@ def __init__(self, other=None, validate=True, mergeDefaults=True, searchPaths=No
 
         # Sometimes we do not want to merge with defaults.
         if mergeDefaults:
-
             # Supplied search paths have highest priority
             fullSearchPath = []
             if searchPaths:

python/lsst/daf/butler/core/configSupport.py

@@ -83,7 +83,6 @@ def __init__(
         self._name = None
 
         if name is not None:
-
             if not isinstance(name, str):
                 raise ValueError(f"Supplied name must be str not: '{name}'")
 

python/lsst/daf/butler/core/datastore.py

@@ -210,7 +210,6 @@ def __init__(
         primaryURI: Optional[ResourcePath] = None,
         componentURIs: Optional[Dict[str, ResourcePath]] = None,
     ):
-
         self.primaryURI = primaryURI
         """The URI to the primary artifact associated with this dataset. If the
         dataset was disassembled within the datastore this may be `None`.

python/lsst/daf/butler/core/datastoreRecordData.py

@@ -39,7 +39,6 @@
 from .storedFileInfo import StoredDatastoreItemInfo
 
 if TYPE_CHECKING:
-
     from ..registry import Registry
 
 _Record = Dict[str, Any]

python/lsst/daf/butler/core/dimensions/_records.py

@@ -237,7 +237,7 @@ def __init__(self, **kwargs: Any):
                 v2 = kwargs.setdefault(self.definition.name, v)
                 if v != v2:
                     raise ValueError(
-                        f"Multiple inconsistent values for "
+                        "Multiple inconsistent values for "
                         f"{self.definition.name}.{self.definition.primaryKey.name}: {v!r} != {v2!r}."
                     )
         for name in self.__slots__:

python/lsst/daf/butler/core/dimensions/_universe.py

@@ -505,10 +505,10 @@ def _unpickle(cls, version: int, namespace: Optional[str] = None) -> DimensionUn
         except KeyError as err:
             raise pickle.UnpicklingError(
                 f"DimensionUniverse with version '{version}' and namespace {namespace!r} "
-                f"not found.  Note that DimensionUniverse objects are not "
-                f"truly serialized; when using pickle to transfer them "
-                f"between processes, an equivalent instance with the same "
-                f"version must already exist in the receiving process."
+                "not found.  Note that DimensionUniverse objects are not "
+                "truly serialized; when using pickle to transfer them "
+                "between processes, an equivalent instance with the same "
+                "version must already exist in the receiving process."
             ) from err
 
     def __reduce__(self) -> tuple:

python/lsst/daf/butler/core/fileTemplates.py

@@ -470,7 +470,6 @@ def format(self, ref: DatasetRef) -> str:
         output = ""
 
         for literal, field_name, format_spec, conversion in parts:
-
             if field_name == "component":
                 usedComponent = True
 
@@ -691,8 +690,7 @@ def validateTemplate(self, entity: Union[DatasetRef, DatasetType, StorageClass,
         # Calculate any field usage that does not match a dimension
         if not required.issubset(maximal):
             raise FileTemplateValidationError(
-                f"Template '{self}' is inconsistent with {entity}:"
-                f" {required} is not a subset of {maximal}."
+                f"Template '{self}' is inconsistent with {entity}: {required} is not a subset of {maximal}."
             )
 
         if not allfields.issuperset(minimal):

python/lsst/daf/butler/core/formatter.py

@@ -123,8 +123,7 @@ def __init__(
         if writeParameters:
             if self.supportedWriteParameters is None:
                 raise ValueError(
-                    "This formatter does not accept any write parameters. "
-                    f"Got: {', '.join(writeParameters)}"
+                    f"This formatter does not accept any write parameters. Got: {', '.join(writeParameters)}"
                 )
             else:
                 given = set(writeParameters)

python/lsst/daf/butler/core/mappingFactory.py

@@ -252,8 +252,9 @@ def placeInRegistry(
                 return
 
             raise KeyError(
-                "Item with key {} already registered with different value"
-                " ({} != {})".format(key, self._registry[key], typeName)
+                "Item with key {} already registered with different value ({} != {})".format(
+                    key, self._registry[key], typeName
+                )
             )
 
         self._registry[key] = {

python/lsst/daf/butler/core/repoRelocation.py

@@ -75,8 +75,7 @@ def replaceRoot(configRoot: str, butlerRoot: Optional[ResourcePathExpression]) -
     # None or empty string indicate a problem
     if not butlerRoot:
         raise ValueError(
-            f"Required to replace {BUTLER_ROOT_TAG} in '{configRoot}' "
-            "but a replacement has not been defined"
+            f"Required to replace {BUTLER_ROOT_TAG} in '{configRoot}' but a replacement has not been defined"
         )
 
     # Use absolute file path if this refers to a local file, else use

python/lsst/daf/butler/core/storageClassDelegate.py

@@ -102,8 +102,8 @@ def _attrNames(componentName: str, getter: bool = True) -> Tuple[str, ...]:
             tail = componentName[1:]
         else:
             tail = ""
-        capitalized = "{}{}{}".format(root, first, tail)
-        return (componentName, "{}_{}".format(root, componentName), capitalized)
+        capitalized = f"{root}{first}{tail}"
+        return (componentName, f"{root}_{componentName}", capitalized)
 
     def assemble(self, components: Dict[str, Any], pytype: Optional[Type] = None) -> Any:
         """Construct an object from components based on storageClass.
@@ -144,7 +144,7 @@ def assemble(self, components: Dict[str, Any], pytype: Optional[Type] = None) ->
         requested = set(components.keys())
         unknown = requested - understood
         if unknown:
-            raise ValueError("Requested component(s) not known to StorageClass: {}".format(unknown))
+            raise ValueError(f"Requested component(s) not known to StorageClass: {unknown}")
 
         # First try to create an instance directly using keyword args
         try:
@@ -172,7 +172,7 @@ def assemble(self, components: Dict[str, Any], pytype: Optional[Type] = None) ->
                         failed.append(name)
 
             if failed:
-                raise ValueError("Unhandled components during assembly ({})".format(failed))
+                raise ValueError(f"Unhandled components during assembly ({failed})")
 
         return obj
 
@@ -242,7 +242,7 @@ def getComponent(self, composite: Any, componentName: str) -> Any:
                     component = component()
                 break
         else:
-            raise AttributeError("Unable to get component {}".format(componentName))
+            raise AttributeError(f"Unable to get component {componentName}")
         return component
 
     def disassemble(
@@ -286,14 +286,13 @@ def disassemble(
         """
         if not self.storageClass.isComposite():
             raise TypeError(
-                "Can not disassemble something that is not a composite"
-                f" (storage class={self.storageClass})"
+                f"Can not disassemble something that is not a composite (storage class={self.storageClass})"
             )
 
         if not self.storageClass.validateInstance(composite):
             raise TypeError(
-                "Unexpected type mismatch between parent and StorageClass"
-                " ({} != {})".format(type(composite), self.storageClass.pytype)
+                "Unexpected type mismatch between parent and StorageClass "
+                f"({type(composite)} != {self.storageClass.pytype})"
             )
 
         requested = set(self.storageClass.components)
@@ -302,7 +301,7 @@ def disassemble(
             subset = set(subset)
             diff = subset - requested
             if diff:
-                raise ValueError("Requested subset is not a subset of supported components: {}".format(diff))
+                raise ValueError(f"Requested subset is not a subset of supported components: {diff}")
             requested = subset
 
         if override is not None:
@@ -326,7 +325,7 @@ def disassemble(
                 requested.remove(c)
 
         if requested:
-            raise ValueError("Unhandled components during disassembly ({})".format(requested))
+            raise ValueError(f"Unhandled components during disassembly ({requested})")
 
         return components
 

python/lsst/daf/butler/core/storedFileInfo.py

@@ -108,7 +108,6 @@ def __init__(
         file_size: int,
         dataset_id: DatasetId,
     ):
-
         # Use these shenanigans to allow us to use a frozen dataclass
         object.__setattr__(self, "path", path)
         object.__setattr__(self, "storageClass", storageClass)

python/lsst/daf/butler/datastores/fileDatastore.py

@@ -627,7 +627,6 @@ def _prepare_for_get(
 
         fileGetInfo = []
         for location, storedFileInfo in fileLocations:
-
             # The storage class used to write the file
             writeStorageClass = storedFileInfo.storageClass
 
@@ -1294,7 +1293,6 @@ def _read_artifact_into_memory(
                     location_updated = True
 
                 with uri.as_local() as local_uri:
-
                     can_be_cached = False
                     if uri != local_uri:
                         # URI was remote and file was downloaded
@@ -1769,7 +1767,6 @@ def _predict_URIs(
         uris = DatasetRefURIs()
 
         if self.composites.shouldBeDisassembled(ref):
-
             for component, _ in ref.datasetType.storageClass.components.items():
                 comp_ref = ref.makeComponentRef(component)
                 comp_location, _ = self._determine_put_formatter_location(comp_ref)
@@ -1779,7 +1776,6 @@ def _predict_URIs(
                 uris.componentURIs[component] = ResourcePath(comp_location.uri.geturl() + "#predicted")
 
         else:
-
             location, _ = self._determine_put_formatter_location(ref)
 
             # Add the "#predicted" URI fragment to indicate this is a guess
@@ -2092,7 +2088,6 @@ def get(
             )
 
         elif isDisassembledReadOnlyComponent:
-
             compositeStorageClass = ref.datasetType.parentStorageClass
             if compositeStorageClass is None:
                 raise RuntimeError(
@@ -2401,7 +2396,6 @@ def emptyTrash(self, ignore_errors: bool = True) -> None:
                 )
 
                 for ref, info in trashed:
-
                     # Mypy needs to know this is not the base class
                     assert isinstance(info, StoredFileInfo), f"Unexpectedly got info of class {type(info)}"
 
@@ -2415,7 +2409,6 @@ def emptyTrash(self, ignore_errors: bool = True) -> None:
                 artifacts_to_keep = set(path_map)
 
             for ref, info in trashed:
-
                 # Should not happen for this implementation but need
                 # to keep mypy happy.
                 assert info is not None, f"Internal logic error in emptyTrash with ref {ref}."