lsst · timj · Dec 10, 2021 · Dec 7, 2021 · Dec 10, 2021 · Dec 10, 2021
diff --git a/doc/changes/DM-13365.feature.rst b/doc/changes/DM-13365.feature.rst
diff --git a/doc/changes/DM-22534.misc.rst b/doc/changes/DM-22534.misc.rst
diff --git a/doc/changes/DM-27241.feature.rst b/doc/changes/DM-27241.feature.rst
diff --git a/doc/changes/DM-28650.feature.rst b/doc/changes/DM-28650.feature.rst
diff --git a/doc/changes/DM-28678.other.rst b/doc/changes/DM-28678.other.rst
diff --git a/doc/changes/DM-29370.feature.rst b/doc/changes/DM-29370.feature.rst
diff --git a/doc/changes/DM-29531.misc.rst b/doc/changes/DM-29531.misc.rst
diff --git a/doc/changes/DM-29708.other.rst b/doc/changes/DM-29708.other.rst
diff --git a/doc/changes/DM-30124.bugfix.md b/doc/changes/DM-30124.bugfix.md
diff --git a/doc/changes/DM-30140.perf b/doc/changes/DM-30140.perf
diff --git a/doc/changes/DM-30291.misc.md b/doc/changes/DM-30291.misc.md
diff --git a/doc/changes/DM-30335.misc.rst b/doc/changes/DM-30335.misc.rst
diff --git a/doc/changes/DM-30372.bugfix.rst b/doc/changes/DM-30372.bugfix.rst
diff --git a/doc/changes/DM-30373.feature.rst b/doc/changes/DM-30373.feature.rst
diff --git a/doc/changes/DM-30685.misc.rst b/doc/changes/DM-30685.misc.rst
diff --git a/doc/changes/DM-30743.bugfix.rst b/doc/changes/DM-30743.bugfix.rst
diff --git a/doc/changes/DM-30784.bugfix.rst b/doc/changes/DM-30784.bugfix.rst
diff --git a/doc/changes/DM-30855.bugfix.rst b/doc/changes/DM-30855.bugfix.rst
diff --git a/doc/changes/DM-30935.feature.rst b/doc/changes/DM-30935.feature.rst
diff --git a/doc/changes/DM-30977.feature.rst b/doc/changes/DM-30977.feature.rst
diff --git a/doc/changes/DM-31120.other.rst b/doc/changes/DM-31120.other.rst
diff --git a/doc/changes/DM-31251.misc.rst b/doc/changes/DM-31251.misc.rst
diff --git a/doc/changes/DM-31337.feature.md b/doc/changes/DM-31337.feature.md
diff --git a/doc/changes/DM-31366.feature.rst b/doc/changes/DM-31366.feature.rst
diff --git a/doc/changes/DM-31367.feature.rst b/doc/changes/DM-31367.feature.rst
diff --git a/doc/changes/DM-31429.bugfix.rst b/doc/changes/DM-31429.bugfix.rst
diff --git a/doc/changes/DM-31548.perf.rst b/doc/changes/DM-31548.perf.rst
diff --git a/doc/changes/DM-31583.feature.rst b/doc/changes/DM-31583.feature.rst
diff --git a/doc/changes/DM-31596.feature.rst b/doc/changes/DM-31596.feature.rst
diff --git a/doc/changes/DM-31623.feature.rst b/doc/changes/DM-31623.feature.rst
diff --git a/doc/changes/DM-31700.bugfix.rst b/doc/changes/DM-31700.bugfix.rst
diff --git a/doc/changes/DM-31785.perf.rst b/doc/changes/DM-31785.perf.rst
diff --git a/doc/changes/DM-31826.bugfix.rst b/doc/changes/DM-31826.bugfix.rst
diff --git a/doc/changes/DM-31884.feature.rst b/doc/changes/DM-31884.feature.rst
diff --git a/doc/changes/DM-31976.feature.rst b/doc/changes/DM-31976.feature.rst
diff --git a/doc/changes/DM-32305.perf.rst b/doc/changes/DM-32305.perf.rst
diff --git a/doc/changes/registry/DM-32403.feature.md → doc/changes/DM-32403.feature.md b/doc/changes/registry/DM-32403.feature.md → doc/changes/DM-32403.feature.md
diff --git a/doc/changes/datastore/.gitignore b/doc/changes/datastore/.gitignore
diff --git a/doc/changes/datastore/DM-29849.perf b/doc/changes/datastore/DM-29849.perf
diff --git a/doc/changes/datastore/DM-30335.api.rst b/doc/changes/datastore/DM-30335.api.rst
diff --git a/doc/changes/registry/.gitignore b/doc/changes/registry/.gitignore
diff --git a/doc/changes/registry/DM-29196.feature b/doc/changes/registry/DM-29196.feature
diff --git a/doc/changes/registry/DM-30200.feature.rst b/doc/changes/registry/DM-30200.feature.rst
diff --git a/doc/changes/registry/DM-30866.feature.rst b/doc/changes/registry/DM-30866.feature.rst
diff --git a/doc/lsst.daf.butler/CHANGES.rst b/doc/lsst.daf.butler/CHANGES.rst
@@ -1,3 +1,105 @@
+Butler v23.0.0 2021-12-10
+=========================
+
+New Features
+------------
+
+- Add ability to cache datasets locally when using a remote file store.
+  This can significantly improve performance when retrieving components from a dataset. (`DM-13365 <https://jira.lsstcorp.org/browse/DM-13365>`_)
+- Add a new ``butler retrieve-artifacts`` command to copy file artifacts from a Butler datastore. (`DM-27241 <https://jira.lsstcorp.org/browse/DM-27241>`_)
+- Add ``butler transfer-datasets`` command-line tool and associated ``Butler.transfer_from()`` API.
+
+  This can be used to transfer datasets between different butlers, with the caveat that dimensions and dataset types must be pre-defined in the receiving butler repository. (`DM-28650 <https://jira.lsstcorp.org/browse/DM-28650>`_)
+- Add ``amp`` parameter to the Exposure StorageClass, allowing single-amplifier subimage reads. (`DM-29370 <https://jira.lsstcorp.org/browse/DM-29370>`_)
+- Add new ``butler collection-chain`` subcommand for creating collection chains from the command line. (`DM-30373 <https://jira.lsstcorp.org/browse/DM-30373>`_)
+- Add ``butler ingest-files`` subcommand to simplify ingest of any external file. (`DM-30935 <https://jira.lsstcorp.org/browse/DM-30935>`_)
+- * Add class representing a collection of log records (``ButlerLogRecords``).
+  * Allow this class to be stored and retrieved from a Butler datastore.
+  * Add special log handler to allow JSON log records to be stored.
+  * Add ``--log-file`` option to command lines to redirect log output to file.
+  * Add ``--no-log-tty`` to disable log output to terminal. (`DM-30977 <https://jira.lsstcorp.org/browse/DM-30977>`_)
+- Registry methods that previously could raise an exception when searching in
+  calibrations collections now have an improved logic that skip those
+  collections if they were not given explicitly but only appeared in chained
+  collections. (`DM-31337 <https://jira.lsstcorp.org/browse/DM-31337>`_)
+- Add a confirmation step to ``butler prune-collection`` to help prevent
+  accidental removal of collections. (`DM-31366 <https://jira.lsstcorp.org/browse/DM-31366>`_)
+- Add ``butler register-dataset-type`` command to register a new dataset type. (`DM-31367 <https://jira.lsstcorp.org/browse/DM-31367>`_)
+- Use cached summary information to simplify queries involving datasets and provide better diagnostics when those queries yield no results. (`DM-31583 <https://jira.lsstcorp.org/browse/DM-31583>`_)
+- Add a new ``butler export-calibs`` command to copy calibrations and write an export.yaml document from a Butler datastore. (`DM-31596 <https://jira.lsstcorp.org/browse/DM-31596>`_)
+- Support rewriting of dataId containing dimension records such as ``day_obs`` and ``seq_num`` in ``butler.put()``.
+  This matches the behavior of ``butler.get()``. (`DM-31623 <https://jira.lsstcorp.org/browse/DM-31623>`_)
+- Add ``--log-label`` option to ``butler`` command to allow extra information to be injected into the log record. (`DM-31884 <https://jira.lsstcorp.org/browse/DM-31884>`_)
+- * The ``Butler.transfer_from`` method no longer registers new dataset types by default.
+  * Add the related option ``--register-dataset-types`` to the ``butler transfer-datasets`` subcommand. (`DM-31976 <https://jira.lsstcorp.org/browse/DM-31976>`_)
+- Support UUIDs as the primary keys in registry and allow for reproducible UUIDs.
+
+  This change will significantly simplify transferring of data between butler repositories. (`DM-29196 <https://jira.lsstcorp.org/browse/DM-29196>`_)
+- Allow registry methods such as ``queryDatasets`` to use a glob-style string when specifying collection or dataset type names. (`DM-30200 <https://jira.lsstcorp.org/browse/DM-30200>`_)
+- Add support for updating and replacing dimension records. (`DM-30866 <https://jira.lsstcorp.org/browse/DM-30866>`_)
+
+
+API Changes
+-----------
+
+- A new method ``Datastore.knows()`` has been added to allow a user to ask the datastore whether it knows about a specific dataset but without requiring a check to see if the artifact itself exists.
+  Use ``Datastore.exists()`` to check that the datastore knows about a dataset and the artifact exists. (`DM-30335 <https://jira.lsstcorp.org/browse/DM-30335>`_)
+
+
+Bug Fixes
+---------
+
+- Fix handling of ingest_date timestamps.
+
+  Previously there was an inconsistency between ingest_date database-native UTC
+  handling and astropy Time used for time literals which resulted in 37 second
+  difference. This updates makes consistent use of database-native time
+  functions to resolve this issue. (`DM-30124 <https://jira.lsstcorp.org/browse/DM-30124>`_)
+- Fix butler repository creation when a seed config has specified a registry manager override.
+
+  Previously only that manager was recorded rather than the full set.
+  We always require a full set to be recorded to prevent breakage of a butler when a default changes. (`DM-30372 <https://jira.lsstcorp.org/browse/DM-30372>`_)
+- Stop writing a temporary datastore cache directory every time a ``Butler`` object was instantiated.
+  Now only create one when one is requested. (`DM-30743 <https://jira.lsstcorp.org/browse/DM-30743>`_)
+- Fix ``Butler.transfer_from()`` such that it registers any missing dataset types and also skips any datasets that do not have associated datastore artifacts. (`DM-30784 <https://jira.lsstcorp.org/browse/DM-30784>`_)
+- Add support for click 8.0. (`DM-30855 <https://jira.lsstcorp.org/browse/DM-30855>`_)
+- Replace UNION ALL with UNION for subqueries for simpler query plans. (`DM-31429 <https://jira.lsstcorp.org/browse/DM-31429>`_)
+- Fix parquet formatter error when reading tables with no indices.
+
+  Previously, this would cause butler.get to fail to read valid parquet tables. (`DM-31700 <https://jira.lsstcorp.org/browse/DM-31700>`_)
+- Fix problem in ButlerURI where transferring a file from one URI to another would overwrite the existing file even if they were the same actual file (for example because of soft links in the directory hierarchy). (`DM-31826 <https://jira.lsstcorp.org/browse/DM-31826>`_)
+
+
+Performance Enhancement
+-----------------------
+
+- Make collection and dataset pruning significantly more efficient. (`DM-30140 <https://jira.lsstcorp.org/browse/DM-30140>`_)
+- Add indexes to make certain spatial join queries much more efficient. (`DM-31548 <https://jira.lsstcorp.org/browse/DM-31548>`_)
+- Made 20x speed improvement for ``Butler.transfer_from``.
+  The main slow down is asking the datastore whether a file artifact exists.
+  This is now parallelized and the result is cached for later. (`DM-31785 <https://jira.lsstcorp.org/browse/DM-31785>`_)
+- Minor efficiency improvements when accessing `lsst.daf.butler.Config` hierarchies. (`DM-32305 <https://jira.lsstcorp.org/browse/DM-32305>`_)
+- FileDatastore: Improve removing of datasets from the trash by at least a factor of 10. (`DM-29849 <https://jira.lsstcorp.org/browse/DM-29849>`_)
+
+Other Changes and Additions
+---------------------------
+
+- Enable serialization of ``DatasetRef`` and related classes to JSON format. (`DM-28678 <https://jira.lsstcorp.org/browse/DM-28678>`_)
+- `ButlerURI` ``http`` schemes can now handle non-WebDAV endpoints.
+  Warnings are only issued if WebDAV functionality is requested. (`DM-29708 <https://jira.lsstcorp.org/browse/DM-29708>`_)
+- Switch logging such that all logging messages are now forwarded to Python ``logging`` from ``lsst.log``.
+  Previously all Python ``logging`` messages were being forwarded to ``lsst.log``. (`DM-31120 <https://jira.lsstcorp.org/browse/DM-31120>`_)
+- Add formatter and storageClass information for FocalPlaneBackground. (`DM-22534 <https://jira.lsstcorp.org/browse/DM-22534>`_)
+- Add formatter and storageClass information for IsrCalib. (`DM-29531 <https://jira.lsstcorp.org/browse/DM-29531>`_)
+- Change release note creation to use [Towncrier](https://towncrier.readthedocs.io/en/actual-freaking-docs/index.html). (`DM-30291 <https://jira.lsstcorp.org/browse/DM-30291>`_)
+- Add a Butler configuration for an execution butler that has pre-defined registry entries but no datastore records.
+
+  The `Butler.put()` will return the pre-existing dataset ref but will still fail if a datastore record is found. (`DM-30335 <https://jira.lsstcorp.org/browse/DM-30335>`_)
+- If an unrecognized dimension is used as a look up key in a configuration file (using the ``+`` syntax) a warning is used suggesting a possible typo rather than a confusing `KeyError`.
+  This is no longer a fatal error and the key will be treated as a name. (`DM-30685 <https://jira.lsstcorp.org/browse/DM-30685>`_)
+- Add ``split`` transfer mode that can be used when some files are inside the datastore and some files are outside the datastore.
+  This is equivalent to using `None` and ``direct`` mode dynamically. (`DM-31251 <https://jira.lsstcorp.org/browse/DM-31251>`_)
+
 Butler v22.0 2021-04-01
 =======================
 

diff --git a/pyproject.toml b/pyproject.toml
@@ -27,29 +27,12 @@
         name = "Performance Enhancement"
         showcontent = true
 
-    [[tool.towncrier.type]]
-        directory = "other"
-        name = "Other Changes and Additions"
-        showcontent = false
-
     [[tool.towncrier.type]]
         directory = "misc"
-        name = "Miscellaneous Changes of Minor Interest"
-        showcontent = false
+        name = "Other Changes and Additions"
+        showcontent = true
 
     [[tool.towncrier.type]]
         directory = "removal"
         name = "An API Removal or Deprecation"
-        showcontent = false
-
-    [[tool.towncrier.section]]
-        name = ""
-        path = ""
-
-    [[tool.towncrier.section]]
-        name = "Butler Datastores"
-        path = "datastore"
-
-    [[tool.towncrier.section]]
-        name = "Butler Registry"
-        path = "registry"
+        showcontent = true