Add run, collection keys to file templates.

config/datastores/posixDatastore.yaml

@@ -5,7 +5,7 @@ datastore:
     table: PosixDatastoreRecords
   create: true
   templates:
-    default: "{datasetType}/{tract:?}/{patch:?}/{filter:?}/{camera:?}_{visit:?}"
+    default: "{collection}/{datasetType}/{tract:?}/{patch:?}/{filter:?}/{camera:?}_{visit:?}"
   formatters:
     TablePersistable: lsst.daf.butler.formatters.fitsCatalogFormatter.FitsCatalogFormatter
     TablePersistableWcs: lsst.daf.butler.formatters.fitsCatalogFormatter.FitsCatalogFormatter

python/lsst/daf/butler/core/fileTemplates.py

@@ -98,9 +98,15 @@ class FileTemplate:
     -----
     The templates use the standard Format Specification Mini-Language
     with the caveat that only named fields can be used. The field names
-    are taken from the DataUnits along with two additional fields:
-    "datasetType" will be replaced with the DatasetType and "component"
-    will be replaced with the component name of a composite.
+    are taken from the DataUnits along with several additional fields:
+
+     - datasetType: `str`, `DatasetType.name`
+     - component: `str`, name of the StorageClass component
+     - collection: `str`, `Run.collection`
+     - run: `int`, `Run.id`
+
+    At least one or both of `run` or `collection` must be provided to ensure
+    unique filenames.
 
     The mini-language is extended to understand a "?" in the format
     specification. This indicates that a field is optional. If that
@@ -142,6 +148,10 @@ def format(self, ref):
         if component is not None:
             fields["component"] = component
 
+        usedRunOrCollection = False
+        fields["collection"] = ref.run.collection
+        fields["run"] = ref.run.id
+
         fmt = string.Formatter()
         parts = fmt.parse(self.template)
         output = ""
@@ -162,6 +172,11 @@ def format(self, ref):
             else:
                 optional = False
 
+            if field_name in ("run", "collection"):
+                usedRunOrCollection = True
+                if optional:
+                    raise KeyError("'run' and 'collection' may not be optional.")
+
             if field_name in fields:
                 value = fields[field_name]
             elif optional:
@@ -183,6 +198,10 @@ def format(self, ref):
             raise KeyError("Component '{}' specified but template {} did not use it".format(component,
                                                                                             self.template))
 
+        # Complain if there's no run or collection
+        if not usedRunOrCollection:
+            raise KeyError("Template does not include 'run' or 'collection'.")
+
         # Since this is known to be a path, normalize it in case some double
         # slashes have crept in
         path = os.path.normpath(output)

tests/config/basic/posixDatastore.yaml

@@ -2,10 +2,10 @@ datastore:
   cls: lsst.daf.butler.datastores.posixDatastore.PosixDatastore
   root: ./butler_test_repository
   templates:
-    default: "{datasetType}/{tract:?}/{patch:?}/{filter:?}/{camera:?}_{visit:?}"
-    calexp: "{datasetType}.{component:?}/{datasetType}_v{visit}_f{filter:?}_{component:?}"
-    metric: "{datasetType}.{component:?}/{datasetType}_v{visit:08d}_f{filter}_{component:?}"
-    test_metric_comp: "{datasetType}.{component:?}/{datasetType}_v{visit:08d}_f{camera}_{component:?}"
+    default: "{collection}/{datasetType}/{tract:?}/{patch:?}/{filter:?}/{camera:?}_{visit:?}"
+    calexp: "{collection}/{datasetType}.{component:?}/{datasetType}_v{visit}_f{filter:?}_{component:?}"
+    metric: "{collection}/{datasetType}.{component:?}/{datasetType}_v{visit:08d}_f{filter}_{component:?}"
+    test_metric_comp: "{collection}/{datasetType}.{component:?}/{datasetType}_v{visit:08d}_f{camera}_{component:?}"
   formatters:
     StructuredDataDictYaml: lsst.daf.butler.formatters.yamlFormatter.YamlFormatter
     StructuredDataListYaml: lsst.daf.butler.formatters.yamlFormatter.YamlFormatter

tests/config/basic/posixDatastore2.yaml

@@ -4,10 +4,10 @@ datastore:
   records:
     table: PosixDatastoreRecords2
   templates:
-    default: "{datasetType}/{tract:?}/{patch:?}/{filter:?}/{camera:?}_{visit:?}"
-    calexp: "{datasetType}.{component:?}/{datasetType}_v{visit}_f{filter:?}_{component:?}"
-    metric: "{datasetType}.{component:?}/{datasetType}_v{visit:08d}_f{filter}_{component:?}"
-    test_metric_comp: "{datasetType}.{component:?}/{datasetType}_v{visit:08d}_f{camera}_{component:?}"
+    default: "{run:02d}/{datasetType}/{tract:?}/{patch:?}/{filter:?}/{camera:?}_{visit:?}"
+    calexp: "{run:02d}/{datasetType}.{component:?}/{datasetType}_v{visit}_f{filter:?}_{component:?}"
+    metric: "{run:02d}/{datasetType}.{component:?}/{datasetType}_v{visit:08d}_f{filter}_{component:?}"
+    test_metric_comp: "{run:02d}/{datasetType}.{component:?}/{datasetType}_v{visit:08d}_f{camera}_{component:?}"
   formatters:
     StructuredDataDictYaml: lsst.daf.butler.formatters.yamlFormatter.YamlFormatter
     StructuredDataListYaml: lsst.daf.butler.formatters.yamlFormatter.YamlFormatter

tests/test_templates.py

@@ -23,7 +23,7 @@
 
 import unittest
 
-from lsst.daf.butler import DatasetType, DatasetRef, FileTemplate, StorageClass
+from lsst.daf.butler import DatasetType, DatasetRef, FileTemplate, StorageClass, Run
 
 
 class TestFileTemplates(unittest.TestCase):
@@ -37,7 +37,7 @@ def makeDatasetRef(self, datasetTypeName, dataId=None):
             self.datasetTypes[datasetTypeName] = DatasetType(datasetTypeName, list(dataId.keys()),
                                                              StorageClass(None))
         datasetType = self.datasetTypes[datasetTypeName]
-        return DatasetRef(datasetType, dataId)
+        return DatasetRef(datasetType, dataId, run=Run(id=2, collection="run2"))
 
     def setUp(self):
         self.dataId = {"visit": 52, "filter": "U"}
@@ -49,40 +49,40 @@ def assertTemplate(self, template, answer, ref):
         self.assertEqual(path, answer)
 
     def testBasic(self):
-        tmplstr = "{datasetType}/{visit:05d}/{filter}"
+        tmplstr = "{run:02d}/{datasetType}/{visit:05d}/{filter}"
         self.assertTemplate(tmplstr,
-                            "calexp/00052/U",
+                            "02/calexp/00052/U",
                             self.makeDatasetRef("calexp"))
-        tmplstr = "{datasetType}/{visit:05d}/{filter}-trail"
+        tmplstr = "{run:02d}/{datasetType}/{visit:05d}/{filter}-trail"
         self.assertTemplate(tmplstr,
-                            "calexp/00052/U-trail",
+                            "02/calexp/00052/U-trail",
                             self.makeDatasetRef("calexp"))
 
     def testOptional(self):
         """Optional units in templates."""
         ref = self.makeDatasetRef("calexp")
-        tmplstr = "{datasetType}/v{visit:05d}_f{filter:?}"
-        self.assertTemplate(tmplstr, "calexp/v00052_fU",
+        tmplstr = "{run:02d}/{datasetType}/v{visit:05d}_f{filter:?}"
+        self.assertTemplate(tmplstr, "02/calexp/v00052_fU",
                             self.makeDatasetRef("calexp"))
 
         du = {"visit": 48, "tract": 265}
-        self.assertTemplate(tmplstr, "calexpT/v00048",
+        self.assertTemplate(tmplstr, "02/calexpT/v00048",
                             self.makeDatasetRef("calexpT", du))
 
         # Ensure that this returns a relative path even if the first field
         # is optional
-        tmplstr = "{tract:?}/{visit:?}/f{filter}"
-        self.assertTemplate(tmplstr, "52/fU", ref)
+        tmplstr = "{run:02d}/{tract:?}/{visit:?}/f{filter}"
+        self.assertTemplate(tmplstr, "02/52/fU", ref)
 
         # Ensure that // from optionals are converted to singles
-        tmplstr = "{datasetType}/{patch:?}/{tract:?}/f{filter}"
-        self.assertTemplate(tmplstr, "calexp/fU", ref)
+        tmplstr = "{run:02d}/{datasetType}/{patch:?}/{tract:?}/f{filter}"
+        self.assertTemplate(tmplstr, "02/calexp/fU", ref)
 
         # Optionals with some text between fields
-        tmplstr = "{datasetType}/p{patch:?}_t{tract:?}/f{filter}"
-        self.assertTemplate(tmplstr, "calexp/p/fU", ref)
-        tmplstr = "{datasetType}/p{patch:?}_t{visit:04d?}/f{filter}"
-        self.assertTemplate(tmplstr, "calexp/p_t0052/fU", ref)
+        tmplstr = "{run:02d}/{datasetType}/p{patch:?}_t{tract:?}/f{filter}"
+        self.assertTemplate(tmplstr, "02/calexp/p/fU", ref)
+        tmplstr = "{run:02d}/{datasetType}/p{patch:?}_t{visit:04d?}/f{filter}"
+        self.assertTemplate(tmplstr, "02/calexp/p_t0052/fU", ref)
 
     def testComponent(self):
         """Test handling of components in templates."""
@@ -91,14 +91,14 @@ def testComponent(self):
         refMaskedImage = self.makeDatasetRef("calexp.maskedimage.variance")
         refWcs = self.makeDatasetRef("calexp.wcs")
 
-        tmplstr = "c_{component}_v{visit}"
-        self.assertTemplate(tmplstr, "c_output_v52", refMetricOutput)
+        tmplstr = "{run:02d}_c_{component}_v{visit}"
+        self.assertTemplate(tmplstr, "02_c_output_v52", refMetricOutput)
 
-        tmplstr = "{component:?}_{visit}"
-        self.assertTemplate(tmplstr, "_52", refMetric)
-        self.assertTemplate(tmplstr, "output_52", refMetricOutput)
-        self.assertTemplate(tmplstr, "maskedimage.variance_52", refMaskedImage)
-        self.assertTemplate(tmplstr, "output_52", refMetricOutput)
+        tmplstr = "{run:02d}_{component:?}_{visit}"
+        self.assertTemplate(tmplstr, "02_52", refMetric)
+        self.assertTemplate(tmplstr, "02_output_52", refMetricOutput)
+        self.assertTemplate(tmplstr, "02_maskedimage.variance_52", refMaskedImage)
+        self.assertTemplate(tmplstr, "02_output_52", refMetricOutput)
 
         # Providing a component but not using it
         tmplstr = "{datasetType}/v{visit:05d}"