Merge pull request #328 from lsst/tickets/DM-27478

DM-27478: Add direct transfer option to butler ingest-raws

python/lsst/obs/base/formatters/fitsExposure.py

@@ -276,11 +276,6 @@ def write(self, inMemoryDataset):
         ----------
         inMemoryDataset : `object`
             The Python object to store.
-
-        Returns
-        -------
-        path : `str`
-            The `URI` where the primary file is stored.
         """
         # Update the location with the formatter-preferred file extension
         self.fileDescriptor.location.updateExtension(self.extension)
@@ -297,7 +292,6 @@ def write(self, inMemoryDataset):
             inMemoryDataset.writeFitsWithOptions(outputPath, options=ps)
         else:
             inMemoryDataset.writeFits(outputPath)
-        return self.fileDescriptor.location.pathInStore
 
     def getImageCompressionSettings(self, recipeName):
         """Retrieve the relevant compression settings for this recipe.

python/lsst/obs/base/ingest.py

@@ -143,6 +143,7 @@ def makeTransferChoiceField(doc="How to transfer files (None for no transfer).",
         allowed={"move": "move",
                  "copy": "copy",
                  "auto": "choice will depend on datastore",
+                 "direct": "use URI to ingested file directly in datastore",
                  "link": "hard link falling back to symbolic link",
                  "hardlink": "hard link",
                  "symlink": "symbolic (soft) link",

python/lsst/obs/base/ingest_tests.py

@@ -31,7 +31,7 @@
 import shutil
 
 import lsst.afw.cameraGeom
-from lsst.daf.butler import Butler
+from lsst.daf.butler import Butler, ButlerURI
 from lsst.daf.butler.cli.butler import cli as butlerCli
 from lsst.daf.butler.cli.utils import LogCliRunner
 import lsst.obs.base
@@ -146,8 +146,13 @@ def verifyIngest(self, files=None, cli=False, fullCheck=False):
         This only really affects files that contain multiple datasets.
         """
         butler = Butler(self.root, run=self.outputRun)
-        datasets = butler.registry.queryDatasets("raw", collections=self.outputRun)
-        self.assertEqual(len(list(datasets)), len(self.dataIds))
+        datasets = list(butler.registry.queryDatasets("raw", collections=self.outputRun))
+        self.assertEqual(len(datasets), len(self.dataIds))
+
+        # Get the URI to the first dataset and check it is inside the
+        # datastore
+        datasetUri = butler.getURI(datasets[0])
+        self.assertIsNotNone(datasetUri.relative_to(butler.datastore.root))
 
         for dataId in self.dataIds:
             # Check that we can read metadata from a raw
@@ -236,6 +241,16 @@ def testSymLink(self):
         self._ingestRaws(transfer="symlink")
         self.verifyIngest()
 
+    def testDirect(self):
+        self._ingestRaws(transfer="direct")
+
+        # Check that it really did have a URI outside of datastore
+        srcUri = ButlerURI(self.file)
+        butler = Butler(self.root, run=self.outputRun)
+        datasets = list(butler.registry.queryDatasets("raw", collections=self.outputRun))
+        datastoreUri = butler.getURI(datasets[0])
+        self.assertEqual(datastoreUri, srcUri)
+
     def testCopy(self):
         self._ingestRaws(transfer="copy")
         # Only test full read of raws for the copy test. No need to do it