Fix butler error relating to calibRepo on ingestion

lsst · Jun 12, 2019 · c823663 · c823663
1 parent 98783bc
commit c823663
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 10 deletions.
diff --git a/python/lsst/ap/verify/dataset.py b/python/lsst/ap/verify/dataset.py
@@ -204,7 +204,7 @@ def _validatePackage(self):
         if not _isRepo(self._stubInputRepo):
             raise RuntimeError('Stub repo at ' + self._stubInputRepo + 'is missing mapper file')
 
-    def makeCompatibleRepo(self, repoDir):
+    def makeCompatibleRepo(self, repoDir, calibRepoDir):
         """Set up a directory as a repository compatible with this dataset.
 
         If the directory already exists, any files required by the dataset will
@@ -214,14 +214,17 @@ def makeCompatibleRepo(self, repoDir):
         ----------
         repoDir : `str`
             The directory where the output repository will be created.
+        calibRepoDir : `str`
+            The directory where the output calibration repository will be created.
         """
+        mapperArgs = {'mapperArgs': {'calibRoot': calibRepoDir}}
         if _isRepo(self.templateLocation):
             # Stub repo is not a parent because can't mix v1 and v2 repositories in parents list
             Butler(inputs=[{"root": self.templateLocation, "mode": "r"}],
-                   outputs=[{"root": repoDir, "mode": "rw"}])
+                   outputs=[{"root": repoDir, "mode": "rw", **mapperArgs}])
         else:
             Butler(inputs=[{"root": self._stubInputRepo, "mode": "r"}],
-                   outputs=[{"root": repoDir, "mode": "rw"}])
+                   outputs=[{"root": repoDir, "mode": "rw", **mapperArgs}])
 
 
 def _isRepo(repoDir):

diff --git a/python/lsst/ap/verify/ingestion.py b/python/lsst/ap/verify/ingestion.py
@@ -141,7 +141,8 @@ def run(self, dataset, workspace):
             If the repositories already exist, they must support the same
             ``obs`` package as this task's subtasks.
         """
-        dataset.makeCompatibleRepo(workspace.dataRepo)
+        # We're assuming ingest tasks always give absolute path to butler
+        dataset.makeCompatibleRepo(workspace.dataRepo, os.path.abspath(workspace.calibRepo))
         self._ingestRaws(dataset, workspace)
         self._ingestCalibs(dataset, workspace)
         self._ingestDefects(dataset, workspace)
@@ -173,12 +174,13 @@ def _ingestRaws(self, dataset, workspace):
             self.log.info("Ingesting raw images...")
             dataFiles = _findMatchingFiles(dataset.rawLocation, self.config.dataFiles)
             if dataFiles:
-                self._doIngestRaws(workspace.dataRepo, dataFiles, self.config.dataBadFiles)
+                self._doIngestRaws(workspace.dataRepo, workspace.calibRepo,
+                                   dataFiles, self.config.dataBadFiles)
                 self.log.info("Images are now ingested in {0}".format(workspace.dataRepo))
             else:
                 raise RuntimeError("No raw files found at %s." % dataset.rawLocation)
 
-    def _doIngestRaws(self, repo, dataFiles, badFiles):
+    def _doIngestRaws(self, repo, calibRepo, dataFiles, badFiles):
         """Ingest raw images into a repository.
 
         ``repo`` shall be populated with *links* to ``dataFiles``.
@@ -187,6 +189,8 @@ def _doIngestRaws(self, repo, dataFiles, badFiles):
         ----------
         repo : `str`
             The output repository location on disk for raw images. Must exist.
+        calibRepo : `str`
+            The output calibration repository location on disk.
         dataFiles : `list` of `str`
             A list of filenames to ingest. May contain wildcards.
         badFiles : `list` of `str`
@@ -201,7 +205,7 @@ def _doIngestRaws(self, repo, dataFiles, badFiles):
         if not dataFiles:
             raise RuntimeError("No raw files to ingest (expected list of filenames, got %r)." % dataFiles)
 
-        args = [repo, "--mode", "link"]
+        args = [repo, "--calib", calibRepo, "--mode", "link"]
         args.extend(dataFiles)
         if badFiles:
             args.append('--badFile')

diff --git a/tests/test_ingestion.py b/tests/test_ingestion.py
@@ -239,7 +239,7 @@ def testDataIngest(self):
         self.setUpRawRegistry()
         files = [os.path.join(self._dataset.rawLocation, datum['file'])
                  for datum in IngestionTestSuite.rawData]
-        self._task._doIngestRaws(self._repo, files, [])
+        self._task._doIngestRaws(self._repo, self._calibRepo, files, [])
 
         self.assertRawRegistryCalls(self._registerTask, IngestionTestSuite.rawData)
 
@@ -311,7 +311,7 @@ def testNoFileIngest(self):
         self.setUpRawRegistry()
 
         with self.assertRaises(RuntimeError):
-            self._task._doIngestRaws(self._repo, files, [])
+            self._task._doIngestRaws(self._repo, self._calibRepo, files, [])
         with self.assertRaises(RuntimeError):
             self._task._doIngestCalibs(self._repo, self._calibRepo, files)
 
@@ -336,7 +336,7 @@ def testBadFileIngest(self):
 
         files = [os.path.join(self._dataset.rawLocation, datum['file'])
                  for datum in IngestionTestSuite.rawData]
-        self._task._doIngestRaws(self._repo, files, badFiles)
+        self._task._doIngestRaws(self._repo, self._calibRepo, files, badFiles)
 
         filteredData = [datum for datum in IngestionTestSuite.rawData if datum['file'] not in badFiles]
         self.assertRawRegistryCalls(self._registerTask, filteredData)