Fix repo creation always trying to overwrite tables.

lsst · Aug 27, 2020 · fb3784c · fb3784c
1 parent 8da2693
commit fb3784c
Show file tree

Hide file tree

Showing 2 changed files with 27 additions and 23 deletions.
diff --git a/python/lsst/ap/verify/dataset.py b/python/lsst/ap/verify/dataset.py
@@ -255,18 +255,21 @@ def makeCompatibleRepo(self, repoDir, calibRepoDir):
     def makeCompatibleRepoGen3(self, repoDir):
         """Set up a directory as a Gen 3 repository compatible with this ap_verify dataset.
 
-        If the directory already exists, any files required by the dataset will
-        be added if absent; otherwise the directory will remain unchanged.
+        If the repository already exists, this call has no effect.
 
         Parameters
         ----------
         repoDir : `str`
             The directory where the output repository will be created.
         """
-        repoConfig = dafButler.Butler.makeRepo(repoDir, overwrite=True)
-        butler = dafButler.Butler(repoConfig, writeable=True)
-        butler.import_(directory=self._preloadedRepo, filename=self._preloadedExport,
-                       transfer="auto")
+        # No way to tell makeRepo "create only what's missing"
+        try:
+            repoConfig = dafButler.Butler.makeRepo(repoDir)
+            butler = dafButler.Butler(repoConfig, writeable=True)
+            butler.import_(directory=self._preloadedRepo, filename=self._preloadedExport,
+                           transfer="auto")
+        except FileExistsError:
+            pass
 
 
 def _isRepo(repoDir):

diff --git a/tests/test_dataset.py b/tests/test_dataset.py
@@ -115,6 +115,20 @@ def testExistingOutput(self):
             if os.path.exists(testDir):
                 shutil.rmtree(testDir, ignore_errors=True)
 
+    def _checkOutputGen3(self, repo):
+        """Perform various integrity checks on a repository.
+
+        Parameters
+        ----------
+        repo : `str`
+            The repository to test. Currently only filesystem repositories
+            are supported.
+        """
+        self.assertTrue(os.path.exists(repo), 'Output directory must exist.')
+        # Call to Butler will fail if repo is corrupted
+        butler = dafButler.Butler(repo)
+        self.assertIn("LSST-ImSim/calib", butler.registry.queryCollections())
+
     def testOutputGen3(self):
         """Verify that a Dataset can create an output repository as desired.
         """
@@ -123,12 +137,7 @@ def testOutputGen3(self):
 
         try:
             self._testbed.makeCompatibleRepoGen3(outputDir)
-            self.assertTrue(os.path.exists(outputDir), 'Output directory must exist.')
-            self.assertTrue(os.listdir(outputDir), 'Output directory must not be empty.')
-            self.assertTrue(os.path.exists(os.path.join(outputDir, 'butler.yaml')),
-                            'Output directory must have a butler.yaml file.')
-            butler = dafButler.Butler(outputDir)
-            self.assertIn("LSST-ImSim/calib", butler.registry.queryCollections())
+            self._checkOutputGen3(outputDir)
         finally:
             if os.path.exists(testDir):
                 shutil.rmtree(testDir, ignore_errors=True)
@@ -141,18 +150,10 @@ def testExistingOutputGen3(self):
         outputDir = os.path.join(testDir, 'badOut')
 
         try:
-            os.makedirs(outputDir)
-            output = os.path.join(outputDir, 'foo.txt')
-            with open(output, 'w') as dummy:
-                dummy.write('This is a test!')
-
             self._testbed.makeCompatibleRepoGen3(outputDir)
-            self.assertTrue(os.path.exists(outputDir), 'Output directory must exist.')
-            self.assertTrue(os.listdir(outputDir), 'Output directory must not be empty.')
-            self.assertTrue(os.path.exists(os.path.join(outputDir, 'butler.yaml')),
-                            'Output directory must have a butler.yaml file.')
-            butler = dafButler.Butler(outputDir)
-            self.assertIn("LSST-ImSim/calib", butler.registry.queryCollections())
+            self._checkOutputGen3(outputDir)
+            self._testbed.makeCompatibleRepoGen3(outputDir)
+            self._checkOutputGen3(outputDir)
         finally:
             if os.path.exists(testDir):
                 shutil.rmtree(testDir, ignore_errors=True)