DM-15372: Use temp directories for datastores

tests/test_butler.py

@@ -24,7 +24,8 @@
 
 import os
 import unittest
-from tempfile import TemporaryDirectory
+import tempfile
+import shutil
 import pickle
 
 import lsst.utils.tests
@@ -55,6 +56,7 @@ class TransactionTestError(Exception):
 class ButlerTests:
     """Tests for Butler.
     """
+    useTempRoot = True
 
     @staticmethod
     def addDatasetType(datasetTypeName, dataUnits, storageClass, registry):
@@ -75,14 +77,28 @@ def assertGetComponents(self, butler, datasetTypeName, dataId, components, refer
             result = butler.get(compTypeName, dataId)
             self.assertEqual(result, getattr(reference, component))
 
+    def setUp(self):
+        """Create a new butler root for each test."""
+        if self.useTempRoot:
+            self.root = tempfile.mkdtemp(dir=TESTDIR)
+            Butler.makeRepo(self.root, config=Config(self.configFile))
+            self.tmpConfigFile = os.path.join(self.root, "butler.yaml")
+        else:
+            self.root = None
+            self.tmpConfigFile = self.configFile
+
+    def tearDown(self):
+        if self.root is not None and os.path.exists(self.root):
+            shutil.rmtree(self.root, ignore_errors=True)
+
     def testConstructor(self):
         """Independent test of constructor.
         """
-        butler = Butler(self.configFile)
+        butler = Butler(self.tmpConfigFile)
         self.assertIsInstance(butler, Butler)
 
     def testBasicPutGet(self):
-        butler = Butler(self.configFile)
+        butler = Butler(self.tmpConfigFile)
         # Create and register a DatasetType
         datasetTypeName = "test_metric"
         dataUnits = ("Camera", "Visit")
@@ -112,7 +128,7 @@ def testBasicPutGet(self):
                                  ("summary", "data", "output"), metric)
 
     def testBasicPutGetWithDatasetRef(self):
-        butler = Butler(self.configFile)
+        butler = Butler(self.tmpConfigFile)
         # Create and register a DatasetType
         datasetTypeName = "test_metric"
         dataUnits = ("Camera", "Visit")
@@ -149,7 +165,7 @@ def testBasicPutGetWithDatasetRef(self):
             butler.get(DatasetRef(ref.datasetType, ref.dataId, id=101))
 
     def testCompositePutGet(self):
-        butler = Butler(self.configFile)
+        butler = Butler(self.tmpConfigFile)
         # Create and register a DatasetType
         datasetTypeName = "test_metric_comp"
         dataUnits = ("Camera", "Visit")
@@ -182,13 +198,13 @@ def testCompositePutGet(self):
     def testPickle(self):
         """Test pickle support.
         """
-        butler = Butler(self.configFile)
+        butler = Butler(self.tmpConfigFile)
         butlerOut = pickle.loads(pickle.dumps(butler))
         self.assertIsInstance(butlerOut, Butler)
         self.assertEqual(butlerOut.config, butler.config)
 
     def testTransaction(self):
-        butler = Butler(self.configFile)
+        butler = Butler(self.tmpConfigFile)
         datasetTypeName = "test_metric"
         dataUnits = ("Camera", "Visit")
         dataUnitEntries = (("Camera", {"camera": "DummyCam"}),
@@ -241,15 +257,16 @@ def testMakeRepo(self):
         if self.fullConfigKey is None:
             return
 
-        with TemporaryDirectory(prefix=TESTDIR + "/") as root:
-            Butler.makeRepo(root, config=Config(self.configFile))
-            limited = Config(self.configFile)
-            print(limited.ppprint())
-            butler1 = Butler(root, collection="ingest")
-            Butler.makeRepo(root, standalone=True, createRegistry=False,
-                            config=Config(self.configFile))
-            full = Config(os.path.join(root, "butler.yaml"))
-            butler2 = Butler(root, collection="ingest")
+        # Remove the file created in setUp
+        os.unlink(self.tmpConfigFile)
+
+        Butler.makeRepo(self.root, config=Config(self.configFile))
+        limited = Config(self.configFile)
+        butler1 = Butler(self.root, collection="ingest")
+        Butler.makeRepo(self.root, standalone=True, createRegistry=False,
+                        config=Config(self.configFile))
+        full = Config(self.tmpConfigFile)
+        butler2 = Butler(self.root, collection="ingest")
         # Butlers should have the same configuration regardless of whether
         # defaults were expanded.
         self.assertEqual(butler1.config, butler2.config)
@@ -271,6 +288,7 @@ class InMemoryDatastoreButlerTestCase(ButlerTests, lsst.utils.tests.TestCase):
     """InMemoryDatastore specialization of a butler"""
     configFile = os.path.join(TESTDIR, "config/basic/butler-inmemory.yaml")
     fullConfigKey = None
+    useTempRoot = False
 
 
 class ChainedDatastoreButlerTestCase(ButlerTests, lsst.utils.tests.TestCase):

tests/test_butlerFits.py

@@ -21,10 +21,12 @@
 
 import os
 import unittest
+import tempfile
+import shutil
 
 import lsst.utils.tests
 
-from lsst.daf.butler import Butler
+from lsst.daf.butler import Butler, Config
 from lsst.daf.butler import StorageClassFactory
 from lsst.daf.butler import DatasetType
 from datasetsHelper import FitsCatalogDatasetsHelper, DatasetTestHelper
@@ -38,6 +40,7 @@
 
 
 class ButlerFitsTests(FitsCatalogDatasetsHelper, DatasetTestHelper):
+    useTempRoot = True
 
     @staticmethod
     def registerDatasetTypes(datasetTypeName, dataUnits, storageClass, registry):
@@ -57,10 +60,24 @@ def setUpClass(cls):
         cls.storageClassFactory = StorageClassFactory()
         cls.storageClassFactory.addFromConfig(cls.configFile)
 
+    def setUp(self):
+        """Create a new butler root for each test."""
+        if self.useTempRoot:
+            self.root = tempfile.mkdtemp(dir=TESTDIR)
+            Butler.makeRepo(self.root, config=Config(self.configFile))
+            self.tmpConfigFile = os.path.join(self.root, "butler.yaml")
+        else:
+            self.root = None
+            self.tmpConfigFile = self.configFile
+
+    def tearDown(self):
+        if self.root is not None and os.path.exists(self.root):
+            shutil.rmtree(self.root, ignore_errors=True)
+
     def testExposureCompositePutGet(self):
         example = os.path.join(TESTDIR, "data", "basic", "small.fits")
         exposure = lsst.afw.image.ExposureF(example)
-        butler = Butler(self.configFile)
+        butler = Butler(self.tmpConfigFile)
         datasetTypeName = "calexp"
         dataUnits = ("Camera", "Visit")
         storageClass = self.storageClassFactory.getStorageClass("ExposureF")
@@ -99,6 +116,7 @@ class PosixDatastoreButlerTestCase(ButlerFitsTests, lsst.utils.tests.TestCase):
 class InMemoryDatastoreButlerTestCase(ButlerFitsTests, lsst.utils.tests.TestCase):
     """InMemoryDatastore specialization of a butler"""
     configFile = os.path.join(TESTDIR, "config/basic/butler-inmemory.yaml")
+    useTempRoot = False
 
 
 class ChainedDatastoreButlerTestCase(ButlerFitsTests, lsst.utils.tests.TestCase):

tests/test_datastore.py

@@ -107,12 +107,12 @@ def setUp(self):
 
         # Some subclasses override the working root directory
         if self.root is not None:
-            def cleanup(path):
-                if path is not None and os.path.exists(path):
-                    shutil.rmtree(path)
-            self.addCleanup(cleanup, self.root)
             self.datastoreType.setConfigRoot(self.root, self.config, self.config.copy())
 
+    def tearDown(self):
+        if self.root is not None and os.path.exists(self.root):
+            shutil.rmtree(self.root, ignore_errors=True)
+
     def testConstructor(self):
         datastore = self.makeDatastore()
         self.assertIsNotNone(datastore)

tests/test_datastoreFits.py

@@ -21,6 +21,8 @@
 
 import os
 import unittest
+import tempfile
+import shutil
 
 import lsst.utils.tests
 
@@ -45,6 +47,7 @@
 
 
 class DatastoreFitsTests(FitsCatalogDatasetsHelper, DatasetTestHelper):
+    root = None
 
     @classmethod
     def setUpClass(cls):
@@ -72,6 +75,16 @@ def setUp(self):
         # for these tests
         self.id = 1
 
+        self.config = DatastoreConfig(self.configFile)
+
+        # Some subclasses override the working root directory
+        if self.root is not None:
+            self.datastoreType.setConfigRoot(self.root, self.config, self.config.copy())
+
+    def tearDown(self):
+        if self.root is not None and os.path.exists(self.root):
+            shutil.rmtree(self.root, ignore_errors=True)
+
     def testConstructor(self):
         datastore = self.datastoreType(config=self.configFile, registry=self.registry)
         self.assertIsNotNone(datastore)
@@ -147,10 +160,12 @@ def testTransfer(self):
         ref = self.makeDatasetRef("calexp", dataUnits, storageClass, dataId)
 
         inputConfig = DatastoreConfig(self.configFile)
-        inputConfig["root"] = os.path.join(self.testDir, "./test_input_datastore")
+        self.datastoreType.setConfigRoot(os.path.join(self.testDir, "test_input_datastore"),
+                                         inputConfig, inputConfig.copy())
         inputPosixDatastore = self.datastoreType(config=inputConfig, registry=self.registry)
         outputConfig = inputConfig.copy()
-        outputConfig["root"] = os.path.join(self.testDir, "./test_output_datastore")
+        self.datastoreType.setConfigRoot(os.path.join(self.testDir, "test_output_datastore"),
+                                         outputConfig, outputConfig.copy())
         outputPosixDatastore = self.datastoreType(config=outputConfig,
                                                   registry=DummyRegistry())
 
@@ -245,6 +260,11 @@ class PosixDatastoreTestCase(DatastoreFitsTests, lsst.utils.tests.TestCase):
     uriScheme = "file:"
     fileExt = ".fits"
 
+    def setUp(self):
+        # Override the working directory before calling the base class
+        self.root = tempfile.mkdtemp(dir=TESTDIR)
+        super().setUp()
+
 
 class InMemoryDatastoreTestCase(DatastoreFitsTests, lsst.utils.tests.TestCase):
     """PosixDatastore specialization"""
@@ -253,11 +273,9 @@ class InMemoryDatastoreTestCase(DatastoreFitsTests, lsst.utils.tests.TestCase):
     fileExt = None
 
 
-class ChainedDatastoreTestCase(DatastoreFitsTests, lsst.utils.tests.TestCase):
+class ChainedDatastoreTestCase(PosixDatastoreTestCase):
     """PosixDatastore specialization"""
     configFile = os.path.join(TESTDIR, "config/basic/chainedDatastore.yaml")
-    uriScheme = "file:"
-    fileExt = ".fits"
 
 
 class MemoryTester(lsst.utils.tests.MemoryTestCase):