Tickets/dm 5452

python/lsst/daf/persistence/__init__.py

@@ -22,16 +22,21 @@
 
 """Python interface to lsst::daf::persistence classes
 """
+from registries import *
+from fsScanner import *
 from persistenceLib import *
+from butlerExceptions import *
 from policy import *
+from registries import *
 from butlerLocation import *
 from readProxy import *
 from butlerSubset import *
 from access import *
+from posixStorage import *
+from mapper import *
+from repositoryMapper import *
 from repository import *
 from butler import *
-from mapper import *
 from butlerFactory import *
 from .version import *
-from butlerExceptions import *
 

python/lsst/daf/persistence/access.py

@@ -24,154 +24,26 @@
 
 import cPickle
 import collections
-import importlib
 import os
 
-from lsst.daf.persistence import LogicalLocation, Policy, butlerExceptions, StorageList, Persistence
-import lsst.pex.policy as pexPolicy
-import lsst.pex.logging as pexLog
-from .safeFileIo import SafeFilename
+from lsst.daf.persistence import Policy
 
-# How to document Storage class protocol? Virtual base class seems somewhat non-pythonic?
-# todo: https://docs.python.org/2/library/abc.html?highlight=abc#module-abc is acceptable.
-class Storage(object):
+import yaml
 
-    @staticmethod
-    def getMapperClass(root):
-        raise NotImplementedError("getMapperClass is not implemented by derived class")
+class AccessCfg(Policy, yaml.YAMLObject):
+    yaml_tag = u"!AccessCfg"
+    def __init__(self, cls, storageCfg):
+        super(AccessCfg, self).__init__({'storageCfg':storageCfg, 'cls':cls})
 
+class Access:
+    """Implements an butler framework interface for Transport, Storage, and Registry
 
-class PosixStorage:
-
-    @classmethod
-    def cfg(cls, root):
-        """Helper func to create a properly formatted Policy to configure a PosixStorage instance.
-
-        :param root: a posix path where the repository is or should be created.
-        :return:
-        """
-        return Policy({'root':root, 'cls':cls})
-
-    def __init__(self, cfg):
-        """Initializer
-
-        :param cfg: a Policy that defines the configuration for this class. It is recommended that the cfg be
-                    created by calling PosixStorage.cfg()
-        :return:
-        """
-        self.log = pexLog.Log(pexLog.Log.getDefaultLog(), "daf.persistence.butler")
-        self.root = cfg['root']
-        if self.root and not os.path.exists(self.root):
-            os.makedirs(self.root)
-
-        # Always use an empty Persistence policy until we can get rid of it
-        persistencePolicy = pexPolicy.Policy()
-        self.persistence = Persistence.getPersistence(persistencePolicy)
-
-    @staticmethod
-    def getMapperClass(root):
-        """Returns the mapper class associated with a repository root.
-
-        Supports the legacy _parent symlink search (which was only ever posix-only. This should not be used by
-        new code and repositories; they should use the Repository parentCfg mechanism."""
-        if not (root):
-            return None
-
-        # Find a "_mapper" file containing the mapper class name
-        basePath = root
-        mapperFile = "_mapper"
-        globals = {}
-        while not os.path.exists(os.path.join(basePath, mapperFile)):
-            # Break abstraction by following _parent links from CameraMapper
-            if os.path.exists(os.path.join(basePath, "_parent")):
-                basePath = os.path.join(basePath, "_parent")
-            else:
-                raise RuntimeError(
-                        "No mapper provided and no %s available" %
-                        (mapperFile,))
-        mapperFile = os.path.join(basePath, mapperFile)
-
-        # Read the name of the mapper class and instantiate it
-        with open(mapperFile, "r") as f:
-            mapperName = f.readline().strip()
-        components = mapperName.split(".")
-        if len(components) <= 1:
-            raise RuntimeError("Unqualified mapper name %s in %s" %
-                    (mapperName, mapperFile))
-        pkg = importlib.import_module(".".join(components[:-1]))
-        return getattr(pkg, components[-1])
-
-    def mapperClass(self):
-        """Get the class object for the mapper specified in the stored repository"""
-        return PosixStorage.getMapperClass(self.root)
-
-    def setCfg(self, repoCfg):
-        """Writes the configuration to root in the repository on disk.
-
-        :param repoCfg: the Policy cfg to be written
-        :return: None
-        """
-        if not self.root:
-            raise RuntimeError("Storage root was declared to be None.")
-        path = os.path.join(self.root, 'repoCfg.yaml')
-        repoCfg.dumpToFile(path)
-
-    def loadCfg(self):
-        """Reads the configuration from the repository on disk at root.
-
-        :return: the Policy cfg
-        """
-        if not self.root:
-            raise RuntimeError("Storage root was declared to be None.")
-        path = os.path.join(self.root, 'repoCfg.yaml')
-        return Policy(filePath=path)
-
-    def write(self, butlerLocation, obj):
-        """Writes an object to a location and persistence format specified by ButlerLocation
+    .. warning::
 
-        :param butlerLocation: the location & formatting for the object to be written.
-        :param obj: the object to be written.
-        :return: None
-        """
-        self.log.log(pexLog.Log.DEBUG, "Put location=%s obj=%s" % (butlerLocation, obj))
-        additionalData = butlerLocation.getAdditionalData()
-        storageName = butlerLocation.getStorageName()
-        locations = butlerLocation.getLocations()
-        with SafeFilename(locations[0]) as locationString:
-            logLoc = LogicalLocation(locationString, additionalData)
-
-            if storageName == "PickleStorage":
-                with open(logLoc.locString(), "wb") as outfile:
-                    cPickle.dump(obj, outfile, cPickle.HIGHEST_PROTOCOL)
-                return
-
-            if storageName == "ConfigStorage":
-                obj.save(logLoc.locString())
-                return
-
-            if storageName == "FitsCatalogStorage":
-                flags = additionalData.getInt("flags", 0)
-                obj.writeFits(logLoc.locString(), flags=flags)
-                return
-
-            # Create a list of Storages for the item.
-            storageList = StorageList()
-            storage = self.persistence.getPersistStorage(storageName, logLoc)
-            storageList.append(storage)
-
-            if storageName == 'FitsStorage':
-                self.persistence.persist(obj, storageList, additionalData)
-                return
-
-            # Persist the item.
-            if hasattr(obj, '__deref__'):
-                # We have a smart pointer, so dereference it.
-                self.persistence.persist(obj.__deref__(), storageList, additionalData)
-            else:
-                self.persistence.persist(obj, storageList, additionalData)
+        Access is 'wet paint' and very likely to change. Use of it in production code other than via the 'old butler'
+        API is strongly discouraged.
 
-class Access:
-    """Implements an butler framework interface for Transport, Storage, and Registry"""
+    """
 
     @classmethod
     def cfg(cls, storageCfg):
@@ -180,7 +52,7 @@ def cfg(cls, storageCfg):
         :param storageCfg: a cfg to instantiate a storage.
         :return:
         """
-        return Policy({'storageCfg': storageCfg})
+        return AccessCfg(cls=cls, storageCfg=storageCfg)
 
     def __init__(self, cfg):
         """Initializer
@@ -191,6 +63,9 @@ def __init__(self, cfg):
         """
         self.storage = cfg['storageCfg.cls'](cfg['storageCfg'])
 
+    def __repr__(self):
+        return 'Access(storage=%s)' % self.storage
+
     def mapperClass(self):
         """Get the mapper class associated with a repository root.
 
@@ -207,6 +82,16 @@ def root(self):
 
         return self.storage.root
 
+    def locationWithRoot(self, location):
+        """Given a location, get a fully qualified handle to location including storage root.
+
+        Note; at the time of this writing the only existing storage type is PosixStorage. This returns the
+        root+location.
+        :param location:
+        :return:
+        """
+        return self.storage.locationWithRoot(location)
+
     def setCfg(self, repoCfg):
         """Writes the repository configuration to Storage.
 
@@ -230,3 +115,28 @@ def write(self, butlerLocation, obj):
         :return: None
         """
         self.storage.write(butlerLocation, obj)
+
+    def read(self, butlerLocation):
+        """Reads an object from storage
+
+        :param butlerLocation: describes the location & how to load the object.
+        :return:
+        """
+        return self.storage.read(butlerLocation=butlerLocation)
+
+    def exists(self, location):
+        """Query if a location exists.
+
+        As of this writing the only storage type is PosixStorage, and it works to say that 'location' is a
+        simple locaiton descriptor. In the case of PosixStorage that's a path. If this needs to become more
+        complex it could be changed to be a butlerLocation, or something else, as needed.
+        :param location: a simple location descriptor, type is dependent on Storage.
+        :return: True if location exists, else False.
+        """
+        return self.storage.exists(location)
+
+    def lookup(self, *args, **kwargs):
+        """Perform a lookup in the registry.
+
+        Returns a list of dataId for each valid lookup (right? TODO VERIFY)"""
+        return self.storage.lookup(*args, **kwargs)