Tickets/dm 6459

python/lsst/daf/persistence/__init__.py

@@ -22,21 +22,25 @@
 
 """Python interface to lsst::daf::persistence classes
 """
-from registries import *
-from fsScanner import *
-from persistenceLib import *
-from butlerExceptions import *
-from policy import *
-from registries import *
-from butlerLocation import *
-from readProxy import *
-from butlerSubset import *
-from access import *
-from posixStorage import *
-from mapper import *
-from repositoryMapper import *
-from repository import *
-from butler import *
-from butlerFactory import *
+from .utils import *
+from .registries import *
+from .fsScanner import *
+from .persistenceLib import *
+from .butlerExceptions import *
+from .policy import *
+from .registries import *
+from .dataId import *
+from .butlerLocation import *
+from .readProxy import *
+from .butlerSubset import *
+from .access import *
+from .repositoryCfg import *
+from .storage import *
+from .posixStorage import *
+from .mapper import *
+from .repositoryMapper import *
+from .repository import *
+from .butler import *
+from .butlerFactory import *
 from .version import *
 

python/lsst/daf/persistence/butlerFactory.py

@@ -26,7 +26,7 @@
 
 """This module defines the ButlerFactory class."""
 
-from lsst.daf.persistence import Butler
+from lsst.daf.persistence import Butler, RepositoryArgs, PosixStorage
 
 class ButlerFactory(object):
     """ButlerFactory creates data Butlers containing data mappers.  Use of it
@@ -70,4 +70,8 @@ def create(self):
         @returns a new Butler.
         """
 
-        return Butler(None, mapper=self.mapper)
+        if hasattr(self.mapper, 'root'):
+            root = self.mapper.root
+        else:
+            root = None
+        return Butler(root=root, mapper=self.mapper)

python/lsst/daf/persistence/butlerLocation.py

@@ -47,12 +47,12 @@ def __repr__(self):
         'ButlerLocation(pythonType=%r, cppType=%r, storageName=%r, locationList=%r, additionalData=%r, mapper=%r)' % \
         (self.pythonType, self.cppType, self.storageName, self.locationList, self.additionalData, self.mapper)
 
-    def __init__(self, pythonType, cppType, storageName, locationList, dataId, mapper, access=None):
+    def __init__(self, pythonType, cppType, storageName, locationList, dataId, mapper, storage=None):
         self.pythonType = pythonType
         self.cppType = cppType
         self.storageName = storageName
         self.mapper = mapper
-        self.access = access
+        self.storage = storage
         if hasattr(locationList, '__iter__'):
             self.locationList = locationList
         else:
@@ -76,7 +76,7 @@ def to_yaml(dumper, obj):
         """
         return dumper.represent_mapping(ButlerLocation.yaml_tag,
             {'pythonType':obj.pythonType, 'cppType':obj.cppType, 'storageName':obj.storageName,
-             'locationList':obj.locationList, 'mapper':obj.mapper, 'access':obj.access, 'dataId':obj.dataId})
+             'locationList':obj.locationList, 'mapper':obj.mapper, 'storage':obj.storage, 'dataId':obj.dataId})
 
     @staticmethod
     def from_yaml(loader, node):

python/lsst/daf/persistence/butlerSubset.py

@@ -29,6 +29,8 @@
 
 from __future__ import with_statement
 
+from . import DataId
+
 class ButlerSubset(object):
 
     """ButlerSubset is a container for ButlerDataRefs.  It represents a
@@ -70,11 +72,11 @@ def __init__(self, butler, datasetType, level, dataId):
         """
         self.butler = butler
         self.datasetType = datasetType
-        self.dataId = dataId
+        self.dataId = DataId(dataId)
         self.cache = []
         self.level = level
 
-        keys = self.butler.getKeys(datasetType, level)
+        keys = self.butler.getKeys(datasetType, level, tag=dataId.tag)
         if keys is None:
             return
         fmt = list(keys.iterkeys())
@@ -218,11 +220,13 @@ def subLevels(self):
 
         return set(
                 self.butlerSubset.butler.getKeys(
-                    self.butlerSubset.datasetType).keys()
+                    self.butlerSubset.datasetType, 
+                    tag=self.butlerSubset.dataId.tag).keys()
             ) - set(
                 self.butlerSubset.butler.getKeys(
                     self.butlerSubset.datasetType,
-                    self.butlerSubset.level).keys()
+                    self.butlerSubset.level,
+                    tag=self.butlerSubset.dataId.tag).keys()
             )
 
     def subItems(self, level=None):
@@ -238,7 +242,10 @@ def subItems(self, level=None):
         """
 
         if level is None:
-            mappers = self.butlerSubset.butler.repository.mappers()
+            mappers = []
+            for repoData in self.butlerSubset.butler._repos.all():
+                if repoData.repo._mapper not in mappers:
+                    mappers.append(repoData.repo._mapper)
             if len(mappers) != 1:
                 raise RuntimeError("Support for multiple repositories not yet implemented!")
             mapper = mappers[0]

python/lsst/daf/persistence/dataId.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+
+#
+# LSST Data Management System
+# Copyright 2016 LSST Corporation.
+#
+# This product includes software developed by the
+# LSST Project (http://www.lsst.org/).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the LSST License Statement and
+# the GNU General Public License along with this program.  If not,
+# see <http://www.lsstcorp.org/LegalNotices/>.
+#
+
+import copy
+import UserDict
+
+class DataId(UserDict.IterableUserDict):
+    """DataId is used to pass scientifically meaningful key-value pairs. It may be tagged as applicable only 
+    to repositories that are tagged with the same value"""
+
+    def __init__(self, initialdata=None, tag=None, **kwargs):
+        """Constructor
+
+        Parameters
+        -----------
+        initialdata : dict or dataId
+            A dict of initial data for the DataId
+        tag : any type, or a container of any type
+            A value or container of values used to restrict the DataId to one or more repositories that 
+            share that tag value. It will be stored in a set for comparison with the set of tags assigned to 
+            repositories.
+        kwargs : any values
+            key-value pairs to be used as part of the DataId's data.
+        """
+        UserDict.UserDict.__init__(self, initialdata)
+        try:
+            self.tag = copy.deepcopy(initialdata.tag)
+        except AttributeError:
+            self.tag = set()
+
+        if tag is not None:
+            if isinstance(tag, basestring):
+                self.tag.update([tag])
+            else:
+                try:
+                    self.tag.update(tag)
+                except TypeError:
+                    self.tag.update([tag])
+
+        self.data.update(kwargs)
+
+    def __repr__(self):
+        return "DataId(initialdata=%s, tag=%s)" %(self.data.__repr__(), self.tag)

python/lsst/daf/persistence/mapper.py

@@ -24,7 +24,7 @@
 
 import yaml
 
-from lsst.daf.persistence import Policy
+from . import Policy
 
 """This module defines the Mapper base class."""
 
@@ -41,14 +41,14 @@ class MapperCfg(Policy):
     yaml_loader = yaml.Loader
     yaml_dumper = yaml.Dumper
 
-    def __init__(self, cls, policy, access):
+    def __init__(self, cls, policy, storage):
         super(MapperCfg, self).__init__()
-        self.update({'cls':cls, 'policy':policy, 'access':access})
+        self.update({'cls':cls, 'policy':policy, 'storage':storage})
 
     @staticmethod
     def to_yaml(dumper, obj):
         return dumper.represent_mapping(RepositoryMapperCfg.yaml_tag,
-                                        {'cls':obj['cls'], 'policy':obj['policy'], 'access':obj['access']})
+                                        {'cls':obj['cls'], 'policy':obj['policy'], 'storage':obj['storage']})
 
     @staticmethod
     def from_yaml(loader, node):

python/lsst/daf/persistence/policy.py

@@ -376,6 +376,36 @@ def getStringArray(self, key):
             val = [val]
         return val
 
+    def __lt__(self, other):
+        if isinstance(other, Policy):
+            other = other.data
+        return self.data < other
+
+    def __le__(self, other):
+        if isinstance(other, Policy):
+            other = other.data
+        return self.data <= other
+
+    def __eq__(self, other):
+        if isinstance(other, Policy):
+            other = other.data
+        return self.data == other
+
+    def __ne__(self, other):
+        if isinstance(other, Policy):
+            other = other.data
+        return self.data != other
+
+    def __gt__(self, other):
+        if isinstance(other, Policy):
+            other = other.data
+        return self.data > other
+
+    def __ge__(self, other):
+        if isinstance(other, Policy):
+            other = other.data
+        return self.data >= other
+
     #######
     # i/o #