Add external resources (#442)

* move NWBTable to Table
* add Row class and RowGetter for working with Table objects
* add new ExternalResources Container class with 4 compound-dtype tables

CHANGELOG.md

@@ -9,6 +9,7 @@
 - Add support for creating and specifying multi-index columns in a `DynamicTable` using `add_column(...)`.
   @bendichter, @rly (#430)
 - Add capability to add a row to a column after IO. @bendichter (#426)
+- Add functionality for storing external resource references @ajtritt (#442)
 - Add method `hdmf.utils.get_docval_macro` to get a tuple of the current values for a docval_macro, e.g., 'array_data'  
   and 'scalar_data'. @rly (#446)
 - Add SimpleMultiContainer, a data_type for storing a Container and Data objects together. @ajtritt (#449)

src/hdmf/__init__.py

@@ -1,20 +1,6 @@
 from . import query  # noqa: F401
 from .container import Container, Data, DataRegion
 from .utils import docval, getargs
-from .region import ListSlicer
-from .backends.hdf5.h5_utils import H5RegionSlicer, H5Dataset
-
-
-@docval({'name': 'dataset', 'type': None, 'doc': 'the HDF5 dataset to slice'},
-        {'name': 'region', 'type': None, 'doc': 'the region reference to use to slice'},
-        is_method=False)
-def get_region_slicer(**kwargs):
-    dataset, region = getargs('dataset', 'region', kwargs)
-    if isinstance(dataset, (list, tuple, Data)):
-        return ListSlicer(dataset, region)
-    elif isinstance(dataset, H5Dataset):
-        return H5RegionSlicer(dataset, region)
-    return None
 
 
 from ._version import get_versions  # noqa: E402

src/hdmf/backends/hdf5/h5_utils.py

@@ -119,6 +119,11 @@ def __init__(self, **kwargs):
                 self.__refgetters[i] = self.__get_regref
             elif t is Reference:
                 self.__refgetters[i] = self._get_ref
+            elif t is str:
+                # we need this for when we read compound data types
+                # that have unicode sub-dtypes since h5py does not
+                # store UTF-8 in compound dtypes
+                self.__refgetters[i] = self._get_utf
         self.__types = types
         tmp = list()
         for i in range(len(self.dataset.dtype)):
@@ -162,6 +167,12 @@ def __swap_refs(self, row):
             getref = self.__refgetters[i]
             row[i] = getref(row[i])
 
+    def _get_utf(self, string):
+        """
+        Decode a dataset element to unicode
+        """
+        return string.decode('utf-8') if isinstance(string, bytes) else string
+
     def __get_regref(self, ref):
         obj = self._get_ref(ref)
         return obj[ref]

src/hdmf/backends/hdf5/h5tools.py

@@ -636,7 +636,7 @@ def __read_dataset(self, h5obj, name=None):
                     kwargs['dtype'] = d.dtype
             elif h5obj.dtype.kind == 'V':    # table / compound data type
                 cpd_dt = h5obj.dtype
-                ref_cols = [check_dtype(ref=cpd_dt[i]) for i in range(len(cpd_dt))]
+                ref_cols = [check_dtype(ref=cpd_dt[i]) or check_dtype(vlen=cpd_dt[i]) for i in range(len(cpd_dt))]
                 d = BuilderH5TableDataset(h5obj, self, ref_cols)
                 kwargs['dtype'] = HDF5IO.__compound_dtype_to_list(h5obj.dtype, d.dtype)
             else:

src/hdmf/build/manager.py

@@ -108,11 +108,11 @@ def type_map(self):
     def get_proxy(self, **kwargs):
         obj = getargs('object', kwargs)
         if isinstance(obj, BaseBuilder):
-            return self.__get_proxy_builder(obj)
+            return self._get_proxy_builder(obj)
         elif isinstance(obj, AbstractContainer):
-            return self.__get_proxy_container(obj)
+            return self._get_proxy_container(obj)
 
-    def __get_proxy_builder(self, builder):
+    def _get_proxy_builder(self, builder):
         dt = self.__type_map.get_builder_dt(builder)
         ns = self.__type_map.get_builder_ns(builder)
         stack = list()
@@ -123,7 +123,7 @@ def __get_proxy_builder(self, builder):
         loc = "/".join(reversed(stack))
         return Proxy(self, builder.source, loc, ns, dt)
 
-    def __get_proxy_container(self, container):
+    def _get_proxy_container(self, container):
         ns, dt = self.__type_map.get_container_ns_dt(container)
         stack = list()
         tmp = container
@@ -273,7 +273,7 @@ def construct(self, **kwargs):
         if result is None:
             parent_builder = self.__get_parent_dt_builder(builder)
             if parent_builder is not None:
-                parent = self.__get_proxy_builder(parent_builder)
+                parent = self._get_proxy_builder(parent_builder)
                 result = self.__type_map.construct(builder, self, parent)
             else:
                 # we are at the top of the hierarchy,

src/hdmf/build/objectmapper.py

@@ -1202,14 +1202,19 @@ def construct(self, **kwargs):
                 continue
             kwargs[argname] = val
         try:
-            obj = cls.__new__(cls, container_source=builder.source, parent=parent,
-                              object_id=builder.attributes.get(self.__spec.id_key()))
-            obj.__init__(**kwargs)
+            obj = self.__new_container__(cls, builder.source, parent, builder.attributes.get(self.__spec.id_key()),
+                                         **kwargs)
         except Exception as ex:
             msg = 'Could not construct %s object due to: %s' % (cls.__name__, ex)
             raise Exception(msg) from ex
         return obj
 
+    def __new_container__(self, cls, container_source, parent, object_id, **kwargs):
+        """A wrapper function for ensuring a container gets everything set appropriately"""
+        obj = cls.__new__(cls, container_source=container_source, parent=parent, object_id=object_id)
+        obj.__init__(**kwargs)
+        return obj
+
     @docval({'name': 'container', 'type': AbstractContainer,
              'doc': 'the AbstractContainer to get the Builder name for'})
     def get_builder_name(self, **kwargs):

src/hdmf/common/__init__.py

@@ -104,6 +104,7 @@ def available_namespaces():
 
     from . import table  # noqa: F401,E402
     from . import sparse  # noqa: F401,E402
+    from . import resources  # noqa: F401,E402
     from . import multi  # noqa: F401,E402
 
     from .. import Data, Container
@@ -121,6 +122,7 @@ def available_namespaces():
 DynamicTableRegion = __TYPE_MAP.get_container_cls(CORE_NAMESPACE, 'DynamicTableRegion')
 VocabData = __TYPE_MAP.get_container_cls(CORE_NAMESPACE, 'VocabData')
 CSRMatrix = __TYPE_MAP.get_container_cls(CORE_NAMESPACE, 'CSRMatrix')
+ExternalResources = __TYPE_MAP.get_container_cls(CORE_NAMESPACE, 'ExternalResources')
 SimpleMultiContainer = __TYPE_MAP.get_container_cls(CORE_NAMESPACE, 'SimpleMultiContainer')
 
 

src/hdmf/common/io/__init__.py

@@ -1,2 +1,3 @@
 from . import table  # noqa: F401
+from . import resources  # noqa: F401
 from . import multi  # noqa: F401

src/hdmf/common/io/resources.py

@@ -0,0 +1,36 @@
+from ...build import ObjectMapper
+from ..resources import ExternalResources, KeyTable, ResourceTable, ObjectTable, ObjectKeyTable
+from .. import register_map
+
+
+@register_map(ExternalResources)
+class ExternalResourcesMap(ObjectMapper):
+
+    def construct_helper(self, name, parent_builder, table_cls, manager):
+        """Create a new instance of table_cls with data from parent_builder[name].
+
+           The DatasetBuilder for name is associated with data_type Data and container class Data,
+           but users should use the more specific table_cls for these datasets.
+        """
+        parent = manager._get_proxy_builder(parent_builder)
+        builder = parent_builder[name]
+        src = builder.source
+        oid = builder.attributes.get(self.spec.id_key())
+        kwargs = dict(name=builder.name, data=builder.data)
+        return self.__new_container__(table_cls, src, parent, oid, **kwargs)
+
+    @ObjectMapper.constructor_arg('keys')
+    def keys(self, builder, manager):
+        return self.construct_helper('keys', builder, KeyTable, manager)
+
+    @ObjectMapper.constructor_arg('resources')
+    def resources(self, builder, manager):
+        return self.construct_helper('resources', builder, ResourceTable, manager)
+
+    @ObjectMapper.constructor_arg('objects')
+    def objects(self, builder, manager):
+        return self.construct_helper('objects', builder, ObjectTable, manager)
+
+    @ObjectMapper.constructor_arg('object_keys')
+    def object_keys(self, builder, manager):
+        return self.construct_helper('object_keys', builder, ObjectKeyTable, manager)