Merge pull request #490 from martindurant/unify

Let users never see Entry objects

intake/catalog/base.py

@@ -297,7 +297,7 @@ def _get_entry(self, name):
         entry = self._entries[name]
         entry._catalog = self
         entry._pmode = self.pmode
-        return entry
+        return entry()
 
     @reload_on_change
     def _get_entries(self):
@@ -380,7 +380,7 @@ def __getitem__(self, key):
             e = self._entries[key]
             e._catalog = self
             e._pmode = self.pmode
-            return e
+            return e()
         if isinstance(key, str) and '.' in key:
             key = key.split('.')
         if isinstance(key, list):
@@ -393,7 +393,7 @@ def __getitem__(self, key):
                     rest = '.'.join(parts)
                     try:
                         out = self._entries[prefix][rest]
-                        return out
+                        return out()
                     except KeyError:
                         # name conflict like "thing" and "think.oi", where it's
                         # the latter we are after
@@ -402,7 +402,7 @@ def __getitem__(self, key):
             out = self
             for part in key:
                 out = self[part]
-            return out
+            return out()
         raise KeyError(key)
 
     def discover(self):
@@ -768,8 +768,7 @@ def _data_to_source(cat, path, **kwargs):
         if not isinstance(cat, Catalog):
             raise NotImplementedError
         out = {}
-        for name in cat:
-            entry = cat[name]
+        for name, entry in cat.items():
             out[name] = entry.__getstate__()
             out[name]['parameters'] = [up._captured_init_kwargs for up
                                        in entry._user_parameters]

intake/catalog/entry.py

@@ -81,11 +81,12 @@ def __call__(self, persist=None, **kwargs):
             s2 = s.get_persisted()
             met = s2.metadata
             if persist == 'always' or not met['ttl']:
-                return s2
-            if met['ttl'] < time.time() - met['timestamp']:
-                return s2
+                s = s2
+            elif met['ttl'] < time.time() - met['timestamp']:
+                s = s2
             else:
-                return store.refresh(s2)
+                s = store.refresh(s2)
+        s._entry = self
         return s
 
     def _get_default_source(self):
@@ -94,6 +95,16 @@ def _get_default_source(self):
             self._default_source = self()
         return self._default_source
 
+    @property
+    def container(self):
+        return getattr(self, '_container', None)
+
+    @container.setter
+    def container(self, cont):
+        # so that .container (which sources always have)  always reflects ._container,
+        # which is the variable name for entries.
+        self._container = cont
+
     @property
     def has_been_persisted(self, **kwargs):
         """For the source created with the given args, has it been persisted?"""
@@ -115,18 +126,6 @@ def _ipython_display_(self):
             'application/json': {'root': contents["name"]}
         }, raw=True)
 
-    def __getattr__(self, attr):
-        if attr in self.__dict__:
-            return self.__dict__[attr]
-        else:
-            return getattr(self._get_default_source(), attr)
-
-    def __dir__(self):
-        selflist = {'describe', 'describe_open', 'get', 'gui',
-                    'has_been_persisted', 'plots'}
-        selflist.update(set(dir(self._get_default_source())))
-        return list(sorted(selflist))
-
     def __iter__(self):
         # If the entry is a catalog, this allows list(cat.entry)
         if self._container == 'catalog':

intake/catalog/tests/catalog1.yml

@@ -41,7 +41,7 @@ sources:
       urlpath: '{{ CATALOG_DIR }}/entry1_{{ part }}.csv'
   remote_env:
     description: env gets interpreted in server
-    driver: csv
+    driver: intake.conftest.TestSource
     args:
       urlpath: 'path-{{intake_test}}'
     parameters:
@@ -72,9 +72,9 @@ sources:
       chunks: 5
   datetime:
     description: datetime parameters
-    driver: csv
+    driver: intake.conftest.TestSource
     args:
-      time: "{{time}}"
+      urlpath: "{{time}}"
     parameters:
       time:
         description: some time

intake/catalog/tests/dot-nest.yaml

@@ -17,9 +17,15 @@ sources:
   leaf:
     description: leaf
     driver: csv
+    args:
+      urlpath: ""
   leafdot.dot:
     description: leaf-dot
     driver: csv
+    args:
+      urlpath: ""
   leaf.dot:
     description: leaf-dot
     driver: csv
+    args:
+      urlpath: ""

intake/catalog/tests/test_local.py

@@ -60,15 +60,16 @@ def test_local_catalog(catalog1):
         'metadata': {'foo': 'baz', 'bar': [2, 4, 6]},
         'plugin': ['csv']
     }
-    assert catalog1['entry1'].get().container == 'dataframe'
-    md = catalog1['entry1'].get().metadata
+    assert catalog1['entry1'].container == 'dataframe'
+    md = catalog1['entry1'].metadata
     md.pop('catalog_dir')
-    assert md == dict(foo='bar', bar=[1, 2, 3])
+    assert md['foo'] == 'bar'
+    assert md['bar'] == [1, 2, 3]
 
     # Use default parameters
-    assert catalog1['entry1_part'].get().container == 'dataframe'
+    assert catalog1['entry1_part'].container == 'dataframe'
     # Specify parameters
-    assert catalog1['entry1_part'].get(part='2').container == 'dataframe'
+    assert catalog1['entry1_part'].configure_new(part='2').container == 'dataframe'
 
 
 def test_get_items(catalog1):
@@ -82,9 +83,8 @@ def test_nested(catalog1):
     assert catalog1.entry1.read().equals(catalog1.nested.nested.entry1.read())
     assert 'nested.nested' not in catalog1.walk(depth=1)
     assert 'nested.nested' in catalog1.walk(depth=2)
-    assert catalog1.nested._catalog == catalog1
-    assert catalog1.nested().cat == catalog1
-    assert catalog1.nested.nested.nested().cat.cat.cat is catalog1
+    assert catalog1.nested.cat == catalog1
+    assert catalog1.nested.nested.nested.cat.cat.cat is catalog1
 
 
 def test_nested_gets_name_from_super(catalog1):
@@ -117,7 +117,7 @@ def test_metadata(catalog1):
 
 
 def test_use_source_plugin_from_config(catalog1):
-    catalog1['use_example1'].get()
+    catalog1['use_example1']
 
 
 def test_get_dir():
@@ -307,7 +307,7 @@ def test_union_catalog():
 
     # Implied creation of data source
     assert union_cat.entry1.container == 'dataframe'
-    md = union_cat.entry1._metadata
+    md = union_cat.entry1.describe()['metadata']
     assert md == dict(foo='bar', bar=[1, 2, 3])
 
     # Use default parameters in explict creation of data source
@@ -541,14 +541,12 @@ def test_multi_plugins():
 def test_no_plugins():
     fn = abspath('multi_plugins.yaml')
     cat = open_catalog(fn)
-    s = cat.tables6
     with pytest.raises(ValueError) as e:
-        s()
+        cat.tables6
     assert 'doesnotexist' in str(e.value)
     assert 'plugin-directory' in str(e.value)
-    s = cat.tables7
     with pytest.raises(ValueError) as e:
-        s()
+        cat.tables7
     assert 'doesnotexist' in str(e.value)
 
 
@@ -580,44 +578,43 @@ def test_getitem_and_getattr():
         catalog.doesnotexit
     with pytest.raises(AttributeError):
         catalog._doesnotexit
-    assert catalog.tables0 is catalog['tables0']
-    assert isinstance(catalog.tables0, LocalCatalogEntry)
+    assert catalog.tables0 == catalog['tables0']
     assert isinstance(catalog.metadata, (dict, type(None)))
 
 
 def test_dot_names():
     fn = abspath('dot-nest.yaml')
     cat = open_catalog(fn)
-    assert cat.self.leaf._description == 'leaf'
-    assert cat.self['leafdot.dot']._description == 'leaf-dot'
-    assert cat['selfdot.dot', 'leafdot.dot']._description == 'leaf-dot'
+    assert cat.self.leaf.description == 'leaf'
+    assert cat.self['leafdot.dot'].description == 'leaf-dot'
+    assert cat['selfdot.dot', 'leafdot.dot'].description == 'leaf-dot'
 
-    assert cat['self.selfdot.dot', 'leafdot.dot']._description == 'leaf-dot'
-    assert cat['self.self.dot', 'leafdot.dot']._description == 'leaf-dot'
-    assert cat['self.self.dot', 'leaf']._description == 'leaf'
-    assert cat['self.self.dot', 'leaf.dot']._description == 'leaf-dot'
+    assert cat['self.selfdot.dot', 'leafdot.dot'].description == 'leaf-dot'
+    assert cat['self.self.dot', 'leafdot.dot'].description == 'leaf-dot'
+    assert cat['self.self.dot', 'leaf'].description == 'leaf'
+    assert cat['self.self.dot', 'leaf.dot'].description == 'leaf-dot'
 
-    assert cat['self.self.dot.leaf.dot']._description == 'leaf-dot'
+    assert cat['self.self.dot.leaf.dot'].description == 'leaf-dot'
 
 
 def test_listing(catalog1):
     assert list(catalog1) == list(catalog1.nested)
-    with pytest.raises(ValueError):
+    with pytest.raises(TypeError):
         list(catalog1.arr)
 
 
 def test_dict_save():
     from intake.catalog.base import Catalog
     fn = os.path.join(tempfile.mkdtemp(), 'mycat.yaml')
     entry = LocalCatalogEntry(name='trial', description='get this back',
-                              driver='csv')
+                              driver='csv', args=dict(urlpath=""))
     cat = Catalog.from_dict({'trial': entry}, name='mycat')
     cat.save(fn)
 
     cat2 = open_catalog(fn)
     assert 'trial' in cat2
     assert cat2.name == 'mycat'
-    assert cat2.trial._driver == 'csv'
+    assert "CSV" in cat2.trial.classname
 
 
 def test_dict_save_complex():
@@ -641,27 +638,27 @@ def test_dict_save_complex():
 def test_dict_adddel():
     from intake.catalog.base import Catalog
     entry = LocalCatalogEntry(name='trial', description='get this back',
-                              driver='csv')
+                              driver='csv', args=dict(urlpath=""))
     cat = Catalog.from_dict({'trial': entry}, name='mycat')
     assert 'trial' in cat
     cat['trial2'] = entry
     assert list(cat) == ['trial', 'trial2']
     cat.pop('trial')
     assert list(cat) == ['trial2']
-    assert cat['trial2'] is entry
+    assert cat['trial2'].describe() == entry.describe()
 
 
 def test_filter():
     from intake.catalog.base import Catalog
     entry1 = LocalCatalogEntry(name='trial', description='get this back',
-                              driver='csv')
+                               driver='csv', args=dict(urlpath=""))
     entry2 = LocalCatalogEntry(name='trial', description='pass this through',
-                              driver='csv')
+                               driver='csv', args=dict(urlpath=""))
     cat = Catalog.from_dict({'trial1': entry1,
                              'trial2': entry2}, name='mycat')
     cat2 = cat.filter(lambda e: 'pass' in e._description)
     assert list(cat2) == ['trial2']
-    assert cat2.trial2 is entry2
+    assert cat2.trial2 == entry2()
 
 
 def test_no_instance():