Rework VOSI parsing using astropy xml handling

docs/pyvo/data_access.rst

@@ -1195,17 +1195,17 @@ There are three types of service metadata:
 >>> print(service.available)
 True
 >>> print(service.up_since)
-datetime.datetime(2000, 0, 0, 0, 0, 0)
+'2000-01-01T00:00:00Z'
 >>> print(service.capabilities)
->>> print(service.tables.keys())
+>>> print(list(service.tables.keys()))
 
 The keys within tables are the fully qualified table names as they can
-be used in queries.  To inspect the column metadata for a table, see the column
+be used in queries.  To inspect the column metadata for a table, see the columns
 property of a give table.
 
 >>> service.tables["rave.main"].columns
 
-See also http://docs.astropy.org/en/stable/table/index.html.
+See also `~pyvo.io.vosi`
 
 .. note::
          Some TAP services have tables metadata of several megabytes.

docs/pyvo/io.rst

@@ -0,0 +1,15 @@
+
+.. py:module:: pyvo.io
+
+=========================
+The pyvo.io Package
+=========================
+
+This package contains io functionallity for pyvo
+
+.. toctree::
+  :maxdepth: 1
+
+  vosi
+
+.. automodapi:: pyvo.io.vosi

docs/pyvo/ref.rst

@@ -14,4 +14,4 @@ Virtual Observatory (VO) using Python.
   mod
   dal
   registry
-
+  io

docs/pyvo/vosi.rst

@@ -0,0 +1,14 @@
+
+.. py:module:: pyvo.io.vosi
+
+=========================
+The pyvo.io.vosi Package
+=========================
+
+This package contains io functionallity for pyvo
+
+.. automodapi:: pyvo.io.vosi.endpoint
+.. automodapi:: pyvo.io.vosi.vodataservice
+.. automodapi:: pyvo.io.vosi.voresource
+.. automodapi:: pyvo.io.vosi.tapregext
+.. automodapi:: pyvo.io.vosi.exceptions

pyvo/dal/mixin.py

@@ -9,7 +9,7 @@
 import requests
 
 from .query import DALServiceError
-from ..tools import vosi
+from ..io import vosi
 
 class AvailabilityMixin(object):
     """
@@ -20,14 +20,8 @@ class AvailabilityMixin(object):
     @property
     def availability(self):
         """
-        returns availability as a tuple in the following form:
-
-        Returns
-        -------
-        [0] : bool
-            whether the service is available or not
-        [1] : datetime
-            the time since the server is running
+        Service Availability as a
+        :py:class:`~pyvo.io.vosi.availability.Availability` object
         """
         if self._availability == (None, None):
             avail_url = '{0}/availability'.format(self.baseurl)
@@ -42,22 +36,22 @@ def availability(self):
             # requests doesn't decode the content by default
             response.raw.read = partial(response.raw.read, decode_content=True)
 
-            self._availability = vosi.parse_availability(response.raw)
+            self._availability = vosi.parse_availability(response.raw.read)
         return self._availability
 
     @property
     def available(self):
         """
         True if the service is available, False otherwise
         """
-        return self.availability[0]
+        return self.availability.available
 
     @property
     def up_since(self):
         """
         datetime the service was started
         """
-        return self.availability[1]
+        return self.availability.upsince
 
 
 class CapabilityMixin(object):
@@ -83,5 +77,5 @@ def capabilities(self):
             # requests doesn't decode the content by default
             response.raw.read = partial(response.raw.read, decode_content=True)
 
-            self._capabilities = vosi.parse_capabilities(response.raw)
+            self._capabilities = vosi.parse_capabilities(response.raw.read)
         return self._capabilities

pyvo/dal/tap.py

@@ -17,7 +17,8 @@
     DALServiceError, DALQueryError)
 from .mixin import AvailabilityMixin, CapabilityMixin
 from .datalink import DatalinkMixin
-from ..tools import vosi, uws
+from ..io import vosi, uws
+from ..io.vosi import tapregext as tr
 
 __all__ = [
     "search", "escape", "TAPService", "TAPQuery", "AsyncTAPJob", "TAPResults"]
@@ -63,6 +64,72 @@ def search(url, query, language="ADQL", maxrec=None, uploads=None, **keywords):
     service = TAPService(url)
     return service.search(query, language, maxrec, uploads, **keywords)
 
+class VOSITables(object):
+    """
+    This class encapsulates access to the VOSITables using a given Endpoint.
+    Access to table names is like accessing dictionary keys. using iterator
+    syntax or `keys()`
+    """
+    def __init__(self, vosi_tables, endpoint_url):
+        self._vosi_tables = vosi_tables
+        self._endpoint_url = endpoint_url
+        self._cache = {}
+
+    def __len__(self):
+        return self._vosi_tables.ntables
+
+    def __getitem__(self, key):
+        return self._get_table(key)
+
+    def __iter__(self):
+        return self.keys()
+
+    def _get_table(self, name):
+        if name in self._cache:
+            return self._cache[name]
+
+        table = self._vosi_tables.get_table_by_name(name)
+
+        if not table.columns and not table.foreignkeys:
+            tables_url = '{}/{}'.format(self._endpoint_url, name)
+            response = requests.get(tables_url, stream=True)
+
+            try:
+                response.raise_for_status()
+            except requests.RequestException as ex:
+                raise DALServiceError.from_except(ex, tables_url)
+
+            # requests doesn't decode the content by default
+            response.raw.read = partial(response.raw.read, decode_content=True)
+
+            table = vosi.parse_tables(response.raw.read).get_first_table()
+            self._cache[name] = table
+
+        return table
+
+    def keys(self):
+        """
+        Iterates over the keys (table names).
+        """
+        for table in self._vosi_tables.iter_tables():
+            yield table.name
+
+    def values(self):
+        """
+        Iterates over the values (tables).
+        Gathers missing values from endpoint if necessary.
+        """
+        for name in self.keys():
+            yield self._get_table(name)
+
+    def items(self):
+        """
+        Iterates over keys and values (table names and tables).
+        Gathers missing values from endpoint if necessary.
+        """
+        for name in self.keys():
+            yield (name, self._get_table(name))
+
 class TAPService(DALService, AvailabilityMixin, CapabilityMixin):
     """
     a representation of a Table Access Protocol service
@@ -84,7 +151,7 @@ def __init__(self, baseurl):
     @property
     def tables(self):
         """
-        returns tables as a flat OrderedDict
+        returns tables as a dict-like object
         """
         if self._tables is None:
             tables_url = '{0}/tables'.format(self.baseurl)
@@ -99,7 +166,8 @@ def tables(self):
             # requests doesn't decode the content by default
             response.raw.read = partial(response.raw.read, decode_content=True)
 
-            self._tables = vosi.parse_tables(response.raw)
+            self._tables = VOSITables(
+                vosi.parse_tables(response.raw.read), tables_url)
         return self._tables
 
     @property
@@ -114,9 +182,9 @@ def maxrec(self):
         """
         try:
             for capa in self.capabilities:
-                if "outputLimit" in capa:
-                    return capa["outputLimit"]["default"]["value"]
-        except KeyError:
+                if isinstance(capa, tr.TableAccess):
+                    return capa.outputlimit.default.value
+        except AttributeError:
             pass
         raise DALServiceError("Default limit not exposed by the service")
 
@@ -132,9 +200,9 @@ def hardlimit(self):
         """
         try:
             for capa in self.capabilities:
-                if "outputLimit" in capa:
-                    return capa["outputLimit"]["hard"]["value"]
-        except KeyError:
+                if isinstance(capa, tr.TableAccess):
+                    return capa.outputlimit.hard.value
+        except AttributeError:
             pass
         raise DALServiceError("Hard limit not exposed by the service")
 
@@ -143,11 +211,11 @@ def upload_methods(self):
         """
         a list of upload methods in form of IVOA identifiers
         """
-        _upload_methods = []
+        upload_methods = []
         for capa in self.capabilities:
-            if "uploadMethods" in capa:
-                _upload_methods += capa["uploadMethods"]
-        return _upload_methods
+            if isinstance(capa, tr.TableAccess):
+                upload_methods += capa.uploadmethods
+        return upload_methods
 
     def run_sync(
             self, query, language="ADQL", maxrec=None, uploads=None,

pyvo/tools/__init__.py → pyvo/io/__init__.py

@@ -1,2 +1 @@
 # Licensed under a 3-clause BSD style license - see LICENSE.rst
-from . import uws, vosi

pyvo/io/vosi/__init__.py

@@ -0,0 +1,2 @@
+# Licensed under a 3-clause BSD style license - see LICENSE.rst
+from .endpoint import parse_tables, parse_capabilities, parse_availability

pyvo/io/vosi/availability.py

@@ -0,0 +1,83 @@
+# Licensed under a 3-clause BSD style license - see LICENSE.rst
+from __future__ import (absolute_import, division, print_function,
+                        unicode_literals)
+
+from astropy.extern import six
+
+from astropy.utils.collections import HomogeneousList
+from astropy.utils.xml import check as xml_check
+from astropy.io.votable.exceptions import vo_raise, vo_warn, warn_or_raise
+
+from .util import (
+    make_add_simplecontent, make_add_complexcontent, Element, ValueMixin)
+from . import voresource as vr
+from .exceptions import W32, W33, W34, W35
+
+__all__ = ["Availability"]
+
+######################################################################
+# FACTORY FUNCTIONS
+def _convert_boolean(value, default=None):
+    return {
+        'false': False,
+        '0': False,
+        'true': True,
+        '1': True
+    }.get(value, default)
+
+######################################################################
+# ELEMENT CLASSES
+class Availability(Element):
+    def __init__(self, config=None, pos=None, **kwargs):
+        super(Availability, self).__init__(config=config, pos=pos, **kwargs)
+
+        self._tag_mapping.update({
+            "available": make_add_simplecontent(
+                self, "available", "available", W32),
+            "upSince": make_add_simplecontent(self, "upSince", "upsince", W33),
+            "downAt": make_add_simplecontent(self, "downAt", "downat", W34),
+            "backAt": make_add_simplecontent(self, "backAt", "backat", W35),
+            "note": make_add_simplecontent(self, "note", "notes")
+        })
+
+        self._available = None
+        self._upsince = None
+        self._downat = None
+        self._backat = None
+        self._notes = HomogeneousList(six.text_type)
+
+    @property
+    def available(self):
+        return self._available
+
+    @available.setter
+    def available(self, available):
+        self._available = _convert_boolean(available)
+
+    @property
+    def upsince(self):
+        return self._upsince
+
+    @upsince.setter
+    def upsince(self, upsince):
+        self._upsince = upsince
+
+    @property
+    def downat(self):
+        return self._downat
+
+    @downat.setter
+    def downat(self, downat):
+        self._downat = downat
+
+    @property
+    def backat(self):
+        return self._backat
+
+    @backat.setter
+    def backat(self, backat):
+        self._backat = backat
+
+    @property
+    def notes(self):
+        return self._notes