Merge branch 'release/0.13.2'

CHANGES.rst

@@ -2,6 +2,16 @@
 Changes
 -------
 
+0.13.2
+~~~~~~
+
+New feature: Support for JSON table schemas [1] for resource indexes.
+
+[1] http://dataprotocols.org/json-table-schema/
+
+Bugfix: Fixed #26 where JSON data column was not serialized correctly in csv export.
+
+
 0.13.1
 ~~~~~~
 

clld/__init__.py

@@ -5,7 +5,7 @@
 from clld import interfaces
 
 
-__version__ = "0.13.1"
+__version__ = "0.13.2"
 _Resource = namedtuple('Resource', 'name model interface with_index with_rdfdump')
 
 

clld/db/meta.py

@@ -8,7 +8,7 @@
 except ImportError:
     import json
 
-from six import string_types, text_type
+from six import string_types, text_type, PY2
 from pytz import UTC
 import sqlalchemy
 from sqlalchemy.pool import Pool
@@ -149,6 +149,10 @@ def value_to_csv(self, attr, ctx=None, req=None):
             rel = attr[-1]
             attr = '__'.join(attr[:-1])
         prop = getattr(self, attr, '')
+        if attr == 'jsondata':
+            prop = json.dumps(prop)
+            if PY2:
+                prop = prop.decode('utf8')
         if rel == 'id':
             return prop.id
         elif rel == 'ids':
@@ -205,7 +209,7 @@ def __tablename__(cls):
     #: the kind of data stored in a table. 'Natural' candidates for primary keys
     #: should be marked with unique constraints instead. This adds flexibility
     #: when it comes to database changes.
-    pk = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True)
+    pk = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True, doc='primary key')
 
     #: To allow for timestamp-based versioning - as opposed or in addition to the version
     #: number approach implemented in :py:class:`clld.db.meta.Versioned` - we store

clld/db/models/common.py

@@ -257,9 +257,9 @@ class Dataset(Base,
     """Each project (e.g. WALS, APiCS) is regarded as one dataset; thus, each app will
     have exactly one Dataset object.
     """
-    published = Column(Date, default=date.today)
-    publisher_name = Column(Unicode)
-    publisher_place = Column(Unicode)
+    published = Column(Date, default=date.today, doc='date of publication')
+    publisher_name = Column(Unicode, doc='publisher')
+    publisher_place = Column(Unicode, doc='place of publication')
     publisher_url = Column(String)
     license = Column(String, default="http://creativecommons.org/licenses/by/3.0/")
     domain = Column(String, nullable=False)
@@ -314,9 +314,13 @@ class Language(Base,
     to them to be able to put them on maps.
     """
     latitude = Column(
-        Float(), CheckConstraint('-90 <= latitude and latitude <= 90'))
+        Float(),
+        CheckConstraint('-90 <= latitude and latitude <= 90'),
+        doc='geographical latitude in WGS84')
     longitude = Column(
-        Float(), CheckConstraint('-180 <= longitude and longitude <= 180 '))
+        Float(),
+        CheckConstraint('-180 <= longitude and longitude <= 180 '),
+        doc='geographical longitude in WGS84')
     identifiers = association_proxy('languageidentifier', 'identifier')
 
     def get_identifier_objs(self, type_):
@@ -364,10 +368,10 @@ class DomainElement(Base,
 
     parameter_pk = Column(Integer, ForeignKey('parameter.pk'))
 
-    number = Column(Integer)
+    number = Column(Integer, doc='numerical value of the domain element')
     """the number is used to sort domain elements within the domain of one parameter"""
 
-    abbr = Column(Unicode)
+    abbr = Column(Unicode, doc='abbreviated name')
     """abbreviated name, e.g. as label for map legends"""
 
 
@@ -638,7 +642,7 @@ class ValueSet(Base,
     language_pk = Column(Integer, ForeignKey('language.pk'))
     parameter_pk = Column(Integer, ForeignKey('parameter.pk'))
     contribution_pk = Column(Integer, ForeignKey('contribution.pk'))
-    source = Column(Unicode)
+    source = Column(Unicode, doc='textual description of the source for the valueset')
 
     parameter = relationship('Parameter', backref='valuesets')
 
@@ -680,10 +684,13 @@ class Value(Base,
     # Values may be taken from a domain.
     domainelement_pk = Column(Integer, ForeignKey('domainelement.pk'))
 
-    frequency = Column(Float)
+    frequency = Column(
+        Float,
+        doc='frequency of the value relative to other values for the same language')
     """Languages may have multiple values for the same parameter. Their relative
     frequency can be stored here."""
-    confidence = Column(Unicode)
+    confidence = Column(
+        Unicode, doc='textual assessment of the reliability of the value assignment')
 
     domainelement = relationship('DomainElement', backref='values')
 

clld/tests/test_db_meta.py

@@ -1,4 +1,5 @@
 from __future__ import unicode_literals
+import json
 
 from sqlalchemy.orm.exc import NoResultFound
 from nose.tools import assert_almost_equal
@@ -31,12 +32,15 @@ def test_CustomModelMixin(self):
             break
 
     def test_CsvMixin(self):
-        l1 = Language(id='abc', name='Name', latitude=12.4)
+        l1 = Language(id='abc', name='Name', latitude=12.4, jsondata=dict(a=None))
         DBSession.add(l1)
         DBSession.flush()
         l1 = Language.csv_query(DBSession).first()
         cols = l1.csv_head()
         row = l1.to_csv()
+        for k, v in zip(cols, row):
+            if k == 'jsondata':
+                self.assertIn('a', json.loads(v))
         l2 = Language.from_csv(row)
         assert_almost_equal(l1.latitude, l2.latitude)
         row[cols.index('latitude')] = '3,5'

clld/tests/test_web_adapters_csv.py

@@ -1,7 +1,8 @@
 from __future__ import unicode_literals, division, absolute_import, print_function
+import json
 
 from clld.web import datatables
-from clld.db.models.common import Language
+from clld.db.models.common import Language, ValueSet
 from clld.tests.util import TestWithEnv
 
 
@@ -15,3 +16,17 @@ def test_CsvAdapter(self):
         self.assert_(res.splitlines())
         self.assert_(adapter.render_to_response(
             datatables.Languages(self.env['request'], Language), self.env['request']))
+
+    def test_JsonTableSchemaAdapter(self):
+        from clld.web.adapters.csv import JsonTableSchemaAdapter
+
+        adapter = JsonTableSchemaAdapter(None)
+        res = adapter.render(
+            datatables.Languages(self.env['request'], Language), self.env['request'])
+        self.assertIn('fields', json.loads(res))
+        res = adapter.render(
+            datatables.Valuesets(self.env['request'], ValueSet), self.env['request'])
+        self.assertIn('foreignKeys', json.loads(res))
+        res = adapter.render_to_response(
+            datatables.Valuesets(self.env['request'], ValueSet), self.env['request'])
+        #self.assertIn('header=present', str(res))

clld/web/adapters/__init__.py

@@ -42,6 +42,11 @@ def includeme(config):
                 (interface,),
                 interfaces.IIndex,
                 name=csv.CsvAdapter.mimetype)
+            config.registry.registerAdapter(
+                csv.JsonTableSchemaAdapter,
+                (interface,),
+                interfaces.IIndex,
+                name=csv.JsonTableSchemaAdapter.mimetype)
 
         # ... as html details page
         specs.append(

clld/web/adapters/base.py

@@ -22,6 +22,7 @@ class Renderable(object):
     extension = None
     send_mimetype = None
     rel = 'alternate'
+    content_type_params = None
 
     def __init__(self, obj):
         self.obj = obj
@@ -44,6 +45,11 @@ def render_to_response(self, ctx, req):
         res.content_type = str(self.send_mimetype or self.mimetype)
         if self.charset:
             res.content_type += str('; charset=') + str(self.charset)
+        if self.content_type_params:
+            d = res.content_type_params
+            for k, v in self.content_type_params.items():
+                d[str(k)] = str(v)
+            res.content_type_params = d
         return res
 
     def template_context(self, ctx, req):

clld/web/adapters/csv.py

@@ -1,5 +1,9 @@
 from __future__ import unicode_literals, print_function, division, absolute_import
 
+from sqlalchemy import types
+from sqlalchemy.inspection import inspect
+from pyramid.renderers import render as pyramid_render
+
 from clld.web.adapters.base import Index
 from clld.lib.dsv import UnicodeWriter
 
@@ -9,6 +13,7 @@ class CsvAdapter(Index):
     """
     extension = 'csv'
     mimetype = 'text/csv'
+    content_type_params = dict(header='present')
 
     def render(self, ctx, req):
         with UnicodeWriter() as writer:
@@ -23,3 +28,70 @@ def render_to_response(self, ctx, req):
         res = super(CsvAdapter, self).render_to_response(ctx, req)
         res.content_disposition = 'attachment; filename="%s.csv"' % repr(ctx)
         return res
+
+
+class JsonTableSchemaAdapter(Index):
+    """renders DataTables as
+    `JSON table schema <http://dataprotocols.org/json-table-schema/>`_
+
+    .. seealso:: http://csvlint.io/about
+    """
+    extension = 'csv.csvm'
+    mimetype = 'application/csvm+json'
+    send_mimetype = 'application/json'
+    rel = 'describedby'
+
+    type_map = [
+        (types.Integer, 'http://www.w3.org/2001/XMLSchema#int'),
+        (types.Float, 'http://www.w3.org/2001/XMLSchema#float'),
+        (types.Boolean, 'http://www.w3.org/2001/XMLSchema#boolean'),
+        (types.DateTime, 'http://www.w3.org/2001/XMLSchema#dateTime'),
+        (types.Date, 'http://www.w3.org/2001/XMLSchema#date'),
+        (types.Unicode, 'http://www.w3.org/2001/XMLSchema#string'),
+        (types.String, 'http://www.w3.org/2001/XMLSchema#string'),
+    ]
+
+    def render(self, ctx, req):
+        fields = []
+        primary_key = None
+        foreign_keys = []
+        item = ctx.get_query(limit=1).first()
+        if item:
+            cls = inspect(item).class_
+
+            for field in item.csv_head():
+                spec = {
+                    'name': field,
+                    'constraints': {'type': 'http://www.w3.org/2001/XMLSchema#string'}}
+                col = getattr(cls, field, None)
+                if col:
+                    try:
+                        col = col.property.columns[0]
+                    except AttributeError:  # pragma: no cover
+                        col = None
+                if col is not None:
+                    if len(col.foreign_keys) == 1:
+                        fk = list(col.foreign_keys)[0]
+                        foreign_keys.append({
+                            'fields': field,
+                            'reference': {
+                                'datapackage': req.route_url(fk.column.table.name + 's'),
+                                'resource': fk.column.table.name,
+                                'fields': fk.column.name}
+                        })
+                    if col.primary_key:
+                        primary_key = field
+                    for t, s in self.type_map:
+                        if isinstance(col.type, t):
+                            spec['constraints']['type'] = s
+                            break
+                    spec['constraints']['unique'] = bool(col.primary_key or col.unique)
+                    if col.doc:
+                        spec['description'] = col.doc
+                fields.append(spec)
+        doc = {'fields': fields}
+        if primary_key:
+            doc['primaryKey'] = primary_key
+        if foreign_keys:
+            doc['foreignKeys'] = foreign_keys
+        return pyramid_render('json', doc, request=req)

docs/conf.py

@@ -51,10 +51,10 @@
 # built documents.
 #
 # The full version, including alpha/beta/rc tags.
-release = '0.13.1'
+release = '0.13.2'
 
 # The short X.Y version.
-version = '0.13.1'
+version = '0.13.2'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

setup.py

@@ -80,7 +80,7 @@
     ]
 
 setup(name='clld',
-      version='0.13.1',
+      version='0.13.2',
       description=(
           'Python library supporting the development of cross-linguistic databases'),
       long_description='',