Merge branch 'release/0.33'

CHANGES.rst

@@ -2,6 +2,12 @@
 Changes
 -------
 
+0.33
+~~~~
+
+See https://github.com/clld/clld/milestones/clld%200.33
+
+
 0.32.1
 ~~~~~~
 

RELEASING.txt

@@ -13,7 +13,7 @@ Releasing clld
 
 - Make sure all scaffold tests pass (Py 2.7, 3.4):
 
-  $ ./venvs/clld/clld/build.sh "<prev-rel-no>"
+  $ ./venvs/clld-clld/clld/build.sh "<prev-rel-no>"
 
 - Make sure javascript tests pass with coverage of clld.js at > 83%::
 

build.sh

@@ -24,3 +24,19 @@ nosetests
 cd $VENVS
 rm -rf testapp
 
+cd $VENVS
+/opt/python3.4/bin/pyvenv testapp
+cd testapp
+. bin/activate
+pip install "$VENVS/cheesecake/clld/dist/clld-$1.tar.gz"
+#pip install "clld==$1"
+pcreate -t clld_app testapp
+cd testapp
+python setup.py develop
+python testapp/scripts/initializedb.py development.ini
+pip install nose
+pip install mock
+nosetests
+cd $VENVS
+rm -rf testapp
+

clld/__init__.py

@@ -5,7 +5,7 @@
 from clld import interfaces
 
 
-__version__ = "0.32.1"
+__version__ = "0.33"
 
 
 class Resource(namedtuple('Resource', 'name model interface with_index with_rdfdump')):

clld/db/models/__init__.py

@@ -1,6 +1,8 @@
 # flake8: noqa
 from __future__ import unicode_literals, print_function, division, absolute_import
 
+from clld.db.meta import (Base, PolymorphicBaseMixin, CustomModelMixin)
+
 from ._mixins import (IdNameDescriptionMixin,
     DataMixin, HasDataMixin, FilesMixin, HasFilesMixin)
 

clld/db/versioned.py

@@ -1,10 +1,8 @@
-"""Support for per-record versioning; based on an sqlalchemy recipe."""
 from sqlalchemy.ext.declarative import declared_attr
-from sqlalchemy.orm import mapper, attributes, object_mapper, deferred
+from sqlalchemy.orm import mapper, attributes, object_mapper
 from sqlalchemy.orm.exc import UnmappedColumnError
 from sqlalchemy import Table, Column, ForeignKeyConstraint, Integer
-from sqlalchemy import event
-from sqlalchemy.exc import CompileError
+from sqlalchemy import event, util
 from sqlalchemy.orm.properties import RelationshipProperty
 
 
@@ -15,6 +13,10 @@ def col_references_table(col, table):
     return False
 
 
+def _is_versioning_col(col):
+    return "version_meta" in col.info
+
+
 def _history_mapper(local_mapper):
     cls = local_mapper.class_
 
@@ -29,46 +31,78 @@ def _history_mapper(local_mapper):
 
     polymorphic_on = None
     super_fks = []
-    if not super_mapper or local_mapper.local_table is not super_mapper.local_table:
+
+    def _col_copy(col):
+        orig = col
+        col = col.copy()
+        orig.info['history_copy'] = col
+        col.unique = False
+        col.default = col.server_default = None
+        return col
+
+    properties = util.OrderedDict()
+    if not super_mapper or \
+            local_mapper.local_table is not super_mapper.local_table:
         cols = []
+        # add column.info to identify columns specific to versioning
+        version_meta = {"version_meta": True}
+
         for column in local_mapper.local_table.c:
-            if column.name == 'version':
+            if _is_versioning_col(column):
                 continue  # pragma: no cover
 
-            col = column.copy()
-            col.unique = False
+            col = _col_copy(column)
 
-            if super_mapper and col_references_table(column, super_mapper.local_table):
+            if super_mapper and \
+                    col_references_table(column, super_mapper.local_table):
                 super_fks.append(
-                    (col.key, list(super_history_mapper.local_table.primary_key)[0]))
+                    (
+                        col.key,
+                        list(super_history_mapper.local_table.primary_key)[0]
+                    )
+                )
 
             cols.append(col)
 
             if column is local_mapper.polymorphic_on:
                 polymorphic_on = col
 
+            orig_prop = local_mapper.get_property_by_column(column)
+            # carry over column re-mappings
+            if len(orig_prop.columns) > 1 or \
+                    orig_prop.columns[0].key != orig_prop.key:
+                properties[orig_prop.key] = tuple(
+                    col.info['history_copy'] for col in orig_prop.columns)
+
         if super_mapper:
             super_fks.append(
-                ('version', super_history_mapper.base_mapper.local_table.c.version))
-            cols.append(Column('version', Integer, primary_key=True))
-        else:
-            cols.append(Column('version', Integer, primary_key=True))
+                (
+                    'version', super_history_mapper.local_table.c.version
+                )
+            )
+
+        # "version" stores the integer version id.  This column is
+        # required.
+        cols.append(
+            Column(
+                'version', Integer, primary_key=True,
+                autoincrement=False, info=version_meta))
 
         if super_fks:
             cols.append(ForeignKeyConstraint(*zip(*super_fks)))
 
         table = Table(
             local_mapper.local_table.name + '_history',
             local_mapper.local_table.metadata,
-            *cols
+            *cols,
+            schema=local_mapper.local_table.schema
         )
     else:  # pragma: no cover
         # single table inheritance.  take any additional columns that may have
         # been added and add them to the history table.
         for column in local_mapper.local_table.c:
             if column.key not in super_history_mapper.local_table.c:
-                col = column.copy()
-                col.unique = False
+                col = _col_copy(column)
                 super_history_mapper.local_table.append_column(col)
         table = None
 
@@ -83,15 +117,17 @@ def _history_mapper(local_mapper):
         table,
         inherits=super_history_mapper,
         polymorphic_on=polymorphic_on,
-        polymorphic_identity=local_mapper.polymorphic_identity
+        polymorphic_identity=local_mapper.polymorphic_identity,
+        properties=properties
     )
     cls.__history_mapper__ = m
 
     if not super_history_mapper:
         local_mapper.local_table.append_column(
             Column('version', Integer, default=1, nullable=False)
         )
-        local_mapper.add_property("version", deferred(local_mapper.local_table.c.version))
+        local_mapper.add_property(
+            "version", local_mapper.local_table.c.version)
 
 
 class Versioned(object):
@@ -121,12 +157,15 @@ def create_version(obj, session, deleted=False):
 
     obj_changed = False
 
-    for om, hm in zip(obj_mapper.iterate_to_root(), history_mapper.iterate_to_root()):
-        if hm.single:
-            continue  # pragma: no cover
+    for om, hm in zip(
+            obj_mapper.iterate_to_root(),
+            history_mapper.iterate_to_root()
+    ):
+        if hm.single:  # pragma: no cover
+            continue
 
         for hist_col in hm.local_table.c:
-            if hist_col.key == 'version':
+            if _is_versioning_col(hist_col):
                 continue
 
             obj_col = om.local_table.c[hist_col.key]
@@ -141,22 +180,22 @@ def create_version(obj, session, deleted=False):
                 # in the case of single table inheritance, there may be
                 # columns on the mapped table intended for the subclass only.
                 # the "unmapped" status of the subclass column on the
-                # base class is a feature of the declarative module as of sqla 0.5.2.
+                # base class is a feature of the declarative module.
                 continue
 
-            # expired object attributes and also deferred cols might not be in the
-            # dict.  force it to load no matter what by using getattr().
+            # expired object attributes and also deferred cols might not
+            # be in the dict.  force it to load no matter what by
+            # using getattr().
             if prop.key not in obj_state.dict:
                 getattr(obj, prop.key)
 
             a, u, d = attributes.get_history(obj, prop.key)
-
             if d:
                 attr[hist_col.key] = d[0]
                 obj_changed = True
             elif u:
                 attr[hist_col.key] = u[0]
-            else:
+            elif a:
                 # if the attribute had no value.
                 attr[hist_col.key] = a[0]
                 obj_changed = True
@@ -165,16 +204,19 @@ def create_version(obj, session, deleted=False):
         # not changed, but we have relationships.  OK
         # check those too
         for prop in obj_mapper.iterate_properties:
-            if isinstance(prop, RelationshipProperty):
-                try:
-                    if attributes.get_history(obj, prop.key).has_changes():
+            if isinstance(prop, RelationshipProperty) and \
+                attributes.get_history(
+                    obj, prop.key,
+                    passive=attributes.PASSIVE_NO_INITIALIZE).has_changes():
+                for p in prop.local_columns:
+                    if p.foreign_keys:  # pragma: no cover
                         obj_changed = True
                         break
-                except CompileError:  # pragma: no cover
-                    pass
+                if obj_changed is True:
+                    break  # pragma: no cover
 
     if not obj_changed and not deleted:
-        return  # pragma: no cover
+        return
 
     attr['version'] = obj.version
     hist = history_cls()

clld/scripts/freeze.py

@@ -22,6 +22,7 @@
 from six import PY2
 from dateutil.parser import parse
 from path import path
+from sqlalchemy import Boolean
 from sqlalchemy.sql import select
 import requests
 try:
@@ -297,6 +298,8 @@ def get_converter(schema, table):
     for col in table.columns:
         if isinstance(col.type, DeclEnumType):
             conv[col.name] = col.type.enum.from_string
+        elif isinstance(col.type, Boolean):
+            conv[col.name] = lambda v: v == 'True'
         elif isinstance(col.type, JSONEncodedDict):
             conv[col.name] = json.loads
     return conv

clld/tests/test_db_models.py

@@ -9,6 +9,12 @@
 
 
 class Tests(TestWithDb):
+    def test_Config(self):
+        from clld.db.models.common import Config
+
+        self.assertEquals(Config.replacement_key('X', 'Y'), '__X_Y__')
+        self.assertEquals(Config.replacement_key(None, 'Y'), '__NoneType_Y__')
+
     def test_Files(self):
         from clld.db.models.common import Sentence, Sentence_files
         from path import path
@@ -35,6 +41,7 @@ def test_Dataset(self):
         d = Dataset(id='abc', domain='test')
         DBSession.add(d)
         DBSession.flush()
+        self.assertEquals(d.jsondata, d.jsondatadict)
         d.get_stats(RESOURCES, source=Source.id == None)
 
     def test_Contributor(self):

clld/tests/test_db_versioned.py

@@ -5,7 +5,7 @@
 
 class Tests(TestWithDb):
     def test_Versioning(self):
-        from clld.db.models.common import Language, Identifier, LanguageIdentifier
+        from clld.db.models.common import Language, Language_data
         from clld.db.meta import VersionedDBSession
 
         l = Language(id='abc', name='Old Name', jsondata={'i': 2})
@@ -22,10 +22,8 @@ def test_Versioning(self):
         res = VersionedDBSession.query(History).filter(History.pk == l.pk).all()
         self.assertEqual(res[0].name, 'Old Name')
 
-        li = LanguageIdentifier(
-            identifier=Identifier(id='asd', type='wals'),
-            language=l)
+        l.data.append(Language_data(key='k', value='v'))
         VersionedDBSession.flush()
-        VersionedDBSession.delete(li)
+        assert l.datadict()
         VersionedDBSession.delete(l)
         VersionedDBSession.flush()

clld/tests/test_scripts_freeze.py

@@ -10,7 +10,7 @@
 
 from clld.tests.util import TestWithEnv
 from clld.db.meta import Base, DBSession
-from clld.db.models.common import Dataset, Language
+from clld.db.models.common import Dataset, Language, Contribution
 
 
 class Tests(TestWithEnv):
@@ -47,4 +47,8 @@ def data_file(self, *comps):
         self.assertEqual(l1.latitude, l2.latitude)
         self.assertEqual(l1.description, l2.description)
 
+        contrib = s2.query(Contribution).filter(Contribution.id == 'contribution').one()
+        self.assert_(contrib.primary_contributors)
+        self.assert_(contrib.secondary_contributors)
+
         tmp.rmtree(ignore_errors=True)