Get migration_package_activity and tests working, by putting revision…

… model in revision_legacy_code.py

* Having removed the revision model (Revision, PackageRevision etc) from the core ckan code, now we add it back into ckan/migration/revision_legacy_code.py (RevisionTableMappings) so it can be used by the migration_package_activity.py and its tests. Also includes associated vdm functions:
    * make_revisioned_table
    * make_revision_table
    * make_Revision
* Fix up some tests in test_migration_package_activity associated with revision_id now being missing from domain objects.
    * migration tests have to make revisions manually now (make_revision()), because previously the revisioned tables (e.g. Package) had a mapper extension vdm.sqlalchemy.Revisioner(package_revision_table) that triggered on table changes and records a copy in the corresponding revision table (e.g. PackageRevision). HOWEVER we can't add this to the mapping for the migration test and then remove it for the rest of the CKAN tests.
* Fix up some tests following test_migration_package_activity - PackageDictizeMonkeyPatch is introduced to monkeypatch package_dictize. Previously it was being patched in test_migration_package_activity but not being put back for the following tests.
* vdm is now only required by migration_package_activity.py and its tests.

ckan/lib/dictization/__init__.py

@@ -43,7 +43,7 @@ def table_dictize(obj, context, **kw):
         name = field
         if name in ('current', 'expired_timestamp', 'expired_id'):
             continue
-        if name == 'continuity_id':
+        if name in ('continuity_id', 'revision_id'):
             continue
         value = getattr(obj, name)
         if value is None:

ckan/migration/migrate_package_activity.py

@@ -77,28 +77,49 @@ def migrate_all_datasets():
     dataset_names = logic.get_action(u'package_list')(get_context(), {})
     num_datasets = len(dataset_names)
     errors = defaultdict(int)
-    for i, dataset_name in enumerate(dataset_names):
-        print(u'\n{}/{} dataset: {}'.format(i + 1, num_datasets, dataset_name))
-        migrate_dataset(dataset_name, errors)
+    with PackageDictizeMonkeyPatch():
+        for i, dataset_name in enumerate(dataset_names):
+            print(u'\n{}/{} dataset: {}'.format(i + 1, num_datasets, dataset_name))
+            migrate_dataset(dataset_name, errors)
     print(u'Migrated:')
     print(u'  {} datasets'.format(len(dataset_names)))
     num_activities = num_activities_migratable()
     print(u'  with {} activities'.format(num_activities))
     print_errors(errors)
 
 
+class PackageDictizeMonkeyPatch(object):
+    '''Patches package_dictize to work with revisions - it will accept the
+    'revision_id' parameter, so you can see past revisions of the package.
+
+    This is a context manager, so invoke it with 'with'.
+    '''
+    # (We could have saved a couple of lines of code by using mock.patch, but
+    # this way we save having to add mock to requirements.txt)
+    def __enter__(self):
+        # package_dictize monkey-patched to package_dictize_with_revisions
+        import ckan.lib.dictization.model_dictize as model_dictize
+        try:
+            import ckan.migration.revision_legacy_code as revision_legacy_code
+        except ImportError:
+            # convenient to look for it in the current directory if you just
+            # download these files because you are upgrading an older ckan
+            import revision_legacy_code
+        self.original_package_dictize = model_dictize.package_dictize
+        model_dictize.package_dictize = \
+            revision_legacy_code.package_dictize_with_revisions
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        import ckan.lib.dictization.model_dictize as model_dictize
+        model_dictize.package_dictize = self.original_package_dictize
+
+
 def migrate_dataset(dataset_name, errors):
-    # monkey patch the legacy versions of code back into CKAN - so it has the
-    # revision functionality needed for this migration
-    import ckan.lib.dictization.model_dictize as model_dictize
-    try:
-        import ckan.migration.revision_legacy_code as revision_legacy_code
-    except ImportError:
-        # convenient to look for it in the current directory if you just
-        # download these files because you are upgrading an older ckan
-        import revision_legacy_code
-    model_dictize.package_dictize = \
-        revision_legacy_code.package_dictize_with_revisions
+    '''
+    Migrates a single dataset.
+
+    NB this function should be run in a `with PackageDictizeMonkeyPatch():`
+    '''
 
     import ckan.logic as logic
     from ckan import model
@@ -142,11 +163,13 @@ def migrate_dataset(dataset_name, errors):
         context = dict(
             get_context(),
             for_view=False,
-            revision_id=activity[u'revision_id'],
+            revision_id=activity_obj.revision_id,
             use_cache=False,  # avoid the cache (which would give us the
                               # latest revision)
         )
         try:
+            assert activity_obj.revision_id, \
+                u'Revision missing on the activity'
             dataset = logic.get_action(u'package_show')(
                 context,
                 {u'id': activity[u'object_id'], u'include_tracking': False})
@@ -156,8 +179,9 @@ def migrate_dataset(dataset_name, errors):
             else:
                 error_msg = text_type(exc)
             print(u'    Error: {}! Skipping this version '
-                  '(revision_id={})'
-                  .format(error_msg, activity[u'revision_id']))
+                  '(revision_id={}, timestamp={})'
+                  .format(error_msg, activity_obj.revision_id,
+                          activity_obj.timestamp))
             errors[error_msg] += 1
             # We shouldn't leave the activity.data['package'] with missing
             # resources, extras & tags, which could cause the package_read
@@ -271,5 +295,6 @@ class Options(object):
         wipe_activity_detail(delete_activity_detail=args.delete)
     else:
         errors = defaultdict(int)
-        migrate_dataset(args.dataset, errors)
+        with PackageDictizeMonkeyPatch():
+            migrate_dataset(args.dataset, errors)
         print_errors(errors)