Allow extensions to contain migrations

As currently managing database tables in extensions is rather painful we
end up with developers either checking and creating the tables at
startup time, or creating paster command specific to their extension
which creates tables. Running actual migrations is often problematic.

These changes provide a new interface called IMigration that allows an
extension to specify where it stored its migrations.  These migrations
are then run whenever the ```db upgrade``` command is executed.

The paster templates for creating a new extension are also updated to
automatically created the required files for running migrations.

ckan/migration/migrate.cfg

@@ -5,16 +5,16 @@ repository_id=Ckan
 
 # The name of the database table used to track the schema version.
 # This name shouldn't already be used by your project.
-# If this is changed once a database is under version control, you'll need to 
-# change the table name in each database too. 
+# If this is changed once a database is under version control, you'll need to
+# change the table name in each database too.
 version_table=migrate_version
 
-# When committing a change script, Migrate will attempt to generate the 
+# When committing a change script, Migrate will attempt to generate the
 # sql for all supported databases; normally, if one of them fails - probably
-# because you don't have that database installed - it is ignored and the 
-# commit continues, perhaps ending successfully. 
-# Databases in this list MUST compile successfully during a commit, or the 
-# entire commit will fail. List the databases your application will actually 
+# because you don't have that database installed - it is ignored and the
+# commit continues, perhaps ending successfully.
+# Databases in this list MUST compile successfully during a commit, or the
+# entire commit will fail. List the databases your application will actually
 # be using to ensure your updates to that database work properly.
 # This must be a list; example: ['postgres','sqlite']
 required_dbs=[]

ckan/model/__init__.py

@@ -3,6 +3,7 @@
 import logging
 import re
 from datetime import datetime
+import ckan.plugins as plugins
 
 import vdm.sqlalchemy
 from vdm.sqlalchemy.base import SQLAlchemySession
@@ -273,6 +274,14 @@ def setup_migration_version_control(self, version=None):
         except migrate.exceptions.DatabaseAlreadyControlledError:
             pass
 
+        for plugin in plugins.PluginImplementations(plugins.IMigration):
+            path = plugin.migrations()
+            try:
+                mig.version_control(self.metadata.bind, path, version=None)
+            except migrate.exceptions.DatabaseAlreadyControlledError:
+                pass
+
+
     def upgrade_db(self, version=None):
         '''Upgrade db using sqlalchemy migrations.
 
@@ -282,7 +291,9 @@ def upgrade_db(self, version=None):
             'Database migration - only Postgresql engine supported (not %s).' \
                 % meta.engine.name
         import migrate.versioning.api as mig
+
         self.setup_migration_version_control()
+
         version_before = mig.db_version(self.metadata.bind, self.migrate_repository)
         mig.upgrade(self.metadata.bind, self.migrate_repository, version=version)
         version_after = mig.db_version(self.metadata.bind, self.migrate_repository)
@@ -291,6 +302,17 @@ def upgrade_db(self, version=None):
         else:
             log.info('CKAN database version remains as: %s', version_after)
 
+
+        for plugin in plugins.PluginImplementations(plugins.IMigration):
+            path = plugin.migrations()
+            version_before = mig.db_version(self.metadata.bind, path)
+            mig.upgrade(self.metadata.bind, path, version=None)
+            version_after = mig.db_version(self.metadata.bind, path)
+            if version_after != version_before:
+                log.info('Extension database (%s) version upgraded: %s -> %s', path, version_before, version_after)
+            else:
+                log.info('Extension database (%s) version remains as: %s', path, version_after)
+
         ##this prints the diffs in a readable format
         ##import pprint
         ##from migrate.versioning.schemadiff import getDiffOfModelAgainstDatabase

ckan/pastertemplates/template/MANIFEST.in_tmpl

@@ -1,2 +1,3 @@
 include README.rst
 recursive-include ckanext/{{ project_shortname }} *.html *.json *.js *.less *.css
+recursive-include ckanext/{{ project_shortname }}/migrations *

ckan/pastertemplates/template/ckanext/+project_shortname+/migrations/migrate.cfg

@@ -0,0 +1,4 @@
+[db_settings]
+repository_id=ckanext-{{ project_shortname }}
+version_table=migrate_version
+required_dbs=[]

ckan/pastertemplates/template/ckanext/+project_shortname+/migrations/versions/001_initial_creation.py

@@ -0,0 +1,7 @@
+
+def upgrade(migrate_engine):
+    pass
+
+
+def downgrade(migrate_engine):
+    pass

ckan/pastertemplates/template/ckanext/+project_shortname+/plugin.py_tmpl

@@ -1,12 +1,20 @@
+import os
+
 import ckan.plugins as plugins
 import ckan.plugins.toolkit as toolkit
 
 
 class {{ plugin_class_name }}(plugins.SingletonPlugin):
     plugins.implements(plugins.IConfigurer)
+    plugins.implements(plugins.IMigration)
 
-    # IConfigurer
+    # IMigration
+    def migrations(self):
+        current_directory = os.path.dirname(os.path.abspath(__file__))
+        plugin_root = '/'.join(current_directory.split('/')[0:-2])
+        return  os.path.join(plugin_root, 'ckanext/{{ project_shortname }}/migrations')
 
+    # IConfigurer
     def update_config(self, config_):
         toolkit.add_template_directory(config_, 'templates')
         toolkit.add_public_directory(config_, 'public')

ckan/plugins/interfaces.py

@@ -23,7 +23,8 @@
     'IFacets',
     'IAuthenticator',
     'ITranslation',
-    'IUploader'
+    'IUploader',
+    'IMigration'
 ]
 
 from inspect import isclass
@@ -1554,3 +1555,15 @@ def get_resource_uploader(self):
         :type id: string
 
         '''
+
+
+class IMigration(Interface):
+
+    def migrations(self):
+        '''Returns the repository name for the migrations.
+
+          This should be a / separated string specifying a path to a module.
+          For instance, in ckan this is 'ckan/migrations'.  In an extension called
+          my_extension it would be 'my_extension/migrations' if the migrations
+          folder contains versions folder.
+        '''