Data migrations v2

.circleci/config.yml

@@ -20,22 +20,23 @@ jobs:
           command: export BASE_BRANCH=$(base_branch)
       - restore_cache:
           keys:
-          - py3-cache-v3-{{ arch }}-{{ checksum "python.deps" }}
-          - py3-cache-v3-{{ arch }}-{{ .Branch }}
-          - py3-cache-v3-{{ arch }}-{{ .Environment.BASE_BRANCH }}
+          - py3-cache-v4-{{ arch }}-{{ checksum "python.deps" }}
+          - py3-cache-v4-{{ arch }}-{{ .Branch }}
+          - py3-cache-v4-{{ arch }}-{{ .Environment.BASE_BRANCH }}
       - run:
           name: Install python dependencies
           command: |
             python -m venv venv
             source venv/bin/activate
+            pip install -U pip
             pip install -e . || pip install -e .
             pip install -r requirements/circleci.pip
       - save_cache:
-          key: py3-cache-v3-{{ arch }}-{{ checksum "python.deps" }}
+          key: py3-cache-v4-{{ arch }}-{{ checksum "python.deps" }}
           paths:
           - venv
       - save_cache:
-          key: py3-cache-v3-{{ arch }}-{{ .Branch }}
+          key: py3-cache-v4-{{ arch }}-{{ .Branch }}
           paths:
           - venv
       - run:

CHANGELOG.md

@@ -2,7 +2,16 @@
 
 ## Current (in progress)
 
-- Nothing yet
+### New features
+
+- New migration system [#1956](https://github.com/opendatateam/udata/pull/1956):
+  - Use python based migrations instead of relying on mongo internal and deprecated `js_exec`
+  - Handle rollback (optionnal)
+  - Detailled history
+
+### Breaking changes
+
+- The new migration system ([#1956](https://github.com/opendatateam/udata/pull/1956)) uses a new python based format. Pre-2.0 migrations are not compatible so you might need to upgrade to the latest `udata` version `<2.0.0`, execute migrations and then upgrade to `udata` 2+.
 
 ## 1.6.2 (2018-11-05)
 

package.json

@@ -42,6 +42,7 @@
     "faker": "^3.1.0",
     "fetch-mock": "^4.6.0",
     "file-loader": "^0.9.0",
+    "har-validator": "5.1.0",
     "image-webpack-loader": "^1.8.0",
     "imports-loader": "^0.6.5",
     "json-loader": "^0.5.4",

udata/commands/db.py

@@ -1,17 +1,13 @@
+import os
 import logging
 
 import click
 
-from os.path import join
-from pkg_resources import resource_isdir, resource_listdir, resource_string
+from udata.commands import cli, green, yellow, cyan, red, magenta, white, echo
+from udata import migrations
 
-from flask import current_app
-
-from pymongo.errors import PyMongoError, OperationFailure
-from mongoengine.connection import get_db
-
-from udata import entrypoints
-from udata.commands import cli, green, yellow, cyan, red, magenta, echo
+# Date format used to for display
+DATE_FORMAT = '%Y-%m-%d %H:%M'
 
 log = logging.getLogger(__name__)
 
@@ -22,138 +18,36 @@ def grp():
     pass
 
 
-# A migration script wrapper recording the stdout lines
-SCRIPT_WRAPPER = '''
-function(plugin, filename, script) {{
-    var stdout = [];
-    function print() {{
-        var args = Array.prototype.slice.call(arguments);
-        stdout.push(args.join(' '));
-    }}
-
-    {0}
-
-    db.migrations.insert({{
-        plugin: plugin,
-        filename: filename,
-        date: ISODate(),
-        script: script,
-        output: stdout.join('\\n')
-    }});
-
-    return stdout;
-}}
-'''
-
-# Only record a migration script
-RECORD_WRAPPER = '''
-function(plugin, filename, script) {
-    db.migrations.insert({
-        plugin: plugin,
-        filename: filename,
-        date: ISODate(),
-        script: script,
-        output: 'Recorded only'
-    });
-}
-'''
-
-# Only record a migration script
-UNRECORD_WRAPPER = '''
-function(oid) {{
-    db.migrations.remove({_id: oid});
-}}
-'''
-
-# Date format used to for display
-DATE_FORMAT = '%Y-%m-%d %H:%M'
+def log_status(migration, status):
+    '''Properly display a migration status line'''
+    name = os.path.splitext(migration.filename)[0]
+    display = ':'.join((migration.plugin, name)) + ' '
+    log.info('%s [%s]', '{:.<70}'.format(display), status)
 
 
-def normalize_migration(plugin_or_specs, filename):
-    if filename is None and ':' in plugin_or_specs:
-        plugin, filename = plugin_or_specs.split(':')
+def status_label(record):
+    if record.ok:
+        return green(record.last_date.strftime(DATE_FORMAT))
+    elif not record.exists():
+        return yellow('Not applied')
     else:
-        plugin = plugin_or_specs
-    if not filename.endswith('.js'):
-        filename += '.js'
-    return plugin, filename
+        return red(record.status)
 
 
-def get_migration(plugin, filename):
-    '''Get an existing migration record if exists'''
-    db = get_db()
-    return db.migrations.find_one({'plugin': plugin, 'filename': filename})
-
-
-def execute_migration(plugin, filename, script, dryrun=False):
-    '''Execute and record a migration'''
-    db = get_db()
-    js = SCRIPT_WRAPPER.format(script)
-    lines = script.splitlines()
-    success = True
-    if not dryrun:
-        try:
-            lines = db.eval(js, plugin, filename, script)
-        except OperationFailure as e:
-            log.error(e.details['errmsg'].replace('\n', '\\n'))
-            success = False
-        except PyMongoError as e:
-            log.error('Unable to apply migration: %s', str(e))
-            success = False
-    echo('│'.encode('utf8'))
-    for line in lines:
-        echo('│ {0}'.format(line).encode('utf8'))
-    echo('│'.encode('utf8'))
-    echo('└──[{0}]'.format(green('OK') if success else red('KO')).encode('utf8'))
+def format_output(output, success=True):
+    echo('  │')
+    for level, msg in output:
+        echo('  │ {0}'.format(msg))
+    echo('  │')
+    echo('  └──[{0}]'.format(green('OK') if success else red('KO')))
     echo('')
-    return success
-
-
-def record_migration(plugin, filename, script, **kwargs):
-    '''Only record a migration without applying it'''
-    db = get_db()
-    db.eval(RECORD_WRAPPER, plugin, filename, script)
-    return True
-
-
-def available_migrations():
-    '''
-    List available migrations for udata and enabled plugins
-
-    Each row is tuple with following signature:
-
-        (plugin, package, filename)
-    '''
-    migrations = []
-    for filename in resource_listdir('udata', 'migrations'):
-        if filename.endswith('.js'):
-            migrations.append(('udata', 'udata', filename))
-
-    plugins = entrypoints.get_enabled('udata.models', current_app)
-    for plugin, module in plugins.items():
-        if resource_isdir(module.__name__, 'migrations'):
-            for filename in resource_listdir(module.__name__, 'migrations'):
-                if filename.endswith('.js'):
-                    migrations.append((plugin, module.__name__, filename))
-    return sorted(migrations, key=lambda r: r[2])
-
-
-def log_status(plugin, filename, status):
-    '''Properly display a migration status line'''
-    display = ':'.join((plugin, filename)) + ' '
-    log.info('%s [%s]', '{:.<70}'.format(display), status)
 
 
 @grp.command()
 def status():
     '''Display the database migrations status'''
-    for plugin, package, filename in available_migrations():
-        migration = get_migration(plugin, filename)
-        if migration:
-            status = green(migration['date'].strftime(DATE_FORMAT))
-        else:
-            status = yellow('Not applied')
-        log_status(plugin, filename, status)
+    for migration in migrations.list_available():
+        log_status(migration, status_label(migration.record))
 
 
 @grp.command()
@@ -163,17 +57,26 @@ def status():
               help='Only print migrations to be applied')
 def migrate(record, dry_run=False):
     '''Perform database migrations'''
-    handler = record_migration if record else execute_migration
     success = True
-    for plugin, package, filename in available_migrations():
-        migration = get_migration(plugin, filename)
-        if migration or not success:
-            log_status(plugin, filename, cyan('Skipped'))
+    for migration in migrations.list_available():
+        if migration.record.ok or not success:
+            log_status(migration, cyan('Skipped'))
         else:
             status = magenta('Recorded') if record else yellow('Apply')
-            log_status(plugin, filename, status)
-            script = resource_string(package, join('migrations', filename))
-            success &= handler(plugin, filename, script, dryrun=dry_run)
+            log_status(migration, status)
+            try:
+                output = migration.execute(recordonly=record, dryrun=dry_run)
+            except migrations.RollbackError as re:
+                format_output(re.migrate_exc.output, False)
+                log_status(migration, red('Rollback'))
+                format_output(re.output, not re.exc)
+                success = False
+            except migrations.MigrationError as me:
+                format_output(me.output, False)
+                success = False
+            else:
+                format_output(output, True)
+    return success
 
 
 @grp.command()
@@ -186,15 +89,38 @@ def unrecord(plugin_or_specs, filename):
     \b
     A record can be expressed with the following syntaxes:
      - plugin filename
-     - plugin fliename.js
+     - plugin filename.js
      - plugin:filename
      - plugin:fliename.js
     '''
-    plugin, filename = normalize_migration(plugin_or_specs, filename)
-    migration = get_migration(plugin, filename)
-    if migration:
-        log.info('Removing migration %s:%s', plugin, filename)
-        db = get_db()
-        db.eval(UNRECORD_WRAPPER, migration['_id'])
+    migration = migrations.get(plugin_or_specs, filename)
+    removed = migration.unrecord()
+    if removed:
+        log.info('Removed migration %s', migration.label)
     else:
-        log.error('Migration not found %s:%s', plugin, filename)
+        log.error('Migration not found %s', migration.label)
+
+
+@grp.command()
+@click.argument('plugin_or_specs')
+@click.argument('filename', default=None, required=False, metavar='[FILENAME]')
+def info(plugin_or_specs, filename):
+    '''
+    Display detailed info about a migration
+    '''
+    migration = migrations.get(plugin_or_specs, filename)
+    log_status(migration, status_label(migration.record))
+    try:
+        echo(migration.module.__doc__)
+    except migrations.MigrationError:
+        echo(yellow('Module not found'))
+
+    for op in migration.record.get('ops', []):
+        display_op(op)
+
+
+def display_op(op):
+    timestamp = white(op['date'].strftime(DATE_FORMAT))
+    label = white(op['type'].title()) + ' '
+    echo('{label:.<70} [{date}]'.format(label=label, date=timestamp))
+    format_output(op['output'], op['success'])

udata/entrypoints.py

@@ -38,6 +38,13 @@ def get_enabled(name, app):
     return dict(_ep_to_kv(e) for e in iter_all(name) if e.name in plugins)
 
 
+def get_plugin_module(name, app, plugin):
+    '''
+    Get the module for a given plugin
+    '''
+    return next((m for p, m in get_enabled(name, app).items() if p == plugin), None)
+
+
 def _ep_to_kv(entrypoint):
     '''
     Transform an entrypoint into a key-value tuple where: