Port to Python3

Brief summary of the modifications:

* Use six for compatibility with both Python 2 and 3;
* Replace UserDict.DictMixin with collections.MutableMapping;
* Fix relative imports;
* Use test-requirements.txt for requirements that are common to both Python 2
  and 3, and test-requirements-py{2,3}.txt for version-specific requirements;
* Miscellaneous fixes.
* Use a specific test_db_py3.cfg file for Python 3, that only runs tests on
  sqlite.

Thanks to Victor Stinner who co-wrote this patch.

Change-Id: Ia6dc536c39d274924c21fd5bb619e8e5721e04c4
Co-Authored-By: Victor Stinner <victor.stinner@enovance.com>

migrate/changeset/ansisql.py

@@ -4,7 +4,6 @@
    At the moment, this isn't so much based off of ANSI as much as
    things that just happen to work with multiple databases.
 """
-import StringIO
 
 import sqlalchemy as sa
 from sqlalchemy.schema import SchemaVisitor
@@ -20,6 +19,7 @@
 import sqlalchemy.sql.compiler
 from migrate.changeset import constraint
 from migrate.changeset import util
+from six.moves import StringIO
 
 from sqlalchemy.schema import AddConstraint, DropConstraint
 from sqlalchemy.sql.compiler import DDLCompiler
@@ -43,11 +43,12 @@ def execute(self):
         try:
             return self.connection.execute(self.buffer.getvalue())
         finally:
-            self.buffer.truncate(0)
+            self.buffer.seek(0)
+            self.buffer.truncate()
 
     def __init__(self, dialect, connection, **kw):
         self.connection = connection
-        self.buffer = StringIO.StringIO()
+        self.buffer = StringIO()
         self.preparer = dialect.identifier_preparer
         self.dialect = dialect
 

migrate/changeset/databases/sqlite.py

@@ -3,7 +3,10 @@
 
    .. _`SQLite`: http://www.sqlite.org/
 """
-from UserDict import DictMixin
+try:  # Python 3
+    from collections import MutableMapping as DictMixin
+except ImportError:  # Python 2
+    from UserDict import DictMixin
 from copy import copy
 
 from sqlalchemy.databases import sqlite as sa_base

migrate/changeset/schema.py

@@ -1,10 +1,14 @@
 """
    Schema module providing common schema operations.
 """
+import abc
+try:  # Python 3
+    from collections import MutableMapping as DictMixin
+except ImportError:  # Python 2
+    from UserDict import DictMixin
 import warnings
 
-from UserDict import DictMixin
-
+import six
 import sqlalchemy
 
 from sqlalchemy.schema import ForeignKeyConstraint
@@ -163,7 +167,39 @@ def _to_index(index, table=None, engine=None):
     return ret
 
 
-class ColumnDelta(DictMixin, sqlalchemy.schema.SchemaItem):
+
+# Python3: if we just use:
+#
+#     class ColumnDelta(DictMixin, sqlalchemy.schema.SchemaItem):
+#         ...
+#
+# We get the following error:
+# TypeError: metaclass conflict: the metaclass of a derived class must be a
+# (non-strict) subclass of the metaclasses of all its bases.
+#
+# The complete inheritance/metaclass relationship list of ColumnDelta can be
+# summarized by this following dot file:
+#
+# digraph test123 {
+#     ColumnDelta -> MutableMapping;
+#     MutableMapping -> Mapping;
+#     Mapping -> {Sized Iterable Container};
+#     {Sized Iterable Container} -> ABCMeta[style=dashed];
+#
+#     ColumnDelta -> SchemaItem;
+#     SchemaItem -> {SchemaEventTarget Visitable};
+#     SchemaEventTarget -> object;
+#     Visitable -> {VisitableType object} [style=dashed];
+#     VisitableType -> type;
+# }
+#
+# We need to use a metaclass that inherits from all the metaclasses of
+# DictMixin and sqlalchemy.schema.SchemaItem. Let's call it "MyMeta".
+class MyMeta(sqlalchemy.sql.visitors.VisitableType, abc.ABCMeta, object):
+    pass
+
+
+class ColumnDelta(six.with_metaclass(MyMeta, DictMixin, sqlalchemy.schema.SchemaItem)):
     """Extracts the differences between two columns/column-parameters
 
         May receive parameters arranged in several different ways:
@@ -229,7 +265,7 @@ def __init__(self, *p, **kw):
                 diffs = self.compare_1_column(*p, **kw)
         else:
             # Zero columns specified
-            if not len(p) or not isinstance(p[0], basestring):
+            if not len(p) or not isinstance(p[0], six.string_types):
                 raise ValueError("First argument must be column name")
             diffs = self.compare_parameters(*p, **kw)
 
@@ -254,6 +290,12 @@ def __setitem__(self, key, value):
     def __delitem__(self, key):
         raise NotImplementedError
 
+    def __len__(self):
+        raise NotImplementedError
+
+    def __iter__(self):
+        raise NotImplementedError
+
     def keys(self):
         return self.diffs.keys()
 
@@ -332,7 +374,7 @@ def _extract_parameters(self, p, k, column):
         """Extracts data from p and modifies diffs"""
         p = list(p)
         while len(p):
-            if isinstance(p[0], basestring):
+            if isinstance(p[0], six.string_types):
                 k.setdefault('name', p.pop(0))
             elif isinstance(p[0], sqlalchemy.types.TypeEngine):
                 k.setdefault('type', p.pop(0))
@@ -370,7 +412,7 @@ def _get_table(self):
         return getattr(self, '_table', None)
 
     def _set_table(self, table):
-        if isinstance(table, basestring):
+        if isinstance(table, six.string_types):
             if self.alter_metadata:
                 if not self.meta:
                     raise ValueError("metadata must be specified for table"
@@ -587,7 +629,7 @@ def remove_from_table(self, table, unset_table=True):
             if isinstance(cons,(ForeignKeyConstraint,
                                 UniqueConstraint)):
                 for col_name in cons.columns:
-                    if not isinstance(col_name,basestring):
+                    if not isinstance(col_name,six.string_types):
                         col_name = col_name.name
                     if self.name==col_name:
                         to_drop.add(cons)
@@ -622,7 +664,7 @@ def _check_sanity_constraints(self, name):
         if (getattr(self, name[:-5]) and not obj):
             raise InvalidConstraintError("Column.create() accepts index_name,"
             " primary_key_name and unique_name to generate constraints")
-        if not isinstance(obj, basestring) and obj is not None:
+        if not isinstance(obj, six.string_types) and obj is not None:
             raise InvalidConstraintError(
             "%s argument for column must be constraint name" % name)
 

migrate/tests/__init__.py

@@ -6,10 +6,11 @@
 
 from unittest import TestCase
 import migrate
+import six
 
 
 class TestVersionDefined(TestCase):
     def test_version(self):
         """Test for migrate.__version__"""
-        self.assertTrue(isinstance(migrate.__version__, basestring))
+        self.assertTrue(isinstance(migrate.__version__, six.string_types))
         self.assertTrue(len(migrate.__version__) > 0)

migrate/tests/changeset/test_changeset.py

@@ -11,6 +11,7 @@
 from migrate.changeset.schema import ColumnDelta
 from migrate.tests import fixture
 from migrate.tests.fixture.warnings import catch_warnings
+import six
 
 class TestAddDropColumn(fixture.DB):
     """Test add/drop column through all possible interfaces
@@ -400,7 +401,7 @@ def _actual_foreign_keys(self):
             if isinstance(cons,ForeignKeyConstraint):
                 col_names = []
                 for col_name in cons.columns:
-                    if not isinstance(col_name,basestring):
+                    if not isinstance(col_name,six.string_types):
                         col_name = col_name.name
                     col_names.append(col_name)
                 result.append(col_names)
@@ -612,7 +613,7 @@ def _setup(self, url):
             self.table.drop()
         try:
             self.table.create()
-        except sqlalchemy.exc.SQLError, e:
+        except sqlalchemy.exc.SQLError:
             # SQLite: database schema has changed
             if not self.url.startswith('sqlite://'):
                 raise
@@ -621,7 +622,7 @@ def _teardown(self):
         if self.table.exists():
             try:
                 self.table.drop(self.engine)
-            except sqlalchemy.exc.SQLError,e:
+            except sqlalchemy.exc.SQLError:
                 # SQLite: database schema has changed
                 if not self.url.startswith('sqlite://'):
                     raise
@@ -843,7 +844,7 @@ def mkcol(self, name='id', type=String, *p, **k):
 
     def verify(self, expected, original, *p, **k):
         self.delta = ColumnDelta(original, *p, **k)
-        result = self.delta.keys()
+        result = list(self.delta.keys())
         result.sort()
         self.assertEqual(expected, result)
         return self.delta

migrate/tests/fixture/__init__.py

@@ -12,7 +12,7 @@ def main(imports=None):
         defaultTest=None
     return testtools.TestProgram(defaultTest=defaultTest)
 
-from base import Base
-from migrate.tests.fixture.pathed import Pathed
-from shell import Shell
-from database import DB,usedb
+from .base import Base
+from .pathed import Pathed
+from .shell import Shell
+from .database import DB,usedb

migrate/tests/fixture/database.py

@@ -3,6 +3,9 @@
 
 import os
 import logging
+import sys
+
+import six
 from decorator import decorator
 
 from sqlalchemy import create_engine, Table, MetaData
@@ -23,7 +26,7 @@
 def readurls():
     """read URLs from config file return a list"""
     # TODO: remove tmpfile since sqlite can store db in memory
-    filename = 'test_db.cfg'
+    filename = 'test_db.cfg' if six.PY2 else "test_db_py3.cfg"
     ret = list()
     tmpfile = Pathed.tmp()
     fullpath = os.path.join(os.curdir, filename)
@@ -46,12 +49,12 @@ def is_supported(url, supported, not_supported):
     db = url.split(':', 1)[0]
 
     if supported is not None:
-        if isinstance(supported, basestring):
+        if isinstance(supported, six.string_types):
             return supported == db
         else:
             return db in supported
     elif not_supported is not None:
-        if isinstance(not_supported, basestring):
+        if isinstance(not_supported, six.string_types):
             return not_supported != db
         else:
             return not (db in not_supported)
@@ -96,7 +99,7 @@ def dec(f, self, *a, **kw):
                 finally:
                     try:
                         self._teardown()
-                    except Exception,e:
+                    except Exception as e:
                         teardown_exception=e
                     else:
                         teardown_exception=None
@@ -106,14 +109,14 @@ def dec(f, self, *a, **kw):
                         'setup: %r\n'
                         'teardown: %r\n'
                         )%(setup_exception,teardown_exception))
-            except Exception,e:
+            except Exception:
                 failed_for.append(url)
-                fail = True
+                fail = sys.exc_info()
         for url in failed_for:
             log.error('Failed for %s', url)
         if fail:
             # cause the failure :-)
-            raise
+            six.reraise(*fail)
     return dec
 
 

migrate/tests/versioning/test_api.py

@@ -1,6 +1,8 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 
+import six
+
 from migrate.exceptions import *
 from migrate.versioning import api
 
@@ -12,7 +14,7 @@
 class TestAPI(Pathed):
 
     def test_help(self):
-        self.assertTrue(isinstance(api.help('help'), basestring))
+        self.assertTrue(isinstance(api.help('help'), six.string_types))
         self.assertRaises(UsageError, api.help)
         self.assertRaises(UsageError, api.help, 'foobar')
         self.assertTrue(isinstance(api.help('create'), str))
@@ -48,7 +50,7 @@ def test_version_control(self):
         repo = self.tmp_repos()
         api.create(repo, 'temp')
         api.version_control('sqlite:///', repo)
-        api.version_control('sqlite:///', unicode(repo))
+        api.version_control('sqlite:///', six.text_type(repo))
 
     def test_source(self):
         repo = self.tmp_repos()

migrate/tests/versioning/test_genmodel.py

@@ -2,6 +2,7 @@
 
 import os
 
+import six
 import sqlalchemy
 from sqlalchemy import *
 
@@ -43,13 +44,12 @@ def _applyLatestModel(self):
     # so the schema diffs on the columns don't work with this test.
     @fixture.usedb(not_supported='ibm_db_sa')
     def test_functional(self):
-
         def assertDiff(isDiff, tablesMissingInDatabase, tablesMissingInModel, tablesWithDiff):
             diff = schemadiff.getDiffOfModelAgainstDatabase(self.meta, self.engine, excludeTables=['migrate_version'])
             self.assertEqual(
                 (diff.tables_missing_from_B,
                  diff.tables_missing_from_A,
-                 diff.tables_different.keys(),
+                 list(diff.tables_different.keys()),
                  bool(diff)),
                 (tablesMissingInDatabase,
                  tablesMissingInModel,
@@ -97,10 +97,11 @@ def assertDiff(isDiff, tablesMissingInDatabase, tablesMissingInModel, tablesWith
         diff = schemadiff.getDiffOfModelAgainstDatabase(MetaData(), self.engine, excludeTables=['migrate_version'])
         src = genmodel.ModelGenerator(diff,self.engine).genBDefinition()
 
-        exec src in locals()
+        namespace = {}
+        six.exec_(src, namespace)
 
         c1 = Table('tmp_schemadiff', self.meta, autoload=True).c
-        c2 = tmp_schemadiff.c
+        c2 = namespace['tmp_schemadiff'].c
         self.compare_columns_equal(c1, c2, ['type'])
         # TODO: get rid of ignoring type
 
@@ -139,19 +140,19 @@ def assertDiff(isDiff, tablesMissingInDatabase, tablesMissingInModel, tablesWith
         decls, upgradeCommands, downgradeCommands = genmodel.ModelGenerator(diff,self.engine).genB2AMigration(indent='')
 
         # decls have changed since genBDefinition
-        exec decls in locals()
+        six.exec_(decls, namespace)
         # migration commands expect a namespace containing migrate_engine
-        migrate_engine = self.engine
+        namespace['migrate_engine'] = self.engine
         # run the migration up and down
-        exec upgradeCommands in locals()
+        six.exec_(upgradeCommands, namespace)
         assertDiff(False, [], [], [])
 
-        exec decls in locals()
-        exec downgradeCommands in locals()
+        six.exec_(decls, namespace)
+        six.exec_(downgradeCommands, namespace)
         assertDiff(True, [], [], [self.table_name])
 
-        exec decls in locals()
-        exec upgradeCommands in locals()
+        six.exec_(decls, namespace)
+        six.exec_(upgradeCommands, namespace)
         assertDiff(False, [], [], [])
 
         if not self.engine.name == 'oracle':

migrate/tests/versioning/test_repository.py

@@ -111,7 +111,6 @@ def test_timestmap_numbering_version(self):
         # Create a script and test again
         now = int(datetime.utcnow().strftime('%Y%m%d%H%M%S'))
         repos.create_script('')
-        print repos.latest
         self.assertEqual(repos.latest, now)
 
     def test_source(self):