Use importlib_metadata; add namespace for mariadb

The ``importlib_metadata`` library is used to scan for setuptools
entrypoints rather than pkg_resources.   as importlib_metadata is a small
library that is included as of Python 3.8, the compatibility library is
installed as a dependency for Python versions older than 3.8.

Unfortunately setuptools "attr:" is broken because it tries to import
the module; seems like this is fixed as part of
pypa/setuptools#1753 however this is too recent
to rely upon for now.

Added a new dialect token "mariadb" that may be used in place of "mysql" in
the :func:`_sa.create_engine` URL.  This will deliver a MariaDB dialect
subclass of the MySQLDialect in use that forces the "is_mariadb" flag to
True.  The dialect will raise an error if a server version string that does
not indicate MariaDB in use is received.   This is useful for
MariaDB-specific testing scenarios as well as to support applications that
are hardcoding to MariaDB-only concepts.  As MariaDB and MySQL featuresets
and usage patterns continue to diverge, this pattern may become more
prominent.

Fixes: #5400
Fixes: #5496
Change-Id: I330815ebe572b6a9818377da56621397335fa702

doc/build/changelog/unreleased_14/5400.rst

@@ -0,0 +1,9 @@
+.. change::
+    :tags: change, installation
+    :tickets: 5400
+
+    The ``importlib_metadata`` library is used to scan for setuptools
+    entrypoints rather than pkg_resources.   as importlib_metadata is a small
+    library that is included as of Python 3.8, the compatibility library is
+    installed as a dependency for Python versions older than 3.8.
+

doc/build/changelog/unreleased_14/5496.rst

@@ -0,0 +1,14 @@
+.. change::
+    :tags: usecase, mysql
+    :tickets: 5496
+
+    Added a new dialect token "mariadb" that may be used in place of "mysql" in
+    the :func:`_sa.create_engine` URL.  This will deliver a MariaDB dialect
+    subclass of the MySQLDialect in use that forces the "is_mariadb" flag to
+    True.  The dialect will raise an error if a server version string that does
+    not indicate MariaDB in use is received.   This is useful for
+    MariaDB-specific testing scenarios as well as to support applications that
+    are hardcoding to MariaDB-only concepts.  As MariaDB and MySQL featuresets
+    and usage patterns continue to diverge, this pattern may become more
+    prominent.
+

doc/build/dialects/mysql.rst

@@ -1,7 +1,7 @@
 .. _mysql_toplevel:
 
-MySQL
-=====
+MySQL and MariaDB
+=================
 
 .. automodule:: sqlalchemy.dialects.mysql.base
 

lib/sqlalchemy/dialects/__init__.py

@@ -15,6 +15,7 @@
     "sybase",
 )
 
+
 from .. import util
 
 
@@ -44,6 +45,15 @@ def _auto_fn(name):
             except ImportError:
                 module = __import__("sqlalchemy.dialects.sybase").dialects
                 module = getattr(module, dialect)
+        elif dialect == "mariadb":
+            # it's "OK" for us to hardcode here since _auto_fn is already
+            # hardcoded.   if mysql / mariadb etc were third party dialects
+            # they would just publish all the entrypoints, which would actually
+            # look much nicer.
+            module = __import__(
+                "sqlalchemy.dialects.mysql.mariadb"
+            ).dialects.mysql.mariadb
+            return module.loader(driver)
         else:
             module = __import__("sqlalchemy.dialects.%s" % (dialect,)).dialects
             module = getattr(module, dialect)

lib/sqlalchemy/dialects/mysql/base.py

@@ -8,19 +8,50 @@
 r"""
 
 .. dialect:: mysql
-    :name: MySQL
+    :name: MySQL / MariaDB
 
 Supported Versions and Features
 -------------------------------
 
-SQLAlchemy supports MySQL starting with version 4.1 through modern releases.
-However, no heroic measures are taken to work around major missing
-SQL features - if your server version does not support sub-selects, for
-example, they won't work in SQLAlchemy either.
+SQLAlchemy supports MySQL starting with version 4.1 through modern releases, as
+well as all modern versions of MariaDB. However, no heroic measures are taken
+to work around major missing SQL features - if your server version does not
+support sub-selects, for example, they won't work in SQLAlchemy either.
 
 See the official MySQL documentation for detailed information about features
 supported in any given server release.
 
+MariaDB Support
+~~~~~~~~~~~~~~~
+
+The MariaDB variant of MySQL retains fundamental compatibility with MySQL's
+protocols however the development of these two products continues to diverge.
+Within the realm of SQLAlchemy, the two databases have a small number of
+syntactical and behavioral differences that SQLAlchemy accommodates automatically.
+To connect to a MariaDB database, no changes to the database URL are required::
+
+
+    engine = create_engine("mysql+pymsql://user:pass@some_mariadb/dbname?charset=utf8mb4")
+
+Upon first connect, the SQLAlchemy dialect employs a
+server version detection scheme that determines if the
+backing database reports as MariaDB.  Based on this flag, the dialect
+can make different choices in those of areas where its behavior
+must be different.
+
+The dialect also supports a "MariaDB-only" mode of connection, which may be
+useful for the case where an application makes use of MariaDB-specific features
+and is not compatible with a MySQL database.    To use this mode of operation,
+replace the "mysql" token in the above URL with "mariadb"::
+
+    engine = create_engine("mariadb+pymsql://user:pass@some_mariadb/dbname?charset=utf8mb4")
+
+The above engine, upon first connect, will raise an error if the server version
+detection detects that the backing database is not MariaDB.
+
+.. versionadded:: 1.4 Added "mariadb" dialect name supporting "MariaDB-only mode"
+   for the MySQL dialect.
+
 .. _mysql_connection_timeouts:
 
 Connection Timeouts and Disconnects
@@ -1943,7 +1974,7 @@ def visit_drop_constraint(self, drop):
             qual = "INDEX "
             const = self.preparer.format_constraint(constraint)
         elif isinstance(constraint, sa_schema.CheckConstraint):
-            if self.dialect._is_mariadb:
+            if self.dialect.is_mariadb:
                 qual = "CONSTRAINT "
             else:
                 qual = "CHECK "
@@ -2352,6 +2383,8 @@ class MySQLDialect(default.DefaultDialect):
     ischema_names = ischema_names
     preparer = MySQLIdentifierPreparer
 
+    is_mariadb = False
+
     # default SQL compilation settings -
     # these are modified upon initialize(),
     # i.e. first connect
@@ -2378,13 +2411,15 @@ def __init__(
         isolation_level=None,
         json_serializer=None,
         json_deserializer=None,
+        is_mariadb=None,
         **kwargs
     ):
         kwargs.pop("use_ansiquotes", None)  # legacy
         default.DefaultDialect.__init__(self, **kwargs)
         self.isolation_level = isolation_level
         self._json_serializer = json_serializer
         self._json_deserializer = json_deserializer
+        self._set_mariadb(is_mariadb, None)
 
     def on_connect(self):
         if self.isolation_level is not None:
@@ -2473,7 +2508,25 @@ def _parse_server_version(self, val):
                     version.extend(g for g in mariadb.groups() if g)
                 else:
                     version.append(n)
-        return tuple(version)
+
+        server_version_info = tuple(version)
+
+        self._set_mariadb(
+            server_version_info and "MariaDB" in server_version_info, val
+        )
+
+        return server_version_info
+
+    def _set_mariadb(self, is_mariadb, server_version_info):
+        if is_mariadb is None:
+            return
+
+        if not is_mariadb and self.is_mariadb:
+            raise exc.InvalidRequestError(
+                "MySQL version %s is not a MariaDB variant."
+                % (server_version_info,)
+            )
+        self.is_mariadb = is_mariadb
 
     def do_commit(self, dbapi_connection):
         """Execute a COMMIT."""
@@ -2677,21 +2730,21 @@ def initialize(self, connection):
         default.DefaultDialect.initialize(self, connection)
 
         self.supports_sequences = (
-            self._is_mariadb and self.server_version_info >= (10, 3)
+            self.is_mariadb and self.server_version_info >= (10, 3)
         )
 
         self.supports_for_update_of = (
             self._is_mysql and self.server_version_info >= (8,)
         )
 
         self._needs_correct_for_88718_96365 = (
-            not self._is_mariadb and self.server_version_info >= (8,)
+            not self.is_mariadb and self.server_version_info >= (8,)
         )
 
         self._warn_for_known_db_issues()
 
     def _warn_for_known_db_issues(self):
-        if self._is_mariadb:
+        if self.is_mariadb:
             mdb_version = self._mariadb_normalized_version_info
             if mdb_version > (10, 2) and mdb_version < (10, 2, 9):
                 util.warn(
@@ -2706,17 +2759,15 @@ def _warn_for_known_db_issues(self):
 
     @property
     def _is_mariadb(self):
-        return (
-            self.server_version_info and "MariaDB" in self.server_version_info
-        )
+        return self.is_mariadb
 
     @property
     def _is_mysql(self):
-        return not self._is_mariadb
+        return not self.is_mariadb
 
     @property
     def _is_mariadb_102(self):
-        return self._is_mariadb and self._mariadb_normalized_version_info > (
+        return self.is_mariadb and self._mariadb_normalized_version_info > (
             10,
             2,
         )
@@ -2726,7 +2777,7 @@ def _mariadb_normalized_version_info(self):
         # MariaDB's wire-protocol prepends the server_version with
         # the string "5.5"; now that we use @@version we no longer see this.
 
-        if self._is_mariadb:
+        if self.is_mariadb:
             idx = self.server_version_info.index("MariaDB")
             return self.server_version_info[idx - 3 : idx]
         else:

lib/sqlalchemy/dialects/mysql/mariadb.py

@@ -0,0 +1,16 @@
+from .base import MySQLDialect
+
+
+class MariaDBDialect(MySQLDialect):
+    is_mariadb = True
+
+
+def loader(driver):
+    driver_mod = __import__(
+        "sqlalchemy.dialects.mysql.%s" % driver
+    ).dialects.mysql
+    driver_cls = getattr(driver_mod, driver).dialect
+
+    return type(
+        "MariaDBDialect_%s" % driver, (MariaDBDialect, driver_cls,), {}
+    )

lib/sqlalchemy/util/compat.py

@@ -15,6 +15,7 @@
 import sys
 
 
+py38 = sys.version_info >= (3, 8)
 py37 = sys.version_info >= (3, 7)
 py36 = sys.version_info >= (3, 6)
 py3k = sys.version_info >= (3, 0)
@@ -90,6 +91,11 @@ def inspect_getfullargspec(func):
     )
 
 
+if py38:
+    from importlib import metadata as importlib_metadata
+else:
+    import importlib_metadata  # noqa
+
 if py3k:
     import base64
     import builtins

lib/sqlalchemy/util/langhelpers.py

@@ -290,12 +290,10 @@ def load(self, name):
                 self.impls[name] = loader
                 return loader()
 
-        try:
-            import pkg_resources
-        except ImportError:
-            pass
-        else:
-            for impl in pkg_resources.iter_entry_points(self.group, name):
+        for impl in compat.importlib_metadata.entry_points().get(
+            self.group, ()
+        ):
+            if impl.name == name:
                 self.impls[name] = impl.load
                 return impl.load()
 

setup.cfg

@@ -1,6 +1,9 @@
 [metadata]
 name = SQLAlchemy
-version = attr: sqlalchemy.__version__
+# version comes from setup.py; setuptools
+# can't read the "attr:" here without importing
+# until version 47.0.0 which is too recent
+
 description = Database Abstraction Library
 long_description = file: README.rst
 long_description_content_type = text/x-rst
@@ -34,6 +37,9 @@ packages = find:
 python_requires = >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*
 package_dir =
     =lib
+install_requires =
+  importlib-metadata;python_version<"3.8"
+
 
 [options.extras_require]
 mssql = pyodbc

setup.py

@@ -4,6 +4,7 @@
 from distutils.errors import DistutilsPlatformError
 import os
 import platform
+import re
 import sys
 
 from setuptools import Distribution as _Distribution
@@ -116,6 +117,16 @@ def status_msgs(*msgs):
     print("*" * 75)
 
 
+with open(
+    os.path.join(os.path.dirname(__file__), "lib", "sqlalchemy", "__init__.py")
+) as v_file:
+    VERSION = (
+        re.compile(r""".*__version__ = ["'](.*?)['"]""", re.S)
+        .match(v_file.read())
+        .group(1)
+    )
+
+
 def run_setup(with_cext):
     kwargs = {}
     if with_cext:
@@ -129,7 +140,7 @@ def run_setup(with_cext):
 
         kwargs["ext_modules"] = []
 
-    setup(cmdclass=cmdclass, distclass=Distribution, **kwargs)
+    setup(version=VERSION, cmdclass=cmdclass, distclass=Distribution, **kwargs)
 
 
 if not cpython:

test/dialect/mysql/test_compiler.py

@@ -150,13 +150,11 @@ def test_drop_constraint_mariadb(self):
         constraint_name = "constraint"
         constraint = CheckConstraint("data IS NOT NULL", name=constraint_name)
         Table(table_name, m, Column("data", String(255)), constraint)
-        dialect = mysql.dialect()
-        dialect.server_version_info = (10, 1, 1, "MariaDB")
         self.assert_compile(
             schema.DropConstraint(constraint),
             "ALTER TABLE %s DROP CONSTRAINT `%s`"
             % (table_name, constraint_name),
-            dialect=dialect,
+            dialect="mariadb",
         )
 
     def test_create_index_with_length_quoted(self):
@@ -354,8 +352,6 @@ def test_concat_compile_kw(self):
         self.assert_compile(expr, "concat('x', 'y')", literal_binds=True)
 
     def test_mariadb_for_update(self):
-        dialect = mysql.dialect()
-        dialect.server_version_info = (10, 1, 1, "MariaDB")
 
         table1 = table(
             "mytable", column("myid"), column("name"), column("description")
@@ -366,7 +362,7 @@ def test_mariadb_for_update(self):
             "SELECT mytable.myid, mytable.name, mytable.description "
             "FROM mytable WHERE mytable.myid = %s "
             "FOR UPDATE",
-            dialect=dialect,
+            dialect="mariadb",
         )
 
         self.assert_compile(
@@ -376,7 +372,7 @@ def test_mariadb_for_update(self):
             "SELECT mytable.myid, mytable.name, mytable.description "
             "FROM mytable WHERE mytable.myid = %s "
             "FOR UPDATE",
-            dialect=dialect,
+            dialect="mariadb",
         )
 
     def test_delete_extra_froms(self):