Migrate from MySQLdb to PyMySQL (#40123)

* Migrate from MySQLdb to PyMySQL

* Deduplicate driver loading and failure message

* Explain requirements

* Apply requirements docs change to proxysql too

* Add changelog

changelogs/fragments/mysql-migrate_to_pymysql.yaml

@@ -0,0 +1,2 @@
+bugfixes:
+- mysql_*, proxysql_* - PyMySQL (a pure-Python MySQL driver) is now a preferred dependency also supporting Python 3.X.

lib/ansible/module_utils/mysql.py

@@ -30,10 +30,14 @@
 import os
 
 try:
-    import MySQLdb
-    mysqldb_found = True
+    import pymysql as mysql_driver
 except ImportError:
-    mysqldb_found = False
+    try:
+        import MySQLdb as mysql_driver
+    except ImportError:
+        mysql_driver = None
+
+mysql_driver_fail_msg = 'The PyMySQL (Python 2.7 and Python 3.X) or MySQL-python (Python 2.X) module is required.'
 
 
 def mysql_connect(module, login_user=None, login_password=None, config_file='', ssl_cert=None, ssl_key=None, ssl_ca=None, db=None, cursor_class=None,
@@ -69,8 +73,8 @@ def mysql_connect(module, login_user=None, login_password=None, config_file='',
     if connect_timeout is not None:
         config['connect_timeout'] = connect_timeout
 
-    db_connection = MySQLdb.connect(**config)
+    db_connection = mysql_driver.connect(**config)
     if cursor_class is not None:
-        return db_connection.cursor(cursorclass=MySQLdb.cursors.DictCursor)
+        return db_connection.cursor(cursorclass=mysql_driver.cursors.DictCursor)
     else:
         return db_connection.cursor()

lib/ansible/modules/database/mysql/mysql_db.py

@@ -67,7 +67,8 @@
    - mysql (command line binary)
    - mysqldump (command line binary)
 notes:
-   - Requires the python-mysqldb package on the remote host, as well as mysql and mysqldump binaries.
+   - Requires the PyMySQL (Python 2.7 and Python 3.X) or MySQL-python (Python 2.X) package on the remote host,
+     as well as mysql and mysqldump binaries.
    - This module is B(not idempotent) when I(state) is C(import), and will import the dump file each time if run more than once.
 extends_documentation_fragment: mysql
 '''
@@ -107,16 +108,9 @@
 import subprocess
 import traceback
 
-try:
-    import MySQLdb
-except ImportError:
-    mysqldb_found = False
-else:
-    mysqldb_found = True
-
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.database import mysql_quote_identifier
-from ansible.module_utils.mysql import mysql_connect, mysqldb_found
+from ansible.module_utils.mysql import mysql_connect, mysql_driver, mysql_driver_fail_msg
 from ansible.module_utils._text import to_native
 
 
@@ -279,8 +273,8 @@ def main():
         supports_check_mode=True
     )
 
-    if not mysqldb_found:
-        module.fail_json(msg="The MySQL-python module is required.")
+    if mysql_driver is None:
+        module.fail_json(msg=mysql_driver_fail_msg)
 
     db = module.params["name"]
     encoding = module.params["encoding"]

lib/ansible/modules/database/mysql/mysql_replication.py

@@ -118,15 +118,8 @@
 import os
 import warnings
 
-try:
-    import MySQLdb
-except ImportError:
-    mysqldb_found = False
-else:
-    mysqldb_found = True
-
 from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils.mysql import mysql_connect
+from ansible.module_utils.mysql import mysql_connect, mysql_driver, mysql_driver_fail_msg
 from ansible.module_utils._text import to_native
 
 
@@ -239,16 +232,16 @@ def main():
     connect_timeout = module.params['connect_timeout']
     config_file = module.params['config_file']
 
-    if not mysqldb_found:
-        module.fail_json(msg="The MySQL-python module is required.")
+    if mysql_driver is None:
+        module.fail_json(msg=mysql_driver_fail_msg)
     else:
-        warnings.filterwarnings('error', category=MySQLdb.Warning)
+        warnings.filterwarnings('error', category=mysql_driver.Warning)
 
     login_password = module.params["login_password"]
     login_user = module.params["login_user"]
 
     try:
-        cursor = mysql_connect(module, login_user, login_password, config_file, ssl_cert, ssl_key, ssl_ca, None, 'MySQLdb.cursors.DictCursor',
+        cursor = mysql_connect(module, login_user, login_password, config_file, ssl_cert, ssl_key, ssl_ca, None, 'mysql_driver.cursors.DictCursor',
                                connect_timeout=connect_timeout)
     except Exception as e:
         if os.path.exists(config_file):
@@ -325,7 +318,7 @@ def main():
             chm.append("MASTER_AUTO_POSITION = 1")
         try:
             changemaster(cursor, chm, chm_params)
-        except MySQLdb.Warning as e:
+        except mysql_driver.Warning as e:
             result['warning'] = to_native(e)
         except Exception as e:
             module.fail_json(msg='%s. Query == CHANGE MASTER TO %s' % (to_native(e), chm))

lib/ansible/modules/database/mysql/mysql_user.py

@@ -196,16 +196,9 @@
 import string
 import traceback
 
-try:
-    import MySQLdb
-except ImportError:
-    mysqldb_found = False
-else:
-    mysqldb_found = True
-
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.database import SQLParseError
-from ansible.module_utils.mysql import mysql_connect, mysqldb_found
+from ansible.module_utils.mysql import mysql_connect, mysql_driver, mysql_driver_fail_msg
 from ansible.module_utils.six import iteritems
 from ansible.module_utils._text import to_native
 
@@ -578,8 +571,8 @@ def main():
     db = 'mysql'
     sql_log_bin = module.params["sql_log_bin"]
 
-    if not mysqldb_found:
-        module.fail_json(msg="The MySQL-python module is required.")
+    if mysql_driver is None:
+        module.fail_json(msg=mysql_driver_fail_msg)
 
     cursor = None
     try:
@@ -618,14 +611,14 @@ def main():
                 else:
                     changed = user_mod(cursor, user, host, host_all, None, encrypted, priv, append_privs, module)
 
-            except (SQLParseError, InvalidPrivsError, MySQLdb.Error) as e:
+            except (SQLParseError, InvalidPrivsError, mysql_driver.Error) as e:
                 module.fail_json(msg=to_native(e), exception=traceback.format_exc())
         else:
             if host_all:
                 module.fail_json(msg="host_all parameter cannot be used when adding a user")
             try:
                 changed = user_add(cursor, user, host, host_all, password, encrypted, priv, module.check_mode)
-            except (SQLParseError, InvalidPrivsError, MySQLdb.Error) as e:
+            except (SQLParseError, InvalidPrivsError, mysql_driver.Error) as e:
                 module.fail_json(msg=to_native(e), exception=traceback.format_exc())
     elif state == "absent":
         if user_exists(cursor, user, host, host_all):

lib/ansible/modules/database/mysql/mysql_variables.py

@@ -50,16 +50,9 @@
 import warnings
 from re import match
 
-try:
-    import MySQLdb
-except ImportError:
-    mysqldb_found = False
-else:
-    mysqldb_found = True
-
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.database import SQLParseError, mysql_quote_identifier
-from ansible.module_utils.mysql import mysql_connect, mysqldb_found
+from ansible.module_utils.mysql import mysql_connect, mysql_driver, mysql_driver_fail_msg
 from ansible.module_utils._text import to_native
 
 
@@ -148,10 +141,10 @@ def main():
         module.fail_json(msg="Cannot run without variable to operate with")
     if match('^[0-9a-z_]+$', mysqlvar) is None:
         module.fail_json(msg="invalid variable name \"%s\"" % mysqlvar)
-    if not mysqldb_found:
-        module.fail_json(msg="The MySQL-python module is required.")
+    if mysql_driver is None:
+        module.fail_json(msg=mysql_driver_fail_msg)
     else:
-        warnings.filterwarnings('error', category=MySQLdb.Warning)
+        warnings.filterwarnings('error', category=mysql_driver.Warning)
 
     try:
         cursor = mysql_connect(module, user, password, config_file, ssl_cert, ssl_key, ssl_ca, db,

lib/ansible/modules/database/proxysql/proxysql_backend_servers.py

@@ -148,18 +148,10 @@
 
 
 from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils.mysql import mysql_connect
+from ansible.module_utils.mysql import mysql_connect, mysql_driver, mysql_driver_fail_msg
 from ansible.module_utils.six import iteritems
 from ansible.module_utils._text import to_native
 
-try:
-    import MySQLdb
-    import MySQLdb.cursors
-except ImportError:
-    MYSQLDB_FOUND = False
-else:
-    MYSQLDB_FOUND = True
-
 # ===========================================
 # proxysql module specific support methods.
 #
@@ -192,10 +184,8 @@ def perform_checks(module):
                 msg="max_replication_lag must be set between 0 and 102400"
             )
 
-    if not MYSQLDB_FOUND:
-        module.fail_json(
-            msg="the python mysqldb module is required"
-        )
+    if mysql_driver is None:
+        module.fail_json(msg=mysql_driver_fail_msg)
 
 
 def save_config_to_disk(cursor):
@@ -457,8 +447,8 @@ def main():
                                login_user,
                                login_password,
                                config_file,
-                               cursor_class=MySQLdb.cursors.DictCursor)
-    except MySQLdb.Error as e:
+                               cursor_class=mysql_driver.cursors.DictCursor)
+    except mysql_driver.Error as e:
         module.fail_json(
             msg="unable to connect to ProxySQL Admin Module.. %s" % to_native(e)
         )
@@ -487,7 +477,7 @@ def main():
                                  " and doesn't need to be updated.")
                 result['server'] = \
                     proxysql_server.get_server_config(cursor)
-        except MySQLdb.Error as e:
+        except mysql_driver.Error as e:
             module.fail_json(
                 msg="unable to modify server.. %s" % to_native(e)
             )
@@ -502,7 +492,7 @@ def main():
                 result['changed'] = False
                 result['msg'] = ("The server is already absent from the" +
                                  " mysql_hosts memory configuration")
-        except MySQLdb.Error as e:
+        except mysql_driver.Error as e:
             module.fail_json(
                 msg="unable to remove server.. %s" % to_native(e)
             )

lib/ansible/modules/database/proxysql/proxysql_global_variables.py

@@ -73,17 +73,9 @@
 
 
 from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils.mysql import mysql_connect
+from ansible.module_utils.mysql import mysql_connect, mysql_driver, mysql_driver_fail_msg
 from ansible.module_utils._text import to_native
 
-try:
-    import MySQLdb
-    import MySQLdb.cursors
-except ImportError:
-    MYSQLDB_FOUND = False
-else:
-    MYSQLDB_FOUND = True
-
 # ===========================================
 # proxysql module specific support methods.
 #
@@ -96,10 +88,8 @@ def perform_checks(module):
             msg="login_port must be a valid unix port number (0-65535)"
         )
 
-    if not MYSQLDB_FOUND:
-        module.fail_json(
-            msg="the python mysqldb module is required"
-        )
+    if mysql_driver is None:
+        module.fail_json(msg=mysql_driver_fail_msg)
 
 
 def save_config_to_disk(variable, cursor):
@@ -211,8 +201,8 @@ def main():
                                login_user,
                                login_password,
                                config_file,
-                               cursor_class=MySQLdb.cursors.DictCursor)
-    except MySQLdb.Error as e:
+                               cursor_class=mysql_driver.cursors.DictCursor)
+    except mysql_driver.Error as e:
         module.fail_json(
             msg="unable to connect to ProxySQL Admin Module.. %s" % to_native(e)
         )
@@ -231,7 +221,7 @@ def main():
                     msg="The variable \"%s\" was not found" % variable
                 )
 
-        except MySQLdb.Error as e:
+        except mysql_driver.Error as e:
             module.fail_json(
                 msg="unable to get config.. %s" % to_native(e)
             )
@@ -264,7 +254,7 @@ def main():
                     msg="The variable \"%s\" was not found" % variable
                 )
 
-        except MySQLdb.Error as e:
+        except mysql_driver.Error as e:
             module.fail_json(
                 msg="unable to set config.. %s" % to_native(e)
             )

lib/ansible/modules/database/proxysql/proxysql_manage_config.py

@@ -96,16 +96,9 @@
 
 
 from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils.mysql import mysql_connect
+from ansible.module_utils.mysql import mysql_connect, mysql_driver, mysql_driver_fail_msg
 from ansible.module_utils._text import to_native
 
-try:
-    import MySQLdb
-except ImportError:
-    MYSQLDB_FOUND = False
-else:
-    MYSQLDB_FOUND = True
-
 # ===========================================
 # proxysql module specific support methods.
 #
@@ -140,10 +133,8 @@ def perform_checks(module):
                           " with the CONFIG config_layer")
             module.fail_json(msg=msg_string % module.params["direction"])
 
-    if not MYSQLDB_FOUND:
-        module.fail_json(
-            msg="the python mysqldb module is required"
-        )
+    if mysql_driver is None:
+        module.fail_json(msg=mysql_driver_fail_msg)
 
 
 def manage_config(manage_config_settings, cursor):
@@ -201,7 +192,7 @@ def main():
                                login_user,
                                login_password,
                                config_file)
-    except MySQLdb.Error as e:
+    except mysql_driver.Error as e:
         module.fail_json(
             msg="unable to connect to ProxySQL Admin Module.. %s" % to_native(e)
         )
@@ -214,7 +205,7 @@ def main():
     try:
         result['changed'] = manage_config(manage_config_settings,
                                           cursor)
-    except MySQLdb.Error as e:
+    except mysql_driver.Error as e:
         module.fail_json(
             msg="unable to manage config.. %s" % to_native(e)
         )