#38 #15 MySQL support in Python 3; fixes #12

Added support for PyMySQL, fixed nasty xml coding auto-detection bug
Refactoring of sample code
#3 Makefile changes now this is closed

HISTORY.rst

@@ -53,6 +53,20 @@ you use the bytes type (and the 'b' prefix on any string constants) when
 initialising OData entity properties of type Edm.Binary.  Failure to do
 so will raise an error in Python 3.
 
+*Build 20161110*
+
+#12 bug when using numeric or named parameters in DB API
+
+Added support for pyformat in DB APIs as part of enabling support for
+PyMySQL.
+
+#38 Python 3 compatibility work (ongoing)
+
+Updated more samples to work in Python 3, including the weather OData
+service using MySQL connected through PyMySQL as MySQLdb is not
+supported in Python 3.
+
+
 *Build 20161109*
 
 #3 PEP-8 driven refactoring (complete)

Makefile

@@ -7,13 +7,11 @@ help:
 	@echo "clean-cov - remove coverage report"
 	@echo "test - run tests with the default Python"
 	@echo "test27 - run tests with python2.7"
-	@echo "test3 - run tests with default python and "-3 -Wd -W module -t" options
+	@echo "test3 - run tests with default python and -3Wd options"
 	@echo "docs - generate Sphinx HTML documentation"
 	@echo "dist - package"
-	@echo "pep8 - PEP-8 compliance check using pep8"
-	@echo "pep8s - PEP-8 compliance statistics using pep8"
-	@echo "flake8 - PEP-8 compliance check using flake8"
-	@echo "flake8s - PEP-8 compliance statistics using flake8"
+	@echo "pep8 - PEP-8 compliance check using flake8"
+	@echo "flake8 - same as pep8"
 	@echo "coverage - run coverage to check test coverage"
 
 clean: clean-build clean-pyc clean-docs clean-cov
@@ -51,24 +49,9 @@ dist: clean
 	ls -l dist
 
 pep8:
-	touch pep8-count.txt
-	date >> pep8-count.txt
-	-pep8 --count -qq pyslet >> pep8-count.txt 2>&1
-	tail -n 4 pep8-count.txt
-
-pep8s: pep8
-	pep8 --statistics -qq pyslet
-
-flake8:
-	autopep8 -i -r pyslet
-	touch flake8-count.txt
-	date >> flake8-count.txt
-	-flake8 --count -qq pyslet >> flake8-count.txt 2>&1
 	./pep8-regression.py
-	tail -n 4 flake8-count.txt
-
-flake8s: flake8
-	flake8 --statistics -qq pyslet
+
+flake8: pep8
 
 coverage:
 	coverage run --source pyslet unittests/test.py

doc/odatav2/memexample.rst

@@ -1,8 +1,8 @@
 Sample Project: InMemory Data Service
 =====================================
 
-The sample code for this service is in the samples directory in the
-Pyslet distribution.
+The sample code for this service is in the samples/memcache directory in
+the Pyslet distribution.
 
 This project demonstrates how to construct a simple OData service based
 on the :py:class:`~pyslet.odata2.memds.InMemoryEntityContainer` class. 

pyslet/mysqldbds.py

@@ -1,8 +1,8 @@
 #! /usr/bin/env python
 """DAL implementation for MySQLdb"""
 
+import logging
 import os
-import string
 import math
 import datetime
 
@@ -11,7 +11,12 @@
 import pyslet.odata2.sqlds as sqlds
 import pyslet.blockstore as blockstore
 
-import MySQLdb
+try:
+    import MySQLdb as dbapi
+except ImportError:
+    dbapi = None
+    logging.warning("MySQLdb not found, will try PyMySQL instead")
+    import pymysql as dbapi
 
 
 class MySQLEntityContainer(sqlds.SQLEntityContainer):
@@ -52,7 +57,7 @@ class MySQLEntityContainer(sqlds.SQLEntityContainer):
     def __init__(self, db, host=None, user=None, passwd=None,
                  prefix=None, mysql_options={}, **kwargs):
         self.prefix = prefix
-        super(MySQLEntityContainer, self).__init__(dbapi=MySQLdb, **kwargs)
+        super(MySQLEntityContainer, self).__init__(dbapi=dbapi, **kwargs)
         self.dbname = db
         self.host = host
         self.user = user
@@ -96,9 +101,9 @@ def quote_identifier(self, identifier):
 
         By default we replace backtick with a single quote and then use
         backticks to quote the identifier. E.g., if the string
-        u'Employee_Name' is passed then the string u'`Employee_Name`' is
+        'Employee_Name' is passed then the string '`Employee_Name`' is
         returned."""
-        return u'`%s`' % identifier.replace('`', '')
+        return '`%s`' % identifier.replace('`', '')
 
     def prepare_sql_type(self, simple_value, params, nullable=None):
         """Performs MySQL custom mappings
@@ -119,18 +124,18 @@ def prepare_sql_type(self, simple_value, params, nullable=None):
         explicit_default = None
         if isinstance(simple_value, edm.BinaryValue):
             if p is None or (p.fixedLength is None and p.maxLength is None):
-                column_def.append(u"BLOB")
+                column_def.append("BLOB")
             else:
                 max = 0
                 if p.fixedLength:
                     max = p.fixedLength
                 if p.maxLength:
                     max = p.maxLength
                 if max > 1024:
-                    column_def.append(u"BLOB")
+                    column_def.append("BLOB")
         elif isinstance(simple_value, edm.StringValue):
             if p is None or (p.fixedLength is None and p.maxLength is None):
-                column_def.append(u"TEXT")
+                column_def.append("TEXT")
         elif isinstance(simple_value, edm.DateTimeValue):
             if p is None or p.precision is None:
                 precision = 0
@@ -140,9 +145,9 @@ def prepare_sql_type(self, simple_value, params, nullable=None):
                 # maximum precision
                 precision = 6
             if precision:
-                column_def.append(u"DATETIME(%i)" % precision)
+                column_def.append("DATETIME(%i)" % precision)
             else:
-                column_def.append(u"DATETIME")
+                column_def.append("DATETIME")
             explicit_null = True
             explicit_default = '0'
         elif isinstance(simple_value, edm.TimeValue):
@@ -154,25 +159,25 @@ def prepare_sql_type(self, simple_value, params, nullable=None):
                 # maximum precision
                 precision = 6
             if precision:
-                column_def.append(u"TIME(%i)" % precision)
+                column_def.append("TIME(%i)" % precision)
             else:
-                column_def.append(u"TIME")
+                column_def.append("TIME")
         if column_def:
             if ((nullable is not None and not nullable) or
                     (nullable is None and p is not None and not p.nullable)):
-                column_def.append(u' NOT NULL')
+                column_def.append(' NOT NULL')
                 # use the explicit default
                 if explicit_default and not simple_value:
-                    column_def.append(u' DEFAULT ')
+                    column_def.append(' DEFAULT ')
                     column_def.append(explicit_default)
             elif explicit_null:
-                column_def.append(u' NULL')
+                column_def.append(' NULL')
             if simple_value:
                 # Format the default
-                column_def.append(u' DEFAULT ')
+                column_def.append(' DEFAULT ')
                 column_def.append(
                     params.add_param(self.prepare_sql_value(simple_value)))
-            return string.join(column_def, '')
+            return ''.join(column_def)
         else:
             return super(
                 MySQLEntityContainer,
@@ -221,7 +226,7 @@ def limit_clause(self, skip, top):
         if skip:
             clause.append('OFFSET %i ' % skip)
             skip = 0
-        return skip, string.join(clause, '')
+        return skip, ''.join(clause)
 
 
 class MySQLStreamStore(blockstore.StreamStore):
@@ -253,7 +258,7 @@ def load_container(self):
         :py:class:`pyslet.blockstore.StreamStore`
         respectively."""
         doc = edmx.Document()
-        with file(os.path.join(os.path.dirname(__file__),
+        with open(os.path.join(os.path.dirname(__file__),
                                'odata2', 'streamstore.xml'), 'r') as f:
             doc.read(f)
         return doc.root.DataServices['StreamStoreSchema.Container']

pyslet/odata2/csdl.py

@@ -707,8 +707,9 @@ def parse_datetime_literal(self):
                     self.require(":")
                     zm = self.require_production(self.parse_integer(0, 24),
                                                  "zminute")
-                    logging.warn("DateTime ignored zone offset: %s%.2i:%.2i",
-                                 z, zh, zm)
+                    logging.warning(
+                        "DateTime ignored zone offset: %s%.2i:%.2i",
+                        z, zh, zm)
             except ValueError:
                 self.setpos(zpos)
                 pass
@@ -4158,7 +4159,7 @@ def update_type_refs(self, scope, stop_on_errors=False):
             if len(plist) > 1:
                 # these are all duplicates!
                 for p in plist:
-                    logging.warn(
+                    logging.warning(
                         "Ambiguous navigation: %s.%s", self.name, p.name)
                     p.mark_as_ambiguous()
         self.Key.update_type_refs(scope, stop_on_errors)
@@ -4592,8 +4593,8 @@ def update_set_refs(self, scope, stop_on_errors=False):
                 raise
 
     def set_unbound_principal(self, aset_end):
-        logging.warn("Entity set %s has an unbound principal: %s",
-                     self.name, aset_end.otherEnd.entity_set.name)
+        logging.warning("Entity set %s has an unbound principal: %s",
+                        self.name, aset_end.otherEnd.entity_set.name)
         if not self.bad_principal and self.unboundPrincipal is None:
             self.unboundPrincipal = aset_end
         else:

pyslet/odata2/sqlds.py

@@ -196,6 +196,25 @@ def add_param(self, value):
         return ":" + name
 
 
+class PyFormatParams(SQLParams):
+
+    """A class for building parameter lists using '%(name)s' syntax."""
+
+    def __init__(self):
+        super(PyFormatParams, self).__init__()
+        self.params = {}
+
+    def add_param(self, value):
+        name = "p%i" % len(self.params)
+        self.params[name] = value
+        return "%%(%s)s" % name
+
+    @classmethod
+    def escape_literal(cls, literal):
+        """Doubles any % characters to prevent formatting errors"""
+        return literal.replace("%", "%%")
+
+
 def retry_decorator(tmethod):
     """Decorates a transaction method with retry handling"""
 
@@ -4073,11 +4092,13 @@ def __init__(self, container, dbapi, streamstore=None, max_connections=10,
             self.ParamsClass = NamedParams
         elif self.dbapi.paramstyle == "format":
             self.ParamsClass = FormatParams
+        elif self.dbapi.paramstyle == "pyformat":
+            self.ParamsClass = PyFormatParams
         else:
             # will fail later when we try and add parameters
-            logging.warn("Unsupported DBAPI params style: %s\n"
-                         "setting to qmark",
-                         self.dbapi.paramstyle)
+            logging.warning("Unsupported DBAPI params style: %s\n"
+                            "setting to qmark",
+                            self.dbapi.paramstyle)
             self.ParamsClass = SQLParams
         self.fk_table = {}
         """A mapping from an entity set name to a FK mapping of the form::
@@ -4498,7 +4519,7 @@ def drop_all_tables(self, out=None):
                 try:
                     nav_class.drop_table(self, aset_name)
                 except SQLError as e:
-                    logging.warn("Ignoring : %s", str(e))
+                    logging.warning("Ignoring : %s", str(e))
             else:
                 query = nav_class.drop_table_query(self, aset_name)
                 out.write(query)
@@ -4515,7 +4536,7 @@ def drop_all_tables(self, out=None):
                     try:
                         collection.drop_table()
                     except SQLError as e:
-                        logging.warn("Ignoring : %s", str(e))
+                        logging.warning("Ignoring : %s", str(e))
                 else:
                     query = collection.drop_table_query()
                     out.write(query)
@@ -4536,7 +4557,7 @@ def acquire_connection(self, timeout=None):
                 self.cpool_lock.wait(timeout)
                 now = time.time()
                 if timeout is not None and now > start + timeout:
-                    logging.warn(
+                    logging.warning(
                         "Thread[%i] timed out waiting for the the database "
                         "module lock", thread_id)
                     return None
@@ -4579,7 +4600,7 @@ def acquire_connection(self, timeout=None):
                 else:
                     now = time.time()
                     if timeout is not None and now > start + timeout:
-                        logging.warn(
+                        logging.warning(
                             "Thread[%i] timed out waiting for a database "
                             "connection", thread_id)
                         break
@@ -4843,7 +4864,7 @@ def close(self, timeout=5):
                     # shrinking, wait for locked connections to be
                     # released
                     nlocked = len(self.cpool_locked)
-                    logging.warn(
+                    logging.warning(
                         "Waiting to break unreleased database connections")
                     self.cpool_lock.wait(timeout)
                     continue
@@ -5230,8 +5251,8 @@ def __init__(self, file_path, sqlite_options={}, **kwargs):
             if (('max_connections' in kwargs and
                     kwargs['max_connections'] != 1) or
                     'max_connections' not in kwargs):
-                logging.warn("Forcing max_connections=1 for in-memory "
-                             "SQLite database")
+                logging.warning("Forcing max_connections=1 for in-memory "
+                                "SQLite database")
             kwargs['max_connections'] = 1
             self.sqlite_memdbc = sqlite3.connect(
                 ":memory:", check_same_thread=False, **sqlite_options)

pyslet/xml/structures.py

@@ -3587,7 +3587,7 @@ def auto_detect_encoding(self, src_file):
             src_file.seek(0)
             return
         while len(magic) < 4:
-            magic = magic + 'Q'
+            magic = magic + b'Q'
         if magic[:2] == b'\xff\xfe' or magic[:2] == b'\xfe\xff':
             if magic[2:] != b'\x00\x00':
                 magic = magic[:2]