Merge pull request #130 from level12/82-db-backend-tests

Test against multiple DB backends

.circleci/config.yml

@@ -1,20 +1,17 @@
-version: 2
-jobs:
-    build:
-        docker:
-            - image: level12/python-test-multi
-
+version: 2.1
+commands:
+    runtests:
+        parameters:
+            toxcommand:
+                type: string
+                default: tox
         steps:
             - checkout
 
-            - run:
-                name: folder listing for debugging
-                command: ls -al
-
             - run:
                 name: install tox
                 command: >
-                    pip install --upgrade --force-reinstall tox
+                    pip install --upgrade --force-reinstall tox pip
 
             - run:
                 name: version checks
@@ -25,11 +22,51 @@ jobs:
 
             - run:
                 name: run tox
-                command: tox
+                command: << parameters.toxcommand >>
 
             - store_test_results:
                 path: .circleci/test-reports/
 
             - run:
                 name: push code coverage
                 command: bash <(curl -s https://codecov.io/bash) -X coveragepy -t "f52ea144-6e93-4cda-b927-1f578a6e814c"
+
+jobs:
+    postgres:
+        docker:
+            - image: level12/python-test-multi
+              environment:
+                SQLALCHEMY_DATABASE_URI: "postgresql://postgres:password@localhost/test"
+            - image: postgres:10.5
+              environment:
+                POSTGRES_USER: postgres
+                POSTGRES_PASSWORD: password
+                POSTGRES_DB: test
+        steps:
+            - runtests
+    sqlite:
+        docker:
+            - image: level12/python-test-multi
+              environment:
+                  SQLALCHEMY_DATABASE_URI: "sqlite://"
+        steps:
+            - runtests
+    mssql:
+        docker:
+            - image: level12/python-test-multi
+              environment:
+                  SQLALCHEMY_DATABASE_URI: "mssql+pyodbc_mssql://SA:Password12!@localhost:1433/tempdb?driver=ODBC+Driver+17+for+SQL+Server"
+            - image: mcr.microsoft.com/mssql/server:2017-latest
+              environment:
+                ACCEPT_EULA: Y
+                SA_PASSWORD: "Password12!"
+        steps:
+            - runtests
+
+workflows:
+    version: 2
+    build:
+        jobs:
+            - postgres
+            - sqlite
+            - mssql

setup.py

@@ -17,6 +17,9 @@
     'SQLAlchemyBWC',
     'mock',
     'nose',  # required to import some blazeweb helpers
+    'psycopg2-binary',
+    # pinned to version in our package index.
+    'pyodbc==4.0.30',
     'pytest',
     'pytest-cov',
     'Flask',
@@ -25,6 +28,7 @@
     'Flask-SQLAlchemy',
     'Flask-WebTest',
     'pyquery',
+    'sqlalchemy_pyodbc_mssql',
     'sqlalchemy_utils',
     'sqlalchemybwc',
     'wrapt',

tox.ini

@@ -3,11 +3,14 @@ envlist = py{36,37,38}-{base,i18n},flake8,i18n
 
 
 [testenv]
+setenv =
+    PIP_EXTRA_INDEX_URL=https://package-index.level12.net
+
 usedevelop = false
 # Always recreate the virtualenv so that we are confident dependencies are specified correctly.
 # This is a bit slow, but due to the wheelhouse, it shouldn't be a lot slower.
 recreate = true
-
+passenv = SQLALCHEMY_DATABASE_URI
 deps =
     py37: formencode>=2.0.0a1
 

webgrid/__init__.py

@@ -995,8 +995,31 @@ def query_sort(self, query):
         else:
             log.debug('No sorts')
 
+        # Special consideration for MSSQL, because if paging is to work, the query
+        # must have an ORDER BY clause. This is problematic, because if an app
+        # does not test for the query case where paging is enabled for page > 1,
+        # the query will not hit the error state. Fix the case if possible.
+        if (
+            self.pager_on
+            and self.manager
+            and self.manager.db.engine.dialect.name == 'mssql'
+            and not query._order_by
+        ):
+            query = self._fix_mssql_order_by(query)
+
         return query
 
+    def _fix_mssql_order_by(self, query):
+        # MSSQL must have an ORDER BY for paging to work. If no sort clause has been
+        # defined, sort by the first column. If that doesn't work, error out.
+        if len(self.columns):
+            query = self.columns[0].apply_sort(query, False)
+            if query._order_by:
+                return query
+        raise Exception(
+            'Paging is enabled, but query does not have ORDER BY clause required for MSSQL'
+        )
+
     def args_have_op(self, args):
         # any of the grid's query string args can be used to
         #   override the session behavior (except export_to)

webgrid/tests/conftest.py

@@ -1,7 +1,9 @@
+import os
+
 
 def pytest_configure(config):
     from webgrid_ta.app import create_app
-    app = create_app(config='Test')
+    app = create_app(config='Test', database_url=os.environ.get('SQLALCHEMY_DATABASE_URI'))
     app.test_request_context().push()
 
     from webgrid_ta.model import load_db

webgrid/tests/helpers.py

@@ -1,5 +1,6 @@
 from __future__ import absolute_import
 from os import path as opath
+from unittest import mock
 
 from blazeutils.testing import assert_equal_txt
 import flask
@@ -51,7 +52,14 @@ def render_literal_value(self, value, type_):
             elif value is None:
                 return 'NULL'
             else:
-                return super(LiteralCompiler, self).render_literal_value(value, type_)
+                # Turn off double percent escaping, since we don't run these strings and
+                # it creates a large number of differences for test cases
+                with mock.patch.object(
+                    dialect.identifier_preparer,
+                    '_double_percents',
+                    False
+                ):
+                    return super(LiteralCompiler, self).render_literal_value(value, type_)
 
         def visit_bindparam(
                 self, bindparam, within_columns_clause=False,
@@ -87,7 +95,6 @@ def query_to_str(statement, bind=None):
 
     dialect = bind.dialect
     compiler = statement._compiler(dialect)
-
     literal_compiler = compiler_instance_factory(compiler, dialect, statement)
     return 'TESTING ONLY BIND: ' + literal_compiler.process(statement)
 

webgrid/tests/test_filters.py

@@ -158,26 +158,42 @@ class MockDialect:
     def test_eq(self):
         tf = self.get_filter()
         tf.set('eq', 'foo')
+        query_term = "'foo'"
+        if db.engine.dialect.name == 'mssql':
+            query_term = f'N{query_term}'
         query = tf.apply(db.session.query(Person.id))
-        self.assert_in_query(query, "WHERE upper(persons.firstname) = upper('foo')")
+        self.assert_in_query(query, f'WHERE upper(persons.firstname) = upper({query_term})')
 
     def test_not_eq(self):
         tf = self.get_filter()
         tf.set('!eq', 'foo')
+        query_term = "'foo'"
+        if db.engine.dialect.name == 'mssql':
+            query_term = f'N{query_term}'
         query = tf.apply(db.session.query(Person.id))
-        self.assert_in_query(query, "WHERE upper(persons.firstname) != upper('foo')")
+        self.assert_in_query(query, f'WHERE upper(persons.firstname) != upper({query_term})')
 
     def test_contains(self):
+        sql_check = {
+            'sqlite': "WHERE lower(persons.firstname) LIKE lower('%foo%')",
+            'postgresql': "WHERE persons.firstname ILIKE '%foo%'",
+            'mssql': "WHERE lower(persons.firstname) LIKE lower('%foo%')",
+        }
         tf = self.get_filter()
         tf.set('contains', 'foo')
         query = tf.apply(db.session.query(Person.id))
-        self.assert_in_query(query, "WHERE lower(persons.firstname) LIKE lower('%foo%')")
+        self.assert_in_query(query, sql_check.get(db.engine.dialect.name))
 
     def test_doesnt_contain(self):
+        sql_check = {
+            'sqlite': "WHERE lower(persons.firstname) NOT LIKE lower('%foo%')",
+            'postgresql': "WHERE persons.firstname NOT ILIKE '%foo%'",
+            'mssql': "WHERE lower(persons.firstname) NOT LIKE lower('%foo%')",
+        }
         tf = self.get_filter()
         tf.set('!contains', 'foo')
         query = tf.apply(db.session.query(Person.id))
-        self.assert_in_query(query, "WHERE lower(persons.firstname) NOT LIKE lower('%foo%')")
+        self.assert_in_query(query, sql_check.get(db.engine.dialect.name))
 
     def test_search_expr(self):
         expr_factory = self.get_filter().get_search_expr()
@@ -1344,45 +1360,59 @@ def test_search_expr_overflow_date(self):
 
 
 class TestTimeFilter(CheckFilterBase):
+    def dialect_time(self, time_str):
+        sql = {
+            'sqlite': f'CAST(\'{time_str}:00.000000\' AS TIME)',
+            'postgresql': f'CAST(\'{time_str}:00.000000\' AS TIME WITHOUT TIME ZONE)',
+            'mssql': f'CAST(\'{time_str}:00.000000\' AS TIME)',
+        }
+        return sql.get(db.engine.dialect.name)
+
     def test_eq(self):
+        time_string = self.dialect_time('11:30')
         filter = TimeFilter(Person.start_time)
         filter.set('eq', '11:30 am')
         self.assert_filter_query(filter,
-                                 "WHERE persons.start_time = CAST('11:30:00.000000' AS TIME)")
+                                 "WHERE persons.start_time = " + time_string)
 
     def test_not_eq(self):
+        time_string = self.dialect_time('23:30')
         filter = TimeFilter(Person.start_time)
         filter.set('!eq', '11:30 pm')
         self.assert_filter_query(filter,
-                                 "WHERE persons.start_time != CAST('23:30:00.000000' AS TIME)")
+                                 "WHERE persons.start_time != " + time_string)
 
     def test_lte(self):
+        time_string = self.dialect_time('09:00')
         filter = TimeFilter(Person.start_time)
         filter.set('lte', '9:00 am')
         self.assert_filter_query(filter,
-                                 "WHERE persons.start_time <= CAST('09:00:00.000000' AS TIME)")
+                                 "WHERE persons.start_time <= " + time_string)
 
     def test_gte(self):
+        time_string = self.dialect_time('10:15')
         filter = TimeFilter(Person.start_time)
         filter.set('gte', '10:15 am')
         self.assert_filter_query(filter,
-                                 "WHERE persons.start_time >= CAST('10:15:00.000000' AS TIME)")
+                                 "WHERE persons.start_time >= " + time_string)
 
     def test_between(self):
+        time_start = self.dialect_time('09:00')
+        time_end = self.dialect_time('17:00')
         filter = TimeFilter(Person.start_time)
         filter.set('between', '9:00 am', '5:00 pm')
         self.assert_filter_query(
             filter,
-            "WHERE persons.start_time BETWEEN CAST('09:00:00.000000' AS TIME) AND "
-            "CAST('17:00:00.000000' AS TIME)")
+            f"WHERE persons.start_time BETWEEN {time_start} AND {time_end}")
 
     def test_not_between(self):
+        time_start = self.dialect_time('09:00')
+        time_end = self.dialect_time('17:00')
         filter = TimeFilter(Person.start_time)
         filter.set('!between', '9:00 am', '5:00 pm')
         self.assert_filter_query(
             filter,
-            "WHERE persons.start_time NOT BETWEEN CAST('09:00:00.000000' AS TIME) AND "
-            "CAST('17:00:00.000000' AS TIME)")
+            f"WHERE persons.start_time NOT BETWEEN {time_start} AND {time_end}")
 
     def test_empty(self):
         filter = TimeFilter(Person.start_time)
@@ -1603,16 +1633,26 @@ def __init__(self, a, b, *vargs, **kwargs):
 
 class TestYesNoFilter(CheckFilterBase):
     def test_y(self):
+        sql_check = {
+            'sqlite': "WHERE persons.boolcol = 1",
+            'postgresql': "WHERE persons.boolcol = true",
+            'mssql': "WHERE persons.boolcol = 1",
+        }
         filterobj = YesNoFilter(Person.boolcol)
         filterobj.set('y', None)
         query = filterobj.apply(db.session.query(Person.boolcol))
-        self.assert_in_query(query, "WHERE persons.boolcol = 1")
+        self.assert_in_query(query, sql_check.get(db.engine.dialect.name))
 
     def test_n(self):
+        sql_check = {
+            'sqlite': "WHERE persons.boolcol = 0",
+            'postgresql': "WHERE persons.boolcol = false",
+            'mssql': "WHERE persons.boolcol = 0",
+        }
         filterobj = YesNoFilter(Person.boolcol)
         filterobj.set('n', None)
         query = filterobj.apply(db.session.query(Person.boolcol))
-        self.assert_in_query(query, "WHERE persons.boolcol = 0")
+        self.assert_in_query(query, sql_check.get(db.engine.dialect.name))
 
     def test_a(self):
         filterobj = YesNoFilter(Person.boolcol)

webgrid/tests/test_rendering.py

@@ -52,6 +52,14 @@
 from .helpers import eq_html, inrequest, render_in_grid
 
 
+def _query_exclude_person(query):
+    # this is pretty limited, but only used in the below couple of grids to
+    # exclude the third Person record
+    persons = Person.query.order_by(Person.id).limit(3).all()
+    exclude_id = persons[2].id if len(persons) >= 3 else -1
+    return query.filter(Person.id != exclude_id)
+
+
 class PeopleGrid(PG):
     def query_prep(self, query, has_sort, has_filters):
         query = PG.query_prep(self, query, True, True)
@@ -62,7 +70,7 @@ def query_prep(self, query, has_sort, has_filters):
 
         # default filter
         if not has_filters:
-            query = query.filter(Person.id != 3)
+            query = _query_exclude_person(query)
 
         return query
 
@@ -79,7 +87,7 @@ def query_prep(self, query, has_sort, has_filters):
 
         # default filter
         if not has_filters:
-            query = query.filter(Person.id != 3)
+            query = _query_exclude_person(query)
 
         return query
 
@@ -225,11 +233,13 @@ def test_car_html(self):
         mg.set_records(key_data)
         eq_html(mg.html.table(), 'basic_table.html')
 
+    @pytest.mark.skipif(db.engine.dialect.name != 'sqlite', reason="IDs will not line up")
     @inrequest('/')
     def test_people_html(self):
         pg = render_in_grid(PeopleGrid, 'html')()
         eq_html(pg.html.table(), 'people_table.html')
 
+    @pytest.mark.skipif(db.engine.dialect.name != 'sqlite', reason="IDs will not line up")
     @inrequest('/')
     def test_stopwatch_html(self):
         # Test Stopwatch grid with column groups.