diff --git a/.circleci/config.yml b/.circleci/config.yml index 70ccbeb..8076407 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,20 +1,17 @@ -version: 2 -jobs: - build: - docker: - - image: level12/python-test-multi - +version: 2.1 +commands: + runtests: + parameters: + toxcommand: + type: string + default: tox steps: - checkout - - run: - name: folder listing for debugging - command: ls -al - - run: name: install tox command: > - pip install --upgrade --force-reinstall tox + pip install --upgrade --force-reinstall tox pip - run: name: version checks @@ -25,7 +22,7 @@ jobs: - run: name: run tox - command: tox + command: << parameters.toxcommand >> - store_test_results: path: .circleci/test-reports/ @@ -33,3 +30,43 @@ jobs: - run: name: push code coverage command: bash <(curl -s https://codecov.io/bash) -X coveragepy -t "f52ea144-6e93-4cda-b927-1f578a6e814c" + +jobs: + postgres: + docker: + - image: level12/python-test-multi + environment: + SQLALCHEMY_DATABASE_URI: "postgresql://postgres:password@localhost/test" + - image: postgres:10.5 + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: password + POSTGRES_DB: test + steps: + - runtests + sqlite: + docker: + - image: level12/python-test-multi + environment: + SQLALCHEMY_DATABASE_URI: "sqlite://" + steps: + - runtests + mssql: + docker: + - image: level12/python-test-multi + environment: + SQLALCHEMY_DATABASE_URI: "mssql+pyodbc_mssql://SA:Password12!@localhost:1433/tempdb?driver=ODBC+Driver+17+for+SQL+Server" + - image: mcr.microsoft.com/mssql/server:2017-latest + environment: + ACCEPT_EULA: Y + SA_PASSWORD: "Password12!" + steps: + - runtests + +workflows: + version: 2 + build: + jobs: + - postgres + - sqlite + - mssql diff --git a/setup.py b/setup.py index 3f5ae82..450e132 100644 --- a/setup.py +++ b/setup.py @@ -17,6 +17,9 @@ 'SQLAlchemyBWC', 'mock', 'nose', # required to import some blazeweb helpers + 'psycopg2-binary', + # pinned to version in our package index. + 'pyodbc==4.0.30', 'pytest', 'pytest-cov', 'Flask', @@ -25,6 +28,7 @@ 'Flask-SQLAlchemy', 'Flask-WebTest', 'pyquery', + 'sqlalchemy_pyodbc_mssql', 'sqlalchemy_utils', 'sqlalchemybwc', 'wrapt', diff --git a/tox.ini b/tox.ini index 7704802..0908a6c 100644 --- a/tox.ini +++ b/tox.ini @@ -3,11 +3,14 @@ envlist = py{36,37,38}-{base,i18n},flake8,i18n [testenv] +setenv = + PIP_EXTRA_INDEX_URL=https://package-index.level12.net + usedevelop = false # Always recreate the virtualenv so that we are confident dependencies are specified correctly. # This is a bit slow, but due to the wheelhouse, it shouldn't be a lot slower. recreate = true - +passenv = SQLALCHEMY_DATABASE_URI deps = py37: formencode>=2.0.0a1 diff --git a/webgrid/__init__.py b/webgrid/__init__.py index 925a73e..b6e0553 100644 --- a/webgrid/__init__.py +++ b/webgrid/__init__.py @@ -995,8 +995,31 @@ def query_sort(self, query): else: log.debug('No sorts') + # Special consideration for MSSQL, because if paging is to work, the query + # must have an ORDER BY clause. This is problematic, because if an app + # does not test for the query case where paging is enabled for page > 1, + # the query will not hit the error state. Fix the case if possible. + if ( + self.pager_on + and self.manager + and self.manager.db.engine.dialect.name == 'mssql' + and not query._order_by + ): + query = self._fix_mssql_order_by(query) + return query + def _fix_mssql_order_by(self, query): + # MSSQL must have an ORDER BY for paging to work. If no sort clause has been + # defined, sort by the first column. If that doesn't work, error out. + if len(self.columns): + query = self.columns[0].apply_sort(query, False) + if query._order_by: + return query + raise Exception( + 'Paging is enabled, but query does not have ORDER BY clause required for MSSQL' + ) + def args_have_op(self, args): # any of the grid's query string args can be used to # override the session behavior (except export_to) diff --git a/webgrid/tests/conftest.py b/webgrid/tests/conftest.py index beaa326..06d4ce7 100644 --- a/webgrid/tests/conftest.py +++ b/webgrid/tests/conftest.py @@ -1,7 +1,9 @@ +import os + def pytest_configure(config): from webgrid_ta.app import create_app - app = create_app(config='Test') + app = create_app(config='Test', database_url=os.environ.get('SQLALCHEMY_DATABASE_URI')) app.test_request_context().push() from webgrid_ta.model import load_db diff --git a/webgrid/tests/helpers.py b/webgrid/tests/helpers.py index c0ab5e8..594fc18 100644 --- a/webgrid/tests/helpers.py +++ b/webgrid/tests/helpers.py @@ -1,5 +1,6 @@ from __future__ import absolute_import from os import path as opath +from unittest import mock from blazeutils.testing import assert_equal_txt import flask @@ -51,7 +52,14 @@ def render_literal_value(self, value, type_): elif value is None: return 'NULL' else: - return super(LiteralCompiler, self).render_literal_value(value, type_) + # Turn off double percent escaping, since we don't run these strings and + # it creates a large number of differences for test cases + with mock.patch.object( + dialect.identifier_preparer, + '_double_percents', + False + ): + return super(LiteralCompiler, self).render_literal_value(value, type_) def visit_bindparam( self, bindparam, within_columns_clause=False, @@ -87,7 +95,6 @@ def query_to_str(statement, bind=None): dialect = bind.dialect compiler = statement._compiler(dialect) - literal_compiler = compiler_instance_factory(compiler, dialect, statement) return 'TESTING ONLY BIND: ' + literal_compiler.process(statement) diff --git a/webgrid/tests/test_filters.py b/webgrid/tests/test_filters.py index 0f41ac9..0cf17eb 100644 --- a/webgrid/tests/test_filters.py +++ b/webgrid/tests/test_filters.py @@ -158,26 +158,42 @@ class MockDialect: def test_eq(self): tf = self.get_filter() tf.set('eq', 'foo') + query_term = "'foo'" + if db.engine.dialect.name == 'mssql': + query_term = f'N{query_term}' query = tf.apply(db.session.query(Person.id)) - self.assert_in_query(query, "WHERE upper(persons.firstname) = upper('foo')") + self.assert_in_query(query, f'WHERE upper(persons.firstname) = upper({query_term})') def test_not_eq(self): tf = self.get_filter() tf.set('!eq', 'foo') + query_term = "'foo'" + if db.engine.dialect.name == 'mssql': + query_term = f'N{query_term}' query = tf.apply(db.session.query(Person.id)) - self.assert_in_query(query, "WHERE upper(persons.firstname) != upper('foo')") + self.assert_in_query(query, f'WHERE upper(persons.firstname) != upper({query_term})') def test_contains(self): + sql_check = { + 'sqlite': "WHERE lower(persons.firstname) LIKE lower('%foo%')", + 'postgresql': "WHERE persons.firstname ILIKE '%foo%'", + 'mssql': "WHERE lower(persons.firstname) LIKE lower('%foo%')", + } tf = self.get_filter() tf.set('contains', 'foo') query = tf.apply(db.session.query(Person.id)) - self.assert_in_query(query, "WHERE lower(persons.firstname) LIKE lower('%foo%')") + self.assert_in_query(query, sql_check.get(db.engine.dialect.name)) def test_doesnt_contain(self): + sql_check = { + 'sqlite': "WHERE lower(persons.firstname) NOT LIKE lower('%foo%')", + 'postgresql': "WHERE persons.firstname NOT ILIKE '%foo%'", + 'mssql': "WHERE lower(persons.firstname) NOT LIKE lower('%foo%')", + } tf = self.get_filter() tf.set('!contains', 'foo') query = tf.apply(db.session.query(Person.id)) - self.assert_in_query(query, "WHERE lower(persons.firstname) NOT LIKE lower('%foo%')") + self.assert_in_query(query, sql_check.get(db.engine.dialect.name)) def test_search_expr(self): expr_factory = self.get_filter().get_search_expr() @@ -1344,45 +1360,59 @@ def test_search_expr_overflow_date(self): class TestTimeFilter(CheckFilterBase): + def dialect_time(self, time_str): + sql = { + 'sqlite': f'CAST(\'{time_str}:00.000000\' AS TIME)', + 'postgresql': f'CAST(\'{time_str}:00.000000\' AS TIME WITHOUT TIME ZONE)', + 'mssql': f'CAST(\'{time_str}:00.000000\' AS TIME)', + } + return sql.get(db.engine.dialect.name) + def test_eq(self): + time_string = self.dialect_time('11:30') filter = TimeFilter(Person.start_time) filter.set('eq', '11:30 am') self.assert_filter_query(filter, - "WHERE persons.start_time = CAST('11:30:00.000000' AS TIME)") + "WHERE persons.start_time = " + time_string) def test_not_eq(self): + time_string = self.dialect_time('23:30') filter = TimeFilter(Person.start_time) filter.set('!eq', '11:30 pm') self.assert_filter_query(filter, - "WHERE persons.start_time != CAST('23:30:00.000000' AS TIME)") + "WHERE persons.start_time != " + time_string) def test_lte(self): + time_string = self.dialect_time('09:00') filter = TimeFilter(Person.start_time) filter.set('lte', '9:00 am') self.assert_filter_query(filter, - "WHERE persons.start_time <= CAST('09:00:00.000000' AS TIME)") + "WHERE persons.start_time <= " + time_string) def test_gte(self): + time_string = self.dialect_time('10:15') filter = TimeFilter(Person.start_time) filter.set('gte', '10:15 am') self.assert_filter_query(filter, - "WHERE persons.start_time >= CAST('10:15:00.000000' AS TIME)") + "WHERE persons.start_time >= " + time_string) def test_between(self): + time_start = self.dialect_time('09:00') + time_end = self.dialect_time('17:00') filter = TimeFilter(Person.start_time) filter.set('between', '9:00 am', '5:00 pm') self.assert_filter_query( filter, - "WHERE persons.start_time BETWEEN CAST('09:00:00.000000' AS TIME) AND " - "CAST('17:00:00.000000' AS TIME)") + f"WHERE persons.start_time BETWEEN {time_start} AND {time_end}") def test_not_between(self): + time_start = self.dialect_time('09:00') + time_end = self.dialect_time('17:00') filter = TimeFilter(Person.start_time) filter.set('!between', '9:00 am', '5:00 pm') self.assert_filter_query( filter, - "WHERE persons.start_time NOT BETWEEN CAST('09:00:00.000000' AS TIME) AND " - "CAST('17:00:00.000000' AS TIME)") + f"WHERE persons.start_time NOT BETWEEN {time_start} AND {time_end}") def test_empty(self): filter = TimeFilter(Person.start_time) @@ -1603,16 +1633,26 @@ def __init__(self, a, b, *vargs, **kwargs): class TestYesNoFilter(CheckFilterBase): def test_y(self): + sql_check = { + 'sqlite': "WHERE persons.boolcol = 1", + 'postgresql': "WHERE persons.boolcol = true", + 'mssql': "WHERE persons.boolcol = 1", + } filterobj = YesNoFilter(Person.boolcol) filterobj.set('y', None) query = filterobj.apply(db.session.query(Person.boolcol)) - self.assert_in_query(query, "WHERE persons.boolcol = 1") + self.assert_in_query(query, sql_check.get(db.engine.dialect.name)) def test_n(self): + sql_check = { + 'sqlite': "WHERE persons.boolcol = 0", + 'postgresql': "WHERE persons.boolcol = false", + 'mssql': "WHERE persons.boolcol = 0", + } filterobj = YesNoFilter(Person.boolcol) filterobj.set('n', None) query = filterobj.apply(db.session.query(Person.boolcol)) - self.assert_in_query(query, "WHERE persons.boolcol = 0") + self.assert_in_query(query, sql_check.get(db.engine.dialect.name)) def test_a(self): filterobj = YesNoFilter(Person.boolcol) diff --git a/webgrid/tests/test_rendering.py b/webgrid/tests/test_rendering.py index 8f90bf3..dad93e6 100644 --- a/webgrid/tests/test_rendering.py +++ b/webgrid/tests/test_rendering.py @@ -52,6 +52,14 @@ from .helpers import eq_html, inrequest, render_in_grid +def _query_exclude_person(query): + # this is pretty limited, but only used in the below couple of grids to + # exclude the third Person record + persons = Person.query.order_by(Person.id).limit(3).all() + exclude_id = persons[2].id if len(persons) >= 3 else -1 + return query.filter(Person.id != exclude_id) + + class PeopleGrid(PG): def query_prep(self, query, has_sort, has_filters): query = PG.query_prep(self, query, True, True) @@ -62,7 +70,7 @@ def query_prep(self, query, has_sort, has_filters): # default filter if not has_filters: - query = query.filter(Person.id != 3) + query = _query_exclude_person(query) return query @@ -79,7 +87,7 @@ def query_prep(self, query, has_sort, has_filters): # default filter if not has_filters: - query = query.filter(Person.id != 3) + query = _query_exclude_person(query) return query @@ -225,11 +233,13 @@ def test_car_html(self): mg.set_records(key_data) eq_html(mg.html.table(), 'basic_table.html') + @pytest.mark.skipif(db.engine.dialect.name != 'sqlite', reason="IDs will not line up") @inrequest('/') def test_people_html(self): pg = render_in_grid(PeopleGrid, 'html')() eq_html(pg.html.table(), 'people_table.html') + @pytest.mark.skipif(db.engine.dialect.name != 'sqlite', reason="IDs will not line up") @inrequest('/') def test_stopwatch_html(self): # Test Stopwatch grid with column groups. diff --git a/webgrid/tests/test_unit.py b/webgrid/tests/test_unit.py index 8db4662..881fae7 100644 --- a/webgrid/tests/test_unit.py +++ b/webgrid/tests/test_unit.py @@ -77,7 +77,12 @@ class CTG(Grid): g = CTG() query = g.build_query() assert_not_in_query(query, 'WHERE') - assert_not_in_query(query, 'ORDER BY') + if db.engine.dialect.name != 'mssql': + assert_not_in_query(query, 'ORDER BY') + else: + # MSSQL queries get an ORDER BY patched in if none is provided, + # else paging doesn't work + assert_in_query(query, 'ORDER BY') with mock.patch('logging.Logger.debug') as m_debug: rs = g.records assert len(rs) > 0, rs @@ -175,7 +180,10 @@ class CTG(Grid): g = CTG() g.set_filter('firstname', 'eq', 'foo') - assert_in_query(g, "WHERE upper(persons.firstname) = upper('foo')") + if db.engine.dialect.name == 'mssql': + assert_in_query(g, "WHERE persons.firstname = 'foo'") + else: + assert_in_query(g, "WHERE upper(persons.firstname) = upper('foo')") with mock.patch('logging.Logger.debug') as m_debug: g.records @@ -204,7 +212,10 @@ class CTG(Grid): g = CTG() g.set_filter('firstname', 'eq', 'foo') - assert_in_query(g, "WHERE upper(persons.last_name) = upper('foo')") + if db.engine.dialect.name == 'mssql': + assert_in_query(g, "WHERE persons.last_name = 'foo'") + else: + assert_in_query(g, "WHERE upper(persons.last_name) = upper('foo')") def test_filter_two_values(self): class CTG(Grid): @@ -241,7 +252,8 @@ class CTG(Grid): g = CTG() g.set_sort('firstname', 'lastname', '-firstname') - assert_in_query(g, 'ORDER BY persons.firstname, persons.last_name\n') + assert_in_query(g, 'ORDER BY persons.firstname, persons.last_name') + assert_not_in_query(g, 'ORDER BY persons.firstname, persons.last_name,') with mock.patch('logging.Logger.debug') as m_debug: g.records expected = [ @@ -257,13 +269,24 @@ class CTG(Grid): def test_paging(self): g = self.TG() - assert_in_query(g, 'LIMIT 50 OFFSET 0') + if db.engine.dialect.name == 'mssql': + assert_in_query(g, 'SELECT TOP 50 ') + else: + assert_in_query(g, 'LIMIT 50 OFFSET 0') g.set_paging(2, 1) - assert_in_query(g, 'LIMIT 2 OFFSET 0') + if db.engine.dialect.name == 'mssql': + assert_in_query(g, 'SELECT TOP 2 ') + else: + assert_in_query(g, 'LIMIT 2 OFFSET 0') g.set_paging(10, 5) - assert_in_query(g, 'LIMIT 10 OFFSET 40') + if db.engine.dialect.name == 'mssql': + assert_in_query(g, 'WHERE mssql_rn > 40 AND mssql_rn <= 10 + 40') + assert_in_query(g, 'SELECT persons.firstname AS firstname, ROW_NUMBER() ' + 'OVER (ORDER BY persons.firstname) AS mssql_rn') + else: + assert_in_query(g, 'LIMIT 10 OFFSET 40') def test_paging_disabled(self): class TG(Grid): @@ -432,8 +455,15 @@ class CTG(Grid): g = CTG() g.search_value = 'foo' - search_where = ("WHERE lower(persons.firstname) LIKE lower('%foo%')" - " OR lower(persons.last_name) LIKE lower('%foo%')") + if db.engine.dialect.name == 'sqlite': + search_where = ("WHERE lower(persons.firstname) LIKE lower('%foo%')" + " OR lower(persons.last_name) LIKE lower('%foo%')") + elif db.engine.dialect.name == 'postgresql': + search_where = ("WHERE persons.firstname ILIKE '%foo%'" + " OR persons.last_name ILIKE '%foo%'") + elif db.engine.dialect.name == 'mssql': + search_where = ("WHERE persons.firstname LIKE '%foo%'" + " OR persons.last_name LIKE '%foo%'") assert_in_query(g, search_where) def test_column_keys_unique(self): @@ -497,7 +527,10 @@ def test_qs_paging(self): assert pg.per_page == 1 # make sure the corret values get applied to the query - assert 'LIMIT 1 OFFSET 1' in query_to_str(pg.build_query()) + if db.engine.dialect.name == 'mssql': + assert 'WHERE mssql_rn > 1 AND mssql_rn <= 1 + 1' in query_to_str(pg.build_query()) + else: + assert 'LIMIT 1 OFFSET 1' in query_to_str(pg.build_query()) @inrequest('/foo?perpage=5&onpage=foo') def test_qs_onpage_invalid(self): @@ -563,16 +596,21 @@ def test_qs_sorting_ignores_emptystring(self): assert pg.order_by == [] assert len(pg.user_warnings) == 0 - @inrequest('/foo?op(firstname)=eq&v1(firstname)=fn001&op(status)=is&v1(status)=1&v1(status)=2') def test_qs_filtering(self): - pg = PeopleGrid() - pg.apply_qs_args() + first_id = Status.query.filter_by(label='pending').one().id + second_id = Status.query.filter_by(label='in process').one().id + with flask.current_app.test_request_context( + '/foo?op(firstname)=eq&v1(firstname)=fn001&op(status)=is' + f'&v1(status)={first_id}&v1(status)={second_id}' + ): + pg = PeopleGrid() + pg.apply_qs_args() assert pg.columns[0].filter.op == 'eq' assert pg.columns[0].filter.value1 == 'fn001' assert pg.columns[0].filter.value2 is None assert pg.columns[4].filter.op == 'is' - assert pg.columns[4].filter.value1 == [1, 2] + assert pg.columns[4].filter.value1 == [first_id, second_id] assert pg.columns[4].filter.value2 is None @inrequest('/foo') @@ -737,9 +775,9 @@ def test_qs_blank_operator(self): @inrequest('/foo?sort1=legacycol1&sort2=legacycol2') def test_sa_expr_sort(self): class AGrid(Grid): - firstname = Column('First Name') - legacycol1 = Column('LC1') - legacycol2 = Column('LC2') + Column('First Name', 'firstname') + Column('LC1', Person.legacycol1) + Column('LC2', Person.legacycol2) def query_base(self, has_sort, has_filters): query = db.session.query( diff --git a/webgrid_ta/app.py b/webgrid_ta/app.py index 8a92ee8..28b95eb 100644 --- a/webgrid_ta/app.py +++ b/webgrid_ta/app.py @@ -27,10 +27,10 @@ webgrid = WebGrid() -def create_app(config): +def create_app(config, database_url=None): app = Flask(__name__) - app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///' + app.config['SQLALCHEMY_DATABASE_URI'] = database_url or 'sqlite:///' app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False # app.config['SQLALCHEMY_ECHO'] = True # app.config['DEBUG'] = True diff --git a/webgrid_ta/model/entities.py b/webgrid_ta/model/entities.py index 96a5aec..1c95f9a 100644 --- a/webgrid_ta/model/entities.py +++ b/webgrid_ta/model/entities.py @@ -54,7 +54,8 @@ class Person(db.Model, DefaultMixin): createdts = sa.Column(sa.DateTime) sortorder = sa.Column(sa.Integer) floatcol = sa.Column(sa.Float) - numericcol = sa.Column(sa.Numeric) + # must specify precision here, as mssql defaults to (18, 0) + numericcol = sa.Column(sa.Numeric(9, 2)) boolcol = sa.Column(sa.Boolean) due_date = sa.Column(sa.Date) start_time = sa.Column(sa.Time) diff --git a/webgrid_ta/views.py b/webgrid_ta/views.py index afddd13..f77da7b 100644 --- a/webgrid_ta/views.py +++ b/webgrid_ta/views.py @@ -15,6 +15,8 @@ def index(): class CurrencyCol(NumericColumn): def format_data(self, data): + if data is None: + return data return data if int(data) % 2 else data * -1 class PeopleGrid(PGBase):