Merge 388ee4a into c29120e

.github/workflows/test-changes.yml

@@ -113,7 +113,7 @@ jobs:
 
       - name: Test python source code for mode ${{ env.testing }}
         if: env.testing == 'simple'
-        run: nosetests -v petl --with-coverage --cover-package=petl
+        run: pytest --cov=petl petl
 
       - name: Test documentation inside source code for mode ${{ env.testing }}
         if: env.testing == 'full'
@@ -122,7 +122,7 @@ jobs:
           python -m pip install -r requirements-formats.txt
           echo "::endgroup::"
           echo "::group::Perform doc test execution with coverage"
-          nosetests -v --with-coverage --cover-package=petl --with-doctest --doctest-options=+NORMALIZE_WHITESPACE petl -I"csv_py2\.py" -I"db\.py"
+          pytest --cov=petl petl
           echo "::endgroup::"
 
       - name: Coveralls

appveyor.yml

@@ -31,6 +31,6 @@ install:
 build: off
 
 test_script:
-  - "%CMD_IN_ENV% python -m pip install -U pip setuptools wheel nose mock"
+  - "%CMD_IN_ENV% python -m pip install -U pip setuptools wheel pytest mock"
   - "%CMD_IN_ENV% python setup.py install"
-  - "%CMD_IN_ENV% python -m nose -v petl"
+  - "%CMD_IN_ENV% python -m pytest petl"

petl/io/db.py

@@ -183,8 +183,10 @@ def _iter_dbapi_cursor(cursor, query, *args, **kwargs):
 
 
 def _iter_sqlalchemy_engine(engine, query, *args, **kwargs):
-    return _iter_sqlalchemy_connection(engine.connect(), query,
-                                       *args, **kwargs)
+    connection = engine.connect()
+    for row in _iter_sqlalchemy_connection(connection, query, *args, **kwargs):
+        yield row
+    connection.close()
 
 
 def _iter_sqlalchemy_connection(connection, query, *args, **kwargs):

petl/test/conftest.py

@@ -0,0 +1,10 @@
+import logging
+
+
+def pytest_configure():
+    org = logging.Logger.debug
+
+    def debug(self, msg, *args, **kwargs):
+        org(self, str(msg), *args, **kwargs)
+
+    logging.Logger.debug = debug

petl/test/failonerror.py

@@ -1,11 +1,11 @@
+import pytest
+
 from petl.test.helpers import ieq, eq_
 import petl.config as config
 
-from nose.tools import nottest
 
 
-@nottest
-def test_failonerror(input_fn, expected_output):
+def assert_failonerror(input_fn, expected_output):
     """In the input rows, the first row should process through the
     transformation cleanly.  The second row should generate an
     exception.  There are no requirements for any other rows."""
@@ -33,13 +33,9 @@ def test_failonerror(input_fn, expected_output):
 
     # When called with failonerror=True, a bad conversion raises an
     # exception
-    try:
+    with pytest.raises(Exception):
         table4 = input_fn(failonerror=True)
         table4.nrows()
-    except Exception:
-        pass
-    else:
-        raise Exception('expected exception not raised')
 
     # When called with failonerror='inline', a bad conversion
     # does not raise an exception, and an Exception for the failed
@@ -60,13 +56,9 @@ def test_failonerror(input_fn, expected_output):
     # When config.failonerror == True, a bad conversion raises an
     # exception
     config.failonerror = True
-    try:
+    with pytest.raises(Exception):
         table6 = input_fn()
         table6.nrows()
-    except Exception:
-        pass
-    else:
-        raise Exception('expected exception not raised')
 
     # When config.failonerror == 'inline', a bad conversion
     # does not raise an exception, and an Exception for the failed
@@ -82,13 +74,9 @@ def test_failonerror(input_fn, expected_output):
     # When config.failonerror is an invalid value, but still truthy, it
     # behaves the same as if == True
     config.failonerror = 'invalid'
-    try:
+    with pytest.raises(Exception):
         table8 = input_fn()
         table8.nrows()
-    except Exception:
-        pass
-    else:
-        raise Exception('expected exception not raised')
 
     # When config.failonerror is None, it behaves the same as if
     # config.failonerror is False
@@ -105,13 +93,9 @@ def test_failonerror(input_fn, expected_output):
 
     # A None keyword parameter uses config.failonerror == True
     config.failonerror = True
-    try:
+    with pytest.raises(Exception):
         table11 = input_fn(failonerror=None)
         table11.nrows()
-    except Exception:
-        pass
-    else:
-        raise Exception('expected exception not raised')
 
     # restore config setting
     config.failonerror = saved_config_failonerror

petl/test/helpers.py

@@ -2,11 +2,19 @@
 
 import sys
 
-from nose.tools import eq_, assert_almost_equal
+import pytest
 
 from petl.compat import izip_longest
 
 
+def eq_(expect, actual, msg=None):
+    assert expect == actual, msg
+
+
+def assert_almost_equal(first, second, places=None, msg=None):
+    abs = None if places is None else 10**-places
+    assert pytest.approx(first, second, abs=abs), msg
+
 def ieq(expect, actual, cast=None):
     '''test when values are equals for eacfh row and column'''
     ie = iter(expect)

petl/test/io/test_avro.py

@@ -1,13 +1,14 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import, print_function, division
 
-import sys
 import math
 
 from datetime import datetime, date
 from decimal import Decimal
 from tempfile import NamedTemporaryFile
 
+import pytest
+
 from petl.compat import PY3
 from petl.transform.basics import cat
 from petl.util.base import dicts
@@ -28,7 +29,7 @@
     # import fastavro dependencies
     import pytz
 except ImportError as e:
-    print('SKIP avro tests: %s' % e, file=sys.stderr)
+    pytest.skip('SKIP avro tests: %s' % e, allow_module_level=True)
 else:
     # region Test Cases
 

petl/test/io/test_bcolz.py

@@ -1,8 +1,8 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import, print_function, division
-import sys
 import tempfile
 
+import pytest
 
 from petl.test.helpers import ieq, eq_
 from petl.io.bcolz import frombcolz, tobcolz, appendbcolz
@@ -11,7 +11,7 @@
 try:
     import bcolz
 except ImportError as e:
-    print('SKIP bcolz tests: %s' % e, file=sys.stderr)
+    pytest.skip('SKIP bcolz tests: %s' % e, allow_module_level=True)
 else:
 
     def test_frombcolz():

petl/test/io/test_db_create.py

@@ -2,11 +2,11 @@
 from __future__ import absolute_import, print_function, division
 
 
-import sys
 import logging
 from datetime import datetime, date
 import sqlite3
 
+import pytest
 
 from petl.io.db import fromdb, todb
 from petl.io.db_create import make_sqlalchemy_column
@@ -98,7 +98,7 @@ def _setup_generic(dbapi_connection):
     # noinspection PyUnresolvedReferences
     import sqlalchemy
 except ImportError as e:
-    print('SKIP generic create tests: %s' % e, file=sys.stderr)
+    pytest.skip('SKIP generic create tests: %s' % e, allow_module_level=True)
 else:
 
     from sqlalchemy import Column, DateTime, Date
@@ -131,7 +131,7 @@ def test_sqlite3_create():
         _test_create(dbapi_cursor)
         dbapi_cursor.close()
 
-
+SKIP_PYMYSQL = False
 try:
     import pymysql
     import sqlalchemy
@@ -140,9 +140,9 @@ def test_sqlite3_create():
                     password=password,
                     database=database)
 except Exception as e:
-    print('SKIP pymysql create tests: %s' % e, file=sys.stderr)
-else:
-
+    SKIP_PYMYSQL = 'SKIP pymysql create tests: %s' % e
+finally:
+    @pytest.mark.skipif(bool(SKIP_PYMYSQL), reason=str(SKIP_PYMYSQL))
     def test_mysql_create():
 
         import pymysql
@@ -183,6 +183,7 @@ def test_mysql_create():
         sqlalchemy_session.close()
 
 
+SKIP_POSTGRES = False
 try:
     import psycopg2
     import sqlalchemy
@@ -191,9 +192,9 @@ def test_mysql_create():
         % (host, database, user, password)
     )
 except Exception as e:
-    print('SKIP psycopg2 create tests: %s' % e, file=sys.stderr)
-else:
-
+    SKIP_POSTGRES = 'SKIP psycopg2 create tests: %s' % e
+finally:
+    @pytest.mark.skipif(bool(SKIP_POSTGRES), reason=str(SKIP_POSTGRES))
     def test_postgresql_create():
         import psycopg2
         import psycopg2.extensions

petl/test/io/test_db_server.py

@@ -1,8 +1,8 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import, print_function, division
-import sys
 import logging
 
+import pytest
 
 import petl as etl
 from petl.test.helpers import ieq
@@ -126,9 +126,14 @@ def _setup_postgresql(dbapi_connection):
     dbapi_connection.commit()
 
 
-host, user, password, database = 'localhost', 'petl', 'test', 'petl'
+def _setup_sqlalchemy_quotes(dbapi_connection, connection_record):
+    cursor = dbapi_connection.cursor()
+    cursor.execute("SET sql_mode = 'ANSI_QUOTES'")
+
 
+host, user, password, database = '127.0.0.1', 'petl', 'test', 'petl'
 
+SKIP_PYMYSQL = False
 try:
     import pymysql
     import sqlalchemy
@@ -137,10 +142,10 @@ def _setup_postgresql(dbapi_connection):
                     password=password,
                     database=database)
 except Exception as e:
-    print('SKIP pymysql tests: %s' % e, file=sys.stderr)
-else:
-
-    def test_mysql():
+    SKIP_PYMYSQL = 'SKIP pymysql tests: %s' % e
+finally:
+    @pytest.mark.skipif(bool(SKIP_PYMYSQL), reason=str(SKIP_PYMYSQL))
+    def test_pymysql():
 
         import pymysql
         connect = pymysql.connect
@@ -166,8 +171,9 @@ def test_mysql():
         from sqlalchemy import create_engine
         sqlalchemy_engine = create_engine('mysql+pymysql://%s:%s@%s/%s' %
                                           (user, password, host, database))
+        from sqlalchemy.event import listen
+        listen(sqlalchemy_engine, "connect", _setup_sqlalchemy_quotes)
         sqlalchemy_connection = sqlalchemy_engine.connect()
-        sqlalchemy_connection.execute('SET SQL_MODE=ANSI_QUOTES')
         _test_dbo(sqlalchemy_connection)
         sqlalchemy_connection.close()
 
@@ -182,8 +188,8 @@ def test_mysql():
         # exercise sqlalchemy engine
         _setup_mysql(dbapi_connection)
         sqlalchemy_engine2 = create_engine('mysql+pymysql://%s:%s@%s/%s' %
-                                           (user, password, host, database))
-        sqlalchemy_engine2.execute('SET SQL_MODE=ANSI_QUOTES')
+                                           (user, password, host, database), echo_pool='debug')
+        listen(sqlalchemy_engine2, "connect", _setup_sqlalchemy_quotes)
         _test_dbo(sqlalchemy_engine2)
         sqlalchemy_engine2.dispose()
 
@@ -195,8 +201,10 @@ def test_mysql():
                                   charset='utf8')
         utf8_connection.cursor().execute('SET SQL_MODE=ANSI_QUOTES')
         _test_unicode(utf8_connection)
+        utf8_connection.close()
 
 
+SKIP_MYSQLDB = False
 try:
     import MySQLdb
     import sqlalchemy
@@ -205,10 +213,10 @@ def test_mysql():
                     passwd=password,
                     db=database)
 except Exception as e:
-    print('SKIP MySQLdb tests: %s' % e, file=sys.stderr)
-else:
-
-    def test_mysql():
+    SKIP_MYSQLDB = 'SKIP MySQLdb tests: %s' % e
+finally:
+    @pytest.mark.skipif(bool(SKIP_MYSQLDB), reason=str(SKIP_MYSQLDB))
+    def test_mysqldb():
 
         import MySQLdb
         connect = MySQLdb.connect
@@ -234,6 +242,8 @@ def test_mysql():
         from sqlalchemy import create_engine
         sqlalchemy_engine = create_engine('mysql+mysqldb://%s:%s@%s/%s' %
                                           (user, password, host, database))
+        from sqlalchemy.event import listen
+        listen(sqlalchemy_engine, "connect", _setup_sqlalchemy_quotes)
         sqlalchemy_connection = sqlalchemy_engine.connect()
         sqlalchemy_connection.execute('SET SQL_MODE=ANSI_QUOTES')
         _test_dbo(sqlalchemy_connection)
@@ -256,7 +266,7 @@ def test_mysql():
         utf8_connection.cursor().execute('SET SQL_MODE=ANSI_QUOTES')
         _test_unicode(utf8_connection)
 
-
+SKIP_TEST_POSTGRES = False
 try:
     import psycopg2
     import sqlalchemy
@@ -265,9 +275,9 @@ def test_mysql():
         % (host, database, user, password)
     )
 except Exception as e:
-    print('SKIP psycopg2 tests: %s' % e, file=sys.stderr)
-else:
-
+    SKIP_TEST_POSTGRES = 'SKIP psycopg2 tests: %s' % e
+finally:
+    @pytest.mark.skipif(bool(SKIP_TEST_POSTGRES), reason=str(SKIP_TEST_POSTGRES))
     def test_postgresql():
 
         import psycopg2

petl/test/io/test_numpy.py

@@ -2,8 +2,7 @@
 from __future__ import absolute_import, print_function, division
 
 
-import sys
-
+import pytest
 
 import petl as etl
 from petl.test.helpers import ieq, eq_, assert_almost_equal
@@ -14,7 +13,7 @@
     # noinspection PyUnresolvedReferences
     import numpy as np
 except ImportError as e:
-    print('SKIP numpy tests: %s' % e, file=sys.stderr)
+    pytest.skip('SKIP numpy tests: %s' % e, allow_module_level=True)
 else:
 
     def test_toarray_nodtype():