Merge pull request #511 from henrykironde/change_regrestion_tests

improve tests platform

.travis.yml

@@ -15,10 +15,10 @@ install:
   - pip install pytest-xdist
 
 
-# command to run tests
+# command to run tests using py.test
 
 script:
-  - py.test --cov=retriever
+  - py.test -v --cov=retriever
 
 # database setup
 before_script:

appveyor.yml

@@ -0,0 +1,66 @@
+environment:
+
+  matrix:
+
+    - PYTHON: "C:\\Python27-x64"
+      PYTHON_VERSION: "2.7.x"
+      PYTHON_ARCH: "64"
+
+  # postgres
+  POSTGRES_PORT: tcp://localhost:5432
+  POSTGRES_ENV_POSTGRES_USER: postgres
+  POSTGRES_ENV_POSTGRES_PASSWORD: Password12!
+  POSTGRES_ENV_POSTGRES_DB: testdb
+  POSTGRES_PATH: C:\Program Files\PostgreSQL\9.4
+  PGUSER: postgres
+  PGPASSWORD: Password12!
+
+  # mysql
+  MYSQL_PORT: tcp://localhost:3306
+  MYSQL_ENV_MYSQL_USER: root
+  MYSQL_ENV_MYSQL_PASSWORD: Password12!
+  MYSQL_ENV_MYSQL_DATABASE: sqlectron
+  MYSQL_PATH: C:\Program Files\MySql\MySQL Server 5.7
+  MYSQL_PWD: Password12!
+
+services:
+  - mysql
+  - postgresql94
+
+init:
+  - SET PATH=%POSTGRES_PATH%\bin;%MYSQL_PATH%\bin;%PATH%
+
+install:
+  - "%PYTHON%\\python.exe -m pip install --upgrade pip"
+  - "%PYTHON%\\python.exe -m pip install wheel"
+  - "%PYTHON%\\python.exe -m pip install -r requirements.txt"
+  - "%PYTHON%\\python.exe -m pip install codecov"
+  - "%PYTHON%\\python.exe -m pip install nose-cov"
+  - "%PYTHON%\\python.exe -m pip install -U pytest"
+
+  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
+  - "%PYTHON%\\python.exe -m  pip --version"
+  - "python --version"
+  - "python -c \"import struct; print(struct.calcsize('P') * 8)\""
+  - echo localhost:*:testdb:postgres:Password12! > C:\Program Files\PostgreSQL\9.4\pgpass.conf
+  - echo "localhost:*:testdb:postgres:Password12!" > pgpass.conf
+  - echo "localhost:*:testdb:postgres:Password12!" > %userprofile%\pgpass.conf
+  - SET PGPASSFILE=C:\Program Files\PostgreSQL\9.4\pgpass.conf
+  - set PGPASSWORD=Password12!
+
+build_script:
+  # postgres
+  - createdb testdb
+  # mysql
+  - mysql -e "create database sqlectron;" --user=root
+  - mysql -e "CREATE USER 'travis'@'localhost';GRANT ALL PRIVILEGES ON *.* TO 'travis'@'localhost';GRANT FILE ON *.* TO 'travis'@'localhost';" -uroot
+  - dir
+  - cd C:\projects\retriever
+  - "python setup.py install"
+  - echo "Installed retriever:"
+  - retriever --version
+  - cd C:\projects\retriever
+
+test_script:
+  - py.test -k "not test_postgres_regression" -v
+#  - py.test -v  currently, we run pytest excluding postgres

docs/developer.rst

@@ -22,8 +22,8 @@ You need to install all the database infrastructures to enable local testing.
 ::
 
   SQLite
-  MySQL 5.6
-  PostgreSQL 9.2
+  MySQL
+  PostgreSQL
   MSAccess
 
 You will also need the following modules:
@@ -81,7 +81,7 @@ you can also install from Git.
   pip install git+git://github.com/myuser/myproject@my_branch # github repository Specific branch
 
 Running tests locally
-^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^
 
 To run tests we use pytest.
 From the source top level directory, run
@@ -100,7 +100,7 @@ In case we want to run tests on a specific test category, we add the path of the
 This will only run test_retriever.py
 
 Tests in the Cloud
-^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^
 
 Currently we support Travis, a cloud testing platform.
 

docs/scripts.rst

@@ -39,7 +39,7 @@ The most basic scripts structure requires only some general metadata about the d
   table: 
 
 Example of a basic script, example.script
-------------------------------------------
+-----------------------------------------
 
 ::
 

engines/__init__.py

@@ -5,7 +5,7 @@
     "postgres",
     "sqlite",
     "msaccess",
-    "csv",
+    "csvengine",
     "download_only",
     "jsonengine",
     "xmlengine"

engines/csv.py → engines/csvengine.py

@@ -1,10 +1,12 @@
 from builtins import str
 from builtins import object
 from builtins import range
+
 import os
-import platform
+
 from retriever.lib.models import Engine, no_cleanup
 from retriever import DATA_DIR
+from retriever.lib.tools import sort_csv
 
 
 class DummyConnection(object):
@@ -100,6 +102,10 @@ def table_exists(self, dbname, tablename):
         tablename = self.table_name(name=tablename, dbname=dbname)
         return os.path.exists(tablename)
 
+    def to_csv(self):
+        """Export sorted version of CSV file"""
+        return sort_csv(self.table_name())
+
     def get_connection(self):
         """Gets the db connection."""
         self.get_input()

engines/jsonengine.py

@@ -9,6 +9,8 @@
 from retriever.lib.models import Engine
 from retriever import DATA_DIR
 from collections import OrderedDict
+from retriever.lib.tools import json2csv, sort_csv
+
 
 class DummyConnection(object):
     def cursor(self):
@@ -120,6 +122,12 @@ def table_exists(self, dbname, tablename):
         tablename = self.table_name(name=tablename, dbname=dbname)
         return os.path.exists(tablename)
 
+    def to_csv(self):
+        """Export table from json engine to CSV file"""
+        keys = [columnname[0] for columnname in self.table.columns]
+        csv_outfile = json2csv(self.table_name(), header_values=keys)
+        return sort_csv(csv_outfile)
+
     def get_connection(self):
         """Gets the db connection."""
         self.get_input()

engines/sqlite.py

@@ -99,6 +99,10 @@ def table_exists(self, dbname, tablename):
                 self.existing_table_names.add(line[0].lower())
         return self.table_name(name=tablename, dbname=dbname).lower() in self.existing_table_names
 
+    def to_csv(self):
+        self.connection.text_factory = str
+        Engine.to_csv(self)
+
     def get_connection(self):
         """Gets the db connection."""
         import sqlite3 as dbapi

engines/xmlengine.py

@@ -5,7 +5,7 @@
 
 from retriever.lib.models import Engine
 from retriever import DATA_DIR
-
+from retriever.lib.tools import xml2csv, sort_csv
 
 class DummyConnection(object):
 
@@ -123,6 +123,12 @@ def table_exists(self, dbname, tablename):
         tablename = self.table_name(name=tablename, dbname=dbname)
         return os.path.exists(tablename)
 
+    def to_csv(self):
+        """Export table from xml engine to CSV file"""
+        keys = [columnname[0] for columnname in self.table.columns]
+        csv_outfile = xml2csv(self.table_name(), header_values=keys)
+        return sort_csv(csv_outfile)
+
     def get_connection(self):
         """Gets the db connection."""
         self.get_input()

lib/cleanup.py

@@ -1,4 +1,6 @@
 from builtins import object
+
+
 def floatable(value):
     """Check if a value can be converted to a float"""
     try:
@@ -7,6 +9,7 @@ def floatable(value):
     except ValueError:
         return False
 
+
 def correct_invalid_value(value, args):
     """This cleanup function replaces null indicators with None."""
     try:

lib/engine.py

@@ -12,8 +12,7 @@
 import tarfile
 import urllib
 import csv
-import itertools
-from decimal import Decimal
+
 from retriever import DATA_SEARCH_PATHS, DATA_WRITE_PATH
 from retriever.lib.cleanup import no_cleanup
 from retriever.lib.warning import Warning
@@ -476,6 +475,24 @@ def exists(self, script):
             )
             for key in list(script.urls.keys()) if key])
 
+    def to_csv(self):
+        # due to Cyclic imports we can not move this import to the top
+        from retriever.lib.tools import sort_csv
+        csvfile_output = (self.table_name() + '.csv')
+        csv_out = open(csvfile_output, "wb")
+        csv_writer = csv.writer(csv_out, dialect='excel')
+        self.get_cursor()
+        self.cursor.execute("SELECT * FROM " + self.table_name() + ";")
+        row = self.cursor.fetchone()
+        colnames = [tuple_i[0] for tuple_i in self.cursor.description]
+        csv_writer.writerow(colnames)
+        while row is not None:
+            csv_writer.writerow(row)
+            row = self.cursor.fetchone()
+        csv_out.close()
+        self.disconnect()
+        return sort_csv(csvfile_output)
+
     def final_cleanup(self):
         """Close the database connection."""
 

lib/templates.py

@@ -14,7 +14,7 @@ class Script(object):
     from this class and execute their code in the download method."""
 
     def __init__(self, name="", description="", shortname="", urls=dict(),
-                 tables=dict(), ref="", public=True, addendum=None, citation="Not currently available",**kwargs):
+                 tables=dict(), ref="", public=True, addendum=None, citation="Not currently available", **kwargs):
         self.name = name
         self.shortname = shortname
         self.filename = __name__
@@ -24,7 +24,7 @@ def __init__(self, name="", description="", shortname="", urls=dict(),
         self.ref = ref
         self.public = public
         self.addendum = addendum
-        self.citation= citation
+        self.citation = citation
         self.tags = []
         for key, item in list(kwargs.items()):
             setattr(self, key, item[0] if isinstance(item, tuple) else item)