Add tests for style on windows (#1468)

* Condition should test if hash_value is None

Clean up style

* Normalize path for Bulk insert

* Use mariadb-client for mysql

* Enforce style as part of tests

Clean up style

* Add tests for style on windows

.travis.yml

@@ -13,12 +13,12 @@ before_install:
 # command to run tests using py.test
 script:
   - docker-compose run --service-ports python_retriever pytest -v --cov=retriever
+  - docker-compose run python_retriever flake8 --ignore=E501,W503,E402,F401,F403,E722,F841,W504 retriever --max-line-length=90 > /dev/null 2>&1
+  - docker-compose run python_retriever yapf -d --recursive retriever/ --style=.style.yapf > /dev/null 2>&1
+  - docker-compose run python_retriever pylint -rn retriever/ -f colorized --rcfile=.pylintrc > /dev/null 2>&1
 
 after_success:
   - codecov
-  - docker-compose run python_retriever flake8 --ignore=E501,W503,E402,F401,F403,E722,F841,W504 retriever --max-line-length=90
-  - docker-compose run python_retriever yapf --recursive retriever/ --style=.style.yapf -d
-  - docker-compose run python_retriever pylint -rn retriever/ -f colorized --rcfile=.pylintrc
 
 notifications:
   slack:

Dockerfile

@@ -9,7 +9,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends apt-utils
 RUN apt-get install -y --force-yes tzdata
 
 RUN apt-get install -y --force-yes build-essential wget git locales locales-all > /dev/null
-RUN apt-get install -y --force-yes postgresql-client mysql-client > /dev/null
+RUN apt-get install -y --force-yes postgresql-client mariadb-client > /dev/null
 
 # Set encoding
 ENV LC_ALL en_US.UTF-8
@@ -36,7 +36,7 @@ RUN pip install pytest-xdist -U
 RUN pip install pytest
 RUN pip install yapf
 RUN pip install pylint
-RUN pip install flake8
+RUN pip install flake8 -U
 
 # Install Postgis after Python is setup
 RUN apt-get install -y --force-yes postgis

appveyor.yml

@@ -37,6 +37,9 @@ install:
   - "%PYTHON%\\python.exe -m pip install codecov"
   - "%PYTHON%\\python.exe -m pip install nose-cov"
   - "%PYTHON%\\python.exe -m pip install -U pytest"
+  - "%PYTHON%\\python.exe -m pip install -U flake8"
+  - "%PYTHON%\\python.exe -m pip install -U yapf"
+  - "%PYTHON%\\python.exe -m pip install -U pylint"
 
   - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
   - "%PYTHON%\\python.exe -m  pip --version"
@@ -63,3 +66,6 @@ build_script:
 
 test_script:
   - py.test -v -k "not test_download_archive and not test_postgres_spatial"
+  - flake8 --ignore=E501,W503,E402,F401,F403,E722,F841,W504 retriever --max-line-length=90
+  - yapf --recursive retriever/ --style=.style.yapf -d
+  - pylint -rn retriever/ -f colorized --rcfile=.pylintrc

retriever/__main__.py

@@ -7,7 +7,7 @@
 
 from retriever.engines import engine_list, choose_engine
 from retriever.lib.datasets import datasets, dataset_names, license
-from retriever.lib.defaults import sample_script, CITATION, SCRIPT_SEARCH_PATHS, LICENSE, VERSION
+from retriever.lib.defaults import sample_script, CITATION, SCRIPT_SEARCH_PATHS, LICENSE
 from retriever.lib.engine_tools import reset_retriever
 from retriever.lib.get_opts import parser
 from retriever.lib.install import _install
@@ -231,7 +231,8 @@ def main():
         else:
             raise Exception("no dataset specified.")
         if scripts:
-            if args.dataset.endswith('.zip') or hasattr(args, 'hash_value'):
+            if args.dataset.endswith('.zip') or (hasattr(args, 'hash_value') and
+                                                 args.hash_value):
                 _install(vars(args), debug=debug, use_cache=use_cache)
                 return
             for dataset in scripts:

retriever/engines/mysql.py

@@ -122,7 +122,8 @@ def get_connection(self):
         self.get_input()
         try:
             return dbapi.connect(charset=self.lookup_encoding(),
-                             read_default_file='~/.my.cnf', **args)
+                                 read_default_file='~/.my.cnf',
+                                 **args)
         except dbapi.err.OperationalError as error:
             raise (error)
         except Exception as e:

retriever/engines/postgres.py

@@ -108,20 +108,20 @@ def insert_data_from_file(self, filename):
         p_bulk = [self.check_bulk_insert(), self.table.delimiter, self.table.header_rows]
         if p_bulk == [True, ",", 1]:
             columns = self.table.get_insert_columns()
-            filename = os.path.abspath(filename)
+            filename = os.path.normpath(os.path.abspath(filename))
             statement = """
 COPY """ + self.table_name() + " (" + columns + """)
-FROM '""" + filename.replace("\\", "\\\\") + """'
+FROM '""" + filename + """'
 WITH DELIMITER ','
 CSV HEADER;"""
             try:
                 self.execute("BEGIN")
                 self.execute(statement)
                 self.execute("COMMIT")
+                print("Bulk insert on .. ", self.table_name())
                 return True
-            except BaseException:
+            except Exception:
                 self.connection.rollback()
-                return None
         return Engine.insert_data_from_file(self, filename)
 
     def insert_statement(self, values):
@@ -166,20 +166,20 @@ def insert_raster(self, path=None, srid=4326):
         if not path:
             path = Engine.format_data_dir(self)
 
-        raster_sql = ("raster2pgsql -Y -M -d -I -s {SRID} \"{path}\""
+        raster_sql = ('raster2pgsql -Y -M -d -I -s {SRID} "{path}"'
                       " -F -t 100x100 {SCHEMA_DBTABLE}".format(
                           SRID=srid,
                           path=os.path.normpath(path),
                           SCHEMA_DBTABLE=self.table_name()))
 
-        cmd_string = " | psql -U {USER} -d {DATABASE} " \
-                     "--port {PORT} --host {HOST} > {nul_dev} ".format(
-            USER=self.opts["user"],
-            DATABASE=self.opts["database"],
-            PORT=self.opts["port"],
-            HOST=self.opts["host"],
-            nul_dev=os.devnull,
-        )
+        cmd_string = (" | psql -U {USER} -d {DATABASE} "
+                      "--port {PORT} --host {HOST} > {nul_dev} ".format(
+                          USER=self.opts["user"],
+                          DATABASE=self.opts["database"],
+                          PORT=self.opts["port"],
+                          HOST=self.opts["host"],
+                          nul_dev=os.devnull,
+                      ))
 
         cmd_stmt = raster_sql + cmd_string
         if self.debug:
@@ -213,16 +213,14 @@ def insert_vector(self, path=None, srid=4326):
          """
         if not path:
             path = Engine.format_data_dir(self)
-        vector_sql = ("shp2pgsql -d -I -W \"{encd}\"  -s {SRID}"
-                      " \"{path}\" \"{SCHEMA_DBTABLE}\"".format(
-                          encd=self.encoding,
-                          SRID=srid,
-                          path=os.path.normpath(path),
-                          SCHEMA_DBTABLE=self.table_name(),
-                      ))
+        vector_sql = 'shp2pgsql -d -I -W "{encd}"  -s {SRID}  "{path}" "{SCHEMA_DBTABLE}"'.format(
+            encd=self.encoding,
+            SRID=srid,
+            path=os.path.normpath(path),
+            SCHEMA_DBTABLE=self.table_name(),
+        )
 
-        cmd_string = " | psql -U {USER} -d {DATABASE} --port {PORT} " \
-                     "--host {HOST} > {nul_dev} ".format(
+        cmd_string = " | psql -U {USER} -d {DATABASE} --port {PORT} --host {HOST} > {nul_dev} ".format(
             USER=self.opts["user"],
             DATABASE=self.opts["database"],
             PORT=self.opts["port"],
@@ -260,18 +258,19 @@ def get_connection(self):
         self.get_input()
         try:
             conn = dbapi.connect(
-            host=self.opts["host"],
-            port=int(self.opts["port"]),
-            user=self.opts["user"],
-            password=self.opts["password"],
-            database=self.opts["database"],)
-        except dbapi.OperationalError as error: 
+                host=self.opts["host"],
+                port=int(self.opts["port"]),
+                user=self.opts["user"],
+                password=self.opts["password"],
+                database=self.opts["database"],
+            )
+        except dbapi.OperationalError as error:
             raise (error)
         except Exception as e:
             print(e)
-        
+
         self.set_engine_encoding()
-        encoding_lookup = {'iso-8859-1': 'Latin1', 'latin-1': 'Latin1', 'utf-8': 'UTF8'}
+        encoding_lookup = {"iso-8859-1": "Latin1", "latin-1": "Latin1", "utf-8": "UTF8"}
         self.db_encoding = encoding_lookup.get(self.encoding)
         conn.set_client_encoding(self.db_encoding)
         return conn

retriever/lib/create_scripts.py

@@ -19,6 +19,7 @@
 
 
 class TabularPk:
+    """Main Tabular data package"""
 
     def __init__(self,
                  name="fill",
@@ -111,8 +112,10 @@ def __init__(self, **kwargs):
         self.extent = ""
         self.geom_type = ""
 
-    def get_source(self, source, driver_name="ESRI Shapefile"):
+    def get_source(self, source, driver_name=None):
         """Open a data source"""
+        if not driver_name:
+            driver_name = self.driver_name
         driver = ogr.GetDriverByName(driver_name)
         return driver.Open(source, 0)
 
@@ -154,8 +157,10 @@ def create_vector_resources(self, path, driver_name):
             layer["schema"]["fields"].append(col_obj)
         return layer
 
-    def get_resources(self, file_path, file_name=None, driver_name=None):
-        return self.create_vector_resources(file_path, driver_name=self.driver_name)
+    def get_resources(self, file_path, driver_name=None):  # pylint: disable=W0221
+        if not driver_name:
+            driver_name = self.driver_name
+        return self.create_vector_resources(file_path, driver_name)
 
 
 class RasterPk(TabularPk):
@@ -179,9 +184,11 @@ def __init__(self, **kwargs):
         self.transform = ""
         self.resources = []
 
-    def get_source(self, sub_dataset_name):
+    def get_source(self, file_path):
         """Read raster data source"""
-        src_ds = gdal.Open(sub_dataset_name, GA_ReadOnly)
+        if not self.driver:
+            # use default open
+            src_ds = gdal.Open(file_path, GA_ReadOnly)
         return src_ds
 
     def set_global(self, src_ds):
@@ -238,7 +245,7 @@ def create_raster_resources(self, file_path):
                 resource_pk.append(bands)
         return resource_pk[0]
 
-    def get_resources(self, file_path, skip_lines=None):
+    def get_resources(self, file_path):  # pylint: disable=W0221
         """Get raster resources"""
         return self.create_raster_resources(file_path)
 
@@ -321,7 +328,6 @@ def create_script_dict(pk_type, path, file, skip_lines, encoding):
     try:
         resources = pk_type.get_resources(path, skip_lines, encoding)
     except:
-        raise
         print("Skipped file: " + file)
         return None
     dict_values.setdefault("resources", []).append(resources)
@@ -343,7 +349,7 @@ def process_dirs(pk_type, sub_dirs_path, out_path, skip_lines, encoding):
                                                      os.path.join(path, file_name),
                                                      file_name, skip_lines, encoding)
                 json_pk.update(try_create_dict)
-    write_out_scripts(json_pk, path, out_path)
+        write_out_scripts(json_pk, path, out_path)
 
 
 def process_singles(pk_type, single_files_path, out_path, skip_lines, encoding):