Merge pull request #1400 from henrykironde/new-utf8-p3

Apply python formatters,  use utf-8 and remove built-ins

.flake8

@@ -0,0 +1,13 @@
+[flake8]
+ignore =
+    E402, # module level import not at top of file
+    E501, # line too long (82 > 79 characters)
+    E722, # do not use bare except, specify exception instead
+    F401, # module imported but unused
+    F403, # ‘from module import *’ used; unable to detect undefined names
+    F841, # local variable name is assigned to but never used
+    W503, # line break before binary operator
+    W504  # line break after binary operator
+
+max-complexity = 90
+max-line-length = 90

.pylintrc

@@ -0,0 +1,121 @@
+[MASTER]
+# Use the template to set up the required/specific configurations.
+# Specify a configuration file.
+#rcfile=
+
+# Add files or directories to the blacklist. They should be base names, not
+# paths.
+ignore=CVS
+
+# Pickle collected data for later comparisons.
+# persistent=yes
+
+# Use multiple processes to speed up Pylint.
+jobs=4
+
+# Allow loading of arbitrary C extensions. Extensions are imported into the
+# active Python interpreter and may run arbitrary code.
+unsafe-load-any-extension=no
+
+
+# [MESSAGES CONTROL]
+
+# # Only show warnings with the listed confidence levels. Leave empty to show
+# # all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
+# confidence=
+
+# # Enable the message, report, category or checker with the given id(s). You can
+# # either give multiple identifier separated by comma (,) or put this option
+# # multiple time. See also the "--disable" option for examples.
+# enable=use-symbolic-message-instead,useless-supression,fixme
+
+# # Disable the message, report, category or checker with the given id(s).
+
+disable=
+    C0413,
+    # C0111, # (all C0114-C0116)
+    # C0115, # missing-class-docstring
+    # W0143: # Comparing against a callable, did you omit the parenthesis? (comparison-with-callable)
+    # W0611, # unused-import
+    C0103, # invalid-name
+    C0114, # missing-module-docstring
+    C0330, # Wrong hanging indentation before block (add 4 spaces)
+    C0411, # wrong-import-order
+    C0413, # wrong-import-position
+    E1101, # no-member
+    R0911, # too-many-return-statements
+    R0914, # too-many-locals
+    W0102, # dangerous-default-value)
+    W0201, # attribute-defined-outside-init
+    W0614, # unused-wildcard-import
+    W0622, # Redefining built-in 'type' (redefined-builtin)
+    W0702, # bare-except
+    W0703, # broad-except
+    W0706, # try-except-raise)
+    R0913, # too-many-arguments
+    E1101, 
+    R1702, # too-many-nested-blocks
+    W0212, # protected-access
+    R0903, # too-few-public-methods
+    R0902, # too-many-instance-attributes
+    W0401, # wildcard-import
+    C0415, # import-outside-toplevel
+    R0912, # too-many-branches
+    R0915, # Too many statements (53/50) (too-many-statements)
+    R0904, # Too many public methods (51/20) (too-many-public-methods)
+    format
+
+
+[REPORTS]
+
+# Set the output format. Available formats are text, parseable, colorized, msvs
+# (visual studio) and html. You can also give a reporter class, eg
+# mypackage.mymodule.MyReporterClass.
+output-format=text
+
+# Put messages in a separate file for each module / package specified on the
+# command line instead of printing them on stdout. Reports (if any) will be
+# written in a file name "pylint_global.[txt|html]".
+files-output=no
+
+# Tells whether to display a full report or only the messages
+reports=no
+
+# Python expression which should return a note less than 10 (10 is the highest
+# note). You have access to the variables errors warning, statement which
+# respectively contain the number of errors / warnings messages and the total
+# number of statements analyzed. This is used by the global evaluation report
+# (RP0004).
+evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+
+# Template used to display messages. This is a python new-style format string
+# used to format the message information. See doc for all details
+#msg-template=
+
+
+# [LOGGING]
+
+# # Logging modules to check that the string format arguments are in logging
+# # function parameter format
+# logging-modules=logging
+
+
+# [MISCELLANEOUS]
+
+# # List of note tags to take in consideration, separated by a comma.
+# notes=FIXME,XXX,TODO
+
+[FORMAT]
+
+# Maximum number of characters on a single line.
+max-line-length=90
+
+# # Regexp for a line that is allowed to be longer than the limit.
+# ignore-long-lines=^\s*(# )?<?https?://\S+>?$
+
+
+# [BASIC]
+
+# # Good variable names which should always be accepted, separated by a comma
+# good-names=i,j,k,f,_
+

.style.yapf

@@ -0,0 +1,4 @@
+[style]
+# YAPF uses the chromium style
+based_on_style = google
+COLUMN_LIMIT=90

.travis.yml

@@ -6,6 +6,9 @@ services:
 
 # database setup
 before_install:
+  # Make dir corresponding to the version of Postgres used
+  - mkdir -p cli_tools/postgis-data/11/main
+  - chmod +x cli_tools/postgis-data/11/main
   - docker-compose ps
 
 # command to run tests using py.test
@@ -14,6 +17,9 @@ script:
 
 after_success:
   - codecov
+  - docker-compose  run python_retriever flake8 --ignore=E501,W503,E402,F401,F403,E722,F841,W504 retriever --max-line-length=90
+  - docker-compose  run python_retriever yapf --recursive retriever/engines --style=.style.yapf -d
+  - docker-compose  run python_retriever pylint -rn retriever/  -f colorized --rcfile=.pylintrc
 
 notifications:
   slack:

Dockerfile

@@ -34,6 +34,9 @@ RUN pip install codecov -U
 RUN pip install pytest-cov -U
 RUN pip install pytest-xdist -U
 RUN pip install pytest
+RUN pip install yapf
+RUN pip install pylint
+RUN pip install flake8
 
 # Install Postgis after Python is setup
 RUN apt-get install -y --force-yes postgis

docs/conf.py

@@ -1,12 +1,4 @@
-# -*- coding: latin-1  -*-
-
-from __future__ import absolute_import
-from __future__ import print_function
-
 import sys
-from builtins import str
-from imp import reload
-
 import sphinx_rtd_theme
 
 from retriever.lib.defaults import ENCODING
@@ -18,17 +10,9 @@
 from retriever.lib.tools import open_fw
 from retriever.lib.repository import check_for_updates
 
-# sys removes the setdefaultencoding method at startup; reload to get it back
-reload(sys)
-if hasattr(sys, 'setdefaultencoding'):
-    # set default encoding to latin-1 to decode source text
-    sys.setdefaultencoding('latin-1')
-
 
 def to_str(object, object_encoding=encoding):
-    if sys.version_info >= (3, 0, 0):
-        return str(object).encode('UTF-8').decode(encoding)
-    return object
+    return str(object).encode('UTF-8').decode(encoding)
 
 
 # Create the .rst file for the available datasets

docs/retriever.engines.rst

@@ -1,67 +1,67 @@
-retriever\.engines package
-==========================
+retriever.engines package
+=========================
 
 Submodules
 ----------
 
-retriever\.engines\.csvengine module
-------------------------------------
+retriever.engines.csvengine module
+----------------------------------
 
 .. automodule:: retriever.engines.csvengine
     :members:
     :undoc-members:
     :show-inheritance:
 
-retriever\.engines\.download\_only module
------------------------------------------
+retriever.engines.download\_only module
+---------------------------------------
 
 .. automodule:: retriever.engines.download_only
     :members:
     :undoc-members:
     :show-inheritance:
 
-retriever\.engines\.jsonengine module
--------------------------------------
+retriever.engines.jsonengine module
+-----------------------------------
 
 .. automodule:: retriever.engines.jsonengine
     :members:
     :undoc-members:
     :show-inheritance:
 
-retriever\.engines\.msaccess module
------------------------------------
+retriever.engines.msaccess module
+---------------------------------
 
 .. automodule:: retriever.engines.msaccess
     :members:
     :undoc-members:
     :show-inheritance:
 
-retriever\.engines\.mysql module
---------------------------------
+retriever.engines.mysql module
+------------------------------
 
 .. automodule:: retriever.engines.mysql
     :members:
     :undoc-members:
     :show-inheritance:
 
-retriever\.engines\.postgres module
------------------------------------
+retriever.engines.postgres module
+---------------------------------
 
 .. automodule:: retriever.engines.postgres
     :members:
     :undoc-members:
     :show-inheritance:
 
-retriever\.engines\.sqlite module
----------------------------------
+retriever.engines.sqlite module
+-------------------------------
 
 .. automodule:: retriever.engines.sqlite
     :members:
     :undoc-members:
     :show-inheritance:
 
-retriever\.engines\.xmlengine module
-------------------------------------
+retriever.engines.xmlengine module
+----------------------------------
 
 .. automodule:: retriever.engines.xmlengine
     :members: