tests: addition of pycodestyle

* Enables to run run-tests.sh inside of the container.

* Adds pycodestyle to the testing procedure and amends
  codebase to comply. (closes #2433)

* Enriches ``run-tests.sh`` with manifest check.

Signed-off-by: Jan Okraska <jan.okraska@cern.ch>

.dockerignore

@@ -1,16 +1,8 @@
-.git
-*.gitignore
-
 *.mo
 *.pyc
 *.pyo
 *.swp
 *.swo
 *.~
 
-.dockerignore
-Dockerfile
-docker-compose.yml
-docker-compose-dev.yml
-
 Procfile*

.travis.yml

@@ -71,9 +71,7 @@ install:
   - "docker-compose build"
 
 script:
-  - "docker-compose run --rm web pydocstyle cernopendata"
-  - "docker-compose run --rm web isort -rc -c -df **/*.py"
-  - "docker-compose run --rm web python setup.py test"
+  - "docker-compose run --rm web ./run-tests.sh"
 
 after_success:
   - coveralls

Dockerfile

@@ -43,6 +43,7 @@ RUN yum update -y && \
         libxml2-devel \
         libxslt-devel \
         npm \
+        jq \
         python-devel \
         python-pip
 
@@ -70,7 +71,7 @@ RUN yum clean -y all
 ENV APP_INSTANCE_PATH=/usr/local/var/cernopendata/var/cernopendata-instance
 
 RUN pip install --upgrade pip==9 setuptools wheel && \
-    npm install -g node-sass@3.8.0 clean-css@3.4.24 requirejs uglify-js
+    npm install -g node-sass@3.8.0 clean-css@3.4.24 requirejs uglify-js jsonlint
 
 ADD requirements-production-local-forks.txt /tmp
 ADD requirements-production.txt /tmp

MANIFEST.in

@@ -11,6 +11,7 @@ include LICENSE
 include babel.ini
 include nginx/Dockerfile
 include pytest.ini
+include sentry/Dockerfile
 recursive-include cernopendata *.css
 recursive-include cernopendata *.csv
 recursive-include cernopendata *.gif
@@ -26,8 +27,10 @@ recursive-include cernopendata *.scss
 recursive-include cernopendata *.svg
 recursive-include cernopendata *.tpl
 recursive-include cernopendata *.md
+recursive-include cernopendata *.xml
 recursive-include nginx *.conf
 recursive-include scripts *.cfg
+recursive-include scripts *.py
 recursive-include scripts *.sh
 recursive-include tests *.py
 recursive-include sentry *.json

cernopendata/modules/fixtures/cli.py

@@ -30,21 +30,19 @@
 import pkg_resources
 from flask import current_app
 from flask.cli import with_appcontext
-from sqlalchemy.orm.attributes import flag_modified
-
 from invenio_db import db
-from invenio_records_files.api import Record
+from invenio_files_rest.models import Bucket, FileInstance, ObjectVersion
 from invenio_indexer.api import RecordIndexer
-from cernopendata.modules.records.minters.recid import \
-    cernopendata_recid_minter
+from invenio_pidstore.errors import PIDDoesNotExistError
+from invenio_pidstore.models import PersistentIdentifier
+from invenio_records_files.api import Record
+from invenio_records_files.models import RecordsBuckets
+from sqlalchemy.orm.attributes import flag_modified
+
 from cernopendata.modules.records.minters.docid import \
     cernopendata_docid_minter
-
-from invenio_files_rest.models import \
-    Bucket, FileInstance, ObjectVersion
-from invenio_records_files.models import RecordsBuckets
-from invenio_pidstore.models import PersistentIdentifier
-from invenio_pidstore.errors import PIDDoesNotExistError
+from cernopendata.modules.records.minters.recid import \
+    cernopendata_recid_minter
 
 
 def get_jsons_from_dir(dir):

cernopendata/modules/pages/views.py

@@ -77,7 +77,7 @@ def index():
     try:
         results = FeaturedArticlesSearch().sort(
             'featured')[:6].execute().hits.hits
-    except:
+    except Exception:
         pass
     return render_template('cernopendata_pages/front/index.html',
                            featured_articles=results)

cernopendata/modules/records/utils.py

@@ -119,7 +119,7 @@ def eos_send_file_or_404(file_path=""):
 
     try:
         return storage.send_file(filename[0])
-    except:
+    except Exception:
         abort(404)
 
 
@@ -138,7 +138,7 @@ def record_file_page(pid, record, page=1, **kwargs):
     items_per_page = request.args.get('perPage', 5)
     try:
         items_per_page = int(items_per_page)
-    except:
+    except Exception:
         items_per_page = 5
 
     if request.args.get('group'):
@@ -245,7 +245,7 @@ def export_json_view(pid, record, template=None, **kwargs):
             serializer = obj_or_import_string(fmt['serializer'])
             data = serializer.serialize(pid, record)
             data = json.loads(data)
-        except:
+        except Exception:
             data = {}
 
         return jsonify(data)

cernopendata/modules/theme/config.py

@@ -31,6 +31,7 @@ def _(x):
     """Identity function for string extraction."""
     return x
 
+
 # Default language and timezone
 BABEL_DEFAULT_LANGUAGE = 'en'
 BABEL_DEFAULT_TIMEZONE = 'Europe/Zurich'

cernopendata/modules/theme/views.py

@@ -58,10 +58,10 @@ def get_record_title(id, type='recid'):
 @blueprint.app_template_filter('get_first_file')
 def get_first_file(file_list):
     """Fetches first file from a list."""
-    l = [f.get('key') for f in file_list
-         if f.get('key', '').endswith('.ig')]
-    if l:
-        return l[0]
+    keys = [f.get('key') for f in file_list
+            if f.get('key', '').endswith('.ig')]
+    if keys:
+        return keys[0]
 
 
 @blueprint.app_template_filter('sort_multi')

run-tests.sh

@@ -71,6 +71,7 @@ if [[ "$@" = *"--check-fixtures-only"* ]]; then
 fi
 
 # check source code style:
+pycodestyle cernopendata
 pydocstyle cernopendata
 isort -rc -c -df **/*.py
 check-manifest --ignore ".travis-*"

setup.cfg

@@ -33,6 +33,13 @@ all_files = 1
 [bdist_wheel]
 universal = 1
 
+[check-manifest]
+ignore =
+    travis-*
+    elasticsearch-*
+    .github
+    .github/*
+
 [compile_catalog]
 directory = cernopendata/translations/
 

setup.py

@@ -44,6 +44,7 @@
     'locustio>=0.8',
     'mock>=1.3.0',
     'pydocstyle>=1.0.0',
+    'pycodestyle>=2.4.0',
     'pytest-cache>=1.0',
     'pytest-cov>=1.8.0',
     'pytest-pep8>=1.0.6',