Merge pull request #4055 from willkg/1377254-lint-in-ci

fixes bug 1377254 - lint in ci [no rush]

docker/fetch_normalization_data.py

@@ -93,7 +93,7 @@ def insert_adu(conn, adu_count, adu_date, product_name, os_name, version_string)
             INSERT INTO product_adu (adu_count, adu_date, os_name, product_version_id)
             VALUES (%s, %s, %s, %s)
         """, (adu_count, adu_date, os_name, product_version_id))
-    except psycopg2.IntegrityError as exc:
+    except psycopg2.IntegrityError:
         # Hitting an IntegrityError here probably means this data is in the table already, so we
         # just skip it.
         pass

docker/run_tests.sh

@@ -25,17 +25,11 @@ DATABASE_URL=${database_url:-"postgres://postgres:aPassword@postgresql:5432/soco
 ELASTICSEARCH_URL=${elasticsearch_url:-"http://elasticsearch:9200"}
 
 export PYTHONPATH=/app/:$PYTHONPATH
+FLAKE8="$(which flake8)"
 PYTEST="$(which pytest)"
 ALEMBIC="$(which alembic)"
 SETUPDB="$(which python) /app/socorro/external/postgresql/setupdb_app.py"
 
-# FIXME(willkg): Tests fail if /app/config/alembic.ini doesn't exist. But
-# hit permission error when creating it.
-# Create necessary .ini files
-#if [ ! -f /app/config/alembic.ini ]; then
-#    cp /app/config/alembic.ini-dist /app/config/alembic.ini
-#fi
-
 # Verify we have __init__.py files everywhere we need them
 errors=0
 while read d; do
@@ -76,11 +70,14 @@ $SETUPDB --database_name=socorro_test --dropdb --no_schema --logging.stderr_erro
 $ALEMBIC -c "${alembic_config}" downgrade -1
 $ALEMBIC -c "${alembic_config}" upgrade heads
 
+# Run linting
+$FLAKE8
+
 # Run tests
 $PYTEST
 
-# Collect static and then test webapp
+# Collect static and then run py.test in the webapp
 pushd webapp-django
 python manage.py collectstatic --noinput
-./bin/ci.sh --docker
+$PYTEST
 popd

scripts/database/load_and_check_partition_data.py

@@ -4,7 +4,6 @@
 import glob
 import os
 import os.path
-import subprocess
 import sys
 
 if len(sys.argv) != 2:
@@ -48,6 +47,6 @@
 
 if len(errors) > 1:
     print "FAILED: some files returned errors:", str(errors)
-    return 1
+    sys.exit(1)
 
-return 0
+sys.exit(0)

scripts/database/partition_dump.py

@@ -1,19 +1,22 @@
 #!/usr/bin/env python
 
 import datetime
+import errno
 import os.path
-import os, errno
+import os
 import re
 import sys
 
 
 def mkdir_p(path):
     try:
         os.makedirs(path)
-    except OSError as exc: # Python >2.5
+    except OSError as exc:  # Python >2.5
         if exc.errno == errno.EEXIST and os.path.isdir(path):
             pass
-        else: raise
+        else:
+            raise
+
 
 def dsToWeeklyPartition(ds):
     d = datetime.datetime.strptime(ds, '%Y-%m-%d').date()

scripts/hive_adi_test.py

@@ -11,7 +11,9 @@
 
 # Example command-line usage:
 # $ . /etc/socorro/socorrorc
-# $ $PYTHON hive_adi_test.py -d '2015-01-21' -s peach-gw.peach.metrics.scl3.mozilla.com -o /tmp/output_adi.txt
+# $ $PYTHON hive_adi_test.py -d '2015-01-21' -s peach-gw.peach.metrics.scl3.mozilla.com \
+#           -o /tmp/output_adi.txt
+
 
 def main():
 
@@ -97,5 +99,6 @@ def remove_control_characters(s):
             )
             f.write("\n")
 
+
 if __name__ == '__main__':
     main()

scripts/test.sh

@@ -5,6 +5,7 @@ echo "this is test.sh"
 source scripts/defaults
 
 PYTEST="$VIRTUAL_ENV/bin/pytest"
+FLAKE8="$VIRTUAL_ENV/bin/flake8"
 SETUPDB="$VIRTUAL_ENV/bin/python ./socorro/external/postgresql/setupdb_app.py"
 JENKINS_CONF=jenkins.py.dist
 
@@ -99,12 +100,15 @@ PYTHONPATH=$PYTHONPATH $SETUPDB --database_name=socorro_migration_test --dropdb
 PYTHONPATH=$PYTHONPATH ${VIRTUAL_ENV}/bin/alembic -c config/alembic.ini downgrade -1
 PYTHONPATH=$PYTHONPATH ${VIRTUAL_ENV}/bin/alembic -c config/alembic.ini upgrade heads
 
+# run flake8
+$FLAKE8
+
 # run tests
 $ENV $FS_RESOURCES $PG_RESOURCES $RMQ_RESOURCES $ES_RESOURCES PYTHONPATH=$PYTHONPATH $PYTEST
 
 # test webapp
 pushd webapp-django
-./bin/ci.sh
+PYTHONPATH=.. $PYTEST
 popd
 
 # lint puppet manifests; bug 976639