Fix tests and refactor them to use Tox

I also found (and fixed) a few bugs on the way.

.gitignore

@@ -1,3 +1,105 @@
-*.git
-*pyc
-.idea
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+# Editors temporary files
+.#*
+*~
+*.swp
+*.swo

.travis.yml

@@ -1,37 +1,20 @@
-
 language: python
 python:
     - "2.7"
     - "3.3"
     - "3.4"
     - "3.5"
 
-addons:
-    postgresql: "9.4"
-
 services:
     - elasticsearch
 
 env:
-    global:
-        - CELERY_CONFIG='sqla+sqlite:///celerydb.sqlite'
-        - CELERY_BACKEND_CONFIG='db+sqlite:///results.db'
-        - OS_ELASTICSEARCH_ADDRESS=localhost:9200
+    - TOXENV="py${PYTHON_VERSION//./}"
+
 install:
-    - python -m pip install -e .
-    - python -m pip install psycopg2 pytest pytest-cov flask-testing coverage future futures
-    - python -m pip install python-dateutil coveralls unicodecsv
-    - python -m pip install jsontableschema datapackage tabulator
-    - python -m pip install -U git+git://github.com/akariv/jsontableschema-sql-py.git@feature/auto-index
-    - python -m pip install -U git+git://github.com/frictionlessdata/datapackage-py.git
-before_script:
-    - psql -c 'create database os;' -U postgres
-    - python -m pytest --version
-    - sleep 30
+    - pip install tox coveralls
 script:
-    - python -m celery -l INFO -A babbage_fiscal.tasks worker &
-    - python -m coverage run --source babbage_fiscal -m pytest tests/
-    - CELERY_ALWAYS_EAGER=true python -m coverage run --source babbage_fiscal -m pytest tests/
+    - tox
 after_success:
     - coveralls
 notifications:

README.md

@@ -19,6 +19,15 @@ To get started (under development):
 $ pip install babbage_fiscal
 ```
 
+### Testing
+
+To run the tests, make sure you have an ElasticSearch running locally on
+http://localhost:9200 and run:
+
+```
+$ tox
+```
+
 ### Command-line interface
 
 You can use the library using a simple command line interface:

babbage_fiscal/api.py

@@ -11,7 +11,7 @@
 def load():
     package = request.args.get('package')
     callback = request.args.get('callback')
-    logging.info("Requested load of %s with callback %s\n" % (package, callback))
+    logging.info('Requested load of "%s" with callback "%s"\n' % (package, callback))
     if package is not None and callback is not None:
         load_fdp_task.delay(package, callback, get_connection_string())
         return ""

babbage_fiscal/config.py

@@ -2,12 +2,16 @@
 
 from sqlalchemy import create_engine
 
-_connection_string = os.environ.get('FISCAL_PACKAGE_ENGINE')#,u'postgresql://osuser:1234@localhost/os')
+_engine = None
+_connection_string = os.environ['FISCAL_PACKAGE_ENGINE']
 
 
 def _set_connection_string(connection_string):
     global _connection_string
-    _connection_string = connection_string
+    global _engine
+    if _connection_string != connection_string:
+        _connection_string = connection_string
+        _engine = None
 
 
 def get_connection_string():
@@ -16,5 +20,8 @@ def get_connection_string():
 
 def get_engine():
     """Return engine singleton"""
-    return create_engine(_connection_string)
+    global _engine
+    if _engine is None:
+        _engine = create_engine(_connection_string)
 
+    return _engine

babbage_fiscal/loader.py

@@ -64,9 +64,11 @@ def check_hashes(self, resource):
             .get('_data_hash')
         logging.info('Loaded resource data hash is %s', current_data_hash)
 
+        new_data_hash = None
         remote_url = resource.remote_data_path
-        response = requests.head(remote_url)
-        new_data_hash = response.headers.get('etag')
+        if remote_url:
+            response = requests.head(remote_url)
+            new_data_hash = response.headers.get('etag')
         logging.info('Loading resource data hash is %s', new_data_hash)
 
         resource.descriptor['_schema_hash'] = new_schema_hash
@@ -125,7 +127,6 @@ def load_fdp_to_db(self, package, callback=noop):
         """
         Load an FDP to the database, create a babbage model and save it as well
         :param package: URL for the datapackage.json
-        :param engine: DB engine
         :param callback: callback to use to send progress updates
         """
 
@@ -230,7 +231,10 @@ def load_fdp_to_db(self, package, callback=noop):
                     self.status_update(status=STATUS_DELETING_TABLE)
                     storage.delete(faux_table_name)
                 self.status_update(status=STATUS_CREATING_TABLE)
-                storage.create(faux_table_name, storage_schema, indexes_fields=[indexes])
+                indexes_fields = None
+                if indexes:
+                    indexes_fields = [indexes]
+                storage.create(faux_table_name, storage_schema, indexes_fields=indexes_fields)
 
                 self.status_update(status=STATUS_LOADING_DATA_READY)
                 row_processor = RowProcessor(resource.iter(), self.status_update,

babbage_fiscal/tasks.py

@@ -7,7 +7,7 @@
 from .callbacks import STATUS_LOADING_DATA
 from .config import get_engine, _set_connection_string
 from .loader import FDPLoader
-from .callbacks import do_request, STATUS_INITIALIZING, STATUS_FAIL
+from .callbacks import do_request, STATUS_INITIALIZING, STATUS_FAIL, STATUS_DONE
 
 app = Celery('fdp_loader')
 app.config_from_object('babbage_fiscal.celeryconfig')

setup.py

@@ -35,16 +35,19 @@
         'click',
         'datapackage',
         'jsontableschema',
-        'celery',
+        'celery>=3.1.25,<4',  # Version >4 removes support for SQLAlchemy.
+                              # They re-added the support on https://github.com/celery/kombu/pull/687.
+                              # When that's released, we can use versions >4
         'elasticsearch>=1.0.0,<2.0.0',
-        'os-package-registry>=0.0.3'
+        'os-package-registry>=0.0.3',
+        'jsontableschema-sql',
+        'os-api-cache'
+    ],
+    dependency_links=[
+        'git+git://github.com/akariv/jsontableschema-sql-py.git@feature/auto-index#egg=jsontableschema-sql'
     ],
     tests_require=[
-        'nose',
-        'coverage',
-        'wheel',
-        'unicodecsv',
-        'jtskit'
+        'tox',
     ],
     entry_points={
       'console_scripts': [

tests/conftest.py

@@ -0,0 +1,44 @@
+import os
+import pytest
+import flask
+from elasticsearch import Elasticsearch
+from babbage_fiscal import config
+from babbage_fiscal.api import FDPLoaderBlueprint
+
+@pytest.fixture
+def app():
+    app = flask.Flask(__name__)
+    app.register_blueprint(FDPLoaderBlueprint, url_prefix='/loader')
+    app.config['DEBUG'] = True
+    app.config['TESTING'] = True
+    app.config['PRESERVE_CONTEXT_ON_EXCEPTION'] = True
+    return app
+
+
+@pytest.fixture
+def conn():
+    _conn = config.get_engine().connect()
+    yield _conn
+    _conn.close()
+
+
+@pytest.fixture
+def elasticsearch_address():
+    return os.environ.get('OS_ELASTICSEARCH_ADDRESS', 'localhost:9200')
+
+
+@pytest.fixture
+def elasticsearch(elasticsearch_address):
+    es = Elasticsearch(hosts=[elasticsearch_address])
+
+    def _delete_indices():
+        indices = [
+            'packages',
+        ]
+
+        for index in indices:
+            es.indices.delete(index=index, ignore=[400, 404])
+
+    _delete_indices()
+    yield es
+    _delete_indices()