Postgres refactor

.gitignore

@@ -1,5 +1,7 @@
+*.pyc
 .coverage
 .tox
 *.egg-info
 build/
 dist/
+.cache

.travis.yml

@@ -1,33 +1,9 @@
-sudo: required
-dist: trusty
-services:
-    - docker
 notifications:
   email: false
 language: python
 env:
-  global:
-    - SOLR_PORT=8983
-    - MONGO_PORT=27018
-    - SOLR_URI=localhost:8983
-    - MONGO_URI=localhost:27018
-  matrix:
-    - TOX_ENV=py27-travis
-    - TOX_ENV=py33-travis
-    - TOX_ENV=py34-travis
-    - TOX_ENV=coveralls
-install:
-  - pip install tox
-before_script:
-    - sleep 15
-before_install:
-    - sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10
-    - echo "deb http://repo.mongodb.org/apt/ubuntu precise/mongodb-org/3.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.0.list
-    - sudo apt-get update
-    - sudo apt-get install -y mongodb-org
-    - docker pull mitlibraries/oastats-solr
-    - docker run -d -p 8983:8983 mitlibraries/oastats-solr
-    - docker pull mongo
-    - docker run -d -p 27018:27017 mongo
-script:
-  - tox -e $TOX_ENV
+  - TOX_ENV=py27
+  - TOX_ENV=py35
+  - TOX_ENV=coveralls
+install: pip install tox
+script: tox -e $TOX_ENV

README.md

@@ -1,6 +1,8 @@
-oastats-backend
+OAStats Backend
 ===============
 
-The Back End and Data Processing Pipeline for the MIT Libraries Open Access Statistics Project.
+This is the back end processing pipeline for the OA statistics project (https://oastats.mit.edu).
 
-This product includes GeoLite2 data created by MaxMind, available from http://www.maxmind.com. The GeoLite2 database included here is only used for unit tests and is not kept up to date. You should manage the GeoLite2 database separately for production.
+See https://mitlibraries.github.io/oastats-backend/ for documentation.
+
+This product includes GeoLite2 data created by MaxMind, available from http://www.maxmind.com.

docs/.buildinfo

@@ -0,0 +1,4 @@
+# Sphinx build info version 1
+# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
+config: cab2898ef00bb67f44c19b4f114cfc50
+tags: 645f666f9bcd5a90fca523b33c5a78b7

docs/_sources/index.txt

@@ -0,0 +1,74 @@
+Welcome to OAStats Backend's documentation!
+===========================================
+
+This is a command line application that will process the Apache logs for Dspace and generate download statistics for the OA collection.
+
+The current version is an intermediate step in moving off Mongo to PostGres and some of the functionality will be removed once the migration is complete. As such, a separate step is still required to generate a summary collection in Mongo, but this is now done using the data from the PostGres databse.
+
+
+Installation
+------------
+
+Use ``pip`` to install into a virtualenv::
+
+    (oastats)$ pip install \
+        https://github.com/MITLibraries/oastats-backend/zipball/master
+
+This will make an ``oastats`` command available when your virtualenv is active.
+
+
+Usage
+-----
+
+The ``oastats`` command has four subcommands: ``db``, ``load``, ``pipeline`` and ``summary``. The full documentation for each command can be accessed with::
+
+    (oastats)$ oastats <subcommand> --help
+
+Each subcommand will need to connect to the PostGres database. This can be done by providing a valid `SQLAlchemy Database URI <http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls>`_ to the ``--oastats-database`` option. You can also pass this as an environment variable instead of as a command line option using the ``OASTATS_DATABASE`` variable.
+
+
+Creating the Database
+~~~~~~~~~~~~~~~~~~~~~
+
+.. autofunction:: pipeline.cli.db()
+
+Full command documentation::
+
+    (oastats)$ oastats db --help
+
+
+Migrating the Mongo Data
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. important::
+    This subcommand will be removed once the data has been migrated.
+
+.. autofunction:: pipeline.cli.load()
+
+Full command documentation::
+
+    (oastats)$ oastats load --help
+
+
+Running the Pipeline
+~~~~~~~~~~~~~~~~~~~~
+
+.. autofunction:: pipeline.cli.pipeline()
+
+Full command documentation::
+
+    (oastats)$ oastats pipeline --help
+
+
+Generating the Summary Collection
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. important::
+    This subcommand will be removed once Mongo is no longer needed for the main OAStats website.
+
+.. autofunction:: pipeline.cli.summary()
+
+Full command documentation::
+
+    (oastats)$ oastats summary --help
+