Implement Pacifica Dispatcher (#7)

* initial commit * update README.md * add travis config and pre-commit stuff Signed-off-by: David Brown <dmlb2000@gmail.com> * some fixes for flake8 Signed-off-by: David Brown <dmlb2000@gmail.com> * Fix pylint issues Signed-off-by: David Brown <dmlb2000@gmail.com> * try doing celery always egar see if that helps Signed-off-by: David Brown <dmlb2000@gmail.com> * didn't get the right config option sometimes this helps with debugging travis issues, forces the tasks to run in the same thread even though the code calls delay() Signed-off-by: David Brown <dmlb2000@gmail.com> * don't need the sqlite db anymore Signed-off-by: David Brown <dmlb2000@gmail.com> * fix regressions To test manually, open two terminals: * In the first terminal, `cd tests` and then run `env DATABASE_URL=sqlite:///../test.sqlite3 ../venv/bin/celery -A receiver_test worker -l info` to start Celery. * In the second terminal, `env DATABASE_URL=sqlite:///test.sqlite3 venv/bin/coverage run --source=pacifica/dispatcher -m pytest -v` to test. * try undoing the celery egar things Signed-off-by: David Brown <dmlb2000@gmail.com> * create tables before calling test methods * remove comment line * add context manager to refresh database connection Signed-off-by: David Brown <dmlb2000@gmail.com> * fix notify server.conf cherrypy config Signed-off-by: David Brown <dmlb2000@gmail.com> * use cherrypy.tools.json_{in|out}() and better context manager Signed-off-by: David Brown <dmlb2000@gmail.com> * try some different json_in/out Signed-off-by: David Brown <dmlb2000@gmail.com> * add some remote downloader testing Signed-off-by: David Brown <dmlb2000@gmail.com> * fixup yield files for remote downloader Signed-off-by: David Brown <dmlb2000@gmail.com> * add bad event testing Signed-off-by: David Brown <dmlb2000@gmail.com> * add some more exception handling Signed-off-by: David Brown <dmlb2000@gmail.com> * add remote uploader testing Signed-off-by: David Brown <dmlb2000@gmail.com> * add travis deploy stage Signed-off-by: David Brown <dmlb2000@gmail.com> * pre-commit fixes Signed-off-by: David Brown <dmlb2000@gmail.com> * change proposal to project Signed-off-by: David Brown <dmlb2000@gmail.com> * fix ingest path to not conflict with other services Signed-off-by: David Brown <dmlb2000@gmail.com> * Fix appveyor config services Signed-off-by: David Brown <dmlb2000@gmail.com> * try to increase converage Signed-off-by: David Brown <dmlb2000@gmail.com> * debug appveyor Signed-off-by: David Brown <dmlb2000@gmail.com> * add some more coverage testing Signed-off-by: David Brown <dmlb2000@gmail.com>
pacifica · Mar 20, 2019 · cbed16f · cbed16f
1 parent 70f899c
commit cbed16f
Show file tree

Hide file tree

Showing 68 changed files with 2,274 additions and 726 deletions.
diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
@@ -1,8 +1,8 @@
 ### Pacifica Dispatcher version
-[Version of the pacifica software where you are encountering the issue]
+[Version of the Pacifica Dispatcher where you are encountering the issue]
 
 ### Pacifica Core Software versions
-[Versions of the pacifica core software components related to this issue]
+[Versions of the Pacifica Core Software components related to this issue]
 
 ### Platform Details
 [Operating system distribution and release version. Cloud provider if running in the cloud]

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,16 +1,18 @@
 repos:
+-   repo: https://github.com/pre-commit/mirrors-autopep8
+    rev: v1.4.3
+    hooks:
+            -   id: autopep8
 -   repo: git://github.com/pre-commit/pre-commit-hooks
-    rev: v1.2.3
+    rev: v2.1.0
     hooks:
-            -   id: autopep8-wrapper
             -   id: fix-encoding-pragma
             -   id: trailing-whitespace
             -   id: flake8
                 args: [--max-line-length=120]
             -   id: check-merge-conflict
             -   id: double-quote-string-fixer
             -   id: end-of-file-fixer
-            -   id: name-tests-test
             -   id: debug-statements
             -   id: check-added-large-files
             -   id: check-ast
@@ -33,7 +35,6 @@ repos:
             -   id: check-yaml
             -   id: detect-private-key
             -   id: requirements-txt-fixer
-            -   id: trailing-whitespace
 -   repo: local
     hooks:
             -   id: pep257
@@ -47,7 +48,7 @@ repos:
                 language: system
                 types: [python]
 -   repo: git://github.com/Lucas-C/pre-commit-hooks
-    rev: v1.1.5
+    rev: v1.1.6
     hooks:
     -   id: remove-tabs
     -   id: remove-crlf
diff --git a/.travis.yml b/.travis.yml
@@ -3,29 +3,104 @@ dist: xenial
 python:
 - 3.6
 services:
+- postgresql
 - redis-server
 stages:
 - lint
 - test
-
-env:
-  BROKER_URL: redis://localhost:6379/0
+- deploy
 
 jobs:
   include:
   - stage: lint
     script: pre-commit run -a
   - stage: test
+    before_script:
+    - psql -c 'create database pacifica_metadata;' -U postgres
+    - psql -c 'create database pacifica_uniqueid;' -U postgres
+    - psql -c 'create database pacifica_cartd;' -U postgres
+    - psql -c 'create database pacifica_ingest;' -U postgres
+    - psql -c 'create database pacifica_notify;' -U postgres
+    - psql -c 'create database pacifica_dispatcher;' -U postgres
+    - export VOLUME_PATH="/tmp/ingest"
+    - export METADATA_CPCONFIG="$PWD/travis/metadata/server.conf"
+    - export METADATA_CONFIG="$PWD/travis/metadata/config.ini"
+    - export POLICY_CPCONFIG="$PWD/travis/policy/server.conf"
+    - export UNIQUEID_CPCONFIG="$PWD/travis/uniqueid/server.conf"
+    - export UNIQUEID_CONFIG="$PWD/travis/uniqueid/config.ini"
+    - export INGEST_CPCONFIG="$PWD/travis/ingest/server.conf"
+    - export INGEST_CONFIG="$PWD/travis/ingest/config.ini"
+    - export CARTD_CPCONFIG="$PWD/travis/cartd/server.conf"
+    - export CARTD_CONFIG="$PWD/travis/cartd/config.ini"
+    - export ARCHIVEINTERFACE_CPCONFIG="$PWD/travis/archivei/server.conf"
+    - export ARCHIVEINTERFACE_CONFIG="$PWD/travis/archivei/config.ini"
+    - export NOTIFICATIONS_CPCONFIG="$PWD/travis/notify/server.conf"
+    - export NOTIFICATIONS_CONFIG="$PWD/travis/notify/config.ini"
+    - pacifica-metadata-cmd dbsync
+    - pacifica-cartd-cmd dbsync
+    - pacifica-uniqueid-cmd dbsync
+    - pacifica-ingest-cmd dbsync
+    - pacifica-notifications-cmd dbsync
+    - pacifica-metadata & echo $! > metadata.pid
+    - pacifica-archiveinterface & echo $! > archivei.pid
+    - pacifica-uniqueid & echo $! > uniqueid.pid
+    - pacifica-cartd & echo $! > cartd.pid
+    - celery -A pacifica.cartd.tasks worker --loglevel=info & echo $! > cartd-celery.pid
+    - pacifica-ingest & echo $! > ingest.pid
+    - celery -A pacifica.ingest.tasks worker --loglevel=info & echo $! > ingest-celery.pid
+    - pacifica-notifications & echo $! > notify.pid
+    - celery -A pacifica.notifications.tasks worker --loglevel=info & echo $! > notify-celery.pid
+    - |
+      MAX_TRIES=60
+      HTTP_CODE=$(curl -sL -w "%{http_code}\\n" localhost:8121/keys -o /dev/null || true)
+      while [[ $HTTP_CODE != 200 && $MAX_TRIES > 0 ]] ; do
+        sleep 1
+        HTTP_CODE=$(curl -sL -w "%{http_code}\\n" localhost:8121/keys -o /dev/null || true)
+        MAX_TRIES=$(( MAX_TRIES - 1 ))
+      done
+    - |
+      TOP_DIR=$PWD
+      MD_TEMP=$(mktemp -d)
+      VERSION=$(pip show pacifica-metadata | grep Version: | awk '{ print $2 }')
+      git clone https://github.com/pacifica/pacifica-metadata.git ${MD_TEMP}
+      pushd ${MD_TEMP}
+      git checkout v${VERSION}
+      python tests/test_files/loadit_test.py
+      popd
+    - pacifica-policy & echo $! > policy.pid
+    - |
+      curl -X PUT -H 'Last-Modified: Sun, 06 Nov 1994 08:49:37 GMT' --upload-file README.md http://127.0.0.1:8080/103
+      curl -X PUT -H 'Last-Modified: Sun, 06 Nov 1994 08:49:37 GMT' --upload-file README.md http://127.0.0.1:8080/104
+      readme_size=$(stat -c '%s' README.md)
+      readme_sha1=$(sha1sum README.md | awk '{ print $1 }')
+      echo '{ "hashsum": "'$readme_sha1'", "hashtype": "sha1", "size": '$readme_size'}' > /tmp/file-104-update.json
+      curl -X POST -H 'content-type: application/json' -T /tmp/file-104-update.json 'http://localhost:8121/files?_id=103'
+      curl -X POST -H 'content-type: application/json' -T /tmp/file-104-update.json 'http://localhost:8121/files?_id=104'
     script:
     - pip install .
-    - export DISPATCHER_CPCONFIG=$PWD/server.conf
+    - export DATABASE_URL=postgres://postgres:@127.0.0.1:5432/pacifica_dispatcher
+    - export BROKER_URL=redis://127.0.0.1:6379/0
     - cd tests
-    - coverage run --include='*/site-packages/pacifica/dispatcher/*' -p -m celery -A pacifica.dispatcher.tasks worker -c 1 -P solo -l info &
+    - coverage run --include='*/site-packages/pacifica/dispatcher/*' -p -m celery -A receiver_test worker -c 1 -P solo -l info &
     - coverage run --include='*/site-packages/pacifica/dispatcher/*' -m pytest -xv
-    - celery -A pacifica.dispatcher.tasks control shutdown || true
+    - celery -A receiver_test control shutdown || true
     - coverage combine -a .coverage*
     - coverage report -m --fail-under 100
-
+  - stage: deploy
+    services: []
+    language: python
+    before_install: skip
+    script: skip
+    python: 3.6
+    deploy:
+      skip_cleanup: true
+      provider: pypi
+      user: dmlb2000
+      distributions: sdist bdist_wheel
+      password:
+        secure: ig70TaXbJ+4N3X9KYxTb0VbUckqPXhHjgOWZtMLfhAw7GdE4BGqBk01YH23GTXwWOe3FDItLUjkvYMHEs7D2xeYlb6NXT07ews4nWBq3XxZ2NSuDcQCHyQ0KAeY2MeKm7hMv/pGeHYWtnozgNckQ2Aqaty8IE72Go93kUGHY7UaiGt8yy4hi5X0b1NJFEybBf0BSA8yvPxyTSCIhhkWKAwheABDa4MmJootuVpeNVzSwfWnguHsnv6U2ZEAMJcOIQAeEIUMiOxIIkFS0Hv1vdeKuaSJdmrLXLXV/tezaEC7o+Ajb1nsBqaZUiCf0ZfUS1MeEiyzA40JNHkszKsN0GnxpzYVuWql4u7YaWkyOcU9Pgd/os0aCBw0e18W4p8qtuHqiGKtpTUxEgeWFG9WfbJOchfcDVo7VI/GZ2viNmIPUnbVi4cp2Rd4btNdrhyG80eLnntME0QoVYRDOL+xvkkp6ShQ7yT/LS6S9pHS/zGtnGnj/mfuzsJNmkwWIdOmUFFILfqBzVDwHlkrrRjESq9IohkU84rPxSodkVB9K4Ky3kX1mErQM+rOtl7Cg+FLaf/h/r+QLCczonJSrvpmsiCb4kp+LG/1mlZztK3nEXcAlTTRgiKYSgfROaRQdRBBwl9pRDmS9fIGgiq9kX8KZKnyOnKDXeyk85bBvseT67+s=
+      on:
+        tags: true
 install:
 - pip install -r requirements-dev.txt
 - pip install 'celery[redis]'
diff --git a/Dockerfile.celery b/Dockerfile.celery
diff --git a/Dockerfile.uwsgi b/Dockerfile.uwsgi
diff --git a/README.md b/README.md
@@ -10,7 +10,7 @@
 [![Frontend Automated build](https://img.shields.io/docker/automated/pacifica/dispatcher-frontend.svg?maxAge=2592000)](https://cloud.docker.com/swarm/pacifica/repository/docker/pacifica/dispatcher-frontend/builds)
 [![Backend Automated build](https://img.shields.io/docker/automated/pacifica/dispatcher-backend.svg?maxAge=2592000)](https://cloud.docker.com/swarm/pacifica/repository/docker/pacifica/dispatcher-backend/builds)
 
-Pacifica dispatcher runs applications based on incoming cloud events.
+Pacifica Dispatcher receives and handles [CloudEvents](https://cloudevents.io/) notifications.
 
 ## Documentation
 
@@ -19,5 +19,3 @@ refer to the [Read the Docs](https://pacifica-dispatcher.readthedocs.io)
 documentation.
 
 * [Installation](docs/installation.md) documentation.
-* [Configuration](docs/configuration.md) documentation.
-* [Examples](docs/exampleusage.md) documentation.
diff --git a/appveyor.yml b/appveyor.yml
@@ -3,10 +3,15 @@ pull_requests:
   do_not_increment_build_number: true
 
 environment:
-  BROKER_URL: redis://localhost:6379/0
+  PGUSER: postgres
+  PGPASSWORD: Password12!
+  PGSQL_PATH: C:\Program Files\PostgreSQL\9.6
   matrix:
     - PYTHON: C:\Python36-x64
 
+services:
+  - postgresql
+
 install:
   - ps: >
       & "$env:PYTHON\python.exe" -m virtualenv C:\pacifica;
@@ -16,22 +21,83 @@ install:
       python -m pip install 'celery[eventlet]' 'celery[redis]' redis eventlet;
 
 before_test:
+  - ps: >
+      $env:PATH = "${env:PGSQL_PATH}\bin;${env:PATH}";
+      createdb pacifica_metadata;
+      createdb pacifica_uniqueid;
+      createdb pacifica_cartd;
+      createdb pacifica_ingest;
+      createdb pacifica_notify;
+      createdb pacifica_dispatcher;
   - ps: >
       nuget install redis-64 -excludeversion;
       redis-64\tools\redis-server.exe --service-install;
       redis-64\tools\redis-server.exe --service-start;
+  - ps: >
+      $env:METADATA_CPCONFIG = "$PWD/travis/metadata/server.conf";
+      $env:METADATA_CONFIG = "$PWD/travis/metadata/ap-config.ini";
+      $env:UNIQUEID_CPCONFIG = "$PWD/travis/uniqueid/server.conf";
+      $env:UNIQUEID_CONFIG = "$PWD/travis/uniqueid/ap-config.ini";
+      $env:INGEST_CPCONFIG = "$PWD/travis/ingest/server.conf";
+      $env:INGEST_CONFIG = "$PWD/travis/ingest/ap-config.ini";
+      $env:CARTD_CPCONFIG = "$PWD/travis/cartd/server.conf";
+      $env:CARTD_CONFIG = "$PWD/travis/cartd/ap-config.ini";
+      $env:ARCHIVEINTERFACE_CPCONFIG = "$PWD/travis/archivei/server.conf";
+      $env:ARCHIVEINTERFACE_CONFIG = "$PWD/travis/archivei/config.ini";
+      $env:NOTIFICATIONS_CPCONFIG = "$PWD/travis/notify/server.conf";
+      $env:NOTIFICATIONS_CONFIG = "$PWD/travis/notify/ap-config.ini";
+      $env:VOLUME_PATH = "/tmp/ingest";
+      C:\pacifica\Scripts\activate.ps1;
+      pacifica-metadata-cmd dbsync;
+      pacifica-cartd-cmd dbsync;
+      pacifica-uniqueid-cmd dbsync;
+      pacifica-ingest-cmd dbsync;
+      pacifica-notifications-cmd dbsync;
+      Start-Process C:\pacifica\Scripts\pacifica-metadata.exe;
+      Start-Process C:\pacifica\Scripts\pacifica-archiveinterface.exe;
+      Start-Process C:\pacifica\Scripts\pacifica-uniqueid.exe;
+      Start-Process C:\pacifica\Scripts\pacifica-cartd.exe;
+      Start-Process C:\pacifica\Scripts\celery.exe -ArgumentList "-A pacifica.cartd.tasks worker --loglevel=info -P solo -c 1" -RedirectStandardOutput ccelery-output.log -RedirectStandardError ccelery-error.log;
+      Start-Process C:\pacifica\Scripts\pacifica-ingest.exe;
+      Start-Process C:\pacifica\Scripts\celery.exe -ArgumentList "-A pacifica.ingest.tasks worker --loglevel=info -P eventlet" -RedirectStandardOutput icelery-output.log -RedirectStandardError icelery-error.log;
+      Start-Process C:\pacifica\Scripts\pacifica-notifications.exe;
+      Start-Process C:\pacifica\Scripts\celery.exe -ArgumentList "-A pacifica.notifications.tasks worker --loglevel=info -P solo -c 1" -RedirectStandardOutput ncelery-output.log -RedirectStandardError ncelery-error.log;
+  - ps: >
+      $MD_VERSION = `pip show pacifica-metadata | grep Version: | awk '{ print $2 }';
+      Invoke-WebRequest https://github.com/pacifica/pacifica-metadata/archive/v${MD_VERSION}.zip -OutFile pacifica-metadata.zip;
+      Expand-Archive pacifica-metadata.zip -DestinationPath C:\pacifica-metadata;
+      sleep 10; Invoke-WebRequest http://localhost:8121/users;
+      pushd C:\pacifica-metadata\pacifica-metadata-${MD_VERSION};
+      python tests\test_files\loadit_test.py;
+      popd;
+  - ps: >
+      $env:POLICY_CPCONFIG = "$PWD/travis/policy/server.conf";
+      Start-Process C:\pacifica\Scripts\pacifica-policy.exe;
+      Invoke-WebRequest -InFile README.md -Method PUT -Headers @{'Last-Modified'='Sun, 06 Nov 1994 08:49:37 GMT'} http://127.0.0.1:8080/103;
+      Invoke-WebRequest -InFile README.md -Method PUT -Headers @{'Last-Modified'='Sun, 06 Nov 1994 08:49:37 GMT'} http://127.0.0.1:8080/104;
+      $size = (Get-Item README.md).length;
+      $hash = (Get-FileHash -Algorithm sha1 readme.md).hash;
+      '{ "hashsum": "'+$hash.ToLower()+'", "hashtype": "sha1", "size": '+$size+'}' | Invoke-WebRequest -Method POST -Headers @{ "content-type" = "application/json" } http://127.0.0.1:8121/files?_id=103;
+      '{ "hashsum": "'+$hash.ToLower()+'", "hashtype": "sha1", "size": '+$size+'}' | Invoke-WebRequest -Method POST -Headers @{ "content-type" = "application/json" } http://127.0.0.1:8121/files?_id=104;
+      Get-Content icelery-output.log; Get-Content icelery-error.log;
+      Invoke-WebRequest http://127.0.0.1:8181/status/users/search/dmlb2001/simple -TimeoutSec 1800;
+      Invoke-WebRequest 'http://127.0.0.1:8066/get_state?job_id=1234';
+      Invoke-WebRequest 'http://127.0.0.1:8051/getid?range=42&mode=test_mode';
+      Invoke-WebRequest http://127.0.0.1:8080/1234;
+      echo 'Done';
 
 build: off
 
 test_script:
   - ps: >
       mkdir C:\tmp; C:\pacifica\Scripts\activate.ps1;
       pre-commit run -a;
-      $env:DISPATCHER_CPCONFIG = "$PWD/server.conf";
       pip install .; cd tests;
-      $celery_proc = Start-Process C:\pacifica\Scripts\coverage.exe -ArgumentList "run --include=*/site-packages/pacifica/dispatcher/* -p -m celery -A pacifica.dispatcher.tasks worker -l info -c 1 -P solo" -RedirectStandardError celery-error.log -RedirectStandardOutput celery-output.log;
-      coverage run --include='*/site-packages/pacifica/dispatcher/*' -m pytest -v;
-      python -m celery -A pacifica.dispatcher.tasks control shutdown;
+      $env:DATABASE_URL = "postgres://postgres:Password12!@localhost/pacifica_dispatcher";
+      $env:BROKER_URL = "redis://127.0.0.1:6379/0";
+      $celery_proc = Start-Process C:\pacifica\Scripts\coverage.exe -ArgumentList "run --include=*/site-packages/pacifica/dispatcher/* -p -m celery -A receiver_test worker -l info -c 1 -P solo" -RedirectStandardError celery-error.log -RedirectStandardOutput celery-output.log;
+      coverage run --include='*/site-packages/pacifica/dispatcher/*' -m pytest -xv;
+      python -m celery -A receiver_test control shutdown;
       $celery_proc | Wait-Process;
       ls .coverage* | %{ python -m coverage combine -a $_.name };
       coverage report -m --fail-under=100;
diff --git a/docker-compose.yml b/docker-compose.yml
diff --git a/docs/conf.py b/docs/conf.py
@@ -16,17 +16,13 @@
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
-from os import environ
-from os.path import abspath, join
 from recommonmark.parser import CommonMarkParser
 
-environ['DISPATCHER_CPCONFIG'] = join(abspath('..'), 'server.conf')
-
 # -- Project information -----------------------------------------------------
 
-project = 'Pacifica Dispatcher Service'
-copyright = '2019, David Brown'
-author = 'David Brown'
+project = 'Pacifica Dispatcher'
+copyright = '2019, Battelle Memorial Institute'
+author = 'Mark Borkum'
 
 # The short X.Y version
 version = ''
@@ -138,7 +134,7 @@
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
     (master_doc, 'PacificaDispatcher.tex', 'Pacifica Dispatcher Documentation',
-     'David Brown', 'manual'),
+     'Mark Borkum', 'manual'),
 ]
 
 
@@ -159,7 +155,7 @@
 #  dir menu entry, description, category)
 texinfo_documents = [
     (master_doc, 'PacificaDispatcher', 'Pacifica Dispatcher Documentation',
-     author, 'PacificaDispatcher', 'One line description of project.',
+     author, 'PacificaDispatcher', 'Pacifica Dispatcher receives and handles CloudEvents notifications.',
      'Miscellaneous'),
 ]