Move the tests to a test module (#41)

This moves the uploader tests to a tests module at the root and
then installs the package and runs the tests afterwards

Signed-off-by: David Brown <dmlb2000@gmail.com>

.pre-commit-config.yaml

@@ -1,8 +1,11 @@
 repos:
+-   repo: https://github.com/pre-commit/mirrors-autopep8
+    rev: v1.4.2
+    hooks:
+            -   id: autopep8
 -   repo: git://github.com/pre-commit/pre-commit-hooks
-    rev: v1.2.3
+    rev: v2.0.0
     hooks:
-            -   id: autopep8-wrapper
             -   id: fix-encoding-pragma
             -   id: trailing-whitespace
             -   id: flake8
@@ -47,7 +50,7 @@ repos:
                 language: system
                 types: [python]
 -   repo: git://github.com/Lucas-C/pre-commit-hooks
-    rev: v1.1.5
+    rev: v1.1.6
     hooks:
     -   id: remove-tabs
     -   id: remove-crlf

.travis.yml

@@ -1,6 +1,5 @@
 services:
 - postgresql
-- mysql
 - elasticsearch
 - rabbitmq
 addons:
@@ -18,44 +17,42 @@ env:
   POSTGRES_ENV_POSTGRES_USER: postgres
   POSTGRES_ENV_POSTGRES_PASSWORD:
 cache: pip
-before_install:
+".before_install": &3
 - pip install --upgrade pip setuptools wheel
 - bash -xe travis/before-install.sh
-install:
+".install": &1
 - pip install -r requirements-dev.txt
-".test_job":
-  language: python
-  stage: test
-  script: &2
-  - coverage run --include='pacifica/*' -m pytest -v
-  - coverage report --show-missing --fail-under 100
-  - if [[ $CODECLIMATE_REPO_TOKEN ]] ; then codeclimate-test-reporter ; fi
-  - pip install .
-  - python setup.py bdist_wheel
-  - python setup.py sdist
-".lint_job":
-  language: python
-  stage: lint
-  script: &1
-  - pre-commit run -a
-  - radon cc pacifica
+".script": &4
+- pip install .
+- cd tests
+- coverage run --include='*/site-packages/pacifica/uploader/*' -m pytest -v
+- coverage report --show-missing --fail-under 100
+".lint_script": &2
+- pre-commit run -a
+- radon cc pacifica
 jobs:
   include:
   - language: python
     stage: lint
-    script: *1
+    install: *1
+    script: *2
     python: 3.6
   - language: python
     stage: lint
-    script: *1
+    install: *1
+    script: *2
     python: 2.7
   - language: python
     stage: test
-    script: *2
+    install: *1
+    before_install: *3
+    script: *4
     python: 3.6
   - language: python
     stage: test
-    script: *2
+    install: *1
+    before_install: *3
+    script: *4
     python: 2.7
   - stage: deploy
     services: []
@@ -69,7 +66,7 @@ jobs:
       user: dmlb2000
       distributions: sdist bdist_wheel
       password:
-        secure: ih2ArW9JXkMHeyWydRU12Fw1YYJGt5axgrlgne5C0QX3/qPu46Mr8X7VVOMt6j6O+EHVP+Ph4PUV2qh3juNidxcIyOr+kSVXDWw0MNRQbyenjO02B7SpisdyfmtuRG/PZWhUWQII1ULYPihsDvxc6JOVqzTAYlj2r4Ley9ZTj929lC/JA4dNq4pVhUEXCrtWx/TuX+FWmMwfUqaF6zw3O3/jGrE3Ml0jtR3Jmb/V9t0zAjLD0fGksaaw75qjyNjTLID7EKCLSfAEX+NHwNjypuxpb6shOl+w005MaZzIfk7Wcw35mgU96SON8bKFts648nwyYeZByUkQ76dqkDB6KjTuTNaeHFsuq93oXGmZCcM0QVJVJSz97SgOCJiI3nD1ushQ9TuhU3tdQW1ty5HO7l1lrzfdPQEiVu3PjTURY5ptNOtuemtfrHQiv64Be5iBDnv/BduUpSrWL84MGhygWLyau052wt4ke54gKwH6xbVYcKx15REXJmdYmKQe4cSfxhrHK2/rgw0zTyGf8qrieg3oL2sgb/NpQQIfDUtLvW0n73qiCF/QZ+d5bn7fpB7MY58CGgfTnL33qnfaJnf6nbkg6cc96hGF6qc96ww+F5WqEOrKTfykuiO7N3fYyzmL0zrBzWifUIXDOwzamW6NXLie5WPag+aUdqVlsTsEk88=
+        secure: iXzIvgJaMdmFT8IJPWNs0XkUi1l2My6kRXmxnT129hrQAiCIHCfJeB898IfFFQM86kwe4p3dA/1t2+IMa4bNqpUZb6Z615MDgWxcpHzn0JMiHMjYHlj9eae/+8IBjSbjoTZPIXRJGZ2NMy5rY8/wkcsXN87J0flhwdUMX8uO5aNDLad8nVYZsU1NeWHKchx07WriFeJaRYo0aCPXRK+kzc2u8eSrOumzKSTaIPnZlknrMJ25BRA8pEhJVlRPMg0Jp+L9ugdwGNtC7o61W5e50XPYC485P0+GwWqgUz2F6NLTzAY0JYNdLgz4XdqdHrpc/lK+TKN2FE9yJq24IFxyyzO7xDPAmhAUiFYnN7URIJyaNcNYGguc668P6Ogs66MVMCLdCQsUMMVb0ok9QqVycYMoZ0h09b2A8tviOnaUVWZbE8Ur0FL3YVADq51L0FWq/Xe1+EnWOlsySyLymh/uNMbL6sHa2dJCkzzwkyFxPTyELWaUC01VQRjyjFJknkX5S0Z+SNiYSQnuFPb4HQKj7hKK4KQ31Yk3hJ2+NDWjvPzviduFHVrxHGRGzd/X1uIPyfubc85YdJn1e7XxjmhMOfxXmzHanlGGKczwatKZe50UqWXQTat2bKnKGeI24jt+ubfznfDqwnF71Lr60+UdD9Xqrqz/KGr9ldE8QbPRcHM=
       on:
         tags: true
   - stage: deploy
@@ -84,6 +81,6 @@ jobs:
       user: dmlb2000
       distributions: bdist_wheel
       password:
-        secure: ih2ArW9JXkMHeyWydRU12Fw1YYJGt5axgrlgne5C0QX3/qPu46Mr8X7VVOMt6j6O+EHVP+Ph4PUV2qh3juNidxcIyOr+kSVXDWw0MNRQbyenjO02B7SpisdyfmtuRG/PZWhUWQII1ULYPihsDvxc6JOVqzTAYlj2r4Ley9ZTj929lC/JA4dNq4pVhUEXCrtWx/TuX+FWmMwfUqaF6zw3O3/jGrE3Ml0jtR3Jmb/V9t0zAjLD0fGksaaw75qjyNjTLID7EKCLSfAEX+NHwNjypuxpb6shOl+w005MaZzIfk7Wcw35mgU96SON8bKFts648nwyYeZByUkQ76dqkDB6KjTuTNaeHFsuq93oXGmZCcM0QVJVJSz97SgOCJiI3nD1ushQ9TuhU3tdQW1ty5HO7l1lrzfdPQEiVu3PjTURY5ptNOtuemtfrHQiv64Be5iBDnv/BduUpSrWL84MGhygWLyau052wt4ke54gKwH6xbVYcKx15REXJmdYmKQe4cSfxhrHK2/rgw0zTyGf8qrieg3oL2sgb/NpQQIfDUtLvW0n73qiCF/QZ+d5bn7fpB7MY58CGgfTnL33qnfaJnf6nbkg6cc96hGF6qc96ww+F5WqEOrKTfykuiO7N3fYyzmL0zrBzWifUIXDOwzamW6NXLie5WPag+aUdqVlsTsEk88=
+        secure: iXzIvgJaMdmFT8IJPWNs0XkUi1l2My6kRXmxnT129hrQAiCIHCfJeB898IfFFQM86kwe4p3dA/1t2+IMa4bNqpUZb6Z615MDgWxcpHzn0JMiHMjYHlj9eae/+8IBjSbjoTZPIXRJGZ2NMy5rY8/wkcsXN87J0flhwdUMX8uO5aNDLad8nVYZsU1NeWHKchx07WriFeJaRYo0aCPXRK+kzc2u8eSrOumzKSTaIPnZlknrMJ25BRA8pEhJVlRPMg0Jp+L9ugdwGNtC7o61W5e50XPYC485P0+GwWqgUz2F6NLTzAY0JYNdLgz4XdqdHrpc/lK+TKN2FE9yJq24IFxyyzO7xDPAmhAUiFYnN7URIJyaNcNYGguc668P6Ogs66MVMCLdCQsUMMVb0ok9QqVycYMoZ0h09b2A8tviOnaUVWZbE8Ur0FL3YVADq51L0FWq/Xe1+EnWOlsySyLymh/uNMbL6sHa2dJCkzzwkyFxPTyELWaUC01VQRjyjFJknkX5S0Z+SNiYSQnuFPb4HQKj7hKK4KQ31Yk3hJ2+NDWjvPzviduFHVrxHGRGzd/X1uIPyfubc85YdJn1e7XxjmhMOfxXmzHanlGGKczwatKZe50UqWXQTat2bKnKGeI24jt+ubfznfDqwnF71Lr60+UdD9Xqrqz/KGr9ldE8QbPRcHM=
       on:
         tags: true

appveyor.yml

@@ -3,18 +3,11 @@ pull_requests:
   do_not_increment_build_number: true
 
 environment:
-  MYSQL_HOST: localhost
-  MYSQL_USER: root
-  MYSQL_PASSWORD: Password12!
   PGUSER: postgres
   PGPASSWORD: Password12!
-  MYSQL_PATH: C:\Program Files\MySQL\MySQL Server 5.7
   PGSQL_PATH: C:\Program Files\PostgreSQL\9.6
-  POSTGRES_ENV_POSTGRES_USER: postgres
-  POSTGRES_ENV_POSTGRES_PASSWORD: Password12!
-  MYSQL_ENV_MYSQL_USER: root
-  MYSQL_ENV_MYSQL_PASSWORD: Password12!
   VOLUME_PATH: C:\ingest
+  PEEWEE_URL: postgres://postgres:Password12!@localhost/pacifica_metadata
   BROKER_URL: redis://127.0.0.1:6379/0
   BACKEND_URL: redis://127.0.0.1:6379/0
 
@@ -23,44 +16,63 @@ environment:
     - PYTHON: C:\Python36-x64
 
 services:
-  - mysql
   - postgresql
 
 before_test:
-  - SET PATH=%MYSQL_PATH%\bin;%PGSQL_PATH%\bin;%PATH%
-  - mysqladmin --host=%MYSQL_HOST% --user=%MYSQL_USER% --password=%MYSQL_PASSWORD% create pacifica_ingest
-  - mysqladmin --host=%MYSQL_HOST% --user=%MYSQL_USER% --password=%MYSQL_PASSWORD% create pacifica_uniqueid
-  - createdb pacifica_metadata
-  - mkdir C:\ingest
-  - mkdir C:\archive
-  - ps: nuget install redis-64 -excludeversion
-  - ps: redis-64\tools\redis-server.exe --service-install
-  - ps: redis-64\tools\redis-server.exe --service-start
-  - ps: Start-Process $env:PYTHON\python.exe -ArgumentList "$env:PYTHON\scripts\ArchiveInterfaceServer.py --prefix C:\archive" -WorkingDirectory "travis/archivei"
-  - ps: Start-Process $env:PYTHON\python.exe -ArgumentList "$env:PYTHON\scripts\UniqueIDServer.py" -WorkingDirectory "travis\uniqueid"
-  - ps: Start-Process $env:PYTHON\python.exe -ArgumentList "$env:PYTHON\scripts\MetadataServer.py" -WorkingDirectory "travis\metadata"
-  - ps: Start-Process $env:PYTHON\python.exe -ArgumentList "$env:PYTHON\scripts\IngestServer.py" -WorkingDirectory "travis\ingest"
-  - ps: Start-Process $env:PYTHON\python.exe -ArgumentList "-m celery -A ingest.backend worker --loglevel=info -P eventlet" -WorkingDirectory "travis\ingest"
-  - ps: Invoke-WebRequest https://github.com/pacifica/pacifica-metadata/archive/master.zip -OutFile pacifica-metadata.zip
-  - ps: Expand-Archive pacifica-metadata.zip -DestinationPath C:\pacifica-metadata
-  - ps: sleep 10
-  - ps: Invoke-WebRequest http://localhost:8121/users
-  - cd C:\pacifica-metadata\pacifica-metadata-master
-  - '%PYTHON%\python.exe -m test_files.loadit'
-  - cd C:\projects\pacifica-python-uploader
-  - ps: Start-Process $env:PYTHON\python.exe -ArgumentList "$env:PYTHON\scripts\PolicyServer.py" -WorkingDirectory "travis\policy"
-  - ps: Invoke-WebRequest http://localhost:8181/status/users/search/dmlb2001/simple -TimeoutSec 1800
+- ps: >
+    $env:PATH = "$env:PGSQL_PATH\bin;$env:PATH";
+    createdb pacifica_metadata;
+    mkdir C:\ingest;
+    mkdir C:\archive;
+    C:\pacifica\Scripts\activate.ps1;
+    $env:METADATA_CPCONFIG = "$PWD/travis/metadata/server.conf";
+    Start-Process C:\pacifica\Scripts\pacifica-metadata.exe -RedirectStandardError metadata-error.log -RedirectStandardOutput metadata-output.log;
+    nuget install redis-64 -excludeversion;
+    redis-64\tools\redis-server.exe --service-install;
+    redis-64\tools\redis-server.exe --service-start;
+    $env:ARCHIVEINTERFACE_CONFIG = "$PWD/travis/archivei/config.cfg";
+    $env:ARCHIVEINTERFACE_CPCONFIG = "$PWD/travis/archivei/server.conf";
+    Start-Process C:\pacifica\Scripts\pacifica-archiveinterface.exe;
+    $env:UNIQUEID_CONFIG = "$PWD/travis/uniqueid/config.cfg";
+    $env:UNIQUEID_CPCONFIG = "$PWD/travis/uniqueid/server.conf";
+    Start-Process C:\pacifica\Scripts\pacifica-uniqueid.exe;
+    $env:INGEST_CONFIG = "$PWD/travis/ingest/config.cfg";
+    $env:INGEST_CPCONFIG = "$PWD/travis/ingest/server.conf";
+    Start-Process C:\pacifica\Scripts\pacifica-ingest.exe;
+    Start-Process C:\pacifica\Scripts\celery.exe -ArgumentList "-A pacifica.ingest.tasks worker --loglevel=info -P eventlet";
+    $MD_VERSION = `pip show pacifica-metadata | grep Version: | awk '{ print $2 }';
+    Invoke-WebRequest https://github.com/pacifica/pacifica-metadata/archive/v${MD_VERSION}.zip -OutFile pacifica-metadata.zip;
+    Expand-Archive pacifica-metadata.zip -DestinationPath C:\pacifica-metadata;
+    sleep 10; Invoke-WebRequest http://localhost:8121/users;
+    pushd C:\pacifica-metadata\pacifica-metadata-${MD_VERSION};
+    python tests\test_files\loadit_test.py;
+    popd;
+    $env:POLICY_CPCONFIG = "$PWD/travis/policy/server.conf";
+    $env:METADATA_URL = "http://127.0.0.1:8121";
+    $env:STATUS_URL = "http://127.0.0.1:8121/keys?_id=104";
+    Start-Process C:\pacifica\Scripts\pacifica-policy.exe -RedirectStandardError policy-error.log -RedirectStandardOutput policy-output.log;
+    sleep 3;
+    Invoke-WebRequest http://127.0.0.1:8181/status/users/search/dmlb2001/simple -TimeoutSec 1800;
+
+build: off
 
 install:
-  - ps: 'Invoke-WebRequest -Uri "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.6.5.zip" -OutFile "elasticsearch.zip"'
-  - ps: 'Expand-Archive "elasticsearch.zip" -DestinationPath "C:\elasticsearch"'
-  - ps: 'Start-Process C:\elasticsearch\elasticsearch-5.6.5\bin\elasticsearch'
-  - '%PYTHON%\python.exe -m pip install pip setuptools wheel --upgrade'
-  - '%PYTHON%\python.exe -m pip install -r requirements-dev.txt'
-  - '%PYTHON%\python.exe -m pip install celery[redis] eventlet'
+    - ps: >
+        $env:PATH = "$env:MYSQL_PATH\bin;${env:PGSQL_PATH}\bin;${env:PYTHON}\Scripts;${env:PATH}";
+        Invoke-WebRequest -Uri "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.6.5.zip" -OutFile "elasticsearch.zip";
+        Expand-Archive "elasticsearch.zip" -DestinationPath "C:\elasticsearch";
+        Start-Process C:\elasticsearch\elasticsearch-5.6.5\bin\elasticsearch;
+        & $env:PYTHON\python.exe -m virtualenv C:\pacifica;
+        C:\pacifica\Scripts\activate.ps1;
+        python -m pip install pip setuptools wheel --upgrade;
+        pip install -r requirements-dev.txt;
+        pip install celery[redis] eventlet;
 
-build: off
 
 test_script:
-  - ps: '& "$env:PYTHON\python.exe" -m coverage run --include="pacifica/*" -m pytest -v pacifica'
-  - ps: '& "$env:PYTHON\python.exe" -m coverage report --show-missing --fail-under 100'
+  - ps: >
+      C:\pacifica\Scripts\activate.ps1;
+      pip install .;
+      cd tests;
+      coverage run --include="*/site-packages/pacifica/uploader/*" -m pytest -v;
+      coverage report --show-missing --fail-under 100;

requirements-dev.txt

@@ -1,16 +1,15 @@
+-r requirements.txt
 celery
 codeclimate-test-reporter
 coverage>=4.0,<4.4
--e git://github.com/pacifica/pacifica-archiveinterface.git#egg=PacificaArchiveInterface
--e git://github.com/pacifica/pacifica-ingest.git#egg=PacificaIngest
--e git://github.com/pacifica/pacifica-metadata.git#egg=PacificaMetadata
--e git://github.com/pacifica/pacifica-policy.git#egg=PacificaPolicy
--e git://github.com/pacifica/pacifica-uniqueid.git#egg=PacificaUniqueID
+pacifica-archiveinterface
+pacifica-ingest
+pacifica-metadata
+pacifica-policy
+pacifica-uniqueid
 peewee
 pep257
 pre-commit
 pylint<2.0
 pytest
 radon
-requests
-six

tests/__init__.py

@@ -0,0 +1,8 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+"""
+Uploader test module.
+
+This module is to help the uploader_test module import
+the bundler_test module.
+"""

pacifica/uploader/bundler/test/test_bundler.py → tests/bundler_test.py

@@ -9,8 +9,8 @@
 from tempfile import NamedTemporaryFile
 from tarfile import TarFile
 from six import PY2
-from ... import bundler
-from ...metadata import MetaData, MetaObj
+from pacifica.uploader import bundler
+from pacifica.uploader.metadata import MetaData, MetaObj
 
 
 # pylint: disable=too-few-public-methods

pacifica/uploader/common/test/test_common.py → tests/common_test.py

@@ -4,7 +4,7 @@
 from __future__ import absolute_import
 import os
 from unittest import TestCase
-from .. import CommonBase
+from pacifica.uploader.common import CommonBase
 
 
 class TestCommonModule(TestCase):

pacifica/uploader/metadata/test/test_metadata.py → tests/metadata_test.py

@@ -3,8 +3,8 @@
 """Test the metadata module."""
 import json
 from unittest import TestCase
-from .. import MetaData, MetaObj, metadata_encode, metadata_decode, FileObj
-from ..metadata import FileObjEncoder
+from pacifica.uploader.metadata import MetaData, MetaObj, metadata_encode, metadata_decode, FileObj
+from pacifica.uploader.metadata.metadata import FileObjEncoder
 
 
 class TestMetaData(TestCase):

pacifica/uploader/metadata/test/test_metaupdate.py → tests/metaupdate_test.py

@@ -2,8 +2,8 @@
 # -*- coding: utf-8 -*-
 """Module used to test the MetaUpdate module."""
 from unittest import TestCase
-from ..metaupdate import MetaUpdate
-from ..metadata import MetaObj
+from pacifica.uploader.metadata.metaupdate import MetaUpdate
+from pacifica.uploader.metadata.metadata import MetaObj
 
 
 class TestMetaUpdate(TestCase):

pacifica/uploader/metadata/test/test_module.py → tests/module_test.py

@@ -3,7 +3,7 @@
 """Test the metadata module."""
 from __future__ import absolute_import
 from unittest import TestCase
-from ... import metadata
+from pacifica.uploader import metadata
 
 
 class TestModule(TestCase):

pacifica/uploader/metadata/test/test_policyquery.py → tests/policyquery_test.py

@@ -3,8 +3,8 @@
 """Test the metadata module."""
 from __future__ import absolute_import
 from unittest import TestCase
-from .. import metadata_decode
-from ..policyquery import PolicyQuery
+from pacifica.uploader.metadata.metadata import metadata_decode
+from pacifica.uploader.metadata.policyquery import PolicyQuery
 
 
 class TestPolicyQuery(TestCase):

pacifica/uploader/test/test_uploader.py → tests/uploader_test.py

@@ -6,10 +6,10 @@
 from time import sleep
 from unittest import TestCase
 from tempfile import NamedTemporaryFile
-from .. import Uploader
-from .. import bundler
-from ..bundler.test.test_bundler import BuildSampleData
-from ..metadata import MetaData, MetaObj
+from pacifica.uploader import Uploader
+from pacifica.uploader import bundler
+from pacifica.uploader.metadata import MetaData, MetaObj
+from .bundler_test import BuildSampleData
 
 
 class TestUploader(TestCase):