Merge branch 'master' into fix-52-file-not-found-error

dmlb2000 · Feb 3, 2020 · 3ce699d · 3ce699d
2 parents db0e2e0 + fade4b7
commit 3ce699d
Show file tree

Hide file tree

Showing 22 changed files with 220 additions and 217 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,10 +1,10 @@
 repos:
 -   repo: https://github.com/pre-commit/mirrors-autopep8
-    rev: v1.4.4
+    rev: v1.5
     hooks:
             -   id: autopep8
 -   repo: git://github.com/pre-commit/pre-commit-hooks
-    rev: v2.2.3
+    rev: v2.4.0
     hooks:
             -   id: fix-encoding-pragma
             -   id: trailing-whitespace
@@ -49,7 +49,7 @@ repos:
                 language: system
                 types: [python]
 -   repo: git://github.com/Lucas-C/pre-commit-hooks
-    rev: v1.1.6
+    rev: v1.1.7
     hooks:
     -   id: remove-tabs
     -   id: remove-crlf
diff --git a/.travis.yml b/.travis.yml
@@ -1,48 +1,44 @@
 sudo: required
-dist: xenial
-services:
-- postgresql
-- redis-server
+dist: bionic
 language: python
+python: 3.6
 stages:
 - lint
 - test
 - test-docker
 - deploy
 cache: pip
-".before_install": &3
-- bash -xe travis/before-install.sh
-".install": &1
+before_script:
+  - pip install ansible
+  - ansible-galaxy install pacifica.ansible_pacifica pacifica.ansible_travis
+  - ansible-playbook ~/.ansible/roles/pacifica.ansible_travis/molecule/common/playbook.yml -i ~/.ansible/roles/pacifica.ansible_travis/molecule/common/inventory.yml -e "ansible_python_interpreter=/home/travis/virtualenv/python${TRAVIS_PYTHON_VERSION}/bin/python travis_virtualenv=python${TRAVIS_PYTHON_VERSION} github_repository=cli"
+  - |
+    curl -X PUT -H 'Last-Modified: Sun, 06 Nov 1994 08:49:37 GMT' --upload-file README.md http://127.0.0.1:8080/103
+    curl -X PUT -H 'Last-Modified: Sun, 06 Nov 1994 08:49:37 GMT' --upload-file README.md http://127.0.0.1:8080/104
+    readme_size=$(stat -c '%s' README.md)
+    readme_sha1=$(sha1sum README.md | awk '{ print $1 }')
+    echo '{ "hashsum": "'$readme_sha1'", "hashtype": "sha1", "size": '$readme_size'}' > /tmp/file-104-update.json
+    curl -X POST -H 'content-type: application/json' -T /tmp/file-104-update.json 'http://localhost:8121/files?_id=103'
+    curl -X POST -H 'content-type: application/json' -T /tmp/file-104-update.json 'http://localhost:8121/files?_id=104'
+  - python tests/generate_ce_stub_test.py
+  - pip install 'redis<3.4'
+  - for x in ingest cartd notify ; do for y in frontend backend; do sudo systemctl restart ${x}_${y}.service; done ; done
+install:
 - pip install -r requirements-dev.txt
-".script": &4
+script:
 - bash -xe travis/unit-tests.sh
 - bash -xe travis/test-deploy.sh
-".lint_script": &2
-- pre-commit run -a
-- radon cc pacifica
 jobs:
   include:
   - stage: lint
-    services: []
-    python: 3.6
-    install: *1
-    script: *2
-  - python: 2.7
-    services: []
-    script: *2
-    install: *1
+    before_script: skip
+    script:
+    - pre-commit run -a
+    - radon cc pacifica
   - stage: test
-    python: 2.7
-    before_install: *3
-    install: *1
-    script: *4
-  - python: 3.6
-    before_install: *3
-    install: *1
-    script: *4
   - stage: test-docker
+    before_script: skip
     sudo: required
-    python: 3.6
     services:
     - docker
     script:
@@ -66,11 +62,8 @@ jobs:
       docker run -t --rm --network=pacifica-cli_default pacifica/cli upload --logon 10 tests;
       docker run -t --rm --network=pacifica-cli_default pacifica/cli download --transaction-id 1 --destination /tmp/foo;
   - stage: deploy
-    services: []
-    language: python
-    before_install: skip
+    before_script: skip
     script: skip
-    python: 3.6
     deploy:
       skip_cleanup: true
       provider: pypi
@@ -80,17 +73,3 @@ jobs:
         secure: AN8nG/iEsZ1Ol+WZyfkf9H0uFYAmU+H04j6chHHFBqWVFZcbITXa6m0KjKXg/gcGcJz4RnEwKD6qBwb8YMBDODmkmah88VgsXR+hiAJlq4cTYMEMrEewnRZ9fH3f0BCle8FSI16VeHDYOdYvMGRDJXKIn/GxNc6xNVgwEcKqf5tBOt6C+LzemlYHVbx+KBQR9kRaRJOcnLnvVS50gCamQ7YIRyBRNDFyE3luLIm5VDGPO11E9be2OrXCrpSf1fPftiqYyzpT8ttLt+0Rar5J69EC65U/ZVtiJ96Y36FNCVpj9NL+UokcLrdnlU4lSsbqttwDNskSCA6CQ1FCMjTZAgJAI/gTUv2gL3NTzGnAQtcKL8KUeMU37Au5MP4BxfMDofUvxNRdScODWyNRU1fWGmj+e22nAx+gODpiTntR+rm8LR16f+CBs0D8VT371bHjUYXu6H2mDH1lJPISGaG5gEoTH0FpVEoOg82dQAy4IikLA09ra2USaXgV4leuXcBHyxmAZTxhP1YIWXszx1XUa1DP3hqkvqecNU1QuQnG5subHan482tDNY61MDgF9o010h/CgqDU1Tui5ozyHYPjHSaErzYeXF8LUTk8f0NiOkERetzCF0vFX+ZKBvjEMwkjozMPNXZvdjJdrTb90p9dxt8/NdWtZ97jSpItoDTAxSo=
       on:
         tags: true
-  - services: []
-    language: python
-    before_install: skip
-    script: skip
-    python: 2.7
-    deploy:
-      skip_cleanup: true
-      provider: pypi
-      user: dmlb2000
-      distributions: bdist_wheel
-      password:
-        secure: AN8nG/iEsZ1Ol+WZyfkf9H0uFYAmU+H04j6chHHFBqWVFZcbITXa6m0KjKXg/gcGcJz4RnEwKD6qBwb8YMBDODmkmah88VgsXR+hiAJlq4cTYMEMrEewnRZ9fH3f0BCle8FSI16VeHDYOdYvMGRDJXKIn/GxNc6xNVgwEcKqf5tBOt6C+LzemlYHVbx+KBQR9kRaRJOcnLnvVS50gCamQ7YIRyBRNDFyE3luLIm5VDGPO11E9be2OrXCrpSf1fPftiqYyzpT8ttLt+0Rar5J69EC65U/ZVtiJ96Y36FNCVpj9NL+UokcLrdnlU4lSsbqttwDNskSCA6CQ1FCMjTZAgJAI/gTUv2gL3NTzGnAQtcKL8KUeMU37Au5MP4BxfMDofUvxNRdScODWyNRU1fWGmj+e22nAx+gODpiTntR+rm8LR16f+CBs0D8VT371bHjUYXu6H2mDH1lJPISGaG5gEoTH0FpVEoOg82dQAy4IikLA09ra2USaXgV4leuXcBHyxmAZTxhP1YIWXszx1XUa1DP3hqkvqecNU1QuQnG5subHan482tDNY61MDgF9o010h/CgqDU1Tui5ozyHYPjHSaErzYeXF8LUTk8f0NiOkERetzCF0vFX+ZKBvjEMwkjozMPNXZvdjJdrTb90p9dxt8/NdWtZ97jSpItoDTAxSo=
-      on:
-        tags: true
diff --git a/appveyor.yml b/appveyor.yml
@@ -12,7 +12,6 @@ environment:
   BACKEND_URL: redis://127.0.0.1:6379/0
 
   matrix:
-    - PYTHON: C:\Python27-x64
     - PYTHON: C:\Python36-x64
 
 services:
@@ -87,14 +86,14 @@ install:
     C:\pacifica\Scripts\activate.ps1;
     python -m pip install pip setuptools wheel --upgrade;
     pip install -r requirements-dev.txt;
-    pip install celery[redis] 'redis';
     echo "Done";
 
 build: off
 
 test_script:
 - ps: >
     C:\pacifica\Scripts\activate.ps1;
+    $home_var = `python -c "from os.path import expanduser; print(expanduser('~'))";
     pip install .;
     cd tests;
     cp -r ../travis .;
@@ -136,14 +135,16 @@ test_script:
     coverage run --include='*/site-packages/pacifica/cli/*' -a -m pacifica.cli upload --dry-run --logon dmlb2001 --project-regex 'expired closed and end';
     Invoke-WebRequest -Method POST -Headers @{ "content-type" = "application/json" } -Body '{ "network_id":"appveyor" }' http://localhost:8121/users?_id=10;
     Invoke-WebRequest -Method POST -Headers @{ "content-type" = "application/json" } -Body '{ "network_id":"someoneelse" }' http://localhost:8121/users?_id=11;
+    $env:PACIFICA_CLI_INI = "$home_var\.pacifica_cli\config.ini";
     coverage run --include='*/site-packages/pacifica/cli/*' -a -m pacifica.cli upload README.md;
     coverage run --include='*/site-packages/pacifica/cli/*' -a -m pacifica.cli upload travis;
     coverage run --include='*/site-packages/pacifica/cli/*' -a -m pacifica.cli upload --tar-in-tar README.md;
     coverage run --include='*/site-packages/pacifica/cli/*' -a -m pacifica.cli upload --local-save retry.tar README.md;
+    tar -xf retry.tar metadata.txt;
     coverage run --include='*/site-packages/pacifica/cli/*' -a -m pacifica.cli upload --local-save retry.tar --local-compress bzip2 README.md;
     coverage run --include='*/site-packages/pacifica/cli/*' -a -m pacifica.cli upload --local-save retry.tar --local-compress gzip README.md;
     coverage run --include='*/site-packages/pacifica/cli/*' -a -m pacifica.cli upload --local-save retry.tar --do-not-upload README.md;
     coverage run --include='*/site-packages/pacifica/cli/*' -a -m pacifica.cli upload --local-retry retry.tar;
     coverage run --include='*/site-packages/pacifica/cli/*' -a -m pacifica.cli upload --nowait README.md;
-    coverage run --include='*/site-packages/pacifica/cli/*' -a -m pytest -xv;
+    coverage run --include='*/site-packages/pacifica/cli/*' -a -m pytest -xsv;
     coverage report --show-missing --fail-under 100
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -137,6 +137,8 @@ services:
     - archiveinterface:archivei
     volumes:
     - cartddata:/shared
+    ports:
+    - 8081:8081
     environment:
       PEEWEE_ADDR: mysql
       BROKER_URL: pyamqp://guest:guest@amqp:5672//

diff --git a/docs/configuration.md b/docs/configuration.md
@@ -14,6 +14,12 @@ The user configuration is processed second, if found. The directory the
 client looks in by default is `~/.pacifica_cli/config.ini`. The `~`
 translates to the users home directory on any platform.
 
+Optionally, users can manage their config files in their home directory
+and set the `--config-ini` command-line argument. This switch will
+try to open the given file in `~/.pacifica_cli/` and merge it with
+the system configurations. Also, setting the environment variable
+`PACIFICA_CLI_INI` can also be sufficient to change the file name.
+
 ## System Metadata
 
 The metadata is managed by a JSON configuration file referenced by an

diff --git a/docs/exampleusage.md b/docs/exampleusage.md
@@ -37,6 +37,7 @@ There are three kinds of authentication types supported.
 
 - clientssl - This is where you have an SSL client key and cert
 - basic     - This is a username and password
+- gssapi    - Use GSSAPI tickets to authenticate
 - None      - Do not perform any authentication
 
 Authentication Type (None): basic

diff --git a/pacifica/__init__.py b/pacifica/__init__.py
@@ -1,4 +1,7 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 """Pacifica Namespace."""
-__import__('pkg_resources').declare_namespace(__name__)
+try:
+    __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+    __path__ = __import__('pkgutil').extend_path(__path__, __name__)
diff --git a/pacifica/cli/__main__.py b/pacifica/cli/__main__.py
@@ -3,8 +3,12 @@
 """The CLI module contains all the logic needed to run a CLI."""
 import sys
 import argparse
+import warnings
 from os import getenv, path
+from json import loads
+from jsonschema import validate
 from pacifica.uploader.metadata import metadata_decode
+from ._version import __version__
 from .methods import upload, configure, download
 from .utils import system_config_path, user_config_path, compressor_generator
 
@@ -18,21 +22,21 @@ def arg_to_compressor_obj(str_obj=None):
     return compressor_generator(str_obj)
 
 
-def mangle_config_argument(argv):
+def mangle_config_argument(argv, default_config):
     """Get the config argument out of argv and return stripped version."""
     config_arg = '--config'
     len_arg = len(config_arg)
     starts_argv = [arg[:len_arg] for arg in argv]
-    if config_arg in starts_argv:
-        if config_arg in argv:
-            config_file = argv[argv.index(config_arg) + 1]
-            del argv[argv.index(config_arg) + 1]
-            del argv[argv.index(config_arg)]
-        else:
-            config_file = argv[starts_argv.index(config_arg)][len_arg + 1:]
-            del argv[starts_argv.index(config_arg)]
-        return (config_file, argv)
-    return (None, argv)
+    if config_arg not in starts_argv:
+        return (default_config, argv)
+    if config_arg in argv:
+        config_file = argv[argv.index(config_arg) + 1]
+        del argv[argv.index(config_arg) + 1]
+        del argv[argv.index(config_arg)]
+    else:
+        config_file = argv[starts_argv.index(config_arg)][len_arg + 1:]
+        del argv[starts_argv.index(config_arg)]
+    return (config_file, argv)
 
 
 def parse_uploader_config(upload_parser):
@@ -42,26 +46,28 @@ def parse_uploader_config(upload_parser):
         'UPLOADER_CONFIG', system_config_path(upload_file_name))
     if default_config == upload_file_name and path.isfile(user_config_path(upload_file_name)):
         default_config = user_config_path(upload_file_name)
-    config_file, argv = mangle_config_argument(sys.argv)
-    if not config_file:
-        config_file = default_config
-    if path.isfile(config_file):
-        config_data = metadata_decode(open(config_file).read())
-        for config_part in config_data:
-            if not config_part.value:
-                upload_parser.add_argument(
-                    '--{}-regex'.format(config_part.metaID), required=False,
-                    dest='{}_regex'.format(config_part.metaID),
-                    help='{} regular expression match.'.format(
-                        config_part.displayTitle)
-                )
-                upload_parser.add_argument(
-                    '--{}'.format(config_part.metaID), '-{}'.format(
-                        config_part.metaID[0]),
-                    help=config_part.displayTitle, required=False
-                )
-        return config_file, argv, config_data
-    return default_config, sys.argv, None
+    config_file, argv = mangle_config_argument(sys.argv, default_config)
+    if not path.isfile(config_file):
+        warnings.warn('Config File {} is not a file or is not accessible'.format(config_file))
+        return default_config, sys.argv, None
+    json_str = open(config_file).read()
+    schema = loads(open(path.join(path.dirname(__file__), 'config_schema.json')).read())
+    validate(loads(json_str), schema)
+    config_data = metadata_decode(json_str)
+    for config_part in config_data:
+        if not config_part.value:
+            upload_parser.add_argument(
+                '--{}-regex'.format(config_part.metaID), required=False,
+                dest='{}_regex'.format(config_part.metaID),
+                help='{} regular expression match.'.format(
+                    config_part.displayTitle)
+            )
+            upload_parser.add_argument(
+                '--{}'.format(config_part.metaID), '-{}'.format(
+                    config_part.metaID[0]),
+                help=config_part.displayTitle, required=False
+            )
+    return config_file, argv, config_data
 
 
 def main():
@@ -80,10 +86,18 @@ def main():
         '--config', dest='config', default=default_config,
         help='Upload configuration metadata.', required=False
     )
+    parser.add_argument(
+        '--config-ini', dest='config_ini', default=getenv('PACIFICA_CLI_INI', 'config.ini'),
+        help='Endpoint configuration ini file.', required=False
+    )
     parser.add_argument(
         '--verbose', dest='verbose', default='info',
         help='Enable verbose logging.', required=False
     )
+    parser.add_argument(
+        '--version', action='version',
+        version='%(prog)s {version}'.format(version=__version__)
+    )
     upload_parser.add_argument(
         '--follow-links', default=False, action='store_true', dest='followlinks',
         help='Follow links to directories when bundling.', required=False

diff --git a/pacifica/cli/_version.py b/pacifica/cli/_version.py
@@ -0,0 +1,10 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+"""Pacifica CLI Version Module."""
+from pkg_resources import get_distribution, DistributionNotFound
+
+try:
+    __version__ = get_distribution('pacifica-cli').version
+except DistributionNotFound:  # pragma: no cover not sure how to test this
+    # package is not installed
+    pass
diff --git a/pacifica/cli/config_schema.json b/pacifica/cli/config_schema.json
@@ -0,0 +1,46 @@
+{
+  "items": {
+    "properties": {
+      "destinationTable": {
+        "type": "string"
+      },
+      "displayFormat": {
+        "type": "string"
+      },
+      "displayTitle": {
+        "type": "string"
+      },
+      "displayType": {
+        "type": "string"
+      },
+      "metaID": {
+        "type": "string"
+      },
+      "queryDependencies": {
+        "type": "object"
+      },
+      "queryFields": {
+        "items": {
+          "type": "string"
+        },
+        "type": "array"
+      },
+      "sourceTable": {
+        "type": "string"
+      },
+      "value": {
+        "type": "string"
+      },
+      "valueField": {
+        "type": "string"
+      }
+    },
+    "required": [
+      "destinationTable",
+      "metaID",
+      "value"
+    ],
+    "type": "object"
+  },
+  "type": "array"
+}