Release v0.11.0

Merge pull request #536 from chaoss/dev

.gitignore

@@ -9,6 +9,7 @@ __pycache__/
 runtime/
 logs/
 frontend/public/api_docs/
+*.err
 
 # C extensions
 *.so
@@ -116,6 +117,7 @@ venv.bak/
 .spyderproject
 .spyproject
 
+
 # Rope project settings
 .ropeproject
 
@@ -127,5 +129,15 @@ venv.bak/
 .dmypy.json
 dmypy.json
 
+
 # Pyre type checker
 .pyre/
+
+# Augur-spdx
+./spdx-scanner/dosocs2.conf
+spdx-scanner/dosocs2.conf
+spdx-scanner/augur-spdx/
+workers/spdx_worker/dosocs2.conf
+workers/spdx_worker/augur-spdx/
+workers/spdx_worker/ex-raw.txt
+workers/spdx_worker/ex.json

.travis.yml

@@ -1,20 +1,35 @@
-matrix:
+os: linux
+dist: xenial
+language: python
+jobs:
   include:
-  - name: Python Datasource Unit Tests
-    install: python setup.py install; pip install .
-    language: python
-    python: '3.6'
-    script:
-    - python -m pytest augur/metrics/**/test_**_functions.py
+    - name: 3.6 Metrics
+      python: 3.6
+      env: TOXENV=py36-metrics
 
-  - name: API Unit Tests
-    install: python setup.py install; pip install .
-    language: python
-    python: '3.6'
-    script:
-    - augur run --no-enable-housekeeper > /dev/null 2>&1 &
-    - sleep 15
-    - python -m pytest -ra augur/metrics/**/test_**_routes.py
+    - name: 3.6 Metrics API
+      python: 3.6
+      env: TOXENV=py36-metrics_api
 
-notifications:
-  slack: augurlabs:WdpwniZ9L5RqJyMfIk1n8DOU
+    - name: 3.7 Metrics
+      python: 3.7
+      env: TOXENV=py37-metrics
+
+    - name: 3.7 Metrics API
+      python: 3.7
+      env: TOXENV=py37-metrics_api
+
+    - name: 3.8 Metrics
+      python: 3.8
+      env: TOXENV=py38-metrics
+
+    - name: 3.8 Metrics API
+      python: 3.8
+      env: TOXENV=py38-metrics_api
+
+install:
+  - pip install -r requirements.txt
+  - pip install tox
+
+script:
+  - tox

CONTRIBUTING.md

@@ -7,14 +7,14 @@ We do our best to follow the standard Git workflow of `fork -> change -> pull re
 In more detail:
 
 0. Fork the [main repo](https://github.com/chaoss/augur), then clone it:
+
 ```bash
 git clone git@github.com:your-username/augur.git
 ```
 1. Follow the instructions outlined in the [README.md](README.md) to get started.
-2. Make your change.
-3. When you are done making changes, run `make docs` to rebuild the documentation 
-4. Push to your fork.
-5. Then, [submit a pull request](https://github.com/chaoss/augur/compare).
+2. Make your change(s). Check [here](https://oss-augur.readthedocs.io/en/master/) for documentation for the whole development process!
+3. Push to your fork.
+4. Then, [submit a pull request](https://github.com/chaoss/augur/compare).
 
 At this point you're waiting on us. We like to at least comment on pull requests
 within three business days (and, typically, one business day). Once one of our maintainers has had a change to review your PR, we will either mark it as "needs review" and while provide specific feedback on your changes, or we will go ahead and complete the pull request.

Makefile

@@ -2,7 +2,6 @@ SERVE_COMMAND=augur run
 ENABLE_HOUSEKEEPER=--enable-housekeeper
 OLDVERSION="null"
 EDITOR?="vi"
-MODEL=**
 AUGUR_PIP?='pip'
 AUGUR_PYTHON?='python'
 
@@ -21,12 +20,12 @@ default:
 	@ echo "    dev-restart                     Runs dev-stop then dev-start"
 	@ echo
 	@ echo "Testing Commands:"
-	@ echo "    test MODEL={model}              Runs all pytest unit tests and API tests for the specified metrics model. Defaults to all"
-	@ echo "    test-functions MODEL={model}    Run pytest unit tests for the specified metrics model. Defaults to all"
-	@ echo "    test-routes MODEL={model}       Run API tests for the specified metrics model. Defaults to all"
+	@ echo "    test                            Runs all unit tests and API tests"
+	@ echo "    test-metrics                    Run unit tests for the specified metrics model"
+	@ echo "    test-metrics-api                Run API tests for the specified metrics model"
 	@ echo
 	@ echo "Documentation Commands:"
-	@ echo "    sphinx-docs                     Generates the documentation using sphinx"
+	@ echo "    library-docs                    Generates the documentation using sphinx"
 	@ echo "    api-docs                        Generates the REST API documentation using apidocjs"
 	@ echo "    docs                            Generates all documentation"
 
@@ -38,8 +37,14 @@ default:
 install:
 	@ ./util/scripts/install/install.sh
 
-install-augur-sbom: 
-	@ ./util/scripts/install/nomos.sh 
+install-spdx:
+	@ ./util/scripts/install/install-spdx.sh
+
+install-spdx-sudo:
+	@ ./util/scripts/install/install-spdx-sudo.sh
+
+install-augur-sbom:
+	@ ./util/scripts/install/nomos.sh
 
 version:
 	$(eval OLDVERSION=$(shell $(AUGUR_PYTHON) ./util/print-version.py))
@@ -50,7 +55,7 @@ config:
 
 clean:
 	@ echo "Removing node_modules, logs, caches, and some other dumb stuff that can be annoying..."
-	@ rm -rf runtime node_modules frontend/node_modules frontend/public augur.egg-info .pytest_cache logs 
+	@ rm -rf runtime node_modules frontend/node_modules frontend/public augur.egg-info .pytest_cache logs
 	@ find . -name \*.pyc -delete
 	@ find . -type f -name "*.lock" -delete
 
@@ -109,28 +114,47 @@ backend-restart: backend-stop backend-start
 
 backend: backend-restart
 
+augur:
+	@ ./util/scripts/control/augur.sh
+
+collect:
+	@ ./util/scripts/control/collect.sh
+
+run:
+	@ ./util/scripts/control/augur.sh
+	@ echo "Waiting for the server to start... (this will take about 3 minutes)"
+	@ echo "In the meantime, consider taking a short break - you've earned it!"
+	@ sleep 180
+	@ ./util/scripts/control/collect.sh
+
+status:
+	@ ./util/scripts/control/status.sh
+
 
 #
 # Testing
 #
-.PHONY: test test-functions test-routes
-test: test-functions test-routes
+.PHONY: test test-metrics test-metrics-api
+test: test-metrics test-metrics-api
+
+test-metrics:
+	@ bash -c 'tox -e py-metrics 2>&1 | tee logs/metrics_test.log'
 
-test-functions:
-	@ bash -c '$(AUGUR_PYTHON) -m pytest -ra -s augur/metrics/$(MODEL)/test_$(MODEL)_functions.py'
+test-metrics-api:
+	@ bash -c 'tox -e py-metrics_api 2>&1 | tee logs/metrics_api_test.log'
 
-test-routes:
-	@ $(AUGUR_PYTHON) test/api/test_api.py $(MODEL)
+test-python-versions:
+	@ bash -c 'tox -e ALL 2>&1 | tee logs/metrics_ALL.log'
 
 
-# 
+#
 # Documentation
-# 
-.PHONY: sphinx-docs sphinx-docs-view api-docs api-docs-view docs
-sphinx-docs:
+#
+.PHONY: library-docs library-docs-view api-docs api-docs-view docs
+library-docs:
 	@ bash -c 'cd docs/ && rm -rf build/ && make html;'
 
-sphinx-docs-view: sphinx-docs
+library-docs-view: library-docs
 	@ bash -c 'open docs/build/html/index.html'
 
 api-docs:
@@ -139,4 +163,4 @@ api-docs:
 api-docs-view: api-docs
 	@ bash -c "open frontend/public/api_docs/index.html"
 
-docs: api-docs sphinx-docs
+docs: api-docs library-docs

augur/application.py

@@ -5,9 +5,8 @@
 
 import os
 import time
-import argparse
-import multiprocessing as mp
 import logging
+import multiprocessing as mp
 import json
 import pkgutil
 from beaker.cache import CacheManager
@@ -18,42 +17,106 @@
 from augur import logger
 from augur.metrics import MetricDefinitions
 import augur.plugins
-import logging
 
 logging.basicConfig(filename='test.log', level=logging.INFO)
 
 class Application(object):
     """Initalizes all classes from Augur using a config file or environment variables"""
 
-    def __init__(self, config_file='augur.config.json', no_config_file=0, description='Augur application', config=None):
+    def __init__(self, default_config_file_path='augur.config.json', no_config_file=0, config=None):
         """
         Reads config, creates DB session, and initializes cache
         """
         # Open the config file
+        self.__config_file_name = 'augur.config.json'
         self.__already_exported = {}
-        self.__default_config = { 'Plugins': [] }
+        self.__default_config = {
+            "Cache": {
+                "config": {
+                    "cache.data_dir": "runtime/cache/",
+                    "cache.lock_dir": "runtime/cache/",
+                    "cache.type": "file"
+                }
+            },
+            "Database": {
+                "connection_string": "sqlite:///:memory:",
+                "database": "augur",
+                "host": "localhost",
+                "key": "key",
+                "password": "password",
+                "port": 5432,
+                "schema": "augur_data",
+                "user": "augur"
+            },
+            "Development": {
+                "developer": "0",
+                "interactive": "0"
+            },
+            "Facade": {
+                "check_updates": 1,
+                "clone_repos": 1,
+                "create_xlsx_summary_files": 1,
+                "delete_marked_repos": 0,
+                "fix_affiliations": 1,
+                "force_analysis": 1,
+                "force_invalidate_caches": 1,
+                "force_updates": 1,
+                "limited_run": 0,
+                "multithreaded": 0,
+                "nuke_stored_affiliations": 0,
+                "pull_repos": 1,
+                "rebuild_caches": 1,
+                "run_analysis": 1
+            },
+            "Housekeeper": {
+                "jobs": []
+            },
+            "Plugins": [],
+            "Server": {
+                "cache_expire": "3600",
+                "host": "0.0.0.0",
+                "port": "5000",
+                "workers": "1"
+            },
+            "Workers": {}
+         }
+
+        _root_augur_dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+        _possible_config_paths = [self.__config_file_name, _root_augur_dir_path + "/" + self.__config_file_name, f"/opt/augur/{self.__config_file_name}"]
+        _config_file_path = default_config_file_path
+
+        for location in _possible_config_paths:
+            try:
+                f = open(location, "r+")
+                _config_file_path = os.path.abspath(location)
+                f.close()
+                logging.info("Using config file at " + os.path.abspath(location))
+                break
+            except FileNotFoundError:
+                pass
+
         self.__using_config_file = True
         self.__config_bad = False
-        self.__config_file_path = os.path.abspath(os.getenv('AUGUR_CONFIG_FILE', config_file))
+        self.__config_file_path = os.path.abspath(os.getenv('AUGUR_CONFIG_FILE', _config_file_path))
         self.__config_location = os.path.dirname(self.__config_file_path)
         self.__runtime_location = 'runtime/'
         self.__export_env = os.getenv('AUGUR_ENV_EXPORT', '0') == '1'
         self.__shell_config = None
+
         if os.getenv('AUGUR_ENV_ONLY', '0') != '1' and no_config_file == 0:
             try:
                 self.__config_file = open(self.__config_file_path, 'r+')
             except:
-                logger.info('Couldn\'t open {}, attempting to create. If you have a augur.cfg, you can convert it to a json file using "make to-json"'.format(config_file))
+                logger.info('Couldn\'t open {}, attempting to create.'.format(self.__config_file_name))
                 if not os.path.exists(self.__config_location):
                     os.makedirs(self.__config_location)
                 self.__config_file = open(self.__config_file_path, 'w+')
                 self.__config_bad = True
-            # Options to export the loaded configuration as environment variables for Docker
-
+
             if self.__export_env:
-                export_filename = os.getenv('AUGUR_ENV_EXPORT_FILE', 'augur.cfg.sh')
+                export_filename = os.getenv('AUGUR_ENV_EXPORT_FILE', 'augur.config.json.sh')
                 self.__export_file = open(export_filename, 'w+')
-                logger.info('Exporting {} to environment variable export statements in {}'.format(config_file, export_filename))
+                logger.info('Exporting {} to environment variable export statements in {}'.format(self.__config_file_name, export_filename))
                 self.__export_file.write('#!/bin/bash\n')
 
             # Load the config file
@@ -93,28 +156,15 @@ def __init__(self, config_file='augur.config.json', no_config_file=0, descriptio
         cache_parsed = parse_cache_config_options(cache_config)
         self.cache = CacheManager(**cache_parsed)
 
-        # Create DB Session
-        self.db = None
-        self.session = None
-        db_str = self.read_config('Database', 'connection_string', 'AUGUR_DATABASE', 'sqlite:///:memory:')
-        self.db = create_engine(db_str)
-        self.__Session = scoped_session(sessionmaker(bind=self.db))
-        self.session = self.__Session()
-        Base.query = self.__Session.query_property()
-
         self.metrics = MetricDefinitions(self)
 
-
         # # Initalize all objects to None
-        # self.__metrics_status = None
         self._loaded_plugins = {}
 
         # Application.default_plugins
         # for plugin_name in Application.default_plugins:
         #     self[plugin_name]
 
-
-
     def __getitem__(self, plugin_name):
         """
         Returns plugin matching the name of the parameter 'plugin_name'

augur/cli/db.py

@@ -4,7 +4,6 @@
 import pandas as pd
 from sqlalchemy import exc
 
-from augur.application import Application
 from augur.runtime import pass_application
 
 @click.group('db', short_help='Database utilities')