Merge pull request #924 from chaoss/dev

Release v0.13.1

.travis.yml

@@ -41,7 +41,8 @@ env:
 before_install:
   - docker run -d -p 5432:5432 --name augur_test_database augurlabs/augur:test_data@sha256:fd2d9a178a9fee7cd548bd40a16e08d4611be22892491e817aafd53502f74cd0
 install:
-  - pip install .[dev]
+  - ./scripts/install/backend.sh
+  - ./scripts/install/workers.sh
   - augur configure generate
 
 script:

Makefile

@@ -30,8 +30,8 @@ default:
 #
 #  Installation
 #
-.PHONY: install install-dev 
-.PHONY: install-spdx install-spdx-sudo install-augur-sbom 
+.PHONY: install install-dev
+.PHONY: install-spdx install-spdx-sudo install-augur-sbom
 .PHONY: clean rebuild
 install:
 	@ ./scripts/install/install.sh prod
@@ -63,12 +63,12 @@ rebuild-dev:
 #
 .PHONY: dev-start dev-stop dev monitor-frontend monitor-backend monitor frontend backend-stop backend-start backend-restart backend clean rebuild
 
-dev-start: 
+dev-start:
 	@ scripts/control/start_augur.sh
 	@ scripts/control/start_frontend.sh
 
-dev-stop: 
-	@ augur util kill
+dev-stop:
+	@ augur util stop
 	@ scripts/control/kill_frontend.sh
 
 dev: dev-stop dev-start
@@ -100,7 +100,7 @@ test-python-versions:
 #
 # Documentation
 #
-.PHONY: docs docs-view 
+.PHONY: docs docs-view
 docs:
 	@ bash -c 'cd docs/ && rm -rf build/ && make html;'
 
@@ -112,7 +112,7 @@ docs-view: docs
 # Docker Shortcuts
 # Do not use these unless you know what they mean.
 .PHONY: compose-run compose-run-database
-.PHONY: build-backend run-backend build-frontend run-frontend build-database run-database 
+.PHONY: build-backend run-backend build-frontend run-frontend build-database run-database
 
 
 compose-run:

README.md

@@ -10,7 +10,7 @@ master | [![Build Status](https://travis-ci.org/chaoss/augur.svg?branch=master)]
 ## What is Augur?
 
 Augur is a tool for collecting and measuring structured data
-about free (https://www.fsf.org/about/) and [open source](https://opensource.org/docs/osd) (FOSS) communities. 
+about [free](https://www.fsf.org/about/) and [open source](https://opensource.org/docs/osd) (FOSS) communities. 
 
 We gather trace data for a group of repositories, normalize
 it into our data model, and provide a variety of metrics about said
@@ -20,8 +20,7 @@ questions about the way these communities evolve.
 
 We are a [CHAOSS](https://chaoss.community>) project, and many of our
 metrics are implementations of the metrics defined by our awesome community. You
-can find more information about how to get involved
-(here)[https://chaoss.community/participate/].
+can find more information about [how to get involved on the CHAOSS website](https://chaoss.community/participate/).
 
 ## Collecting Data
 
@@ -37,7 +36,7 @@ This data is collected by dedicated data collection workers controlled by Augur,
 
 ## Getting Started
 
-If you're interested in collecting data with our tool, the Augur team has worked hard to develop a detailed guide to getting started with our project, which can be found [here](https://oss-augur.readthedocs.io/en/master/getting-started/toc.html) alongside our main documentation.
+If you're interested in collecting data with our tool, the Augur team has worked hard to develop a detailed guide to getting started with our project, which can be found [in our documentation](https://oss-augur.readthedocs.io/en/master/getting-started/toc.html).
 
 If you're looking to contribute to Augur's code, you can find installation instructions, development guides, architecture references (coming soon), best practices and more in our [developer documentation](https://oss-augur.readthedocs.io/en/master/development-guide/toc.html).
 

augur/cli/configure.py

@@ -7,15 +7,15 @@
 import click
 import json
 import logging
+from pathlib import Path
 
-from augur.config import default_config, ENVVAR_PREFIX
+from augur.config import default_config, ENVVAR_PREFIX, CONFIG_HOME
 from augur.cli import initialize_logging
 from augur.logging import ROOT_AUGUR_DIRECTORY
 
 logger = logging.getLogger(__name__)
 ENVVAR_PREFIX = "AUGUR_"
 
-
 @click.group('configure', short_help='Generate an augur.config.json')
 def cli():
     pass
@@ -30,14 +30,16 @@ def cli():
 @click.option('--facade_repo_directory', help="Directory on the database server where Facade should clone repos", envvar=ENVVAR_PREFIX + 'FACADE_REPO_DIRECTORY')
 @click.option('--rc-config-file', help="File containing existing config whose values will be used as the defaults", type=click.Path(exists=True))
 @click.option('--gitlab_api_key', help="GitLab API key for data collection from the GitLab API", envvar=ENVVAR_PREFIX + 'GITLAB_API_KEY')
+@click.option('--write-to-src', is_flag=True, help="Write generated config file to the source code tree instead of default (for development use only)")
 @initialize_logging
-def generate(db_name, db_host, db_user, db_port, db_password, github_api_key, facade_repo_directory, rc_config_file, gitlab_api_key):
+def generate(db_name, db_host, db_user, db_port, db_password, github_api_key, facade_repo_directory, rc_config_file, gitlab_api_key, write_to_src=False):
     """
     Generate an augur.config.json
     """
 
     config = default_config
     rc_config = None
+    Path(CONFIG_HOME).mkdir(exist_ok=True)
 
     if rc_config_file != None:
         try:
@@ -82,9 +84,13 @@ def generate(db_name, db_host, db_user, db_port, db_password, github_api_key, fa
     if facade_repo_directory is not None:
         config['Workers']['facade_worker']['repo_directory'] = facade_repo_directory
 
+    config_path = CONFIG_HOME + '/augur.config.json'
+    if write_to_src is True:
+        config_path = ROOT_AUGUR_DIRECTORY + '/augur.config.json'
+
     try:
-        with open(os.path.abspath(ROOT_AUGUR_DIRECTORY + '/augur.config.json'), 'w') as f:
+        with open(os.path.abspath(config_path), 'w') as f:
             json.dump(config, f, indent=4)
-            logger.info('augur.config.json successfully created')
+            logger.info('Config written to ' + config_path)
     except Exception as e:
         logger.error("Error writing augur.config.json " + str(e))

augur/cli/db.py

@@ -11,6 +11,7 @@
 import click
 import sqlalchemy as s
 import pandas as pd
+import requests
 from sqlalchemy import exc
 
 from augur.cli import pass_config, pass_application
@@ -32,8 +33,8 @@ def add_repos(augur_app, filename):
     repo_group_IDs = [group[0] for group in df.fetchall()]
 
     insertSQL = s.sql.text("""
-        INSERT INTO augur_data.repo(repo_group_id, repo_git, repo_status, 
-        tool_source, tool_version, data_source, data_collection_date) 
+        INSERT INTO augur_data.repo(repo_group_id, repo_git, repo_status,
+        tool_source, tool_version, data_source, data_collection_date)
         VALUES (:repo_group_id, :repo_git, 'New', 'CLI', 1.0, 'Git', CURRENT_TIMESTAMP)
     """)
 
@@ -81,6 +82,46 @@ def add_repo_groups(augur_app, filename):
             else:
                 logger.info(f"Repo group with ID {row[1]} for repo group {row[1]} already exists, skipping...")
 
+@cli.command('add-github-org')
+@click.argument('organization_name')
+@pass_application
+def add_github_org(augur_app, organization_name):
+    """
+    Create new repo groups in Augur's database
+    """
+    org_query_response = requests.get(f"https://api.github.com/orgs/{organization_name}").json()
+    if "login" in org_query_response:
+        logger.info(f"Organization \"{organization_name}\" found")
+    else:
+        logger.fatal(f"No organization with name {organization_name} could be found")
+        exit(1)
+
+    all_repos = []
+    page = 1
+    repo_query_response = None
+    headers = {'Authorization': 'token %s' % augur_app.config.get_value("Database", "key")}
+    while repo_query_response != []:
+        repo_query_response = requests.get(org_query_response['repos_url'] + f"?per_page=100&page={page}", headers=headers).json()
+        for repo in repo_query_response:
+            all_repos.append(repo)
+        page+=1
+
+    insert_repo_group_sql = s.sql.text("""
+    INSERT INTO "augur_data"."repo_groups"("rg_name", "rg_description", "rg_website", "rg_recache", "rg_last_modified", "rg_type", "tool_source", "tool_version", "data_source", "data_collection_date") VALUES (:repo_group_name, '', '', 0, CURRENT_TIMESTAMP, 'Unknown', 'Loaded by user', '1.0', 'Git', CURRENT_TIMESTAMP) RETURNING repo_group_id;
+    """)
+    new_repo_group_id = augur_app.database.execute(insert_repo_group_sql, repo_group_name=organization_name).fetchone()[0]
+
+    insert_repo_sql = s.sql.text("""
+        INSERT INTO augur_data.repo(repo_group_id, repo_git, repo_status,
+        tool_source, tool_version, data_source, data_collection_date)
+        VALUES (:repo_group_id, :repo_git, 'New', 'CLI', 1.0, 'Git', CURRENT_TIMESTAMP)
+    """)
+    logger.info(f"{organization_name} repo group created")
+
+    for repo in all_repos:
+        logger.info(f"Adding {organization_name}/{repo['name']} ({repo['clone_url']})")
+        result = augur_app.database.execute(insert_repo_sql, repo_group_id=new_repo_group_id, repo_git=repo['clone_url'])
+
 @cli.command('update-repo-directory')
 @click.argument('repo_directory')
 @pass_application
@@ -240,7 +281,7 @@ def check_pgpass(config):
 @click.option('--port', default='5432')
 def init_database(default_db_name, default_user, default_password, target_db_name, target_user, target_password, host, port):
     """
-    Create database with the given credentials using the given maintenance database 
+    Create database with the given credentials using the given maintenance database
     """
     config = {
         'Database': {

augur/cli/run.py

@@ -11,7 +11,7 @@
 
 from augur.housekeeper import Housekeeper
 from augur.server import Server
-from augur.cli.util import kill_processes
+from augur.cli.util import stop_processes
 from augur.application import Application
 
 logger = logging.getLogger("augur")
@@ -25,9 +25,10 @@ def cli(disable_housekeeper, skip_cleanup):
     """
     augur_app = Application()
     logger.info("Augur application initialized")
+    logger.info(f"Using config file: {augur_app.config.config_file_location}")
     if not skip_cleanup:
         logger.debug("Cleaning up old Augur processes...")
-        kill_processes()
+        stop_processes()
         time.sleep(2)
     else:
         logger.debug("Skipping process cleanup")
@@ -98,7 +99,6 @@ def exit(augur_app, worker_processes, master):
         if master is not None:
             logger.debug("Shutting down Gunicorn server")
             master.halt()
-            master = None
 
         logger.info("Shutdown complete")
         sys.exit(0)

augur/cli/util.py

@@ -7,7 +7,6 @@
 import signal
 import logging
 from subprocess import call, run
-import time
 
 import psutil
 import click
@@ -43,76 +42,54 @@ def export_env(config):
     export_file.close()
     env_file.close()
 
-@cli.command('kill')
-@initialize_logging
-def cli_kill_processes():
-    """
-    Terminates all currently running backend Augur processes, including any workers. Will only work in a virtual environment.    
-    """
+def _stop_processes_handler(attach_handler=False):
+    if attach_handler is True:
+        _logger = logging.getLogger("augur")
+    else:
+        _logger = logger
     processes = get_augur_processes()
     if processes != []:
         for process in processes:
             if process.pid != os.getpid():
-                logger.info(f"Terminating process {process.pid}")
+                logger.info(f"Stopping process {process.pid}")
                 try:
                     process.send_signal(signal.SIGTERM)
-                    logger.info(f"sending SIGTERM Signal to {process.pid}")
                 except psutil.NoSuchProcess as e:
                     pass
 
-            logger.info(f"Waiting to check if processes terminated.")
+@cli.command('stop')
+@initialize_logging
+def cli_stop_processes():
+    """
+    Terminates all currently running backend Augur processes, including any workers. Will only work in a virtual environment.
+    """
+    _stop_processes_handler()
 
-    time.sleep(15)
-    logger.info(f"Checking on process termination.")
 
-    processes = get_augur_processes()
-
-    if processes != []:
-        for process in processes:
-
-            if process.pid != os.getpid():
-                logger.info(f"Killing process {process.pid}")
-                try:
-                    process.send_signal(signal.SIGKILL)
-                    logger.info(f"sending SIGKILL Signal to {process.pid}")
-                except psutil.NoSuchProcess as e:
-                    pass
+def stop_processes():
+    _stop_processes_handler(attach_handler=True)
 
+@cli.command('kill')
+@initialize_logging
 def kill_processes():
-    logger = logging.getLogger("augur")
-    processes = get_augur_processes()
-    if processes != []:
-        for process in processes:
-            if process.pid != os.getpid():
-                logger.info(f"Terminating process {process.pid}")
-                try:
-                    process.send_signal(signal.SIGTERM)
-                    logger.info(f"sending SIGTERM Signal to {process.pid}")
-                except psutil.NoSuchProcess as e:
-                    logger.warning(e)
-            logger.info(f"Waiting to check if processes terminated.")
-
-    time.sleep(15)
-    logger.info(f"Checking on process termination.")
-
+    """
+    Terminates all currently running backend Augur processes, including any workers. Will only work in a virtual environment.
+    """
     processes = get_augur_processes()
-
     if processes != []:
         for process in processes:
             if process.pid != os.getpid():
-                logger.info(f"Killing process {process.pid}")
                 logger.info(f"Killing process {process.pid}")
                 try:
                     process.send_signal(signal.SIGKILL)
-                    logger.info(f"sending SIGKILL Signal to {process.pid}")
                 except psutil.NoSuchProcess as e:
                     pass
 
 @cli.command('list',)
 @initialize_logging
 def list_processes():
     """
-    Outputs the name and process ID (PID) of all currently running backend Augur processes, including any workers. Will only work in a virtual environment.    
+    Outputs the name and process ID (PID) of all currently running backend Augur processes, including any workers. Will only work in a virtual environment.
     """
     processes = get_augur_processes()
     for process in processes: