Frontend fixes

README.md

@@ -1,21 +1,21 @@
 
-# Augur NEW Release v0.44.0
+# Augur NEW Release v0.44.2
 
 [![first-timers-only](https://img.shields.io/badge/first--timers--only-friendly-blue.svg?style=flat-square)](https://www.firsttimersonly.com/) We follow the [First Timers Only](https://www.firsttimersonly.com/) philosophy of tagging issues for first timers only, and walking one newcomer through the resolution process weekly. [You can find these issues tagged with "first timers only" on our issues list.](https://github.com/chaoss/augur/labels/first-timers-only).
 
 [![standard-readme compliant](https://img.shields.io/badge/standard--readme-OK-green.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme) [![Build Docker images](https://github.com/chaoss/augur/actions/workflows/build_docker.yml/badge.svg)](https://github.com/chaoss/augur/actions/workflows/build_docker.yml) [![Hits-of-Code](https://hitsofcode.com/github/chaoss/augur?branch=main)](https://hitsofcode.com/github/chaoss/augur/view?branch=main) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/2788/badge)](https://bestpractices.coreinfrastructure.org/projects/2788)
 
 ## NEW RELEASE ALERT!
-[If you want to jump right in, updated docker build/compose and bare metal installation instructions are available here](docs/new-install.md)
+### [If you want to jump right in, updated docker build/compose and bare metal installation instructions are available here](docs/new-install.md)
 
 
-Augur is now releasing a dramatically improved new version to the main branch. It is also available here: https://github.com/chaoss/augur/releases/tag/v0.44.0
+Augur is now releasing a dramatically improved new version to the main branch. It is also available here: https://github.com/chaoss/augur/releases/tag/v0.44.2
 - The `main` branch is a stable version of our new architecture, which features:
-  - Dramatic improvement in the speed of large scale data collection (10,000+ repos). All data is obtained for 10k+ repos within a week
+  - Dramatic improvement in the speed of large scale data collection (100,000+ repos). All data is obtained for 100k+ repos within 2 weeks.
   - A new job management architecture that uses Celery and Redis to manage queues, and enables users to run a Flower job monitoring dashboard
   - Materialized views to increase the snappiness of API’s and Frontends on large scale data
   - Changes to primary keys, which now employ a UUID strategy that ensures unique keys across all Augur instances
-  - Support for https://github.com/chaoss/sandiego-rh dashboards (view a sample here: https://eightknot.osci.io/). (beautification coming soon!)
+  - Support for https://github.com/oss-aspen/8knot dashboards (view a sample here: https://eightknot.osci.io/). (beautification coming soon!)
   - Data collection completeness assurance enabled by a structured, relational data set that is easily compared with platform API Endpoints
 - The next release of the new version will include a hosted version of Augur where anyone can create an account and add repos “they care about”. If the hosted instance already has a requested organization or repository it will be added to a user’s view. If its a new repository or organization, the user will be notified that collection will take (time required for the scale of repositories added). 
 

augur/api/routes/pull_request_reports.py

@@ -1345,10 +1345,8 @@ def mean_days_between_PR_comments():
 
     plot_width = 950
     p1 = figure(x_axis_type="datetime",
-                title="{}: Mean {} Between Comments by Month Closed for {} Pull Requests".format(repo_dict[repo_id],
-                                                                                                    time_unit,
-                                                                                                    description),
-                plot_width=plot_width, x_range=(pr_all[x_axis].min(), pr_all[x_axis].max()), plot_height=500,
+                title="{}: Mean {} Between Comments by Month Closed for {} Pull Requests".format(repo_dict[repo_id], time_unit, description),
+                plot_width=plot_width, x_range=(data_dict["All"][x_axis].min(), data_dict["All"][x_axis].max()), plot_height=500,
                 toolbar_location=None)
     colors = Category20[10][6:]
     color_index = 0
@@ -1379,11 +1377,9 @@ def mean_days_between_PR_comments():
             possible_maximums.append(
                 max(driver_df_mean.loc[driver_df_mean[line_group] == line_group_value][y_axis].dropna()))
     for repo, num_outliers in num_outliers_repo_map.items():
-        # FIXME repo_name is not defined
-        if repo_name == repo:
-            p1.add_layout(
-                Title(text="** {} outliers for {} were removed".format(num_outliers, repo), align="center"),
-                "below")
+        p1.add_layout(
+            Title(text="** {} outliers for {} were removed".format(num_outliers, repo), align="center"),
+            "below")
 
     p1.grid.grid_line_alpha = 0.3
     p1.xaxis.axis_label = 'Month Closed'

augur/api/view/init.py

@@ -1,6 +1,7 @@
 from pathlib import Path
 from .server import Environment
-import logging, sqlite3, secrets, hashlib, yaml
+from augur.application.logs import AugurLogger
+import logging, secrets, yaml
 
 env = Environment()
 
@@ -52,7 +53,7 @@ def update_from(old):
 
         current_settings["version"] = version
         write_settings(current_settings)
-        logging.info(f"Configuration updated from {to_version_string(old)} to {to_version_string(version)}")
+        logger.info(f"Configuration updated from {to_version_string(old)} to {to_version_string(version)}")
 
     def compare_versions(old, new):
         if old["major"] < new["major"]:
@@ -141,7 +142,5 @@ def compare_versions(old, new):
 
 # Initialize logging
 def init_logging():
-    format = "%(asctime)s: %(message)s"
     global logger
-    logger = logging.getLogger("augur view")
-    logger.setLevel("DEBUG")
+    logger = AugurLogger("augur_view", reset_logfiles=True).get_logger()

augur/api/view/utils.py

@@ -1,10 +1,18 @@
 from pathlib import Path
 from concurrent.futures import ThreadPoolExecutor
-from flask import render_template, flash, url_for
+from flask import render_template, flash, url_for, Flask
 from .init import *
 from .server import User
+from ..server import app, db_session
+from augur.application.config import AugurConfig
 import urllib.request, urllib.error, json, os, math, yaml, urllib3, time, logging, re
 
+init_logging()
+
+from .init import logger
+
+config = AugurConfig(logger, db_session)
+
 def parse_url(url):
     from urllib.parse import urlparse
 
@@ -47,7 +55,7 @@ def is_status_ok():
                 if "status" in response:
                     return request.url
         except Exception as e:
-            logging.error(f"Error during serving URL verification: {str(e)}")
+            logger.error(f"Error during serving URL verification: {str(e)}")
 
         return False
 
@@ -88,9 +96,9 @@ def loadSettings():
     if not configFilePath.is_file():
         init_settings()
         with open(configFile, 'w') as file:
-            logging.info(f"Generating default configuration file: {configFile}")
+            logger.info(f"Generating default configuration file: {configFile}")
             yaml.dump(settings, file)
-            logging.info("Default configuration file successfully generated.")
+            logger.info("Default configuration file successfully generated.")
     else:
         with open(configFilePath) as file:
             settings = yaml.load(file, Loader=yaml.FullLoader)
@@ -103,7 +111,7 @@ def loadSettings():
         else:
             try:
                 cachePath.mkdir(parents=True)
-                logging.info("cache directory initialized")
+                logger.info("cache directory initialized")
             except Exception as err:
                 raise Exception(f"Cannot initialize caching: could not create cache directory [{cachePath}]")
 
@@ -120,17 +128,16 @@ def loadSettings():
 
 """ ----------------------------------------------------------------
 """
-def getSetting(key):
-    if key == "serving":
-        return "http://127.0.0.1:5000/api/unstable"
-    return settings[key]
-
-init_logging()
+def getSetting(key, section = "View"):
+    if section == "View":
+        if key == "serving":
+            return "http://127.0.0.1:5000/api/unstable"
+        return settings[key]
+    else:
+        return config.get_value(section, key)
 
 loadSettings()
 
-from .init import logger
-
 User.api = getSetting("serving")
 User.logger = logger
 
@@ -149,16 +156,16 @@ def loadReports():
                     image['id'] = id = id + 1
         return True
     except Exception as err:
-        logging.error(f"An exception occurred reading reports endpoints from [{getSetting('reports')}]:")
-        logging.error(err)
+        logger.error(f"An exception occurred reading reports endpoints from [{getSetting('reports')}]:")
+        logger.error(err)
         try:
             with open(getSetting("reports"), 'w') as file:
-                logging.info("Attempting to generate default reports.yml")
+                logger.info("Attempting to generate default reports.yml")
                 yaml.dump(reports, file)
-                logging.info("Default reports file successfully generated.")
+                logger.info("Default reports file successfully generated.")
         except Exception as ioErr:
-            logging.error("Error creating default report configuration:")
-            logging.error(ioErr)
+            logger.error("Error creating default report configuration:")
+            logger.error(ioErr)
         return False
 
 if not loadReports():
@@ -176,11 +183,11 @@ def cacheFileExists(filename):
             if(cache_file_age > getSetting('cache_expiry')):
                 try:
                     cache_file.unlink()
-                    logging.info(f"Cache file {filename} removed due to expiry")
+                    logger.info(f"Cache file {filename} removed due to expiry")
                     return False
                 except Exception as e:
-                    logging.error("Error: cache file age exceeds expiry limit, but an exception occurred while attempting to remove")
-                    logging.error(e)
+                    logger.error("Error: cache file age exceeds expiry limit, but an exception occurred while attempting to remove")
+                    logger.error(e)
         return True
     else:
         return False
@@ -220,7 +227,7 @@ def toCacheURL(endpoint):
 def requestJson(endpoint, cached = True):
     filename = toCacheFilepath(endpoint)
     requestURL = getSetting('serving') + "/" + endpoint
-    logging.info(f'requesting json from: {endpoint}')
+    logger.info(f'requesting json from: {endpoint}')
     try:
         if cached and cacheFileExists(filename):
             with open(filename) as f:
@@ -239,8 +246,8 @@ def requestJson(endpoint, cached = True):
             cache_files_requested.remove(filename)
         return data
     except Exception as err:
-        logging.error("An exception occurred while fulfilling a json request")
-        logging.error(err)
+        logger.error("An exception occurred while fulfilling a json request")
+        logger.error(err)
         return False, str(err)
 
 """ ----------------------------------------------------------------
@@ -257,8 +264,8 @@ def requestPNG(endpoint):
             cache_files_requested.remove(filename)
         return toCacheURL(endpoint)
     except Exception as err:
-        logging.error("An exception occurred while fulfilling a png request")
-        logging.error(err)
+        logger.error("An exception occurred while fulfilling a png request")
+        logger.error(err)
 
 """ ----------------------------------------------------------------
 """
@@ -269,20 +276,26 @@ def download(url, cmanager, filename, image_cache, image_id, repo_id = None):
     if cacheFileExists(filename):
         image_cache[image_id]['exists'] = True
         return
-    response = cmanager.request('GET', url)
+    try:
+        response = cmanager.request('GET', url)
+    except Exception as e:
+        logger.error("Could not make request: " + str(e))
+        raise e
+
     if "json" in response.headers['Content-Type']:
-        logging.warn(f"repo {repo_id}: unexpected json response in image request")
-        logging.warn(f"  response: {response.data.decode('utf-8')}")
+        logger.warn(f"repo {repo_id}: unexpected json response in image request")
+        logger.warn(f"  response: {response.data.decode('utf-8')}")
         image_cache[image_id]['exists'] = False
         return
     if response and response.status == 200:
         image_cache[image_id]['exists'] = True
         try:
             with open(filename, 'wb') as f:
+                logger.info("Writing image: " + filename)
                 f.write(response.data)
         except Exception as err:
-            logging.error("An exception occurred writing a cache file to disk")
-            logging.error(err)
+            logger.error("An exception occurred writing a cache file to disk")
+            logger.error(err)
 
 """ ----------------------------------------------------------------
 """
@@ -295,6 +308,9 @@ def requestReports(repo_id):
     report_requests[repo_id] = {}
     report_requests[repo_id]['complete'] = False
 
+    host = getSetting("host", "Server")
+    port = getSetting("port", "Server")
+
     """ ----------
         If the report definition could not be loaded, we cannot determine what
         files to request from the backend to compose the report. Returning here
@@ -319,7 +335,7 @@ def requestReports(repo_id):
             # Where should the downloaded image be stored (in cache)
             filename = toCacheFilename(f"{image['url']}?repo_id={repo_id}")
             # Where are we downloading the image from
-            image_url = url_for(image['url'], repo_id = repo_id)
+            image_url = f"{host}:{port}" + url_for(image['url'], repo_id = repo_id)
             # f"{getSetting('serving')}/{image['url']}?repo_id={repo_id}"
 
             # Add a request for this image to the thread pool using the download function

augur/application/cli/backend.py

@@ -23,10 +23,12 @@
 from augur.tasks.init.redis_connection import redis_connection 
 from augur.application.db.models import Repo, CollectionStatus
 from augur.application.db.session import DatabaseSession
+from augur.application.db.util import execute_session_query
 from augur.application.logs import AugurLogger
 from augur.application.config import AugurConfig
 from augur.application.cli import test_connection, test_db_connection 
 import sqlalchemy as s
+from sqlalchemy import or_, and_
 
 
 logger = AugurLogger("augur", reset_logfiles=True).get_logger()
@@ -91,7 +93,7 @@ def start(disable_collection, development, port):
 
     time.sleep(3)
     logger.info('Gunicorn webserver started...')
-    logger.info(f'Augur is running at: http://127.0.0.1:{port}')
+    logger.info(f'Augur is running at: {"http" if development else "https"}://{host}:{port}')
 
     scheduling_worker_process = None
     core_worker_process = None
@@ -116,17 +118,24 @@ def start(disable_collection, development, port):
         create_collection_status(logger)
 
         with DatabaseSession(logger) as session:
-            collection_status_list = session.query(CollectionStatus).filter(CollectionStatus.core_status == CollectionState.COLLECTING.value
-                or CollectionStatus.secondary_status == CollectionState.COLLECTING.value)
+            primaryCollecting = CollectionStatus.core_status == CollectionState.COLLECTING.value
+            secondaryCollecting = CollectionStatus.secondary_status == CollectionState.COLLECTING.value
+
+            query = session.query(CollectionStatus).filter(or_(primaryCollecting,secondaryCollecting))
+
+            collection_status_list = execute_session_query(query,'all')
 
             for status in collection_status_list:
                 repo = status.repo
                 repo.repo_name = None
                 repo.repo_path = None
                 repo.repo_status = "New"
+
+                status.core_status = "Pending"
+                status.secondary_status = "Pending"
 
-            collection_status_list.update({CollectionStatus.core_status: "Pending"})
-            collection_status_list.update({CollectionStatus.secondary_status: "Pending"})
+            #collection_status_list.update({CollectionStatus.core_status: "Pending"})
+            #collection_status_list.update({CollectionStatus.secondary_status: "Pending"})
             session.commit()
 
         augur_collection_monitor.si().apply_async()

augur/application/db/models/augur_data.py

@@ -858,7 +858,7 @@ def get_by_repo_git(session, repo_git):
         return session.query(Repo).filter(Repo.repo_git == repo_git).first()
 
     @staticmethod
-    def is_valid_github_repo(session, url: str) -> bool:
+    def is_valid_github_repo(gh_session, url: str) -> bool:
         """Determine whether repo url is valid.
 
         Args:
@@ -871,7 +871,7 @@ def is_valid_github_repo(session, url: str) -> bool:
 
         REPO_ENDPOINT = "https://api.github.com/repos/{}/{}"
 
-        if not session.oauths.list_of_keys:
+        if not gh_session.oauths.list_of_keys:
             return False, {"status": "No valid github api keys to retrieve data with"}
 
         owner, repo = Repo.parse_github_repo_url(url)
@@ -882,7 +882,7 @@ def is_valid_github_repo(session, url: str) -> bool:
 
         attempts = 0
         while attempts < 10:
-            result = hit_api(session.oauths, url, logger)
+            result = hit_api(gh_session.oauths, url, logger)
 
             # if result is None try again
             if not result: