From 607f4f23e2a8f18f801b74ac85247f8c12d0e102 Mon Sep 17 00:00:00 2001 From: Michael Scherer Date: Fri, 27 Jan 2023 14:43:21 +0100 Subject: [PATCH 01/20] Install only the required nltk data Signed-off-by: Michael Scherer --- scripts/docker/install-workers-deps.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/docker/install-workers-deps.sh b/scripts/docker/install-workers-deps.sh index cdd18a5c76..90d4d44e0e 100755 --- a/scripts/docker/install-workers-deps.sh +++ b/scripts/docker/install-workers-deps.sh @@ -10,4 +10,7 @@ do done # install nltk -/opt/venv/bin/python -m nltk.downloader all +# taken from ./scripts/install/nltk_dictionaries.sh +for i in stopwords punkt popular universal_tagset ; do + /opt/venv/bin/python -m nltk.downloader $i +done From fe298cad81e18508416cadc4a00d707b6eeaaedd Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Mon, 13 Feb 2023 20:41:37 -0600 Subject: [PATCH 02/20] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index eb9890d4c9..ccde98990a 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Augur is now releasing a dramatically improved new version to the main branch. I - A new job management architecture that uses Celery and Redis to manage queues, and enables users to run a Flower job monitoring dashboard - Materialized views to increase the snappiness of API’s and Frontends on large scale data - Changes to primary keys, which now employ a UUID strategy that ensures unique keys across all Augur instances - - Support for https://github.com/chaoss/sandiego-rh dashboards (view a sample here: https://eightknot.osci.io/). (beautification coming soon!) + - Support for https://github.com/oss-aspen/8knot dashboards (view a sample here: https://eightknot.osci.io/). (beautification coming soon!) - Data collection completeness assurance enabled by a structured, relational data set that is easily compared with platform API Endpoints - The next release of the new version will include a hosted version of Augur where anyone can create an account and add repos “they care about”. If the hosted instance already has a requested organization or repository it will be added to a user’s view. If its a new repository or organization, the user will be notified that collection will take (time required for the scale of repositories added). From adf1e7a465b118363fc4c96b47aac58f1e3c46a4 Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Mon, 13 Feb 2023 20:43:28 -0600 Subject: [PATCH 03/20] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ccde98990a..fe2a2c227c 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![standard-readme compliant](https://img.shields.io/badge/standard--readme-OK-green.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme) [![Build Docker images](https://github.com/chaoss/augur/actions/workflows/build_docker.yml/badge.svg)](https://github.com/chaoss/augur/actions/workflows/build_docker.yml) [![Hits-of-Code](https://hitsofcode.com/github/chaoss/augur?branch=main)](https://hitsofcode.com/github/chaoss/augur/view?branch=main) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/2788/badge)](https://bestpractices.coreinfrastructure.org/projects/2788) ## NEW RELEASE ALERT! -[If you want to jump right in, updated docker build/compose and bare metal installation instructions are available here](docs/new-install.md) +#### [If you want to jump right in, updated docker build/compose and bare metal installation instructions are available here](docs/new-install.md) Augur is now releasing a dramatically improved new version to the main branch. It is also available here: https://github.com/chaoss/augur/releases/tag/v0.44.0 From 655a66d8f84f182a161469a2adee53e21d1b9944 Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Tue, 14 Feb 2023 08:44:36 -0600 Subject: [PATCH 04/20] Update Dockerfile Version bump --- docker/backend/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/backend/Dockerfile b/docker/backend/Dockerfile index b536873d0e..3cf69afd23 100644 --- a/docker/backend/Dockerfile +++ b/docker/backend/Dockerfile @@ -2,7 +2,7 @@ FROM python:3.8.11-slim-buster LABEL maintainer="outdoors@acm.org" -LABEL version="0.42.0" +LABEL version="0.44.0" ENV DEBIAN_FRONTEND=noninteractive From e5b4c36080afff5890df9961f62e21e512227fbe Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Tue, 14 Feb 2023 08:44:57 -0600 Subject: [PATCH 05/20] Update Dockerfile Version Bump --- docker/database/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/database/Dockerfile b/docker/database/Dockerfile index a8a725a71a..6b0e64b253 100644 --- a/docker/database/Dockerfile +++ b/docker/database/Dockerfile @@ -2,7 +2,7 @@ FROM postgres:12 LABEL maintainer="outdoors@acm.org" -LABEL version="0.42.0" +LABEL version="0.44.0" ENV POSTGRES_DB "test" ENV POSTGRES_USER "augur" From 504b80964ee6ee8e663d44da945eea898517658d Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Tue, 14 Feb 2023 08:45:18 -0600 Subject: [PATCH 06/20] Update Dockerfile Version bump --- docker/frontend/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/frontend/Dockerfile b/docker/frontend/Dockerfile index eb29636940..06d5bc43de 100644 --- a/docker/frontend/Dockerfile +++ b/docker/frontend/Dockerfile @@ -2,7 +2,7 @@ FROM node:16 as build-stage LABEL maintainer="outdoors@acm.org" -LABEL version="0.42.0" +LABEL version="0.44.0" WORKDIR /augur/frontend/ COPY ./docker/frontend/docker.config.json frontend.config.json From 3b39574f81176e02c81f5d1f3b683ac75796f580 Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Tue, 14 Feb 2023 08:45:39 -0600 Subject: [PATCH 07/20] Update Dockerfile Version BUmp --- docker/augurface/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/augurface/Dockerfile b/docker/augurface/Dockerfile index d495808b8e..c99d089fa9 100644 --- a/docker/augurface/Dockerfile +++ b/docker/augurface/Dockerfile @@ -2,7 +2,7 @@ FROM node:16 as build-stage LABEL maintainer="outdoors@acm.org" -LABEL version="0.42.0" +LABEL version="0.44.0" WORKDIR /augur/frontend/ COPY ./docker/frontend/docker.config.json frontend.config.json From eb3eadf6ad67f6dd3c84577aa8f3cbdd9aecc9a5 Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Tue, 14 Feb 2023 09:34:15 -0600 Subject: [PATCH 08/20] Deprecated Facade `nuke_affiliations` and `fix_empty_affiliations` because they no longer are necessary, and they cause performance issues. --- augur/tasks/git/facade_tasks.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/augur/tasks/git/facade_tasks.py b/augur/tasks/git/facade_tasks.py index 1b1a90f6dd..99bdbc08c7 100644 --- a/augur/tasks/git/facade_tasks.py +++ b/augur/tasks/git/facade_tasks.py @@ -599,11 +599,21 @@ def generate_non_repo_domain_facade_tasks(logger): facade_sequence = [] if nuke_stored_affiliations: - facade_sequence.append(nuke_affiliations_facade_task.si().on_error(facade_error_handler.s()))#nuke_affiliations(session.cfg) + #facade_sequence.append(nuke_affiliations_facade_task.si().on_error(facade_error_handler.s()))#nuke_affiliations(session.cfg) + logger.info("Nuke stored affiliations is deprecated.") + # deprecated because the UI component of facade where affiliations would be + # nuked upon change no longer exists, and this information can easily be derived + # from queries and materialized views in the current version of Augur. + # This method is also a major performance bottleneck with little value. #session.logger.info(session.cfg) if not limited_run or (limited_run and fix_affiliations): - facade_sequence.append(fill_empty_affiliations_facade_task.si().on_error(facade_error_handler.s()))#fill_empty_affiliations(session) + #facade_sequence.append(fill_empty_affiliations_facade_task.si().on_error(facade_error_handler.s()))#fill_empty_affiliations(session) + logger.info("Fill empty affiliations is deprecated.") + # deprecated because the UI component of facade where affiliations would need + # to be fixed upon change no longer exists, and this information can easily be derived + # from queries and materialized views in the current version of Augur. + # This method is also a major performance bottleneck with little value. if force_invalidate_caches: facade_sequence.append(invalidate_caches_facade_task.si().on_error(facade_error_handler.s()))#invalidate_caches(session.cfg) From f895b032f7a7ccfeeb124b0b04cfbe6f9c8fb69c Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Tue, 14 Feb 2023 09:37:55 -0600 Subject: [PATCH 09/20] Update Facade Version. --- augur/tasks/git/facade_tasks.py | 2 ++ augur/tasks/git/util/facade_worker/facade_worker/__init__.py | 2 +- .../git/util/facade_worker/facade_worker/facade01config.py | 2 +- augur/tasks/git/util/facade_worker/setup.py | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/augur/tasks/git/facade_tasks.py b/augur/tasks/git/facade_tasks.py index 99bdbc08c7..49c3677e79 100644 --- a/augur/tasks/git/facade_tasks.py +++ b/augur/tasks/git/facade_tasks.py @@ -1,3 +1,5 @@ +#SPDX-License-Identifier: MIT + import sys import json import time diff --git a/augur/tasks/git/util/facade_worker/facade_worker/__init__.py b/augur/tasks/git/util/facade_worker/facade_worker/__init__.py index 697b69d329..1753083d3f 100644 --- a/augur/tasks/git/util/facade_worker/facade_worker/__init__.py +++ b/augur/tasks/git/util/facade_worker/facade_worker/__init__.py @@ -1,6 +1,6 @@ #SPDX-License-Identifier: MIT """augur_worker_github - Augur Worker that collects GitHub data""" -__version__ = '1.2.4' +__version__ = '1.3.0' __author__ = 'Augur Team ' __all__ = [] diff --git a/augur/tasks/git/util/facade_worker/facade_worker/facade01config.py b/augur/tasks/git/util/facade_worker/facade_worker/facade01config.py index a405aadcca..352d30f1c3 100644 --- a/augur/tasks/git/util/facade_worker/facade_worker/facade01config.py +++ b/augur/tasks/git/util/facade_worker/facade_worker/facade01config.py @@ -305,7 +305,7 @@ def __init__(self, logger: Logger): sys.exit(1) self.tool_source = '\'Facade \'' - self.tool_version = '\'1.2.4\'' + self.tool_version = '\'1.3.0\'' self.data_source = '\'Git Log\'' self.worker_options = worker_options diff --git a/augur/tasks/git/util/facade_worker/setup.py b/augur/tasks/git/util/facade_worker/setup.py index 5bc4f99af1..298baff49d 100644 --- a/augur/tasks/git/util/facade_worker/setup.py +++ b/augur/tasks/git/util/facade_worker/setup.py @@ -14,7 +14,7 @@ def read(filename): setup( name="facade_worker", - version="1.2.4", + version="1.3.0", url="https://github.com/chaoss/augur", license='MIT', author="Augurlabs", From f899a6caeb126c860c9b3535871efb3a2083c55a Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Tue, 14 Feb 2023 11:19:43 -0600 Subject: [PATCH 10/20] Update README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index fe2a2c227c..121f5a6a2b 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,17 @@ -# Augur NEW Release v0.44.0 +# Augur NEW Release v0.44.1 [![first-timers-only](https://img.shields.io/badge/first--timers--only-friendly-blue.svg?style=flat-square)](https://www.firsttimersonly.com/) We follow the [First Timers Only](https://www.firsttimersonly.com/) philosophy of tagging issues for first timers only, and walking one newcomer through the resolution process weekly. [You can find these issues tagged with "first timers only" on our issues list.](https://github.com/chaoss/augur/labels/first-timers-only). [![standard-readme compliant](https://img.shields.io/badge/standard--readme-OK-green.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme) [![Build Docker images](https://github.com/chaoss/augur/actions/workflows/build_docker.yml/badge.svg)](https://github.com/chaoss/augur/actions/workflows/build_docker.yml) [![Hits-of-Code](https://hitsofcode.com/github/chaoss/augur?branch=main)](https://hitsofcode.com/github/chaoss/augur/view?branch=main) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/2788/badge)](https://bestpractices.coreinfrastructure.org/projects/2788) ## NEW RELEASE ALERT! -#### [If you want to jump right in, updated docker build/compose and bare metal installation instructions are available here](docs/new-install.md) +### [If you want to jump right in, updated docker build/compose and bare metal installation instructions are available here](docs/new-install.md) -Augur is now releasing a dramatically improved new version to the main branch. It is also available here: https://github.com/chaoss/augur/releases/tag/v0.44.0 +Augur is now releasing a dramatically improved new version to the main branch. It is also available here: https://github.com/chaoss/augur/releases/tag/v0.44.1 - The `main` branch is a stable version of our new architecture, which features: - - Dramatic improvement in the speed of large scale data collection (10,000+ repos). All data is obtained for 10k+ repos within a week + - Dramatic improvement in the speed of large scale data collection (100,000+ repos). All data is obtained for 100k+ repos within 2 weeks. - A new job management architecture that uses Celery and Redis to manage queues, and enables users to run a Flower job monitoring dashboard - Materialized views to increase the snappiness of API’s and Frontends on large scale data - Changes to primary keys, which now employ a UUID strategy that ensures unique keys across all Augur instances From bcc71d62bd85ab8db45f7f6a97e80b8015861bfb Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Tue, 14 Feb 2023 11:20:11 -0600 Subject: [PATCH 11/20] Update metadata.py --- metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metadata.py b/metadata.py index 525dffaad1..75d5c8fa8b 100644 --- a/metadata.py +++ b/metadata.py @@ -5,8 +5,8 @@ __short_description__ = "Python 3 package for free/libre and open-source software community metrics, models & data collection" -__version__ = "0.44.0" -__release__ = "v0.44.0 (Brussel Sprouts)" +__version__ = "0.44.1" +__release__ = "v0.44.1 (Brussel Sprouts)" __license__ = "MIT" __copyright__ = "University of Missouri, University of Nebraska-Omaha, CHAOSS, Brian Warner & Augurlabs 2023" From 4cd761c589a573e0c9b4b65da3f51064e9df2890 Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Tue, 14 Feb 2023 11:20:42 -0600 Subject: [PATCH 12/20] Update Dockerfile --- docker/augurface/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/augurface/Dockerfile b/docker/augurface/Dockerfile index c99d089fa9..ec09ac325b 100644 --- a/docker/augurface/Dockerfile +++ b/docker/augurface/Dockerfile @@ -2,7 +2,7 @@ FROM node:16 as build-stage LABEL maintainer="outdoors@acm.org" -LABEL version="0.44.0" +LABEL version="0.44.1" WORKDIR /augur/frontend/ COPY ./docker/frontend/docker.config.json frontend.config.json From 1e388afba5593f7adca58f7a4ab5d28268cb58e8 Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Tue, 14 Feb 2023 11:21:05 -0600 Subject: [PATCH 13/20] Update Dockerfile --- docker/backend/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/backend/Dockerfile b/docker/backend/Dockerfile index 3cf69afd23..8c8c35b993 100644 --- a/docker/backend/Dockerfile +++ b/docker/backend/Dockerfile @@ -2,7 +2,7 @@ FROM python:3.8.11-slim-buster LABEL maintainer="outdoors@acm.org" -LABEL version="0.44.0" +LABEL version="0.44.1" ENV DEBIAN_FRONTEND=noninteractive From d9404e58366c4b5f433e12d590677c61b8798070 Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Tue, 14 Feb 2023 11:21:22 -0600 Subject: [PATCH 14/20] Update Dockerfile --- docker/database/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/database/Dockerfile b/docker/database/Dockerfile index 6b0e64b253..b385f90454 100644 --- a/docker/database/Dockerfile +++ b/docker/database/Dockerfile @@ -2,7 +2,7 @@ FROM postgres:12 LABEL maintainer="outdoors@acm.org" -LABEL version="0.44.0" +LABEL version="0.44.1" ENV POSTGRES_DB "test" ENV POSTGRES_USER "augur" From 618aa6bebfc7362ef0a17e837e6fe3ea5f9573dc Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Tue, 14 Feb 2023 11:21:42 -0600 Subject: [PATCH 15/20] Update Dockerfile --- docker/frontend/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/frontend/Dockerfile b/docker/frontend/Dockerfile index 06d5bc43de..8959e23c92 100644 --- a/docker/frontend/Dockerfile +++ b/docker/frontend/Dockerfile @@ -2,7 +2,7 @@ FROM node:16 as build-stage LABEL maintainer="outdoors@acm.org" -LABEL version="0.44.0" +LABEL version="0.44.1" WORKDIR /augur/frontend/ COPY ./docker/frontend/docker.config.json frontend.config.json From bc89e1bcae49beed36b21374fea516c99c37b9de Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Wed, 15 Feb 2023 09:43:05 -0600 Subject: [PATCH 16/20] Release version bump. --- README.md | 4 ++-- docker/augurface/Dockerfile | 2 +- docker/backend/Dockerfile | 2 +- docker/database/Dockerfile | 2 +- docker/frontend/Dockerfile | 2 +- metadata.py | 4 ++-- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 121f5a6a2b..ab5342dac7 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -# Augur NEW Release v0.44.1 +# Augur NEW Release v0.44.2 [![first-timers-only](https://img.shields.io/badge/first--timers--only-friendly-blue.svg?style=flat-square)](https://www.firsttimersonly.com/) We follow the [First Timers Only](https://www.firsttimersonly.com/) philosophy of tagging issues for first timers only, and walking one newcomer through the resolution process weekly. [You can find these issues tagged with "first timers only" on our issues list.](https://github.com/chaoss/augur/labels/first-timers-only). @@ -9,7 +9,7 @@ ### [If you want to jump right in, updated docker build/compose and bare metal installation instructions are available here](docs/new-install.md) -Augur is now releasing a dramatically improved new version to the main branch. It is also available here: https://github.com/chaoss/augur/releases/tag/v0.44.1 +Augur is now releasing a dramatically improved new version to the main branch. It is also available here: https://github.com/chaoss/augur/releases/tag/v0.44.2 - The `main` branch is a stable version of our new architecture, which features: - Dramatic improvement in the speed of large scale data collection (100,000+ repos). All data is obtained for 100k+ repos within 2 weeks. - A new job management architecture that uses Celery and Redis to manage queues, and enables users to run a Flower job monitoring dashboard diff --git a/docker/augurface/Dockerfile b/docker/augurface/Dockerfile index ec09ac325b..c758d6c253 100644 --- a/docker/augurface/Dockerfile +++ b/docker/augurface/Dockerfile @@ -2,7 +2,7 @@ FROM node:16 as build-stage LABEL maintainer="outdoors@acm.org" -LABEL version="0.44.1" +LABEL version="0.44.2" WORKDIR /augur/frontend/ COPY ./docker/frontend/docker.config.json frontend.config.json diff --git a/docker/backend/Dockerfile b/docker/backend/Dockerfile index 8c8c35b993..ca569339bc 100644 --- a/docker/backend/Dockerfile +++ b/docker/backend/Dockerfile @@ -2,7 +2,7 @@ FROM python:3.8.11-slim-buster LABEL maintainer="outdoors@acm.org" -LABEL version="0.44.1" +LABEL version="0.44.2" ENV DEBIAN_FRONTEND=noninteractive diff --git a/docker/database/Dockerfile b/docker/database/Dockerfile index b385f90454..6cf9517cad 100644 --- a/docker/database/Dockerfile +++ b/docker/database/Dockerfile @@ -2,7 +2,7 @@ FROM postgres:12 LABEL maintainer="outdoors@acm.org" -LABEL version="0.44.1" +LABEL version="0.44.2" ENV POSTGRES_DB "test" ENV POSTGRES_USER "augur" diff --git a/docker/frontend/Dockerfile b/docker/frontend/Dockerfile index 8959e23c92..317e3a0cd2 100644 --- a/docker/frontend/Dockerfile +++ b/docker/frontend/Dockerfile @@ -2,7 +2,7 @@ FROM node:16 as build-stage LABEL maintainer="outdoors@acm.org" -LABEL version="0.44.1" +LABEL version="0.44.2" WORKDIR /augur/frontend/ COPY ./docker/frontend/docker.config.json frontend.config.json diff --git a/metadata.py b/metadata.py index 75d5c8fa8b..77f950c14b 100644 --- a/metadata.py +++ b/metadata.py @@ -5,8 +5,8 @@ __short_description__ = "Python 3 package for free/libre and open-source software community metrics, models & data collection" -__version__ = "0.44.1" -__release__ = "v0.44.1 (Brussel Sprouts)" +__version__ = "0.44.2" +__release__ = "v0.44.2 (Brussels Sprouts)" __license__ = "MIT" __copyright__ = "University of Missouri, University of Nebraska-Omaha, CHAOSS, Brian Warner & Augurlabs 2023" From b914ae3497a4919110485b7658ad423dbc5955d7 Mon Sep 17 00:00:00 2001 From: Andrew Brain <61482022+ABrain7710@users.noreply.github.com> Date: Fri, 17 Feb 2023 12:22:46 -0600 Subject: [PATCH 17/20] Fix add repo and org Signed-off-by: Andrew Brain <61482022+ABrain7710@users.noreply.github.com> --- augur/application/db/models/augur_data.py | 6 +++--- augur/application/db/models/augur_operations.py | 8 ++++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/augur/application/db/models/augur_data.py b/augur/application/db/models/augur_data.py index 5365dbf37f..e1234cda2d 100644 --- a/augur/application/db/models/augur_data.py +++ b/augur/application/db/models/augur_data.py @@ -858,7 +858,7 @@ def get_by_repo_git(session, repo_git): return session.query(Repo).filter(Repo.repo_git == repo_git).first() @staticmethod - def is_valid_github_repo(session, url: str) -> bool: + def is_valid_github_repo(gh_session, url: str) -> bool: """Determine whether repo url is valid. Args: @@ -871,7 +871,7 @@ def is_valid_github_repo(session, url: str) -> bool: REPO_ENDPOINT = "https://api.github.com/repos/{}/{}" - if not session.oauths.list_of_keys: + if not gh_session.oauths.list_of_keys: return False, {"status": "No valid github api keys to retrieve data with"} owner, repo = Repo.parse_github_repo_url(url) @@ -882,7 +882,7 @@ def is_valid_github_repo(session, url: str) -> bool: attempts = 0 while attempts < 10: - result = hit_api(session.oauths, url, logger) + result = hit_api(gh_session.oauths, url, logger) # if result is None try again if not result: diff --git a/augur/application/db/models/augur_operations.py b/augur/application/db/models/augur_operations.py index ab7c7d45cd..2a75e65a79 100644 --- a/augur/application/db/models/augur_operations.py +++ b/augur/application/db/models/augur_operations.py @@ -410,8 +410,10 @@ def remove_group(self, group_name): return result def add_repo(self, group_name, repo_url): + + from augur.tasks.github.util.github_task_session import GithubTaskSession - with DatabaseSession(logger) as session: + with GithubTaskSession(logger) as session: result = UserRepo.add(session, repo_url, self.user_id, group_name) return result @@ -425,7 +427,9 @@ def remove_repo(self, session, group_name, repo_id): def add_org(self, group_name, org_url): - with DatabaseSession(logger) as session: + from augur.tasks.github.util.github_task_session import GithubTaskSession + + with GithubTaskSession(logger) as session: result = UserRepo.add_org_repos(session, org_url, self.user_id, group_name) return result From 8143dfe6df88060f622eb84ecbbcbabda840f986 Mon Sep 17 00:00:00 2001 From: Andrew Brain Date: Fri, 17 Feb 2023 15:19:07 -0600 Subject: [PATCH 18/20] Populate db with frontend repo group Signed-off-by: Andrew Brain --- .../application/db/models/augur_operations.py | 12 +++++-- .../6_add_repo_group_for_frontend_repos.py | 36 +++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) create mode 100644 augur/application/schema/alembic/versions/6_add_repo_group_for_frontend_repos.py diff --git a/augur/application/db/models/augur_operations.py b/augur/application/db/models/augur_operations.py index 2a75e65a79..bba812c947 100644 --- a/augur/application/db/models/augur_operations.py +++ b/augur/application/db/models/augur_operations.py @@ -9,12 +9,14 @@ import logging import secrets +import importlib -from augur.application.db.models import Repo +from augur.application.db.models import Repo, RepoGroup from augur.application.db.session import DatabaseSession from augur.application.db.models.base import Base -DEFAULT_REPO_GROUP_ID = 1 +schema_6_revision = importlib.import_module('augur.application.schema.alembic.versions.6_add_repo_group_for_frontend_repos') +FRONTEND_REPO_GROUP_NAME = schema_6_revision.repo_group_name logger = logging.getLogger(__name__) def retrieve_org_repos(session, url: str) -> List[str]: @@ -728,7 +730,11 @@ def add(session, url: List[str], user_id: int, group_name=None, group_id=None, v if not result[0]: return False, {"status": result[1]["status"], "repo_url": url} - repo_id = Repo.insert(session, url, DEFAULT_REPO_GROUP_ID, "Frontend") + frontend_repo_group = session.query(RepoGroup).filter(RepoGroup.rg_name == FRONTEND_REPO_GROUP_NAME).first() + if not frontend_repo_group: + return False, {"status": "Could not find repo group with name 'Frontend Repos'", "repo_url": url} + + repo_id = Repo.insert(session, url, frontend_repo_group.repo_group_id, "Frontend") if not repo_id: return False, {"status": "Repo insertion failed", "repo_url": url} diff --git a/augur/application/schema/alembic/versions/6_add_repo_group_for_frontend_repos.py b/augur/application/schema/alembic/versions/6_add_repo_group_for_frontend_repos.py new file mode 100644 index 0000000000..eebc821d85 --- /dev/null +++ b/augur/application/schema/alembic/versions/6_add_repo_group_for_frontend_repos.py @@ -0,0 +1,36 @@ +"""Add repo group for frontend repos + +Revision ID: 6 +Revises: 5 +Create Date: 2023-02-17 14:52:35.095070 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql +from sqlalchemy.sql import text + + +# revision identifiers, used by Alembic. +revision = '6' +down_revision = '5' +branch_labels = None +depends_on = None + +repo_group_name = "Frontend Repos" + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + conn = op.get_bind() + conn.execute(f"""INSERT INTO "augur_data"."repo_groups" ("rg_name", "rg_description", "rg_website", "rg_recache", "rg_last_modified", "rg_type", "tool_source", "tool_version", "data_source", "data_collection_date") VALUES ('{repo_group_name}', 'DO NOT DELETE OR FRONTEND REPOS WILL BREAK', '', 0, '2023-02-17 15:00:00', NULL, NULL, NULL, NULL, NULL);""") + + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + conn = op.get_bind() + conn.execute(text(f"""DELETE FROM "augur_data"."repo_groups" WHERE rg_name='{repo_group_name}';""")) + # ### end Alembic commands ### From f0b5c6f5da2a083a329e4f38f024370cab637f4b Mon Sep 17 00:00:00 2001 From: Andrew Brain Date: Fri, 17 Feb 2023 16:08:29 -0600 Subject: [PATCH 19/20] Fix syntax error Signed-off-by: Andrew Brain --- augur/api/routes/pull_request_reports.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/augur/api/routes/pull_request_reports.py b/augur/api/routes/pull_request_reports.py index c5e936af5e..02f6e235cd 100644 --- a/augur/api/routes/pull_request_reports.py +++ b/augur/api/routes/pull_request_reports.py @@ -1345,10 +1345,8 @@ def mean_days_between_PR_comments(): plot_width = 950 p1 = figure(x_axis_type="datetime", - title="{}: Mean {} Between Comments by Month Closed for {} Pull Requests".format(repo_dict[repo_id], - time_unit, - description), - plot_width=plot_width, x_range=(pr_all[x_axis].min(), pr_all[x_axis].max()), plot_height=500, + title="{}: Mean {} Between Comments by Month Closed for {} Pull Requests".format(repo_dict[repo_id], time_unit, description), + plot_width=plot_width, x_range=(data_dict["All"][x_axis].min(), data_dict["All"][x_axis].max()), plot_height=500, toolbar_location=None) colors = Category20[10][6:] color_index = 0 @@ -1379,11 +1377,9 @@ def mean_days_between_PR_comments(): possible_maximums.append( max(driver_df_mean.loc[driver_df_mean[line_group] == line_group_value][y_axis].dropna())) for repo, num_outliers in num_outliers_repo_map.items(): - # FIXME repo_name is not defined - if repo_name == repo: - p1.add_layout( - Title(text="** {} outliers for {} were removed".format(num_outliers, repo), align="center"), - "below") + p1.add_layout( + Title(text="** {} outliers for {} were removed".format(num_outliers, repo), align="center"), + "below") p1.grid.grid_line_alpha = 0.3 p1.xaxis.axis_label = 'Month Closed' From b63f517c899e02b47c48c26ac9578290061ecff4 Mon Sep 17 00:00:00 2001 From: Ulincsys <28362836a@gmail.com> Date: Mon, 27 Feb 2023 18:52:34 -0600 Subject: [PATCH 20/20] Fix report image loading not working Signed-off-by: Ulincsys <28362836a@gmail.com> --- augur/api/view/init.py | 9 ++-- augur/api/view/utils.py | 82 +++++++++++++++++++------------- augur/application/cli/backend.py | 2 +- 3 files changed, 54 insertions(+), 39 deletions(-) diff --git a/augur/api/view/init.py b/augur/api/view/init.py index fd98e12338..210dc60e08 100644 --- a/augur/api/view/init.py +++ b/augur/api/view/init.py @@ -1,6 +1,7 @@ from pathlib import Path from .server import Environment -import logging, sqlite3, secrets, hashlib, yaml +from augur.application.logs import AugurLogger +import logging, secrets, yaml env = Environment() @@ -52,7 +53,7 @@ def update_from(old): current_settings["version"] = version write_settings(current_settings) - logging.info(f"Configuration updated from {to_version_string(old)} to {to_version_string(version)}") + logger.info(f"Configuration updated from {to_version_string(old)} to {to_version_string(version)}") def compare_versions(old, new): if old["major"] < new["major"]: @@ -141,7 +142,5 @@ def compare_versions(old, new): # Initialize logging def init_logging(): - format = "%(asctime)s: %(message)s" global logger - logger = logging.getLogger("augur view") - logger.setLevel("DEBUG") + logger = AugurLogger("augur_view", reset_logfiles=True).get_logger() diff --git a/augur/api/view/utils.py b/augur/api/view/utils.py index e6926dc3b4..82ac3ea7de 100644 --- a/augur/api/view/utils.py +++ b/augur/api/view/utils.py @@ -1,10 +1,18 @@ from pathlib import Path from concurrent.futures import ThreadPoolExecutor -from flask import render_template, flash, url_for +from flask import render_template, flash, url_for, Flask from .init import * from .server import User +from ..server import app, db_session +from augur.application.config import AugurConfig import urllib.request, urllib.error, json, os, math, yaml, urllib3, time, logging, re +init_logging() + +from .init import logger + +config = AugurConfig(logger, db_session) + def parse_url(url): from urllib.parse import urlparse @@ -47,7 +55,7 @@ def is_status_ok(): if "status" in response: return request.url except Exception as e: - logging.error(f"Error during serving URL verification: {str(e)}") + logger.error(f"Error during serving URL verification: {str(e)}") return False @@ -88,9 +96,9 @@ def loadSettings(): if not configFilePath.is_file(): init_settings() with open(configFile, 'w') as file: - logging.info(f"Generating default configuration file: {configFile}") + logger.info(f"Generating default configuration file: {configFile}") yaml.dump(settings, file) - logging.info("Default configuration file successfully generated.") + logger.info("Default configuration file successfully generated.") else: with open(configFilePath) as file: settings = yaml.load(file, Loader=yaml.FullLoader) @@ -103,7 +111,7 @@ def loadSettings(): else: try: cachePath.mkdir(parents=True) - logging.info("cache directory initialized") + logger.info("cache directory initialized") except Exception as err: raise Exception(f"Cannot initialize caching: could not create cache directory [{cachePath}]") @@ -120,17 +128,16 @@ def loadSettings(): """ ---------------------------------------------------------------- """ -def getSetting(key): - if key == "serving": - return "http://127.0.0.1:5000/api/unstable" - return settings[key] - -init_logging() +def getSetting(key, section = "View"): + if section == "View": + if key == "serving": + return "http://127.0.0.1:5000/api/unstable" + return settings[key] + else: + return config.get_value(section, key) loadSettings() -from .init import logger - User.api = getSetting("serving") User.logger = logger @@ -149,16 +156,16 @@ def loadReports(): image['id'] = id = id + 1 return True except Exception as err: - logging.error(f"An exception occurred reading reports endpoints from [{getSetting('reports')}]:") - logging.error(err) + logger.error(f"An exception occurred reading reports endpoints from [{getSetting('reports')}]:") + logger.error(err) try: with open(getSetting("reports"), 'w') as file: - logging.info("Attempting to generate default reports.yml") + logger.info("Attempting to generate default reports.yml") yaml.dump(reports, file) - logging.info("Default reports file successfully generated.") + logger.info("Default reports file successfully generated.") except Exception as ioErr: - logging.error("Error creating default report configuration:") - logging.error(ioErr) + logger.error("Error creating default report configuration:") + logger.error(ioErr) return False if not loadReports(): @@ -176,11 +183,11 @@ def cacheFileExists(filename): if(cache_file_age > getSetting('cache_expiry')): try: cache_file.unlink() - logging.info(f"Cache file {filename} removed due to expiry") + logger.info(f"Cache file {filename} removed due to expiry") return False except Exception as e: - logging.error("Error: cache file age exceeds expiry limit, but an exception occurred while attempting to remove") - logging.error(e) + logger.error("Error: cache file age exceeds expiry limit, but an exception occurred while attempting to remove") + logger.error(e) return True else: return False @@ -220,7 +227,7 @@ def toCacheURL(endpoint): def requestJson(endpoint, cached = True): filename = toCacheFilepath(endpoint) requestURL = getSetting('serving') + "/" + endpoint - logging.info(f'requesting json from: {endpoint}') + logger.info(f'requesting json from: {endpoint}') try: if cached and cacheFileExists(filename): with open(filename) as f: @@ -239,8 +246,8 @@ def requestJson(endpoint, cached = True): cache_files_requested.remove(filename) return data except Exception as err: - logging.error("An exception occurred while fulfilling a json request") - logging.error(err) + logger.error("An exception occurred while fulfilling a json request") + logger.error(err) return False, str(err) """ ---------------------------------------------------------------- @@ -257,8 +264,8 @@ def requestPNG(endpoint): cache_files_requested.remove(filename) return toCacheURL(endpoint) except Exception as err: - logging.error("An exception occurred while fulfilling a png request") - logging.error(err) + logger.error("An exception occurred while fulfilling a png request") + logger.error(err) """ ---------------------------------------------------------------- """ @@ -269,20 +276,26 @@ def download(url, cmanager, filename, image_cache, image_id, repo_id = None): if cacheFileExists(filename): image_cache[image_id]['exists'] = True return - response = cmanager.request('GET', url) + try: + response = cmanager.request('GET', url) + except Exception as e: + logger.error("Could not make request: " + str(e)) + raise e + if "json" in response.headers['Content-Type']: - logging.warn(f"repo {repo_id}: unexpected json response in image request") - logging.warn(f" response: {response.data.decode('utf-8')}") + logger.warn(f"repo {repo_id}: unexpected json response in image request") + logger.warn(f" response: {response.data.decode('utf-8')}") image_cache[image_id]['exists'] = False return if response and response.status == 200: image_cache[image_id]['exists'] = True try: with open(filename, 'wb') as f: + logger.info("Writing image: " + filename) f.write(response.data) except Exception as err: - logging.error("An exception occurred writing a cache file to disk") - logging.error(err) + logger.error("An exception occurred writing a cache file to disk") + logger.error(err) """ ---------------------------------------------------------------- """ @@ -295,6 +308,9 @@ def requestReports(repo_id): report_requests[repo_id] = {} report_requests[repo_id]['complete'] = False + host = getSetting("host", "Server") + port = getSetting("port", "Server") + """ ---------- If the report definition could not be loaded, we cannot determine what files to request from the backend to compose the report. Returning here @@ -319,7 +335,7 @@ def requestReports(repo_id): # Where should the downloaded image be stored (in cache) filename = toCacheFilename(f"{image['url']}?repo_id={repo_id}") # Where are we downloading the image from - image_url = url_for(image['url'], repo_id = repo_id) + image_url = f"{host}:{port}" + url_for(image['url'], repo_id = repo_id) # f"{getSetting('serving')}/{image['url']}?repo_id={repo_id}" # Add a request for this image to the thread pool using the download function diff --git a/augur/application/cli/backend.py b/augur/application/cli/backend.py index 008afe8b26..dfbc8281ff 100644 --- a/augur/application/cli/backend.py +++ b/augur/application/cli/backend.py @@ -93,7 +93,7 @@ def start(disable_collection, development, port): time.sleep(3) logger.info('Gunicorn webserver started...') - logger.info(f'Augur is running at: http://127.0.0.1:{port}') + logger.info(f'Augur is running at: {"http" if development else "https"}://{host}:{port}') scheduling_worker_process = None core_worker_process = None