Add weight calculations for repo scheduling in repo collection

augur/application/cli/backend.py

@@ -16,8 +16,7 @@
 import uuid
 import traceback
 from urllib.parse import urlparse
-from sqlalchemy import update
-
+from datetime import datetime
 
 from augur import instance_id
 from augur.tasks.start_tasks import augur_collection_monitor, CollectionState
@@ -187,13 +186,15 @@ def augur_stop(signal, logger):
     """
 
     augur_processes = get_augur_processes()
-    _broadcast_signal_to_processes(augur_processes, broadcast_signal=signal, given_logger=logger)
-
     # if celery is running, run the cleanup function
     process_names = [process.name() for process in augur_processes]
+
+    _broadcast_signal_to_processes(augur_processes, broadcast_signal=signal, given_logger=logger)
+
     if "celery" in process_names:
         cleanup_after_collection_halt(logger)
 
+
 def cleanup_after_collection_halt(logger):
     clear_redis_caches()
     connection_string = ""
@@ -441,6 +442,8 @@ def order_repos(repos):
 
     return repo_git_urls
 
+
+
 # def initialize_components(augur_app, disable_housekeeper):
 #     master = None
 #     manager = None

augur/application/cli/collection.py

@@ -36,8 +36,6 @@
 
 logger = AugurLogger("augur", reset_logfiles=True).get_logger()
 
-def get_page_count()
-
 
 def check_collection(owner, repo, key_manager, session):
 

augur/application/cli/db.py

@@ -22,6 +22,9 @@
 from augur.application.db.session import DatabaseSession
 from augur.application.logs import AugurLogger
 from augur.application.db.engine import DatabaseEngine
+from sqlalchemy import update
+from datetime import datetime
+from augur.application.db.models import Repo
 
 logger = logging.getLogger(__name__)
 
@@ -373,7 +376,6 @@ def init_database(
 def test_db_connection():
     pass
 
-
 # TODO: Fix this function
 def run_psql_command_in_database(target_type, target):
     if target_type not in ["-f", "-c"]:
@@ -456,3 +458,19 @@ def check_pgpass_credentials(config):
             pgpass_file.write(credentials_string + "\n")
         else:
             print("Credentials found in $HOME/.pgpass")
+
+
+#NOTE: For some reason when I try to add function decorators to this function 
+#click thinks it's an argument and tries to parse it but it errors since a function 
+#isn't an iterable. 
+@cli.command("reset-repo-age")
+def reset_repo_age():
+
+    with DatabaseSession(logger) as session:
+        update_query = (
+            update(Repo)
+            .values(repo_added=datetime.now())
+        )
+
+        session.execute(update_query)
+        session.commit()

augur/application/db/models/augur_operations.py

@@ -10,13 +10,15 @@
 
 import logging 
 import secrets
+import traceback
 import importlib
 
 from augur.application.db.models import Repo, RepoGroup
 from augur.application.db.session import DatabaseSession
 from augur.application.db.models.base import Base
 
 
+
 FRONTEND_REPO_GROUP_NAME = "Frontend Repos"
 logger = logging.getLogger(__name__)
 
@@ -1009,14 +1011,48 @@ class CollectionStatus(Base):
     facade_status = Column(String,nullable=False, server_default=text("'Pending'"))
     facade_data_last_collected = Column(TIMESTAMP)
     facade_task_id = Column(String)
+
+    core_weight = Column(BigInteger)
+    facade_weight = Column(BigInteger)
+    secondary_weight = Column(BigInteger)
+
+    issue_pr_sum = Column(BigInteger)
+    commit_sum = Column(BigInteger)
 
     repo = relationship("Repo", back_populates="collection_status")
 
     @staticmethod
     def insert(session, repo_id):
+        from augur.tasks.github.util.util import get_repo_weight_by_issue
+        from augur.tasks.util.worker_util import calculate_date_weight_from_timestamps
+
+        repo = Repo.get_by_id(session, repo_id)
+        repo_git = repo.repo_git
 
         collection_status_unique = ["repo_id"]
-        result = session.insert_data({"repo_id": repo_id}, CollectionStatus, collection_status_unique, on_conflict_update=False)
+
+        try:
+            pr_issue_count = get_repo_weight_by_issue(session.logger, repo_git)
+            #session.logger.info(f"date weight: {calculate_date_weight_from_timestamps(repo.repo_added, None)}")
+            github_weight = pr_issue_count - calculate_date_weight_from_timestamps(repo.repo_added, None)
+        except Exception as e:
+            pr_issue_count = None
+            github_weight = None
+            session.logger.error(
+                    ''.join(traceback.format_exception(None, e, e.__traceback__)))
+
+
+        record = {
+            "repo_id": repo_id,
+            "issue_pr_sum": pr_issue_count,
+            "core_weight": github_weight,
+            "secondary_weight": github_weight
+        }
+
+        result = session.insert_data(record, CollectionStatus, collection_status_unique, on_conflict_update=False)
+
+        session.logger.info(f"Trying to insert repo \n issue and pr sum: {record['issue_pr_sum']}")
+
         if not result:
             return False
 

augur/application/schema/alembic/versions/16_add_weight_data_to_collection_status_to_.py

@@ -0,0 +1,38 @@
+"""Add weight data to collection status to determine collection order of repos
+
+Revision ID: 16
+Revises: 15
+Create Date: 2023-04-10 18:28:12.460522
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '16'
+down_revision = '15'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('collection_status', sa.Column('core_weight', sa.BigInteger()), schema='augur_operations')
+    op.add_column('collection_status', sa.Column('facade_weight', sa.BigInteger()), schema='augur_operations')
+    op.add_column('collection_status', sa.Column('secondary_weight', sa.BigInteger()), schema='augur_operations')
+
+    op.add_column('collection_status', sa.Column('issue_pr_sum', sa.BigInteger()), schema='augur_operations')
+    op.add_column('collection_status', sa.Column('commit_sum', sa.BigInteger()), schema='augur_operations')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('collection_status', 'facade_weight', schema='augur_operations')
+    op.drop_column('collection_status', 'core_weight', schema='augur_operations')
+    op.drop_column('collection_status', 'secondary_weight', schema='augur_operations')
+
+    op.drop_column('collection_status', 'issue_pr_sum', schema='augur_operations')
+    op.drop_column('collection_status', 'commit_sum', schema='augur_operations')
+    # ### end Alembic commands ###

augur/application/schema/alembic/versions/5_add_collection_status_table.py

@@ -36,12 +36,7 @@ def upgrade():
 
     # add collection status for any existing repos
     conn = op.get_bind()
-    repos = conn.execute(text("""SELECT repo_id from repo""")).fetchall()
-
-    for repo in repos:
-        repo_id = repo[0]
-        conn.execute(text(f"""INSERT INTO "augur_operations"."collection_status" ("repo_id") VALUES ({repo_id});"""))
-
+
     conn.execute(text("""
         UPDATE augur_operations.config
         SET value = '600'

augur/application/schema/alembic/versions/legacy/80.3-sample-data.sql

@@ -1072,13 +1072,13 @@ INSERT INTO "augur_data"."contributor_affiliations"("ca_id", "ca_domain", "tool_
 
 INSERT INTO "augur_data"."repo_groups"("repo_group_id", "rg_name", "rg_description", "rg_website", "rg_recache", "rg_last_modified", "rg_type", "tool_source", "tool_version", "data_source", "data_collection_date") VALUES (10, 'Default Repo Group', 'The default repo group created by the schema generation script', '', 0, '2021-06-03 15:55:20', 'GitHub Organization', 'load', 'one', 'git', '2019-06-05 13:36:25');
 
-INSERT INTO "augur_data"."repo" ("repo_id", "repo_group_id", "repo_git", "repo_path", "repo_name", "repo_added", "repo_status", "repo_type", "url", "owner_id", "description", "primary_language", "created_at", "forked_from", "updated_at", "tool_source", "tool_version", "data_source", "data_collection_date", "repo_archived", "repo_archived_date_collected") VALUES (25452, 10, 'https://github.com/chaoss/whitepaper', 'github.com/chaoss/', 'whitepaper', '2021-04-17 21:40:42', 'Complete', '', NULL, NULL, NULL, NULL, NULL, 'Parent not available', NULL, 'CLI', '1.0', 'Git', '2021-04-17 21:40:42', 0, NULL);
-INSERT INTO "augur_data"."repo" ("repo_id", "repo_group_id", "repo_git", "repo_path", "repo_name", "repo_added", "repo_status", "repo_type", "url", "owner_id", "description", "primary_language", "created_at", "forked_from", "updated_at", "tool_source", "tool_version", "data_source", "data_collection_date", "repo_archived", "repo_archived_date_collected") VALUES (24441, 10, 'https://github.com/operate-first/operate-first-twitter', 'github.com/operate-first/', 'operate-first-twitter', '2021-08-25 16:47:47', 'Complete', '', NULL, NULL, NULL, NULL, NULL, 'Parent not available', NULL, 'CLI', '1.0', 'Git', '2021-08-25 16:47:47', 0, NULL);
-INSERT INTO "augur_data"."repo" ("repo_id", "repo_group_id", "repo_git", "repo_path", "repo_name", "repo_added", "repo_status", "repo_type", "url", "owner_id", "description", "primary_language", "created_at", "forked_from", "updated_at", "tool_source", "tool_version", "data_source", "data_collection_date", "repo_archived", "repo_archived_date_collected") VALUES (24442, 10, 'https://github.com/operate-first/blueprint', 'github.com/operate-first/', 'blueprint', '2021-08-25 16:47:47', 'Complete', '', NULL, NULL, NULL, NULL, NULL, 'Parent not available', NULL, 'CLI', '1.0', 'Git', '2021-08-25 16:47:47', 0, NULL);
-INSERT INTO "augur_data"."repo" ("repo_id", "repo_group_id", "repo_git", "repo_path", "repo_name", "repo_added", "repo_status", "repo_type", "url", "owner_id", "description", "primary_language", "created_at", "forked_from", "updated_at", "tool_source", "tool_version", "data_source", "data_collection_date", "repo_archived", "repo_archived_date_collected") VALUES (25445, 10, 'https://github.com/chaoss/grimoirelab-perceval-opnfv', 'github.com/chaoss/', 'grimoirelab-perceval-opnfv', '2020-04-17 21:40:39', 'Complete', '', NULL, NULL, NULL, NULL, NULL, 'Parent not available', NULL, 'CLI', '1.0', 'Git', '2021-04-17 21:40:39', 0, NULL);
-INSERT INTO "augur_data"."repo" ("repo_id", "repo_group_id", "repo_git", "repo_path", "repo_name", "repo_added", "repo_status", "repo_type", "url", "owner_id", "description", "primary_language", "created_at", "forked_from", "updated_at", "tool_source", "tool_version", "data_source", "data_collection_date", "repo_archived", "repo_archived_date_collected") VALUES (1, 1, 'https://github.com/chaoss/augur', 'github.com/chaoss/', 'augur', '2021-08-10 14:28:44', 'Complete', '', NULL, NULL, NULL, NULL, NULL, 'Parent not available', NULL, 'data load', 'one', 'git', '2021-06-05 18:41:14', 0, NULL);
-INSERT INTO "augur_data"."repo" ("repo_id", "repo_group_id", "repo_git", "repo_path", "repo_name", "repo_added", "repo_status", "repo_type", "url", "owner_id", "description", "primary_language", "created_at", "forked_from", "updated_at", "tool_source", "tool_version", "data_source", "data_collection_date", "repo_archived", "repo_archived_date_collected") VALUES (25430, 10, 'https://github.com/SociallyCompute/update-test', 'github.com/SociallyCompute/', 'update-test', '2021-10-07 08:50:13', 'Complete', '', NULL, NULL, NULL, NULL, NULL, 'Parent not available', NULL, NULL, NULL, NULL, NULL, 0, NULL);
-INSERT INTO "augur_data"."repo" ("repo_id", "repo_group_id", "repo_git", "repo_path", "repo_name", "repo_added", "repo_status", "repo_type", "url", "owner_id", "description", "primary_language", "created_at", "forked_from", "updated_at", "tool_source", "tool_version", "data_source", "data_collection_date", "repo_archived", "repo_archived_date_collected") VALUES (25450, 10, 'https://github.com/chaoss/grimoirelab-hatstall', 'github.com/chaoss/', 'grimoirelab-hatstall', '2021-04-17 21:40:42', 'Complete', '', NULL, NULL, NULL, NULL, NULL, 'Parent not available', NULL, 'CLI', '1.0', 'Git', '2021-04-17 21:40:42', 0, NULL);
+INSERT INTO "augur_data"."repo" ("repo_id", "repo_group_id", "repo_git", "repo_path", "repo_name", "repo_status", "repo_type", "url", "owner_id", "description", "primary_language", "created_at", "forked_from", "updated_at", "tool_source", "tool_version", "data_source", "data_collection_date", "repo_archived", "repo_archived_date_collected") VALUES (25452, 10, 'https://github.com/chaoss/whitepaper', 'github.com/chaoss/', 'whitepaper', 'Complete', '', NULL, NULL, NULL, NULL, NULL, 'Parent not available', NULL, 'CLI', '1.0', 'Git', '2021-04-17 21:40:42', 0, NULL);
+INSERT INTO "augur_data"."repo" ("repo_id", "repo_group_id", "repo_git", "repo_path", "repo_name", "repo_status", "repo_type", "url", "owner_id", "description", "primary_language", "created_at", "forked_from", "updated_at", "tool_source", "tool_version", "data_source", "data_collection_date", "repo_archived", "repo_archived_date_collected") VALUES (24441, 10, 'https://github.com/operate-first/operate-first-twitter', 'github.com/operate-first/', 'operate-first-twitter', 'Complete', '', NULL, NULL, NULL, NULL, NULL, 'Parent not available', NULL, 'CLI', '1.0', 'Git', '2021-08-25 16:47:47', 0, NULL);
+INSERT INTO "augur_data"."repo" ("repo_id", "repo_group_id", "repo_git", "repo_path", "repo_name", "repo_status", "repo_type", "url", "owner_id", "description", "primary_language", "created_at", "forked_from", "updated_at", "tool_source", "tool_version", "data_source", "data_collection_date", "repo_archived", "repo_archived_date_collected") VALUES (24442, 10, 'https://github.com/operate-first/blueprint', 'github.com/operate-first/', 'blueprint', 'Complete', '', NULL, NULL, NULL, NULL, NULL, 'Parent not available', NULL, 'CLI', '1.0', 'Git', '2021-08-25 16:47:47', 0, NULL);
+INSERT INTO "augur_data"."repo" ("repo_id", "repo_group_id", "repo_git", "repo_path", "repo_name", "repo_status", "repo_type", "url", "owner_id", "description", "primary_language", "created_at", "forked_from", "updated_at", "tool_source", "tool_version", "data_source", "data_collection_date", "repo_archived", "repo_archived_date_collected") VALUES (25445, 10, 'https://github.com/chaoss/grimoirelab-perceval-opnfv', 'github.com/chaoss/', 'grimoirelab-perceval-opnfv', 'Complete', '', NULL, NULL, NULL, NULL, NULL, 'Parent not available', NULL, 'CLI', '1.0', 'Git', '2021-04-17 21:40:39', 0, NULL);
+INSERT INTO "augur_data"."repo" ("repo_id", "repo_group_id", "repo_git", "repo_path", "repo_name", "repo_status", "repo_type", "url", "owner_id", "description", "primary_language", "created_at", "forked_from", "updated_at", "tool_source", "tool_version", "data_source", "data_collection_date", "repo_archived", "repo_archived_date_collected") VALUES (1, 1, 'https://github.com/chaoss/augur', 'github.com/chaoss/', 'augur', 'Complete', '', NULL, NULL, NULL, NULL, NULL, 'Parent not available', NULL, 'data load', 'one', 'git', '2021-06-05 18:41:14', 0, NULL);
+INSERT INTO "augur_data"."repo" ("repo_id", "repo_group_id", "repo_git", "repo_path", "repo_name", "repo_status", "repo_type", "url", "owner_id", "description", "primary_language", "created_at", "forked_from", "updated_at", "tool_source", "tool_version", "data_source", "data_collection_date", "repo_archived", "repo_archived_date_collected") VALUES (25430, 10, 'https://github.com/SociallyCompute/update-test', 'github.com/SociallyCompute/', 'update-test', 'Complete', '', NULL, NULL, NULL, NULL, NULL, 'Parent not available', NULL, NULL, NULL, NULL, NULL, 0, NULL);
+INSERT INTO "augur_data"."repo" ("repo_id", "repo_group_id", "repo_git", "repo_path", "repo_name", "repo_status", "repo_type", "url", "owner_id", "description", "primary_language", "created_at", "forked_from", "updated_at", "tool_source", "tool_version", "data_source", "data_collection_date", "repo_archived", "repo_archived_date_collected") VALUES (25450, 10, 'https://github.com/chaoss/grimoirelab-hatstall', 'github.com/chaoss/', 'grimoirelab-hatstall', 'Complete', '', NULL, NULL, NULL, NULL, NULL, 'Parent not available', NULL, 'CLI', '1.0', 'Git', '2021-04-17 21:40:42', 0, NULL);
 
 UPDATE "augur_data"."repo" set repo_name = NULL, repo_path = NULL, repo_status = 'New'; 
 

augur/tasks/git/facade_tasks.py

@@ -24,10 +24,13 @@
 from datetime import timedelta
 import sqlalchemy as s
 
+from sqlalchemy import or_, and_, update
 
 from augur.tasks.git.util.facade_worker.facade_worker.facade02utilitymethods import update_repo_log, trim_commit, store_working_author, trim_author
 from augur.tasks.git.util.facade_worker.facade_worker.facade02utilitymethods import get_absolute_repo_path, get_parent_commits_set, get_existing_commits_set
 from augur.tasks.git.util.facade_worker.facade_worker.facade03analyzecommit import analyze_commit
+from augur.tasks.git.util.facade_worker.facade_worker.facade02utilitymethods import get_facade_weight_time_factor, get_repo_commit_count
+
 from augur.tasks.github.facade_github.tasks import *
 
 from augur.tasks.util.worker_util import create_grouped_task_load
@@ -347,6 +350,31 @@ def git_repo_initialize_facade_task(repo_git):
 #    with FacadeSession(logger) as session:
 #        check_for_repo_updates(session, repo_git)
 
+@celery.task
+def git_update_commit_count_weight(repo_git):
+
+    from augur.tasks.init.celery_app import engine
+    logger = logging.getLogger(git_update_commit_count_weight.__name__)
+
+    with FacadeSession(logger) as session:
+        commit_count = get_repo_commit_count(session, repo_git)
+        date_factor = get_facade_weight_time_factor(session, repo_git)
+
+    weight = commit_count - date_factor
+    logger.info(f"Repo {repo_git} has a weight of {weight} and a commit count of {commit_count}")
+
+    with DatabaseSession(logger,engine=engine) as session:
+        repo = Repo.get_by_repo_git(session, repo_git)
+
+        update_query = (
+            update(CollectionStatus)
+            .where(CollectionStatus.repo_id == repo.repo_id)
+            .values(facade_weight=weight,commit_sum=commit_count)
+        )
+
+        session.execute(update_query)
+        session.commit()
+
 
 @celery.task
 def git_repo_updates_facade_task(repo_git):
@@ -460,7 +488,9 @@ def facade_clone_update_phase(repo_git):
 
         if not limited_run or (limited_run and pull_repos):
             facade_sequence.append(git_repo_updates_facade_task.si(repo_git))
-
+
+        facade_sequence.append(git_update_commit_count_weight.si(repo_git))
+
         return chain(*facade_sequence)
 
 
@@ -507,7 +537,8 @@ def facade_phase(repo_git):
             group(
                 chain(*facade_core_collection),
                 process_dependency_metrics.si(repo_git),
-                process_libyear_dependency_metrics.si(repo_git)
+                process_libyear_dependency_metrics.si(repo_git),
+                git_update_commit_count_weight.si(repo_git)
             )
         )