Performance updates and misc.

README.md

@@ -1,5 +1,4 @@
-
-# Augur NEW Release v0.50.0
+# Augur NEW Release v0.50.1
 
 [![first-timers-only](https://img.shields.io/badge/first--timers--only-friendly-blue.svg?style=flat-square)](https://www.firsttimersonly.com/) We follow the [First Timers Only](https://www.firsttimersonly.com/) philosophy of tagging issues for first timers only, and walking one newcomer through the resolution process weekly. [You can find these issues tagged with "first timers only" on our issues list.](https://github.com/chaoss/augur/labels/first-timers-only).
 

augur/application/cli/backend.py

@@ -107,8 +107,8 @@ def start(disable_collection, development, port):
             os.remove("celerybeat-schedule.db")
 
         scheduling_worker = f"celery -A augur.tasks.init.celery_app.celery_app worker -l info --concurrency=1 -n scheduling:{uuid.uuid4().hex}@%h -Q scheduling"
-        core_worker = f"celery -A augur.tasks.init.celery_app.celery_app worker -l info --concurrency=14 -n core:{uuid.uuid4().hex}@%h"
-        secondary_worker = f"celery -A augur.tasks.init.celery_app.celery_app worker -l info --concurrency=5 -n secondary:{uuid.uuid4().hex}@%h -Q secondary"
+        core_worker = f"celery -A augur.tasks.init.celery_app.celery_app worker -l info --concurrency=70 -n core:{uuid.uuid4().hex}@%h"
+        secondary_worker = f"celery -A augur.tasks.init.celery_app.celery_app worker -l info --concurrency=10 -n secondary:{uuid.uuid4().hex}@%h -Q secondary"
 
         scheduling_worker_process = subprocess.Popen(scheduling_worker.split(" "))
         core_worker_process = subprocess.Popen(core_worker.split(" "))

augur/application/schema/alembic/versions/13_no_null_commit_contributors.py

@@ -0,0 +1,36 @@
+"""No null author affiliation and committer affiliation in commits table
+
+Revision ID: 7
+Revises: 6
+Create Date: 2023-02-23 10:14:08.787528
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+from sqlalchemy.sql import text
+import re
+
+
+# revision identifiers, used by Alembic.
+revision = '13'
+down_revision = '12'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    conn = op.get_bind()
+    conn.execute(text(f"""
+          ALTER TABLE "augur_data"."commits" 
+          ALTER COLUMN "cmt_author_affiliation" SET DEFAULT NULL,
+          ALTER COLUMN "cmt_committer_affiliation" SET DEFAULT NULL;
+        """))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

augur/application/schema/alembic/versions/14_commits_auto_vacuum.py

@@ -0,0 +1,34 @@
+"""Frequent vaccuming on high update commits table
+
+Revision ID: 14
+Revises: 13
+Create Date: 2023-02-23 10:14:08.787528
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+from sqlalchemy.sql import text
+import re
+
+
+# revision identifiers, used by Alembic.
+revision = '14'
+down_revision = '13'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    conn = op.get_bind()
+    conn.execute(text(f"""
+        ALTER TABLE augur_data.commits SET (autovacuum_vacuum_scale_factor = 0, autovacuum_vacuum_threshold = 1000);
+        """))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

augur/application/schema/alembic/versions/15_commit_performance_update.py

@@ -0,0 +1,42 @@
+"""Commit performance update
+
+Revision ID: 14
+Revises: 13
+Create Date: 2023-02-23 10:14:08.787528
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+from sqlalchemy.sql import text
+import re
+
+
+# revision identifiers, used by Alembic.
+revision = '15'
+down_revision = '14'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    conn = op.get_bind()
+    conn.execute(text(f"""
+          ALTER TABLE "augur_data"."commits" 
+          DROP CONSTRAINT "cmt_ght_author_cntrb_id_fk",
+          ADD CONSTRAINT "cmt_ght_author_cntrb_id_fk" FOREIGN KEY ("cmt_ght_author_id") REFERENCES "augur_data"."contributors" ("cntrb_id") ON DELETE RESTRICT ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED;
+        """))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    conn = op.get_bind()
+    conn.execute(text(f"""
+          ALTER TABLE "augur_data"."commits" 
+          DROP CONSTRAINT "cmt_ght_author_cntrb_id_fk",
+          ADD CONSTRAINT "cmt_ght_author_cntrb_id_fk" FOREIGN KEY ("cmt_ght_author_id") REFERENCES "augur_data"."contributors" ("cntrb_id");
+        """))
+    pass
+    # ### end Alembic commands ###

augur/tasks/github/facade_github/tasks.py

@@ -190,8 +190,9 @@ def link_commits_to_contributor(session,contributorQueue):
                 UPDATE commits 
                 SET cmt_ght_author_id=:cntrb_id
                 WHERE 
-                cmt_author_raw_email=:cntrb_email
-                OR cmt_author_email=:cntrb_email
+                (cmt_author_raw_email=:cntrb_email
+                OR cmt_author_email=:cntrb_email)
+                AND cmt_ght_author_id is NULL
         """).bindparams(cntrb_id=cntrb["cntrb_id"],cntrb_email=cntrb["email"])
 
         #engine.execute(query, **data)

augur/tasks/start_tasks.py

@@ -365,14 +365,14 @@ def augur_collection_monitor():
         enabled_phase_names = get_enabled_phase_names_from_config(session.logger, session)
 
         if primary_repo_collect_phase.__name__ in enabled_phase_names:
-            start_primary_collection(session, max_repo=50, days=30)
+            start_primary_collection(session, max_repo=60, days=30)
 
         if secondary_repo_collect_phase.__name__ in enabled_phase_names:
             start_secondary_collection(session, max_repo=30, days=30)
 
         if facade_phase.__name__ in enabled_phase_names:
             #Schedule facade collection before clone/updates as that is a higher priority
-            start_facade_collection(session, max_repo=30, days=30)
+            start_facade_collection(session, max_repo=20, days=30)
             start_facade_clone_update(session,max_repo=15,days=30)