Test traffic Merge Into Dev: Request for Feedback

README.md

@@ -20,7 +20,6 @@ Augur is now releasing a dramatically improved new version to the main branch. I
 - The next release of the new version will include a hosted version of Augur where anyone can create an account and add repos “they care about”. If the hosted instance already has a requested organization or repository it will be added to a user’s view. If its a new repository or organization, the user will be notified that collection will take (time required for the scale of repositories added). 
 
 ## What is Augur?
-
 Augur is a software suite for collecting and measuring structured data
 about [free](https://www.fsf.org/about/) and [open-source](https://opensource.org/docs/osd) software (FOSS) communities.
 

augur/application/db/data_parse.py

@@ -466,8 +466,22 @@ def extract_needed_contributor_data(contributor, tool_source, tool_version, data
 
     return contributor
 
+def extract_needed_clone_history_data(clone_history_data:List[dict], repo_id:int):
 
+    if len(clone_history_data) == 0:
+        return []
+
+    clone_data_dicts = []
+    for clone in clone_history_data:
+        clone_data_dict = {
+            'repo_id': repo_id,
+            'clone_data_timestamp': clone['timestamp'],
+            'count_clones': clone['count'],
+            'unique_clones': clone['uniques']
+        }
+        clone_data_dicts.append(clone_data_dict)
 
+    return clone_data_dicts
 
 def extract_needed_pr_review_data(review, pull_request_id, repo_id, platform_id, tool_version, data_source):
 

augur/application/db/models/__init__.py

@@ -63,6 +63,7 @@
     PullRequestTeam,
     PullRequestRepo,
     PullRequestReviewMessageRef,
+    RepoClone,
 )
 
 from augur.application.db.models.spdx import (

augur/application/db/models/augur_data.py

@@ -3348,3 +3348,31 @@ class PullRequestReviewMessageRef(Base):
     msg = relationship("Message")
     pr_review = relationship("PullRequestReview")
     repo = relationship("Repo")
+
+
+class RepoClone(Base):
+    __tablename__ = "repo_clones_data"
+    __table_args__ = {"schema": "augur_data"}
+
+    repo_clone_data_id = Column(
+        BigInteger,
+        primary_key=True,
+        server_default=text(
+            "nextval('augur_data.repo_clones_data_id_seq'::regclass)"
+        ),
+    )
+    repo_id = Column(
+        ForeignKey(
+            "augur_data.repo.repo_id",
+            ondelete="RESTRICT",
+            onupdate="CASCADE",
+            deferrable=True,
+            initially="DEFERRED",
+        ),
+        nullable=False,
+    )
+    unique_clones = Column(BigInteger)
+    count_clones = Column(BigInteger)
+    clone_data_timestamp = Column(TIMESTAMP(precision=6))
+
+    repo = relationship("Repo")

augur/application/schema/alembic/versions/12_traffic_additions.py

@@ -0,0 +1,63 @@
+"""traffic additions
+
+Revision ID: 3
+Revises: 2
+Create Date: 2022-12-30 19:23:17.997570
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '3'
+down_revision = '2'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+
+    add_repo_clone_data_table_1()
+
+def downgrade():
+
+    upgrade = False
+
+    add_repo_clone_data_table_1(upgrade)
+
+
+def add_repo_clone_data_table_1(upgrade = True):
+
+    if upgrade:
+
+        op.create_table('repo_clones_data',
+        sa.Column('repo_clone_data_id', sa.BigInteger(), server_default=sa.text("nextval('augur_data.repo_clones_data_id_seq'::regclass)"), nullable=False),
+        sa.Column('repo_id', sa.BigInteger(), nullable=False),
+        sa.Column('unique_clones', sa.BigInteger(), nullable=True),
+        sa.Column('count_clones', sa.BigInteger(), nullable=True),
+        sa.Column('clone_data_timestamp', postgresql.TIMESTAMP(precision=6), nullable=True),
+        sa.ForeignKeyConstraint(['repo_id'], ['augur_data.repo.repo_id'], onupdate='CASCADE', ondelete='RESTRICT', initially='DEFERRED', deferrable=True),
+        sa.PrimaryKeyConstraint('repo_clone_data_id'),
+        schema='augur_data'
+        )
+        op.alter_column('releases', 'release_id',
+                existing_type=sa.CHAR(length=256),
+                type_=sa.CHAR(length=128),
+                existing_nullable=False,
+                existing_server_default=sa.text('nextval(\'"augur_data".releases_release_id_seq\'::regclass)'),
+                schema='augur_data')
+        op.drop_constraint('user_repos_repo_id_fkey', 'user_repos', schema='augur_operations', type_='foreignkey')
+        op.create_foreign_key(None, 'user_repos', 'repo', ['repo_id'], ['repo_id'], source_schema='augur_operations', referent_schema='augur_data')
+
+    else:
+
+        op.drop_constraint(None, 'user_repos', schema='augur_operations', type_='foreignkey')
+        op.create_foreign_key('user_repos_repo_id_fkey', 'user_repos', 'repo', ['repo_id'], ['repo_id'], source_schema='augur_operations')
+        op.alter_column('releases', 'release_id',
+                existing_type=sa.CHAR(length=128),
+                type_=sa.CHAR(length=256),
+                existing_nullable=False,
+                existing_server_default=sa.text('nextval(\'"augur_data".releases_release_id_seq\'::regclass)'),
+                schema='augur_data')
+        op.drop_table('repo_clones_data', schema='augur_data')

augur/application/schema/augur_full.sql

@@ -2777,6 +2777,35 @@ CREATE TABLE augur_data.working_commits (
 
 ALTER TABLE augur_data.working_commits OWNER TO augur;
 
+--
+-- Name: repo_clones_id_seq; Type: SEQUENCE; Schema: augur_data; Owner: augur
+--
+
+CREATE SEQUENCE augur_data.repo_clones_data_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+ALTER TABLE augur_data.repo_clones_data_id_seq OWNER TO augur;
+
+--
+-- Name: repo_clones; Type: TABLE; Schema: augur_data; Owner: augur
+--
+
+CREATE TABLE augur_data.repo_clones_data (
+    repo_clone_data_id bigint DEFAULT nextval('augur_data.repo_clones_data_id_seq'::regclass) NOT NULL,
+    repo_id integer NOT NULL,
+    unique_clones integer NOT NULL,
+    count_clones integer NOT NULL,
+    clone_data_timestamp timestamp(0) without time zone
+);
+
+
+ALTER TABLE augur_data.repo_clones_data OWNER TO augur;
+
 --
 -- Name: affiliations_corp_id_seq; Type: SEQUENCE; Schema: augur_operations; Owner: augur
 --

augur/tasks/github/traffic/tasks.py

@@ -0,0 +1,76 @@
+import time
+import logging
+
+from augur.tasks.init.celery_app import celery_app as celery, engine
+from augur.application.db.data_parse import *
+from augur.tasks.github.util.github_paginator import GithubPaginator
+from augur.tasks.github.util.github_task_session import GithubTaskSession
+from augur.tasks.util.worker_util import remove_duplicate_dicts
+from augur.tasks.github.util.util import get_owner_repo
+from augur.application.db.models import RepoClone, Repo
+from augur.application.db.util import execute_session_query
+
+@celery.task
+def collect_github_repo_clones_data(repo_git: str) -> None:
+
+    logger = logging.getLogger(collect_github_repo_clones_data.__name__)
+
+    # using GithubTaskSession to get our repo_obj for which we will store data of clones
+    with GithubTaskSession(logger) as session:
+
+        query = session.query(Repo).filter(Repo.repo_git == repo_git)
+        repo_obj = execute_session_query(query, 'one')
+        repo_id = repo_obj.repo_id
+
+        owner, repo = get_owner_repo(repo_git)
+
+        logger.info(f"Collecting Github repository clone data for {owner}/{repo}")
+
+    clones_data = retrieve_all_clones_data(repo_git, logger)
+
+    if clones_data:
+        process_clones_data(clones_data, f"{owner}/{repo}: Traffic task", repo_id, logger)
+    else:
+        logger.info(f"{owner}/{repo} has no clones")
+
+
+def retrieve_all_clones_data(repo_git: str, logger):
+    owner, repo = get_owner_repo(repo_git)
+
+    url = f"https://api.github.com/repos/{owner}/{repo}/traffic/clones"
+
+    # define GithubTaskSession to handle insertions, and store oauth keys 
+    with GithubTaskSession(logger, engine) as session:
+
+        clones = GithubPaginator(url, session.oauths, logger)
+
+    num_pages = clones.get_num_pages()
+    all_data = []
+    for page_data, page in clones.iter_pages():
+
+        if page_data is None:
+            return all_data
+
+        elif len(page_data) == 0:
+            logger.debug(f"{repo.capitalize()} Traffic Page {page} contains no data...returning")
+            logger.info(f"Traffic Page {page} of {num_pages}")
+            return all_data
+
+        logger.info(f"{repo} Traffic Page {page} of {num_pages}")
+
+        all_data += page_data
+
+    return all_data
+
+
+def process_clones_data(clones_data, task_name, repo_id, logger) -> None:
+    clone_history_data = clones_data[0]['clones']
+
+    clone_history_data_dicts = extract_needed_clone_history_data(clone_history_data, repo_id)
+
+    with GithubTaskSession(logger, engine) as session:
+
+        clone_history_data = remove_duplicate_dicts(clone_history_data_dicts, 'clone_data_timestamp')
+        logger.info(f"{task_name}: Inserting {len(clone_history_data_dicts)} clone history records")
+
+        session.insert_data(clone_history_data_dicts, RepoClone, ['repo_id'])

augur/tasks/init/celery_app.py

@@ -46,7 +46,8 @@ class CollectionState(Enum):
                 'augur.tasks.github.repo_info.tasks',
                 'augur.tasks.github.detect_move.tasks',
                 'augur.tasks.github.pull_requests.files_model.tasks',
-                'augur.tasks.github.pull_requests.commits_model.tasks']
+                'augur.tasks.github.pull_requests.commits_model.tasks',
+                'augur.tasks.github.traffic.tasks']
 
 git_tasks = ['augur.tasks.git.facade_tasks',
             'augur.tasks.git.dependency_tasks.tasks',

augur/tasks/start_tasks.py

@@ -22,7 +22,7 @@
 from augur.tasks.github.pull_requests.files_model.tasks import process_pull_request_files
 from augur.tasks.github.pull_requests.commits_model.tasks import process_pull_request_commits
 from augur.tasks.git.dependency_tasks.tasks import process_ossf_scorecard_metrics
-
+from augur.tasks.github.traffic.tasks import collect_github_repo_clones_data
 from augur.tasks.git.facade_tasks import *
 from augur.tasks.db.refresh_materialized_views import *
 # from augur.tasks.data_analysis import *
@@ -74,6 +74,34 @@ def primary_repo_collect_phase(repo_git):
     #A chain is needed for each repo.
     repo_info_task = collect_repo_info.si(repo_git)#collection_task_wrapper(self)
 
+### Section from traffic metric merge that may need to be changed
+
+    with DatabaseSession(logger) as session:
+        query = session.query(Repo)
+        repos = execute_session_query(query, 'all')
+        #Just use list comprehension for simple group
+        repo_info_tasks = [collect_repo_info.si(repo.repo_git) for repo in repos]
+
+        for repo in repos:
+            first_tasks_repo = group(collect_issues.si(repo.repo_git),collect_pull_requests.si(repo.repo_git),collect_github_repo_clones_data.si(repo.repo_git))
+            second_tasks_repo = group(collect_events.si(repo.repo_git),
+                collect_github_messages.si(repo.repo_git),process_pull_request_files.si(repo.repo_git), process_pull_request_commits.si(repo.repo_git))
+
+            repo_chain = chain(first_tasks_repo,second_tasks_repo)
+            issue_dependent_tasks.append(repo_chain)
+
+        repo_task_group = group(
+            *repo_info_tasks,
+            chain(group(*issue_dependent_tasks),process_contributors.si()),
+            generate_facade_chain(logger),
+            collect_releases.si()
+        )
+
+    chain(repo_task_group, refresh_materialized_views.si()).apply_async()
+
+#### End of section from traffic metric merge that may need to be changed
+
+
     primary_repo_jobs = group(
         collect_issues.si(repo_git),
         collect_pull_requests.si(repo_git)

frontend/frontend.config.json

@@ -0,0 +1,7 @@
+{
+    "Frontend": {
+        "host": "ebay.chaoss.io",
+        "port": 5000,
+        "ssl": false
+    }
+}