Grow 1352 API: Data Sync for OSS

apiserver/app.py

@@ -1,4 +1,5 @@
 from flask import Flask
+
 from dora.store import configure_db_with_app
 from env import load_app_env
 
@@ -10,6 +11,7 @@
 from dora.api.incidents import app as incidents_api
 from dora.api.integrations import app as integrations_api
 from dora.api.deployment_analytics import app as deployment_analytics_api
+from dora.api.sync import app as sync_api
 
 from dora.store.initialise_db import initialize_database
 
@@ -21,10 +23,10 @@
 app.register_blueprint(incidents_api)
 app.register_blueprint(deployment_analytics_api)
 app.register_blueprint(integrations_api)
+app.register_blueprint(sync_api)
 
 configure_db_with_app(app)
 initialize_database(app)
 
-
 if __name__ == "__main__":
     app.run()

apiserver/dora/api/sync.py

@@ -0,0 +1,12 @@
+from flask import Blueprint
+
+from dora.service.sync_data import trigger_data_sync
+from dora.utils.time import time_now
+
+app = Blueprint("sync", __name__)
+
+
+@app.route("/sync", methods=["POST"])
+def sync():
+    trigger_data_sync()
+    return {"message": "sync started", "time": time_now().isoformat()}

apiserver/dora/service/code/sync/etl_code_analytics.py

@@ -19,6 +19,9 @@ def create_pr_metrics(
         pr_events: List[PullRequestEvent],
         pr_commits: List[PullRequestCommit],
     ) -> PullRequest:
+        if pr.state == PullRequestState.OPEN:
+            return pr
+
         pr_performance = self.get_pr_performance(pr, pr_events)
 
         pr.first_response_time = (

apiserver/dora/service/code/sync/etl_github_handler.py

@@ -149,7 +149,6 @@ def get_repo_pull_requests_data(
             pr_commits += pr_commit_models
             prs_added.add(github_pr.number)
 
-        sorted(pull_requests, key=lambda x: x.state_changed_at)
         return pull_requests, pr_commits, pr_events
 
     def process_pr(

apiserver/dora/sync_data.py → apiserver/dora/service/sync_data.py

@@ -16,12 +16,16 @@
 def trigger_data_sync():
     default_org = get_query_validator().get_default_org()
     org_id = str(default_org.id)
+    LOG.info(f"Starting data sync for org {org_id}")
 
     for sync_func in sync_sequence:
         try:
             sync_func(org_id)
+            LOG.info(f"Data sync for {sync_func.__name__} completed successfully")
         except Exception as e:
-            LOG.error(f"Error syncing data for org {org_id}: {str(e)}")
+            LOG.error(
+                f"Error syncing {sync_func.__name__} data for org {org_id}: {str(e)}"
+            )
             continue
     LOG.info(f"Data sync for org {org_id} completed successfully")
 

cronjob.txt

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
 
 # Every minute 
-*/30 * * * * PYTHONPATH=/app/backend/apiserver/ /opt/venv/bin/python3 /app/backend/apiserver/dora/sync_data.py >> /var/log/cron/cron.log 2>&1
+*/30 * * * * PYTHONPATH=/app/backend/apiserver/ /opt/venv/bin/python3 /app/backend/apiserver/dora/service/sync_data.py >> /var/log/cron/cron.log 2>&1
 

docker-compose.yml

@@ -55,8 +55,8 @@ services:
           target: /app/backend/apiserver/.env.local
 
         - action: sync+restart
-          path: ./apiserver/dora/sync_data.py
-          target: /app/backend/apiserver/dora/sync_data.py
+          path: ./apiserver/dora/service/sync_data.py
+          target: /app/backend/apiserver/dora/service/sync_data.py
 
         - action: sync
           path: ./web-server