From a30a89cc6485e7edae0ba6badaaced435933a42e Mon Sep 17 00:00:00 2001 From: amoghjalan Date: Fri, 19 Apr 2024 16:01:29 +0530 Subject: [PATCH 1/2] Move sync_data to service layer --- apiserver/dora/{ => service}/sync_data.py | 6 +++++- cronjob.txt | 2 +- docker-compose.yml | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) rename apiserver/dora/{ => service}/sync_data.py (75%) diff --git a/apiserver/dora/sync_data.py b/apiserver/dora/service/sync_data.py similarity index 75% rename from apiserver/dora/sync_data.py rename to apiserver/dora/service/sync_data.py index 47a748b1c..0d263db9c 100644 --- a/apiserver/dora/sync_data.py +++ b/apiserver/dora/service/sync_data.py @@ -16,12 +16,16 @@ def trigger_data_sync(): default_org = get_query_validator().get_default_org() org_id = str(default_org.id) + LOG.info(f"Starting data sync for org {org_id}") for sync_func in sync_sequence: try: sync_func(org_id) + LOG.error(f"Data sync for {sync_func.__name__} completed successfully") except Exception as e: - LOG.error(f"Error syncing data for org {org_id}: {str(e)}") + LOG.error( + f"Error syncing {sync_func.__name__} data for org {org_id}: {str(e)}" + ) continue LOG.info(f"Data sync for org {org_id} completed successfully") diff --git a/cronjob.txt b/cronjob.txt index c8694c7c0..b9b553d11 100644 --- a/cronjob.txt +++ b/cronjob.txt @@ -1,5 +1,5 @@ #!/usr/bin/env python3 # Every minute -*/30 * * * * PYTHONPATH=/app/backend/apiserver/ /opt/venv/bin/python3 /app/backend/apiserver/dora/sync_data.py >> /var/log/cron/cron.log 2>&1 +*/30 * * * * PYTHONPATH=/app/backend/apiserver/ /opt/venv/bin/python3 /app/backend/apiserver/dora/service/sync_data.py >> /var/log/cron/cron.log 2>&1 diff --git a/docker-compose.yml b/docker-compose.yml index 0dcfc53fd..09d1a184d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -55,8 +55,8 @@ services: target: /app/backend/apiserver/.env.local - action: sync+restart - path: ./apiserver/dora/sync_data.py - target: /app/backend/apiserver/dora/sync_data.py + path: ./apiserver/dora/service/sync_data.py + target: /app/backend/apiserver/dora/service/sync_data.py - action: sync path: ./web-server From b113b58de2ae00c9c1192a5539a7ce7afb30ccc4 Mon Sep 17 00:00:00 2001 From: amoghjalan Date: Fri, 19 Apr 2024 16:07:25 +0530 Subject: [PATCH 2/2] Create sync data API --- apiserver/app.py | 4 +++- apiserver/dora/api/sync.py | 12 ++++++++++++ .../dora/service/code/sync/etl_code_analytics.py | 3 +++ .../dora/service/code/sync/etl_github_handler.py | 1 - apiserver/dora/service/sync_data.py | 2 +- 5 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 apiserver/dora/api/sync.py diff --git a/apiserver/app.py b/apiserver/app.py index 3f6d9ce65..5c9ae4454 100644 --- a/apiserver/app.py +++ b/apiserver/app.py @@ -1,4 +1,5 @@ from flask import Flask + from dora.store import configure_db_with_app from env import load_app_env @@ -10,6 +11,7 @@ from dora.api.incidents import app as incidents_api from dora.api.integrations import app as integrations_api from dora.api.deployment_analytics import app as deployment_analytics_api +from dora.api.sync import app as sync_api from dora.store.initialise_db import initialize_database @@ -21,10 +23,10 @@ app.register_blueprint(incidents_api) app.register_blueprint(deployment_analytics_api) app.register_blueprint(integrations_api) +app.register_blueprint(sync_api) configure_db_with_app(app) initialize_database(app) - if __name__ == "__main__": app.run() diff --git a/apiserver/dora/api/sync.py b/apiserver/dora/api/sync.py new file mode 100644 index 000000000..ce75d2bd6 --- /dev/null +++ b/apiserver/dora/api/sync.py @@ -0,0 +1,12 @@ +from flask import Blueprint + +from dora.service.sync_data import trigger_data_sync +from dora.utils.time import time_now + +app = Blueprint("sync", __name__) + + +@app.route("/sync", methods=["POST"]) +def sync(): + trigger_data_sync() + return {"message": "sync started", "time": time_now().isoformat()} diff --git a/apiserver/dora/service/code/sync/etl_code_analytics.py b/apiserver/dora/service/code/sync/etl_code_analytics.py index 1e5477493..942d8b574 100644 --- a/apiserver/dora/service/code/sync/etl_code_analytics.py +++ b/apiserver/dora/service/code/sync/etl_code_analytics.py @@ -19,6 +19,9 @@ def create_pr_metrics( pr_events: List[PullRequestEvent], pr_commits: List[PullRequestCommit], ) -> PullRequest: + if pr.state == PullRequestState.OPEN: + return pr + pr_performance = self.get_pr_performance(pr, pr_events) pr.first_response_time = ( diff --git a/apiserver/dora/service/code/sync/etl_github_handler.py b/apiserver/dora/service/code/sync/etl_github_handler.py index a039fa108..7d9f88d51 100644 --- a/apiserver/dora/service/code/sync/etl_github_handler.py +++ b/apiserver/dora/service/code/sync/etl_github_handler.py @@ -149,7 +149,6 @@ def get_repo_pull_requests_data( pr_commits += pr_commit_models prs_added.add(github_pr.number) - sorted(pull_requests, key=lambda x: x.state_changed_at) return pull_requests, pr_commits, pr_events def process_pr( diff --git a/apiserver/dora/service/sync_data.py b/apiserver/dora/service/sync_data.py index 0d263db9c..d5accbfa5 100644 --- a/apiserver/dora/service/sync_data.py +++ b/apiserver/dora/service/sync_data.py @@ -21,7 +21,7 @@ def trigger_data_sync(): for sync_func in sync_sequence: try: sync_func(org_id) - LOG.error(f"Data sync for {sync_func.__name__} completed successfully") + LOG.info(f"Data sync for {sync_func.__name__} completed successfully") except Exception as e: LOG.error( f"Error syncing {sync_func.__name__} data for org {org_id}: {str(e)}"