diff --git a/apiserver/app.py b/apiserver/app.py index 3f6d9ce65..5c9ae4454 100644 --- a/apiserver/app.py +++ b/apiserver/app.py @@ -1,4 +1,5 @@ from flask import Flask + from dora.store import configure_db_with_app from env import load_app_env @@ -10,6 +11,7 @@ from dora.api.incidents import app as incidents_api from dora.api.integrations import app as integrations_api from dora.api.deployment_analytics import app as deployment_analytics_api +from dora.api.sync import app as sync_api from dora.store.initialise_db import initialize_database @@ -21,10 +23,10 @@ app.register_blueprint(incidents_api) app.register_blueprint(deployment_analytics_api) app.register_blueprint(integrations_api) +app.register_blueprint(sync_api) configure_db_with_app(app) initialize_database(app) - if __name__ == "__main__": app.run() diff --git a/apiserver/dora/api/sync.py b/apiserver/dora/api/sync.py new file mode 100644 index 000000000..ce75d2bd6 --- /dev/null +++ b/apiserver/dora/api/sync.py @@ -0,0 +1,12 @@ +from flask import Blueprint + +from dora.service.sync_data import trigger_data_sync +from dora.utils.time import time_now + +app = Blueprint("sync", __name__) + + +@app.route("/sync", methods=["POST"]) +def sync(): + trigger_data_sync() + return {"message": "sync started", "time": time_now().isoformat()} diff --git a/apiserver/dora/service/code/sync/etl_code_analytics.py b/apiserver/dora/service/code/sync/etl_code_analytics.py index 1e5477493..942d8b574 100644 --- a/apiserver/dora/service/code/sync/etl_code_analytics.py +++ b/apiserver/dora/service/code/sync/etl_code_analytics.py @@ -19,6 +19,9 @@ def create_pr_metrics( pr_events: List[PullRequestEvent], pr_commits: List[PullRequestCommit], ) -> PullRequest: + if pr.state == PullRequestState.OPEN: + return pr + pr_performance = self.get_pr_performance(pr, pr_events) pr.first_response_time = ( diff --git a/apiserver/dora/service/code/sync/etl_github_handler.py b/apiserver/dora/service/code/sync/etl_github_handler.py index a039fa108..7d9f88d51 100644 --- a/apiserver/dora/service/code/sync/etl_github_handler.py +++ b/apiserver/dora/service/code/sync/etl_github_handler.py @@ -149,7 +149,6 @@ def get_repo_pull_requests_data( pr_commits += pr_commit_models prs_added.add(github_pr.number) - sorted(pull_requests, key=lambda x: x.state_changed_at) return pull_requests, pr_commits, pr_events def process_pr( diff --git a/apiserver/dora/sync_data.py b/apiserver/dora/service/sync_data.py similarity index 75% rename from apiserver/dora/sync_data.py rename to apiserver/dora/service/sync_data.py index 47a748b1c..d5accbfa5 100644 --- a/apiserver/dora/sync_data.py +++ b/apiserver/dora/service/sync_data.py @@ -16,12 +16,16 @@ def trigger_data_sync(): default_org = get_query_validator().get_default_org() org_id = str(default_org.id) + LOG.info(f"Starting data sync for org {org_id}") for sync_func in sync_sequence: try: sync_func(org_id) + LOG.info(f"Data sync for {sync_func.__name__} completed successfully") except Exception as e: - LOG.error(f"Error syncing data for org {org_id}: {str(e)}") + LOG.error( + f"Error syncing {sync_func.__name__} data for org {org_id}: {str(e)}" + ) continue LOG.info(f"Data sync for org {org_id} completed successfully") diff --git a/cronjob.txt b/cronjob.txt index c8694c7c0..b9b553d11 100644 --- a/cronjob.txt +++ b/cronjob.txt @@ -1,5 +1,5 @@ #!/usr/bin/env python3 # Every minute -*/30 * * * * PYTHONPATH=/app/backend/apiserver/ /opt/venv/bin/python3 /app/backend/apiserver/dora/sync_data.py >> /var/log/cron/cron.log 2>&1 +*/30 * * * * PYTHONPATH=/app/backend/apiserver/ /opt/venv/bin/python3 /app/backend/apiserver/dora/service/sync_data.py >> /var/log/cron/cron.log 2>&1 diff --git a/docker-compose.yml b/docker-compose.yml index 0dcfc53fd..09d1a184d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -55,8 +55,8 @@ services: target: /app/backend/apiserver/.env.local - action: sync+restart - path: ./apiserver/dora/sync_data.py - target: /app/backend/apiserver/dora/sync_data.py + path: ./apiserver/dora/service/sync_data.py + target: /app/backend/apiserver/dora/service/sync_data.py - action: sync path: ./web-server